PLearn 0.1
|
00001 00002 // -*- C++ -*- 00003 00004 // TextFilesVMatrix.h 00005 // 00006 // Copyright (C) 2003-2004 ApSTAT Technologies Inc. 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 00033 /* ******************************************************* 00034 * $Id: TextFilesVMatrix.cc 10132 2009-04-20 19:36:56Z nouiz $ 00035 ******************************************************* */ 00036 00037 // Author: Pascal Vincent, Christian Hudon 00038 00040 #include "TextFilesVMatrix.h" 00041 #include <plearn/base/PDate.h> 00042 #include <plearn/base/ProgressBar.h> 00043 #include <plearn/base/stringutils.h> 00044 #include <plearn/io/load_and_save.h> 00045 #include <plearn/io/fileutils.h> 00046 #define PL_LOG_MODULE_NAME "TextFilesVMatrix" 00047 #include <plearn/io/pl_log.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 00053 TextFilesVMatrix::TextFilesVMatrix(): 00054 idxfile(0), 00055 delimiter("\t"), 00056 quote_delimiter(""), 00057 auto_build_map(false), 00058 auto_extend_map(true), 00059 build_vmatrix_stringmap(false), 00060 reorder_fieldspec_from_headers(false), 00061 partial_match(false) 00062 {} 00063 00064 PLEARN_IMPLEMENT_OBJECT( 00065 TextFilesVMatrix, 00066 "Parse and represent a text file as a VMatrix", 00067 "This VMatrix contains a plethora of options for parsing text files,\n" 00068 "interpreting the fields (including arbitrary string fields), and\n" 00069 "representing the result as a numerical VMatrix. It can be used to parse\n" 00070 "both SAS and CSV files.\n" 00071 "\n" 00072 "The metadatadir option should probably be specified.\n" 00073 "\n" 00074 "Internally, the metadata directory contains the following files:\n" 00075 " - a txtmat.idx binary index file (which will be automatically rebuilt if any of the raw text files is newer)\n" 00076 " - a txtmat.idx.log file reporting problems encountered while building the .idx file\n" 00077 "\n" 00078 "The txtmat.idx file is a binary file structured as follows\n" 00079 "- 1 byte indicating endianness: 'L' or 'B'\n" 00080 "- 4 byte int for length (number of data rows in the raw text file)\n" 00081 "- (unsigned char fileno, int pos) indicating in which raw text file and at what position each row starts\n" 00082 ); 00083 00084 00085 void TextFilesVMatrix::getFileAndPos(int i, unsigned char& fileno, int& pos) const 00086 { 00087 PLASSERT(idxfile!=0); 00088 if(i<0 || i>=length()) 00089 PLERROR("TextFilesVMatrix::getFileAndPos out of range row %d (only %d rows)", i, length()); 00090 fseek(idxfile, 5+i*5, SEEK_SET); 00091 fileno = fgetc(idxfile); 00092 fread(&pos, sizeof(int), 1, idxfile); 00093 } 00094 00095 int TextFilesVMatrix::getIndexOfTextField(const string& fieldname) const 00096 { 00097 int n = fieldspec.size(); 00098 for(int i=0; i<n; i++) 00099 if(fieldspec[i].first==fieldname) 00100 return i; 00101 PLERROR("In TextFilesVMatrix::getIndexOfTextField unknown field %s",fieldname.c_str()); 00102 return -1; // to make the compiler happy 00103 } 00104 00105 void TextFilesVMatrix::buildIdx() 00106 { 00107 perr << "Building the index file. Please be patient..." << endl; 00108 00109 if(idxfile) 00110 fclose(idxfile); 00111 PPath ft=getMetaDataDir()/"txtmat.idx.tmp"; 00112 PPath f=getMetaDataDir()/"txtmat.idx"; 00113 idxfile = fopen(ft.c_str(),"wb"); 00114 FILE* logfile = fopen((getMetaDataDir()/"txtmat.idx.log").c_str(),"a"); 00115 00116 if (! idxfile) 00117 PLERROR("TextFilesVMatrix::buildIdx: could not open index file '%s'", 00118 ( getMetaDataDir()/"txtmat.idx").c_str()); 00119 if (! logfile) 00120 PLERROR("TextFilesVMatrix::buildIdx: could not open log file '%s'", 00121 (getMetaDataDir()/"txtmat.idx.log").c_str()); 00122 00123 // write endianness 00124 fputc(byte_order(), idxfile); 00125 // We don't know length yet, 00126 length_ = 0; 00127 fwrite(&length_, 4, 1, idxfile); 00128 00129 TVec<string> fields; 00130 char buf[50000]; 00131 00132 int lineno = 0; 00133 for(unsigned char fileno=0; fileno<txtfiles.length(); fileno++) 00134 { 00135 FILE* fi = txtfiles[(int)fileno]; 00136 fseek(fi,0,SEEK_SET); 00137 00138 int nskip = 0; // number of header lines to skip 00139 if(!skipheader.isEmpty()) 00140 nskip = skipheader[int(fileno)]; 00141 00142 // read the data rows and build the index 00143 for(;;) 00144 { 00145 long pos_long = ftell(fi); 00146 if (pos_long > INT_MAX) 00147 PLERROR("In TextFilesVMatrix::buildIdx - 'pos_long' cannot be " 00148 "more than %d", INT_MAX); 00149 int pos = int(pos_long); 00150 if(!fgets(buf, sizeof(buf), fi)) 00151 break; 00152 00153 #ifdef CYGWIN_FGETS_BUGFIX 00154 // Bugfix for CYGWIN carriage return bug. 00155 // Should be safe to enable in all case, but need to be tester more widely. 00156 long new_pos = ftell(fi); 00157 long lbuf = long(strlen(buf)); 00158 if (lbuf+pos != new_pos){ 00159 if(lbuf+1+pos==new_pos && buf[lbuf-1]=='\n' && buf[lbuf-2]!='\r') 00160 { 00161 //bug under windows. fgets return the good string if unix end of lines, but 00162 //change the position suppossing the use of \r\n as carrige return. 00163 //So if their is only a \n, we are a caractere too far. 00164 //if dos end of lines, return \n as end of lines in the strings and put the pos correctly. 00165 00166 fseek(fi,-1,SEEK_CUR); 00167 00168 //if unix end of lines 00169 if(fgetc(fi)!='\n') 00170 fseek(fi,-1,SEEK_CUR); 00171 } 00172 //in the eof case? 00173 else if(lbuf-1+pos==new_pos && buf[lbuf-1]=='\n' && buf[lbuf-2]!='\r') 00174 fseek(fi,+1,SEEK_CUR); 00175 else 00176 PLERROR("In TextFilesVMatrix::buildId - The number of characters read " 00177 "does not match the position in the file."); 00178 } 00179 #endif 00180 00181 buf[sizeof(buf)-1] = '\0'; // ensure null-terminated 00182 lineno++; 00183 if(nskip>0) 00184 --nskip; 00185 else if(!isBlank(buf)) 00186 { 00187 fields = splitIntoFields(buf); 00188 int nf = fields.length(); 00189 if(nf!=fieldspec.size()){ 00190 fprintf(logfile, "ERROR In file %d line %d: Found %d fields (should be %d):\n %s",fileno,lineno,nf,fieldspec.size(),buf); 00191 PLWARNING("In file %d line %d: Found %d fields (should be %d):\n %s",fileno,lineno,nf,fieldspec.size(),buf); 00192 } 00193 else // Row OK! append it to index 00194 { 00195 fputc(fileno, idxfile); 00196 fwrite(&pos, 4, 1, idxfile); 00197 length_++; 00198 } 00199 } 00200 else 00201 PLWARNING("In TextFilesVMatrix::buildIdx() - The line %d is blank",lineno); 00202 } // end of loop over lines of file 00203 } // end of loop over files 00204 00205 // Write true length and width 00206 fseek(idxfile, 1, SEEK_SET); 00207 fwrite(&length_, 4, 1, idxfile); 00208 00209 // close files 00210 fclose(logfile); 00211 fclose(idxfile); 00212 mvforce(ft,f); 00213 perr << "Index file built." << endl; 00214 } 00215 00217 // isValidNonSkipFieldType // 00219 bool TextFilesVMatrix::isValidNonSkipFieldType(const string& ftype) const { 00220 return (ftype=="auto" || ftype=="num" || ftype=="date" || ftype=="jdate" || 00221 ftype=="postal" || ftype=="dollar" || ftype=="dollar-comma" || 00222 ftype=="YYYYMM" || ftype=="sas_date" || ftype == "bell_range" || 00223 ftype == "char" || ftype=="num-comma" || ftype=="auto-num"); 00224 } 00225 00226 void TextFilesVMatrix::setColumnNamesAndWidth() 00227 { 00228 width_ = 0; 00229 TVec<string> fnames; 00230 TVec<string> fnames_header;//field names take in the header of source file 00231 char buf[50000]; 00232 00233 00234 //select witch delimiter we will use for all the files. 00235 if(delimiter.size()>1){ 00236 FILE* f = txtfiles[0]; 00237 fseek(f,0,SEEK_SET); 00238 if(!fgets(buf, sizeof(buf), f)) 00239 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - " 00240 "Couldn't read the fields names from file '%s'", 00241 txtfilenames[0].c_str()); 00242 00243 string s1 = string(buf); 00244 if(!fgets(buf, sizeof(buf), f)) 00245 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - " 00246 "Couldn't read the fields names from file '%s'", 00247 txtfilenames[0].c_str()); 00248 string s2 = string(buf); 00249 TVec<int> nbs1(delimiter.size()); 00250 TVec<int> nbs2(delimiter.size()); 00251 00252 string old_delimiter = delimiter; 00253 for(uint i=0;i<old_delimiter.size();i++){ 00254 delimiter = old_delimiter[i]; 00255 TVec<string> fields1 = splitIntoFields(s1); 00256 TVec<string> fields2 = splitIntoFields(s2); 00257 nbs1[i]=fields1.size(); 00258 nbs2[i]=fields2.size(); 00259 } 00260 delimiter=old_delimiter; 00261 for(uint i=0;i<old_delimiter.size();i++){ 00262 if(nbs1[i]==nbs2[i]&& nbs1[i]>0){ 00263 delimiter = old_delimiter[i]; 00264 } 00265 } 00266 MODULE_LOG << "Selected delimiter: <" << delimiter << ">" << endl; 00267 if(delimiter.size()!=1){ 00268 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - We can't" 00269 " automatically determine the delimiter to use as the two" 00270 " first row don't have a common delimiter with the same" 00271 " number of occurence. nbs1=%s, nbs2=%s", 00272 tostring(nbs1).c_str(),tostring(nbs2).c_str()); 00273 } 00274 } 00275 PLCHECK(delimiter.size()==1); 00276 00277 //read the fieldnames from the files. 00278 for(int i=0; i<txtfiles.size(); i++){ 00279 FILE* f = txtfiles[i]; 00280 fseek(f,0,SEEK_SET); 00281 if(!fgets(buf, sizeof(buf), f)) 00282 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - " 00283 "Couldn't read the fields names from file '%s'", 00284 txtfilenames[i].c_str()); 00285 fseek(f,0,SEEK_SET); 00286 00287 TVec<string> fields = splitIntoFields(buf); 00288 00289 //check that we have the good delimiter 00290 if(fields.size()==1 && fieldspec.size()>1) 00291 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() -" 00292 " We found only 1 column in the first line, but" 00293 " their is %d fieldspec. Meaby the delimiter '%s'" 00294 " is not the right one. The line is %s", 00295 fieldspec.size(),delimiter.c_str(), 00296 string(buf).c_str()); 00297 00298 if(reorder_fieldspec_from_headers || partial_match){ 00299 fields.append(removeblanks(fields.pop())); 00300 00301 fnames_header.append(fields); 00302 } 00303 } 00304 if(partial_match) 00305 { 00306 TVec< pair<string, string> > new_fieldspec; 00307 TVec<string> no_expended_fields; 00308 PLCHECK_MSG(reorder_fieldspec_from_headers, 00309 "In TextFilesVMatrix::setColumnNamesAndWidth - " 00310 "when partial_match is true, reorder_fieldspec_from_headers" 00311 " must be true."); 00312 for(int i=0;i<fieldspec.size();i++) 00313 { 00314 bool expended = false; 00315 string fname=fieldspec[i].first; 00316 if(fname[fname.size()-1]!='*') 00317 { 00318 new_fieldspec.append(fieldspec[i]); 00319 continue; 00320 } 00321 fname.resize(fname.size()-1);//remove the last caracter (*) 00322 for(int j=0;j<fnames_header.size();j++) 00323 { 00324 if(string_begins_with(fnames_header[j],fname)) 00325 { 00326 pair<string,string> n=make_pair(fnames_header[j], 00327 fieldspec[i].second); 00328 // perr<<"expanding "<<fieldspec[i] << " to " << n <<endl; 00329 00330 new_fieldspec.append(n); 00331 expended = true; 00332 } 00333 } 00334 if(!expended) 00335 no_expended_fields.append(fieldspec[i].first); 00336 } 00337 if(no_expended_fields.length()>0){ 00338 NORMAL_LOG<<"In TextFilesVMatrix::setColumnNamesAndWidth - " 00339 <<"Did not find any partial match for " 00340 <<no_expended_fields.length()<<" spec:"; 00341 for(int i=0;i<no_expended_fields.length();i++) 00342 NORMAL_LOG<<" "<<no_expended_fields[i]; 00343 NORMAL_LOG<<endl; 00344 } 00345 00346 fieldspec = new_fieldspec; 00347 } 00348 00349 if(reorder_fieldspec_from_headers) 00350 { 00351 //check that all field names from the header have a spec 00352 TVec<string> not_used_fn; 00353 for(int i=0;i<fnames_header.size();i++) 00354 { 00355 string name=fnames_header[i]; 00356 int j=0; 00357 for(;j<fieldspec.size();j++) 00358 if(fieldspec[j].first==name) 00359 break; 00360 if(j>=fieldspec.size()){ 00361 if(default_spec!=""){ 00362 fieldspec.append(make_pair(name,default_spec)); 00363 }else 00364 not_used_fn.append(name); 00365 } 00366 } 00367 //check that all fieldspec names are also in the header 00368 TVec<string> not_used_fs; 00369 for(int i=0;i<fieldspec.size();i++) 00370 { 00371 string name=fieldspec[i].first; 00372 int j=0; 00373 for(;j<fnames_header.size();j++) 00374 if(fnames_header[j]==name) 00375 break; 00376 if(j>=fnames_header.size()) 00377 not_used_fs.append(name); 00378 } 00379 //check that we have the good number of fieldspec 00380 //if partial match is true, we don't want to generate the warning everytime 00381 if(fnames_header.size()!=fieldspec.size() && !partial_match) 00382 { 00383 PLWARNING("In TextFilesVMatrix::setColumnNamesAndWidth() - " 00384 "We read %d field names from the header but have %d" 00385 "fieldspec",fnames_header.size(),fieldspec.size()); 00386 } 00387 00388 if(not_used_fs.size()!=0) 00389 PLWARNING("TextFilesVMatrix::setColumnNamesAndWidth() - " 00390 "%d fieldspecs exists for field(s) that are not in the source: %s\n" 00391 "They will be skipped.", 00392 not_used_fs.length(), tostring(not_used_fs).c_str()); 00393 if(not_used_fn.size()!=0) 00394 PLWARNING("TextFilesVMatrix::setColumnNamesAndWidth() - " 00395 "%d fieldnames in source that don't have fieldspec: %s\n" 00396 "They will be skipped.", 00397 not_used_fn.length(), tostring(not_used_fn).c_str()); 00398 00399 00400 //the new order for fieldspecs 00401 TVec< pair<string, string> > fs(fnames_header.size()); 00402 for(int i=0;i<fnames_header.size();i++) 00403 { 00404 string name=fnames_header[i]; 00405 int j=0; 00406 for(;j<fieldspec.size();j++) 00407 if(fieldspec[j].first==name) 00408 break; 00409 if(j>=fieldspec.size()) 00410 fs[i]=pair<string,string>(name,"skip"); 00411 else 00412 fs[i]=fieldspec[j]; 00413 } 00414 fieldspec=fs; 00415 } 00416 for(int k=0; k<fieldspec.length(); k++) 00417 { 00418 string fname = fieldspec[k].first; 00419 string ftype = fieldspec[k].second; 00420 if(isValidNonSkipFieldType(ftype)) 00421 { 00422 // declare the column name 00423 fnames.push_back(fname); 00424 colrange.push_back( pair<int,int>(width_,1) ); 00425 ++width_; 00426 } 00427 else if(ftype=="skip") 00428 { 00429 colrange.push_back( pair<int,int>(width_,0) ); 00430 } 00431 else 00432 PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth, Invalid field type specification for field %s: %s",fname.c_str(), ftype.c_str()); 00433 } 00434 for(int j=0; j<width_; j++) 00435 declareField(j, fnames[j]); 00436 } 00437 00438 void TextFilesVMatrix::build_() 00439 { 00440 if (!default_spec.empty() && !reorder_fieldspec_from_headers) 00441 PLERROR("In TextFilesVMatrix::build_() when the option default_spec is used, reorder_fieldspec_from_headers must be true"); 00442 if (metadatapath != "") { 00443 PLWARNING("In TextFilesVMatrix::build_() metadatapath option is deprecated. " 00444 "You should use metadatadir instead.\n"); 00445 00446 metadatadir = metadatapath; 00447 setMetaDataDir(metadatapath); 00448 } 00449 } 00451 // setMetaDataDir // 00453 void TextFilesVMatrix::setMetaDataDir(const PPath& the_metadatadir){ 00454 inherited::setMetaDataDir(the_metadatadir); 00455 00456 if(getMetaDataDir().empty()) 00457 PLERROR("In TextFilesVMatrix::setMetaDataDir() - We need a metadatadir"); 00458 if(!force_mkdir(getMetaDataDir())) 00459 PLERROR("In TextFilesVMatrix::setMetaDataDir() - could not create" 00460 " directory '%s'", 00461 getMetaDataDir().absolute().c_str()); 00462 00463 for(int i=0;i<txtfilenames.length();i++) 00464 updateMtime(txtfilenames[i]); 00465 00466 PPath metadir = getMetaDataDir(); 00467 PPath idxfname = metadir/"txtmat.idx"; 00468 00469 // Now open txtfiles 00470 int nf = txtfilenames.length(); 00471 txtfiles.resize(nf); 00472 for(int k=0; k<nf; k++) 00473 { 00474 string fnam = txtfilenames[k]; 00475 txtfiles[k] = fopen(fnam.c_str(),"r"); 00476 if(txtfiles[k]==NULL){ 00477 perror("Can't open file"); 00478 PLERROR("In TextFilesVMatrix::setMetaDataDir - Can't open file %s", 00479 fnam.c_str()); 00480 } 00481 } 00482 00483 setColumnNamesAndWidth(); 00484 00485 // open the index file 00486 if(!isUpToDate(idxfname) || isemptyFile(idxfname)) 00487 buildIdx(); // (re)build it first! 00488 idxfile = fopen(idxfname.c_str(),"rb"); 00489 if(fgetc(idxfile) != byte_order()) 00490 PLERROR("In TextFilesVMatrix::setMetaDataDir - Wrong endianness." 00491 " Remove the index file %s for it to be automatically rebuilt", 00492 idxfname.c_str()); 00493 fread(&length_, 4, 1, idxfile); 00494 00495 // Initialize some sizes 00496 int n = fieldspec.size(); 00497 mapping.resize(n); 00498 mapfiles.resize(n); 00499 mapfiles.fill(0); 00500 00501 // Handle string mapping 00502 loadMappings(); 00503 00504 if (auto_build_map) 00505 autoBuildMappings(); 00506 00507 if(build_vmatrix_stringmap) 00508 buildVMatrixStringMapping(); 00509 00510 // Sanity checking 00511 } 00512 00513 00514 string TextFilesVMatrix::getTextRow(int i) const 00515 { 00516 unsigned char fileno; 00517 int pos; 00518 getFileAndPos(i, fileno, pos); 00519 FILE* f = txtfiles[(int)fileno]; 00520 fseek(f,pos,SEEK_SET); 00521 char buf[50000]; 00522 00523 if(!fgets(buf, sizeof(buf), f)) 00524 PLERROR("In TextFilesVMatrix::getTextRow - fgets for row %d returned NULL",i); 00525 return removenewline(buf); 00526 } 00527 00528 void TextFilesVMatrix::loadMappings() 00529 { 00530 int n = fieldspec.size(); 00531 for(int k=0; k<n; k++) 00532 { 00533 string fname = getMapFilePath(k); 00534 if (isfile(fname)) { 00535 updateMtime(fname); 00536 vector<string> all_lines = getNonBlankLines(loadFileAsString(fname)); 00537 for (size_t i = 0; i < all_lines.size(); i++) { 00538 string map_line = all_lines[i]; 00539 size_t start_of_string = map_line.find('"'); 00540 size_t end_of_string = map_line.rfind('"'); 00541 string strval = map_line.substr(start_of_string + 1, end_of_string - start_of_string - 1); 00542 string real_val_str = map_line.substr(end_of_string + 1); 00543 real real_val; 00544 if (!pl_isnumber(real_val_str, &real_val)) 00545 PLERROR("In TextFilesVMatrix::loadMappings - Found a mapping to something that is not a number (%s) in file %s at non-black line %ld", map_line.c_str(), fname.c_str(), long(i)); 00546 mapping[k][strval] = real_val; 00547 } 00548 } 00549 } 00550 } 00551 00553 // autoBuildMappings // 00555 void TextFilesVMatrix::autoBuildMappings() { 00556 // TODO We should somehow check the date of existing mappings to see if they need to be built. 00557 // For now we just create them if they do not exist yet. 00558 00559 // First make sure there is no existing mapping. 00560 int nb_already_exist = 0; 00561 int nb_type_no_mapping = 0; 00562 for (int i = 0; i < mapping.length(); i++) { 00563 if (!mapping[i].empty()) 00564 nb_already_exist++; 00565 else if(fieldspec[i].second!="char")//should add auto when it is char that are selected 00566 nb_type_no_mapping++; 00567 } 00568 if(nb_already_exist == 0){ 00569 // Mappings need to be built. 00570 // We do this by reading the whole data. 00571 Vec row(width()); 00572 bool auto_extend_map_backup = auto_extend_map; 00573 auto_extend_map = true; 00574 ProgressBar pb("Building mappings", length()); 00575 for (int i = 0; i < length(); i++) { 00576 getRow(i, row); 00577 pb.update(i + 1); 00578 } 00579 auto_extend_map = auto_extend_map_backup; 00580 }else if (nb_already_exist+nb_type_no_mapping < mapping.length()) { 00581 for (int i = 0; i < mapping.length(); i++) 00582 if(fieldspec[i].second=="char" && mapping[i].empty())//should add auto when it is char that are selected 00583 PLWARNING("In TextFilesVMatrix::autoBuildMappings - mapping already existing but not for field %d (%s)",i,fieldspec[i].first.c_str()); 00584 00585 PLWARNING("In TextFilesVMatrix::autoBuildMappings - The existing " 00586 "mapping is not complete! There are %d fields with build " 00587 "mapping and there are %d fields that do not need mapping " 00588 "in a total of %d fields. Erase the mapping directory in " 00589 "the metadatadir to have it regenerated next time!", 00590 nb_already_exist,nb_type_no_mapping,mapping.length()); 00591 }//else already build 00592 } 00593 00594 void TextFilesVMatrix::generateMapCounts() 00595 { 00596 int n = fieldspec.size(); 00597 TVec< hash_map<string, int> > counts(n); 00598 for(int k=0; k<n; k++) 00599 { 00600 if(!mapping[k].empty()) 00601 { 00602 hash_map<string, real>& mapping_k = mapping[k]; 00603 hash_map<string, int>& counts_k = counts[k]; 00604 hash_map<string, real>::const_iterator it = mapping_k.begin(); 00605 hash_map<string, real>::const_iterator itend = mapping_k.end(); 00606 while(it!=itend) 00607 { 00608 counts_k[ it->first ] = 0; 00609 ++it; 00610 } 00611 } 00612 } 00613 00614 int l = length(); 00615 ProgressBar pg("Generating counts of mappings",l); 00616 for(int i=0; i<l; i++) 00617 { 00618 TVec<string> fields = getTextFields(i); 00619 for(int k=0; k<fields.length(); k++) 00620 { 00621 if(mapping[k].find(fields[k])!=mapping[k].end()) 00622 ++counts[k][fields[k]]; 00623 } 00624 pg(i); 00625 } 00626 00627 // Save the counts 00628 for(int k=0; k<n; k++) 00629 { 00630 if(!counts[k].empty()) 00631 PLearn::save( getMetaDataDir() / "counts" / fieldspec[k].first+".count", counts[k] ); 00632 } 00633 00634 } 00635 00636 void TextFilesVMatrix::buildVMatrixStringMapping() 00637 { 00638 int n = fieldspec.size(); 00639 for(int k=0; k<n; k++) 00640 { 00641 if(mapping[k].size()>0) 00642 { 00643 // get the corresponding VMatrix column range and add the VMatrix mapping 00644 int colstart = colrange[k].first; 00645 int ncols = colrange[k].second; 00646 hash_map<string,real>::const_iterator it = mapping[k].begin(); 00647 hash_map<string,real>::const_iterator itend = mapping[k].end(); 00648 while(it!=itend) 00649 { 00650 for(int j=colstart; j<colstart+ncols; j++) 00651 addStringMapping(j, it->first, it->second); 00652 ++it; 00653 } 00654 } 00655 } 00656 } 00657 00658 real TextFilesVMatrix::getMapping(int fieldnum, const string& strval) const 00659 { 00660 hash_map<string, real>& m = mapping[fieldnum]; 00661 hash_map<string, real>::const_iterator found = m.find(strval); 00662 if(found!=m.end()) // found it! 00663 return found->second; 00664 00665 // strval not found 00666 if(!auto_extend_map) 00667 PLERROR("In TextFilesVMatrix::getMapping - No mapping found for field %d (%s) string-value \"%s\" ", fieldnum, fieldspec[fieldnum].first.c_str(), strval.c_str()); 00668 00669 // OK, let's extend the mapping... 00670 real val = real(-1000 - int(m.size())); 00671 m[strval] = val; 00672 00673 if(!mapfiles[fieldnum]) 00674 { 00675 string fname = getMapFilePath(fieldnum); 00676 force_mkdir_for_file(fname); 00677 mapfiles[fieldnum] = fopen(fname.c_str(),"a"); 00678 if(!mapfiles[fieldnum]) 00679 PLERROR("In TextFilesVMatrix::getMapping - Could not open map file %s\n for appending\n",fname.c_str()); 00680 } 00681 00682 fprintf(mapfiles[fieldnum],"\n\"%s\" %f", strval.c_str(), val); 00683 return val; 00684 } 00685 00686 TVec<string> TextFilesVMatrix::splitIntoFields(const string& raw_row) const 00687 { 00688 return split_quoted_delimiter(removeblanks(raw_row), delimiter,quote_delimiter); 00689 } 00690 00691 TVec<string> TextFilesVMatrix::getTextFields(int i) const 00692 { 00693 string rowi = getTextRow(i); 00694 TVec<string> fields = splitIntoFields(rowi); 00695 if(fields.size() != fieldspec.size()) 00696 PLERROR("In TextFilesVMatrix::getTextFields - In getting fields of row %d, wrong number of fields: %d (should be %d):\n%s\n",i,fields.size(),fieldspec.size(),rowi.c_str()); 00697 for(int k=0; k<fields.size(); k++) 00698 fields[k] = removeblanks(fields[k]); 00699 return fields; 00700 } 00701 00702 real TextFilesVMatrix::getPostalEncoding(const string& strval, bool display_warning) const 00703 { 00704 if(strval=="") 00705 return MISSING_VALUE; 00706 00707 char first_char = strval[0]; 00708 int second_digit = strval[1]; 00709 real val = 0; 00710 if(first_char=='A') 00711 val = 30 + second_digit; 00712 else if(first_char=='B') 00713 val = 40 + second_digit; 00714 else if(first_char=='C') 00715 val = 50 + second_digit; 00716 else if(first_char=='E') 00717 val = 60 + second_digit; 00718 else if(first_char=='G') 00719 val = 0 + second_digit; 00720 else if(first_char=='H') 00721 val = 10 + second_digit; 00722 else if(first_char=='J') 00723 val = 20 + second_digit; 00724 else if(first_char=='K') 00725 val = 70 + second_digit; 00726 else if(first_char=='L') 00727 val = 80 + second_digit; 00728 else if(first_char=='M') 00729 val = 90 + second_digit; 00730 else if(first_char=='N') 00731 val = 100 + second_digit; 00732 else if(first_char=='P') 00733 val = 110 + second_digit; 00734 else if(first_char=='R') 00735 val = 120 + second_digit; 00736 else if(first_char=='S') 00737 val = 130 + second_digit; 00738 else if(first_char=='T') 00739 val = 140 + second_digit; 00740 else if(first_char=='V') 00741 val = 150 + second_digit; 00742 else if(first_char=='W') 00743 val = 160 + second_digit; 00744 else if(first_char=='X') 00745 val = 170 + second_digit; 00746 else if(first_char=='Y') 00747 val = 180 + second_digit; 00748 else if(first_char=='0' || first_char=='1' || first_char=='2' || first_char=='3' || 00749 first_char=='4' || first_char=='5' || first_char=='6' || first_char=='7' || 00750 first_char=='8' || first_char=='9') { 00751 // That would be a C.P. 00752 int first_digit = strval[0]; 00753 val = 260 + first_digit * 10 + second_digit; 00754 } 00755 else { 00756 //http://en.wikipedia.org/wiki/Canadian_postal_code 00757 //No postal code includes the letters D, F, I, O, Q, or U, 00758 //as the OCR equipment used in automated sorting could easily 00759 //confuse them with other letters and digits, 00760 if (display_warning) { 00761 string errmsg; 00762 if(first_char=='D' ||first_char=='F' ||first_char=='I' || 00763 first_char=='O' ||first_char=='Q' ||first_char=='U') 00764 errmsg = "Postal code don't use letters D, F, I, O, Q, or U: "; 00765 else 00766 errmsg = "Currently only some postal codes are supported: "; 00767 00768 errmsg += "can't process " + strval + ", value will be set to 0."; 00769 PLWARNING(errmsg.c_str()); 00770 } 00771 val = 0; 00772 } 00773 00774 return val; 00775 } 00776 00777 void TextFilesVMatrix::transformStringToValue(int k, string strval, Vec dest) const 00778 { 00779 strval = removeblanks(strval); 00780 string fieldname = fieldspec[k].first; 00781 string fieldtype = fieldspec[k].second; 00782 real val; 00783 00784 if(dest.length() != colrange[k].second) 00785 PLERROR("In TextFilesVMatrix::transformStringToValue, destination vec for field %d should be of length %d, not %d",k,colrange[k].second, dest.length()); 00786 00787 00788 if(fieldtype=="skip") 00789 { 00790 // do nothing, simply skip it 00791 return; 00792 } 00793 00794 if(strval=="") // missing 00795 dest[0] = MISSING_VALUE; 00796 else if(fieldtype=="auto") 00797 { 00798 if(pl_isnumber(strval,&val)) 00799 dest[0] = real(val); 00800 else 00801 dest[0] = getMapping(k, strval); 00802 } 00803 else if(fieldtype=="auto-num") 00804 {//We suppose the decimal point is '.' 00805 string s=strval; 00806 if(strval[0]=='$') 00807 s.erase(0,1); 00808 if(strval[strval.size()-1]=='$') 00809 s.erase(s.end()); 00810 00811 for(unsigned int pos=0; pos<strval.size(); pos++) 00812 if(s[pos]==',') 00813 s.erase(pos--,1); 00814 if(pl_isnumber(s,&val)) 00815 dest[0] = real(val); 00816 else 00817 PLERROR("In TextFilesVMatrix::transformStringToValue -" 00818 " expedted [$]number[$] as the value for field %d(%s)." 00819 " Got %s", k, fieldname.c_str(), strval.c_str()); 00820 } 00821 else if(fieldtype=="char") 00822 { 00823 dest[0] = getMapping(k, strval); 00824 } 00825 else if(fieldtype=="num") 00826 { 00827 if(pl_isnumber(strval,&val)) 00828 dest[0] = real(val); 00829 else 00830 PLERROR("In TextFilesVMatrix::transformStringToValue - expedted a number as the value for field %d(%s). Got %s",k,fieldname.c_str(),strval.c_str()); 00831 00832 } 00833 else if(fieldtype=="date") 00834 { 00835 dest[0] = date_to_float(PDate(strval)); 00836 } 00837 else if(fieldtype=="jdate") 00838 { 00839 dest[0] = PDate(strval).toJulianDay(); 00840 } 00841 else if(fieldtype=="sas_date") 00842 { 00843 if(strval == "0") // missing 00844 dest[0] = MISSING_VALUE; 00845 else if(pl_isnumber(strval,&val)) { 00846 dest[0] = val; 00847 if (val <= 0) { 00848 PLERROR("In TextFilesVMatrix::transformStringToValue - " 00849 "I didn't know a sas_date could be negative"); 00850 } 00851 } 00852 else 00853 PLERROR("In TextFilesVMatrix::transformStringToValue - " 00854 "Error while parsing a sas_date"); 00855 } 00856 else if(fieldtype=="YYYYMM") 00857 { 00858 if(!pl_isnumber(strval) || toint(strval)<197000) 00859 dest[0] = MISSING_VALUE; 00860 else 00861 dest[0] = PDate(strval+"01").toJulianDay(); 00862 } 00863 else if(fieldtype=="postal") 00864 { 00865 dest[0] = getPostalEncoding(strval); 00866 } 00867 else if(fieldtype=="dollar" || fieldtype=="dollar-comma") 00868 { 00869 char char_torm = ' '; 00870 if(fieldtype=="dollar-comma") 00871 char_torm = ','; 00872 if(strval[0]=='$') 00873 { 00874 string s = ""; 00875 for(unsigned int pos=1; pos<strval.size(); pos++) 00876 if(strval[pos]!=char_torm) 00877 s += strval[pos]; 00878 00879 if(pl_isnumber(s,&val)) 00880 dest[0] = real(val); 00881 else 00882 PLERROR("In TextFilesVMatrix::transformStringToValue - Goat as value '%s' while parsing field %d (%s) with fieldtype %s",strval.c_str(),k,fieldname.c_str(),fieldtype.c_str()); 00883 } 00884 else 00885 PLERROR("In TextFilesVMatrix::transformStringToValue - Got as value '%s' while expecting a value beggining with '$' while parsing field %d (%s) with fieldtype %s",strval.c_str(),k,fieldname.c_str(),fieldtype.c_str()); 00886 } 00887 else if(fieldtype=="bell_range") { 00888 if (strval == "Negative Value") { 00889 // We put an arbitrary negative value since we don't have more info. 00890 dest[0] = -100; 00891 } else { 00892 // A range of the kind "A: $0- 250". 00893 string s = ""; 00894 unsigned int pos; 00895 unsigned int end; 00896 for (pos=0; pos<strval.size() && (strval[pos] == ' ' || !pl_isnumber(strval.substr(pos,1))); pos++) {} 00897 for (end=pos; end<strval.size() && strval[end] != ' ' && pl_isnumber(strval.substr(end,1)); end++) { 00898 s += strval[end]; 00899 } 00900 real number_1,number_2; 00901 if (!pl_isnumber(s,&number_1) || is_missing(number_1)) { 00902 PLERROR(("TextFilesVMatrix::transformStringToValue: " + strval + 00903 " is not a well formatted Bell range").c_str()); 00904 } 00905 s = ""; 00906 for (pos=end; pos<strval.size() && (strval[pos] == ' ' || !pl_isnumber(strval.substr(pos,1))); pos++) {} 00907 for (end=pos; end<strval.size() && strval[end] != ' ' && pl_isnumber(strval.substr(end,1)); end++) { 00908 s += strval[end]; 00909 } 00910 if (!pl_isnumber(s,&number_2) || is_missing(number_2)) { 00911 PLERROR(("TextFilesVMatrix::transformStringToValue: " + strval + 00912 " is not a well formatted Bell range").c_str()); 00913 } 00914 dest[0] = (number_1 + number_2) / (real) 2; 00915 } 00916 } 00917 else if(fieldtype=="num-comma") 00918 { 00919 string s=""; 00920 for(uint i=0;i<strval.length();i++) 00921 { 00922 if(strval[i]!=',') 00923 s=s+strval[i]; 00924 } 00925 if(pl_isnumber(s,&val)) 00926 dest[0] = real(val); 00927 else 00928 PLERROR("In TextFilesVMatrix::transformStringToValue - expedted a number as the value for field %d(%s). Got %s",k,fieldname.c_str(),strval.c_str()); 00929 00930 } 00931 else 00932 { 00933 PLERROR("TextFilesVMatrix::TextFilesVMatrix::transformStringToValue, Invalid field type specification for field %s: %s",fieldname.c_str(), fieldtype.c_str()); 00934 } 00935 } 00936 00937 void TextFilesVMatrix::getNewRow(int i, const Vec& v) const 00938 { 00939 TVec<string> fields = getTextFields(i); 00940 int n = fields.size(); 00941 00942 for(int k=0; k<n; k++) 00943 { 00944 string fieldname = fieldspec[k].first; 00945 string fieldtype = fieldspec[k].second; 00946 string strval = fields[k]; 00947 Vec dest = v.subVec(colrange[k].first, colrange[k].second); 00948 00949 try 00950 { transformStringToValue(k, strval, dest); } 00951 catch(const PLearnError& e) 00952 { 00953 PLERROR("In TextFilesVMatrix, while parsing field %d (%s) of row %d: \n%s", 00954 k,fieldname.c_str(),i,e.message().c_str()); 00955 } 00956 } 00957 } 00958 00959 void TextFilesVMatrix::declareOptions(OptionList& ol) 00960 { 00961 declareOption(ol, "metadatapath", &TextFilesVMatrix::metadatapath, OptionBase::buildoption, 00962 "Path of the metadata directory (in which to store the index, ...)\n" 00963 "DEPRECATED: use metadatadir instead.\n"); 00964 00965 declareOption(ol, "txtfilenames", &TextFilesVMatrix::txtfilenames, OptionBase::buildoption, 00966 "A list of paths to raw text files containing the records"); 00967 00968 declareOption(ol, "delimiter", &TextFilesVMatrix::delimiter, OptionBase::buildoption, 00969 "Delimiter to use to split the fields. Common delimiters are:\n" 00970 "- \"\\t\" : used for SAS files (the default)\n" 00971 "- \",\" : used for CSV files\n" 00972 "- \";\" : used for a variant of CSV files\n" 00973 "If more then 1 delimiter, we will select one based on the" 00974 " first two line\n"); 00975 00976 declareOption(ol, "quote_delimiter", &TextFilesVMatrix::quote_delimiter, OptionBase::buildoption, 00977 "The escape character to indicate the delimiter is not considered.\n" 00978 "For instance, '\"' is frequently used."); 00979 00980 declareOption(ol, "skipheader", &TextFilesVMatrix::skipheader, OptionBase::buildoption, 00981 "An (optional) list of integers, one for each of the txtfilenames,\n" 00982 "indicating the number of header lines at the top of the file to be skipped."); 00983 00984 declareOption(ol, "fieldspec", &TextFilesVMatrix::fieldspec, OptionBase::buildoption, 00985 "Specification of field names and types (type indicates how the text field is to be mapped to one or more reals)\n" 00986 "Currently supported types: \n" 00987 "- skip : Ignore the content of the field, won't be inserted in the resulting VMat\n" 00988 "- auto : If a numeric value, keep it as is, if not, look it up in the mapping (possibly inserting a new mapping if it's not there) \n" 00989 "- auto-num : take any float value with the decimal separator as the dot. If there is a $\n" 00990 " at the start or end it is removed. If there are commas they are removed.\n" 00991 "- num : numeric value, keep as is\n" 00992 "- num-comma : numeric value where thousands are separeted by comma\n" 00993 "- char : look it up in the mapping (possibly inserting a new mapping if it's not there)\n" 00994 "- date : date of the form 25DEC2003 or 25-dec-2003 or 2003/12/25 or 20031225, will be mapped to float date format 1031225\n" 00995 "- jdate : date of the form 25DEC2003 or 25-dec-2003 or 2003/12/25 or 20031225, will be mapped to *julian* date format\n" 00996 "- sas_date : date used by SAS = number of days since Jan. 1st, 1960 (with 0 = missing)\n" 00997 "- YYYYMM : date of the form YYYYMM (e.g: 200312), will be mapped to the julian date of the corresponding month. Everthing "\ 00998 " other than a number or lower than 197000 is considered as nan\n" 00999 "- postal : canadian postal code \n" 01000 "- dollar : strangely formatted field with dollar amount. Format is sth like '$12 003'\n" 01001 "- dollar-comma : strangely formatted field with dollar amount. Format is sth like '$12,003'\n" 01002 "- bell_range : a range like \"A: $0- 250\", replaced by the average of the two bounds;\n" 01003 " if the \"Negative Value\" string is found, it is replaced by -100\n" 01004 ); 01005 01006 declareOption(ol, "auto_extend_map", &TextFilesVMatrix::auto_extend_map, OptionBase::buildoption, 01007 "If true, new strings for fields of type AUTO will automatically appended to the mapping (in the metadata/mappings/fieldname.map file)"); 01008 01009 declareOption(ol, "auto_build_map", &TextFilesVMatrix::auto_build_map, OptionBase::buildoption, 01010 "If true, all mappings will be automatically computed at build time if they do not exist yet\n"); 01011 01012 declareOption(ol, "build_vmatrix_stringmap", &TextFilesVMatrix::build_vmatrix_stringmap, 01013 OptionBase::buildoption, 01014 "If true, standard vmatrix stringmap will be built from the txtmat specific stringmap"); 01015 01016 declareOption(ol, "reorder_fieldspec_from_headers", 01017 &TextFilesVMatrix::reorder_fieldspec_from_headers, 01018 OptionBase::buildoption, 01019 "If true, will reorder the fieldspec in the order given " 01020 "by the field names taken from txtfilenames."); 01021 01022 declareOption(ol, "partial_match", 01023 &TextFilesVMatrix::partial_match, 01024 OptionBase::buildoption, 01025 "If true, will repeatedly expand all fieldspec name ending " 01026 "with * to the full name from header." 01027 "The expansion is equivalent to the regex 'field_spec_name*'." 01028 "The option reorder_fieldspec_from_headers must be true"); 01029 01030 declareOption(ol, "default_spec", 01031 &TextFilesVMatrix::default_spec, 01032 OptionBase::buildoption, 01033 "If there is no fieldspec for a fieldname, we will use this" 01034 "value. reorder_fieldspec_from_headers must be true."); 01035 01036 // Now call the parent class' declareOptions 01037 inherited::declareOptions(ol); 01038 } 01039 01040 void TextFilesVMatrix::readAndCheckOptionName(PStream& in, const string& optionname, char buf[]) 01041 { 01042 in.skipBlanksAndComments(); 01043 in.readUntil(buf, sizeof(buf), "= "); 01044 string option = removeblanks(buf); 01045 in.skipBlanksAndComments(); 01046 char eq = in.get(); 01047 if(option!=optionname || eq!='=') 01048 PLERROR("In TextFilesVMatrix::readAndCheckOptionName - " 01049 "Bad syntax in .txtmat file.\n" 01050 "Expected option %s = ...\n" 01051 "Read %s %c\n", optionname.c_str(), option.c_str(), eq); 01052 } 01053 01054 01055 // ### Nothing to add here, simply calls build_ 01056 void TextFilesVMatrix::build() 01057 { 01058 inherited::build(); 01059 build_(); 01060 } 01061 01062 TextFilesVMatrix::~TextFilesVMatrix() 01063 { 01064 if(idxfile) 01065 fclose(idxfile); 01066 for(int k=0; k<txtfiles.length(); k++) 01067 fclose(txtfiles[k]); 01068 01069 for(int k=0; k<mapfiles.size(); k++) 01070 { 01071 if(mapfiles[k]) 01072 fclose(mapfiles[k]); 01073 } 01074 } 01075 01076 void TextFilesVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 01077 { 01078 inherited::makeDeepCopyFromShallowCopy(copies); 01079 idxfile=0; 01080 txtfiles.resize(0); 01081 //the map should be already build. 01082 auto_build_map=false; 01083 build(); 01084 } 01085 01086 } // end of namespace PLearn 01087 01088 01089 /* 01090 Local Variables: 01091 mode:c++ 01092 c-basic-offset:4 01093 c-file-style:"stroustrup" 01094 c-file-offsets:((innamespace . 0)(inline-open . 0)) 01095 indent-tabs-mode:nil 01096 fill-column:79 01097 End: 01098 */ 01099 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :