PLearn 0.1
TextFilesVMatrix.cc
Go to the documentation of this file.
00001 
00002 // -*- C++ -*-
00003 
00004 // TextFilesVMatrix.h
00005 //
00006 // Copyright (C) 2003-2004 ApSTAT Technologies Inc.
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 //
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 //
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 //
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 //
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 
00033 /* *******************************************************
00034  * $Id: TextFilesVMatrix.cc 10132 2009-04-20 19:36:56Z nouiz $
00035  ******************************************************* */
00036 
00037 // Author: Pascal Vincent, Christian Hudon
00038 
00040 #include "TextFilesVMatrix.h"
00041 #include <plearn/base/PDate.h>
00042 #include <plearn/base/ProgressBar.h>
00043 #include <plearn/base/stringutils.h>
00044 #include <plearn/io/load_and_save.h>
00045 #include <plearn/io/fileutils.h>
00046 #define PL_LOG_MODULE_NAME "TextFilesVMatrix"
00047 #include <plearn/io/pl_log.h>
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 
00053 TextFilesVMatrix::TextFilesVMatrix():
00054     idxfile(0),
00055     delimiter("\t"),
00056     quote_delimiter(""),
00057     auto_build_map(false),
00058     auto_extend_map(true),
00059     build_vmatrix_stringmap(false),
00060     reorder_fieldspec_from_headers(false),
00061     partial_match(false)
00062 {}
00063 
00064 PLEARN_IMPLEMENT_OBJECT(
00065     TextFilesVMatrix,
00066     "Parse and represent a text file as a VMatrix",
00067     "This VMatrix contains a plethora of options for parsing text files,\n"
00068     "interpreting the fields (including arbitrary string fields), and\n"
00069     "representing the result as a numerical VMatrix.  It can be used to parse\n"
00070     "both SAS and CSV files.\n"
00071     "\n"
00072     "The metadatadir option should probably be specified.\n"
00073     "\n"
00074     "Internally, the metadata directory contains the following files:\n"
00075     " - a txtmat.idx binary index file (which will be automatically rebuilt if any of the raw text files is newer)\n"
00076     " - a txtmat.idx.log file reporting problems encountered while building the .idx file\n"
00077     "\n"
00078     "The txtmat.idx file is a binary file structured as follows\n"
00079     "- 1 byte indicating endianness: 'L' or 'B'\n"
00080     "- 4 byte int for length (number of data rows in the raw text file)\n"
00081     "- (unsigned char fileno, int pos) indicating in which raw text file and at what position each row starts\n"
00082     );
00083 
00084 
00085 void TextFilesVMatrix::getFileAndPos(int i, unsigned char& fileno, int& pos) const
00086 {
00087     PLASSERT(idxfile!=0);
00088     if(i<0 || i>=length())
00089         PLERROR("TextFilesVMatrix::getFileAndPos out of range row %d (only %d rows)", i, length());
00090     fseek(idxfile, 5+i*5, SEEK_SET);
00091     fileno = fgetc(idxfile);
00092     fread(&pos, sizeof(int), 1, idxfile);
00093 }
00094 
00095 int TextFilesVMatrix::getIndexOfTextField(const string& fieldname) const
00096 {
00097     int n = fieldspec.size();
00098     for(int i=0; i<n; i++)
00099         if(fieldspec[i].first==fieldname)
00100             return i;
00101     PLERROR("In TextFilesVMatrix::getIndexOfTextField unknown field %s",fieldname.c_str());
00102     return -1; // to make the compiler happy
00103 }
00104 
00105 void TextFilesVMatrix::buildIdx()
00106 {
00107     perr << "Building the index file. Please be patient..." << endl;
00108 
00109     if(idxfile)
00110         fclose(idxfile);
00111     PPath ft=getMetaDataDir()/"txtmat.idx.tmp";
00112     PPath f=getMetaDataDir()/"txtmat.idx";
00113     idxfile = fopen(ft.c_str(),"wb");
00114     FILE* logfile = fopen((getMetaDataDir()/"txtmat.idx.log").c_str(),"a");
00115 
00116     if (! idxfile)
00117         PLERROR("TextFilesVMatrix::buildIdx: could not open index file '%s'",
00118                 ( getMetaDataDir()/"txtmat.idx").c_str());
00119     if (! logfile)
00120         PLERROR("TextFilesVMatrix::buildIdx: could not open log file '%s'",
00121                 (getMetaDataDir()/"txtmat.idx.log").c_str());
00122 
00123     // write endianness
00124     fputc(byte_order(), idxfile);
00125     // We don't know length yet,
00126     length_ = 0;
00127     fwrite(&length_, 4, 1, idxfile);
00128 
00129     TVec<string> fields;
00130     char buf[50000];
00131 
00132     int lineno = 0;
00133     for(unsigned char fileno=0; fileno<txtfiles.length(); fileno++)
00134     {
00135         FILE* fi = txtfiles[(int)fileno];
00136         fseek(fi,0,SEEK_SET);
00137 
00138         int nskip = 0; // number of header lines to skip
00139         if(!skipheader.isEmpty())
00140             nskip = skipheader[int(fileno)];
00141 
00142         // read the data rows and build the index
00143         for(;;)
00144         {
00145             long pos_long = ftell(fi);
00146             if (pos_long > INT_MAX)
00147                 PLERROR("In TextFilesVMatrix::buildIdx - 'pos_long' cannot be "
00148                         "more than %d", INT_MAX);
00149             int pos = int(pos_long);
00150             if(!fgets(buf, sizeof(buf), fi))
00151                 break;
00152 
00153 #ifdef CYGWIN_FGETS_BUGFIX
00154             // Bugfix for CYGWIN carriage return bug.
00155             // Should be safe to enable in all case, but need to be tester more widely.
00156             long new_pos = ftell(fi);
00157             long lbuf = long(strlen(buf));
00158             if (lbuf+pos != new_pos){
00159                 if(lbuf+1+pos==new_pos && buf[lbuf-1]=='\n' && buf[lbuf-2]!='\r')
00160                 {
00161                     //bug under windows. fgets return the good string if unix end of lines, but
00162                     //change the position suppossing the use of \r\n as carrige return.
00163                     //So if their is only a \n, we are a caractere too far.
00164                     //if dos end of lines, return \n as end of lines in the strings and put the pos correctly.
00165                     
00166                     fseek(fi,-1,SEEK_CUR);
00167                     
00168                     //if unix end of lines
00169                     if(fgetc(fi)!='\n')
00170                         fseek(fi,-1,SEEK_CUR);
00171                 }
00172                 //in the eof case?
00173                 else if(lbuf-1+pos==new_pos && buf[lbuf-1]=='\n' && buf[lbuf-2]!='\r')
00174                     fseek(fi,+1,SEEK_CUR);
00175                 else
00176                     PLERROR("In TextFilesVMatrix::buildId - The number of characters read "
00177                             "does not match the position in the file.");
00178             }
00179 #endif
00180 
00181             buf[sizeof(buf)-1] = '\0';         // ensure null-terminated
00182             lineno++;
00183             if(nskip>0)
00184                 --nskip;
00185             else if(!isBlank(buf))
00186             {
00187                 fields = splitIntoFields(buf);
00188                 int nf = fields.length();
00189                 if(nf!=fieldspec.size()){
00190                     fprintf(logfile, "ERROR In file %d line %d: Found %d fields (should be %d):\n %s",fileno,lineno,nf,fieldspec.size(),buf);
00191                     PLWARNING("In file %d line %d: Found %d fields (should be %d):\n %s",fileno,lineno,nf,fieldspec.size(),buf);
00192                 }
00193                 else  // Row OK! append it to index
00194                 {
00195                     fputc(fileno, idxfile);
00196                     fwrite(&pos, 4, 1, idxfile);
00197                     length_++;
00198                 }
00199             }
00200             else
00201                 PLWARNING("In TextFilesVMatrix::buildIdx() - The line %d is blank",lineno);
00202         } // end of loop over lines of file
00203     } // end of loop over files
00204 
00205     // Write true length and width
00206     fseek(idxfile, 1, SEEK_SET);
00207     fwrite(&length_, 4, 1, idxfile);
00208 
00209     // close files
00210     fclose(logfile);
00211     fclose(idxfile);
00212     mvforce(ft,f);
00213     perr << "Index file built." << endl;
00214 }
00215 
00217 // isValidNonSkipFieldType //
00219 bool TextFilesVMatrix::isValidNonSkipFieldType(const string& ftype) const {
00220     return (ftype=="auto" || ftype=="num" || ftype=="date" || ftype=="jdate" ||
00221             ftype=="postal" || ftype=="dollar" || ftype=="dollar-comma" ||
00222             ftype=="YYYYMM" || ftype=="sas_date" || ftype == "bell_range" ||
00223             ftype == "char" || ftype=="num-comma" || ftype=="auto-num");
00224 }
00225 
00226 void TextFilesVMatrix::setColumnNamesAndWidth()
00227 {
00228     width_ = 0;
00229     TVec<string> fnames;
00230     TVec<string> fnames_header;//field names take in the header of source file
00231     char buf[50000];
00232 
00233 
00234     //select witch delimiter we will use for all the files.
00235     if(delimiter.size()>1){
00236         FILE* f = txtfiles[0];
00237         fseek(f,0,SEEK_SET);
00238         if(!fgets(buf, sizeof(buf), f))
00239             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - "
00240                     "Couldn't read the fields names from file '%s'",
00241                     txtfilenames[0].c_str());
00242 
00243         string s1 = string(buf);
00244         if(!fgets(buf, sizeof(buf), f))
00245             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - "
00246                     "Couldn't read the fields names from file '%s'",
00247                     txtfilenames[0].c_str());
00248         string s2 = string(buf);
00249         TVec<int> nbs1(delimiter.size());
00250         TVec<int> nbs2(delimiter.size());
00251         
00252         string old_delimiter = delimiter;
00253         for(uint i=0;i<old_delimiter.size();i++){
00254             delimiter = old_delimiter[i];
00255             TVec<string> fields1 = splitIntoFields(s1);
00256             TVec<string> fields2 = splitIntoFields(s2);
00257             nbs1[i]=fields1.size();
00258             nbs2[i]=fields2.size();
00259         }
00260         delimiter=old_delimiter;
00261         for(uint i=0;i<old_delimiter.size();i++){
00262             if(nbs1[i]==nbs2[i]&& nbs1[i]>0){
00263                 delimiter = old_delimiter[i];
00264             }
00265         }
00266         MODULE_LOG << "Selected delimiter: <" << delimiter << ">" << endl;
00267         if(delimiter.size()!=1){
00268             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - We can't"
00269                     " automatically determine the delimiter to use as the two"
00270                     " first row don't have a common delimiter with the same"
00271                     " number of occurence. nbs1=%s, nbs2=%s",
00272                     tostring(nbs1).c_str(),tostring(nbs2).c_str());
00273         }
00274     }
00275     PLCHECK(delimiter.size()==1);
00276 
00277     //read the fieldnames from the files.
00278     for(int i=0; i<txtfiles.size(); i++){
00279         FILE* f = txtfiles[i];
00280         fseek(f,0,SEEK_SET);
00281         if(!fgets(buf, sizeof(buf), f))
00282             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() - "
00283                     "Couldn't read the fields names from file '%s'",
00284                     txtfilenames[i].c_str());
00285         fseek(f,0,SEEK_SET);
00286 
00287         TVec<string> fields = splitIntoFields(buf);
00288 
00289         //check that we have the good delimiter
00290         if(fields.size()==1 && fieldspec.size()>1)
00291             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth() -"
00292                     " We found only 1 column in the first line, but"
00293                     " their is %d fieldspec. Meaby the delimiter '%s'"
00294                     " is not the right one. The line is %s",
00295                     fieldspec.size(),delimiter.c_str(),
00296                     string(buf).c_str());
00297         
00298         if(reorder_fieldspec_from_headers || partial_match){
00299             fields.append(removeblanks(fields.pop()));
00300             
00301             fnames_header.append(fields);
00302         }
00303     }
00304     if(partial_match)
00305     {
00306         TVec< pair<string, string> > new_fieldspec;
00307         TVec<string> no_expended_fields;
00308         PLCHECK_MSG(reorder_fieldspec_from_headers,
00309                     "In TextFilesVMatrix::setColumnNamesAndWidth - "
00310                     "when partial_match is true, reorder_fieldspec_from_headers"
00311                     " must be true.");
00312         for(int i=0;i<fieldspec.size();i++)
00313         {
00314             bool expended = false;
00315             string fname=fieldspec[i].first;
00316             if(fname[fname.size()-1]!='*')
00317             {
00318                 new_fieldspec.append(fieldspec[i]);
00319                 continue;
00320             }
00321             fname.resize(fname.size()-1);//remove the last caracter (*)
00322             for(int j=0;j<fnames_header.size();j++)
00323             {
00324                 if(string_begins_with(fnames_header[j],fname))
00325                 {
00326                     pair<string,string> n=make_pair(fnames_header[j],
00327                                                     fieldspec[i].second);
00328 //                    perr<<"expanding "<<fieldspec[i] << " to " << n <<endl;
00329                     
00330                     new_fieldspec.append(n);
00331                     expended = true;
00332                 }
00333             }
00334             if(!expended)
00335                 no_expended_fields.append(fieldspec[i].first);
00336         }
00337         if(no_expended_fields.length()>0){
00338             NORMAL_LOG<<"In TextFilesVMatrix::setColumnNamesAndWidth - "
00339                       <<"Did not find any partial match for "
00340                       <<no_expended_fields.length()<<" spec:";
00341             for(int i=0;i<no_expended_fields.length();i++)
00342                 NORMAL_LOG<<" "<<no_expended_fields[i];
00343             NORMAL_LOG<<endl;
00344         }
00345             
00346         fieldspec = new_fieldspec;
00347     }
00348 
00349     if(reorder_fieldspec_from_headers)
00350     {
00351         //check that all field names from the header have a spec
00352         TVec<string> not_used_fn;
00353         for(int i=0;i<fnames_header.size();i++)
00354         {
00355             string name=fnames_header[i];
00356             int j=0;
00357             for(;j<fieldspec.size();j++)
00358                 if(fieldspec[j].first==name)
00359                     break;
00360             if(j>=fieldspec.size()){
00361                 if(default_spec!=""){
00362                     fieldspec.append(make_pair(name,default_spec));
00363                 }else
00364                     not_used_fn.append(name);
00365             }
00366         }
00367         //check that all fieldspec names are also in the header
00368         TVec<string> not_used_fs;
00369         for(int i=0;i<fieldspec.size();i++)
00370         {
00371             string name=fieldspec[i].first;
00372             int j=0;
00373             for(;j<fnames_header.size();j++)
00374                 if(fnames_header[j]==name)
00375                     break;
00376             if(j>=fnames_header.size())
00377                 not_used_fs.append(name);
00378         }
00379         //check that we have the good number of fieldspec
00380         //if partial match is true, we don't want to generate the warning everytime
00381         if(fnames_header.size()!=fieldspec.size() && !partial_match)
00382         {
00383             PLWARNING("In TextFilesVMatrix::setColumnNamesAndWidth() - "
00384                     "We read %d field names from the header but have %d"
00385                     "fieldspec",fnames_header.size(),fieldspec.size());
00386         }
00387 
00388         if(not_used_fs.size()!=0)
00389             PLWARNING("TextFilesVMatrix::setColumnNamesAndWidth() - "
00390                       "%d fieldspecs exists for field(s) that are not in the source: %s\n"
00391                       "They will be skipped.",
00392                       not_used_fs.length(), tostring(not_used_fs).c_str());
00393         if(not_used_fn.size()!=0)
00394             PLWARNING("TextFilesVMatrix::setColumnNamesAndWidth() - "
00395                       "%d fieldnames in source that don't have fieldspec: %s\n"
00396                       "They will be skipped.",
00397                       not_used_fn.length(), tostring(not_used_fn).c_str());
00398     
00399 
00400         //the new order for fieldspecs
00401         TVec< pair<string, string> > fs(fnames_header.size());
00402         for(int i=0;i<fnames_header.size();i++)
00403         {
00404             string name=fnames_header[i];
00405             int j=0;
00406             for(;j<fieldspec.size();j++)
00407                 if(fieldspec[j].first==name)
00408                     break;
00409             if(j>=fieldspec.size())
00410                 fs[i]=pair<string,string>(name,"skip");
00411             else
00412                 fs[i]=fieldspec[j];
00413         }
00414         fieldspec=fs;
00415     }
00416     for(int k=0; k<fieldspec.length(); k++)
00417     {
00418         string fname = fieldspec[k].first;
00419         string ftype = fieldspec[k].second;
00420         if(isValidNonSkipFieldType(ftype))
00421         {
00422             // declare the column name
00423             fnames.push_back(fname);
00424             colrange.push_back( pair<int,int>(width_,1) );
00425             ++width_;
00426         }
00427         else if(ftype=="skip")
00428         {
00429             colrange.push_back( pair<int,int>(width_,0) );
00430         }
00431         else
00432             PLERROR("In TextFilesVMatrix::setColumnNamesAndWidth, Invalid field type specification for field %s: %s",fname.c_str(), ftype.c_str());
00433     }
00434     for(int j=0; j<width_; j++)
00435         declareField(j, fnames[j]);
00436 }
00437 
00438 void TextFilesVMatrix::build_()
00439 {
00440     if (!default_spec.empty() && !reorder_fieldspec_from_headers)
00441         PLERROR("In TextFilesVMatrix::build_() when the option default_spec is used, reorder_fieldspec_from_headers must be true");
00442     if (metadatapath != "") {
00443         PLWARNING("In TextFilesVMatrix::build_() metadatapath option is deprecated. "
00444                   "You should use metadatadir instead.\n");
00445 
00446         metadatadir = metadatapath;
00447         setMetaDataDir(metadatapath);
00448     }
00449 }
00451 // setMetaDataDir //
00453 void TextFilesVMatrix::setMetaDataDir(const PPath& the_metadatadir){
00454     inherited::setMetaDataDir(the_metadatadir);
00455 
00456     if(getMetaDataDir().empty())
00457         PLERROR("In TextFilesVMatrix::setMetaDataDir() - We need a metadatadir");
00458     if(!force_mkdir(getMetaDataDir()))
00459         PLERROR("In TextFilesVMatrix::setMetaDataDir() - could not create"
00460                 " directory '%s'",
00461                 getMetaDataDir().absolute().c_str());
00462 
00463     for(int i=0;i<txtfilenames.length();i++)
00464         updateMtime(txtfilenames[i]);
00465 
00466     PPath metadir = getMetaDataDir();
00467     PPath idxfname = metadir/"txtmat.idx";
00468 
00469     // Now open txtfiles
00470     int nf = txtfilenames.length();
00471     txtfiles.resize(nf);
00472     for(int k=0; k<nf; k++)
00473     {
00474         string fnam = txtfilenames[k];
00475         txtfiles[k] = fopen(fnam.c_str(),"r");
00476         if(txtfiles[k]==NULL){
00477             perror("Can't open file");
00478             PLERROR("In TextFilesVMatrix::setMetaDataDir - Can't open file %s",
00479                     fnam.c_str());
00480         }
00481     }
00482 
00483     setColumnNamesAndWidth();
00484 
00485     // open the index file
00486     if(!isUpToDate(idxfname) || isemptyFile(idxfname))
00487         buildIdx(); // (re)build it first!
00488     idxfile = fopen(idxfname.c_str(),"rb");
00489     if(fgetc(idxfile) != byte_order())
00490         PLERROR("In TextFilesVMatrix::setMetaDataDir - Wrong endianness."
00491                 " Remove the index file %s for it to be automatically rebuilt",
00492                 idxfname.c_str());
00493     fread(&length_, 4, 1, idxfile);
00494 
00495     // Initialize some sizes
00496     int n = fieldspec.size();
00497     mapping.resize(n);
00498     mapfiles.resize(n);
00499     mapfiles.fill(0);
00500 
00501     // Handle string mapping
00502     loadMappings();
00503 
00504     if (auto_build_map)
00505         autoBuildMappings();
00506 
00507     if(build_vmatrix_stringmap)
00508         buildVMatrixStringMapping();
00509 
00510     // Sanity checking
00511 }
00512 
00513 
00514 string TextFilesVMatrix::getTextRow(int i) const
00515 {
00516     unsigned char fileno;
00517     int pos;
00518     getFileAndPos(i, fileno, pos);
00519     FILE* f = txtfiles[(int)fileno];
00520     fseek(f,pos,SEEK_SET);
00521     char buf[50000];
00522 
00523     if(!fgets(buf, sizeof(buf), f))
00524         PLERROR("In TextFilesVMatrix::getTextRow - fgets for row %d returned NULL",i);
00525     return removenewline(buf);
00526 }
00527 
00528 void TextFilesVMatrix::loadMappings()
00529 {
00530     int n = fieldspec.size();
00531     for(int k=0; k<n; k++)
00532     {
00533         string fname = getMapFilePath(k);
00534         if (isfile(fname)) {
00535             updateMtime(fname);
00536             vector<string> all_lines = getNonBlankLines(loadFileAsString(fname));
00537             for (size_t i = 0; i < all_lines.size(); i++) {
00538                 string map_line = all_lines[i];
00539                 size_t start_of_string = map_line.find('"');
00540                 size_t end_of_string = map_line.rfind('"');
00541                 string strval = map_line.substr(start_of_string + 1, end_of_string - start_of_string - 1);
00542                 string real_val_str = map_line.substr(end_of_string + 1);
00543                 real real_val;
00544                 if (!pl_isnumber(real_val_str, &real_val))
00545                     PLERROR("In TextFilesVMatrix::loadMappings - Found a mapping to something that is not a number (%s) in file %s at non-black line %ld", map_line.c_str(), fname.c_str(), long(i));
00546                 mapping[k][strval] = real_val;
00547             }
00548         }
00549     }
00550 }
00551 
00553 // autoBuildMappings //
00555 void TextFilesVMatrix::autoBuildMappings() {
00556     // TODO We should somehow check the date of existing mappings to see if they need to be built.
00557     // For now we just create them if they do not exist yet.
00558 
00559     // First make sure there is no existing mapping.
00560     int nb_already_exist = 0;
00561     int nb_type_no_mapping = 0;
00562     for (int i = 0;  i < mapping.length(); i++) {
00563         if (!mapping[i].empty())
00564             nb_already_exist++;
00565         else if(fieldspec[i].second!="char")//should add auto when it is char that are selected
00566             nb_type_no_mapping++;
00567     }
00568     if(nb_already_exist == 0){
00569         // Mappings need to be built.
00570         // We do this by reading the whole data.
00571         Vec row(width());
00572         bool auto_extend_map_backup = auto_extend_map;
00573         auto_extend_map = true;
00574         ProgressBar pb("Building mappings", length());
00575         for (int i = 0; i < length(); i++) {
00576             getRow(i, row);
00577             pb.update(i + 1);
00578         }
00579         auto_extend_map = auto_extend_map_backup;
00580     }else if (nb_already_exist+nb_type_no_mapping < mapping.length()) {
00581         for (int i = 0;  i < mapping.length(); i++) 
00582             if(fieldspec[i].second=="char" && mapping[i].empty())//should add auto when it is char that are selected
00583                 PLWARNING("In TextFilesVMatrix::autoBuildMappings - mapping already existing but not for field %d (%s)",i,fieldspec[i].first.c_str());
00584 
00585         PLWARNING("In TextFilesVMatrix::autoBuildMappings - The existing "
00586                 "mapping is not complete! There are %d fields with build "
00587                 "mapping and there are %d fields that do not need mapping "
00588                 "in a total of %d fields. Erase the mapping directory in "
00589                 "the metadatadir to have it regenerated next time!",
00590                 nb_already_exist,nb_type_no_mapping,mapping.length());
00591     }//else already build
00592 }
00593 
00594 void TextFilesVMatrix::generateMapCounts()
00595 {
00596     int n = fieldspec.size();
00597     TVec< hash_map<string, int> > counts(n);
00598     for(int k=0; k<n; k++)
00599     {
00600         if(!mapping[k].empty())
00601         {
00602             hash_map<string, real>& mapping_k = mapping[k];
00603             hash_map<string, int>& counts_k = counts[k];
00604             hash_map<string, real>::const_iterator it = mapping_k.begin();
00605             hash_map<string, real>::const_iterator itend = mapping_k.end();
00606             while(it!=itend)
00607             {
00608                 counts_k[ it->first ] = 0;
00609                 ++it;
00610             }
00611         }
00612     }
00613 
00614     int l = length();
00615     ProgressBar pg("Generating counts of mappings",l);
00616     for(int i=0; i<l; i++)
00617     {
00618         TVec<string> fields = getTextFields(i);
00619         for(int k=0; k<fields.length(); k++)
00620         {
00621             if(mapping[k].find(fields[k])!=mapping[k].end())
00622                 ++counts[k][fields[k]];
00623         }
00624         pg(i);
00625     }
00626 
00627     // Save the counts
00628     for(int k=0; k<n; k++)
00629     {
00630         if(!counts[k].empty())
00631             PLearn::save( getMetaDataDir() / "counts" / fieldspec[k].first+".count", counts[k] );
00632     }
00633 
00634 }
00635 
00636 void TextFilesVMatrix::buildVMatrixStringMapping()
00637 {
00638     int n = fieldspec.size();
00639     for(int k=0; k<n; k++)
00640     {
00641         if(mapping[k].size()>0)
00642         {
00643             // get the corresponding VMatrix column range and add the VMatrix mapping
00644             int colstart = colrange[k].first;
00645             int ncols = colrange[k].second;
00646             hash_map<string,real>::const_iterator it = mapping[k].begin();
00647             hash_map<string,real>::const_iterator itend = mapping[k].end();
00648             while(it!=itend)
00649             {
00650                 for(int j=colstart; j<colstart+ncols; j++)
00651                     addStringMapping(j, it->first, it->second);
00652                 ++it;
00653             }
00654         }
00655     }
00656 }
00657 
00658 real TextFilesVMatrix::getMapping(int fieldnum, const string& strval) const
00659 {
00660     hash_map<string, real>& m = mapping[fieldnum];
00661     hash_map<string, real>::const_iterator found = m.find(strval);
00662     if(found!=m.end()) // found it!
00663         return found->second;
00664 
00665     // strval not found
00666     if(!auto_extend_map)
00667         PLERROR("In TextFilesVMatrix::getMapping - No mapping found for field %d (%s) string-value \"%s\" ", fieldnum, fieldspec[fieldnum].first.c_str(), strval.c_str());
00668 
00669     // OK, let's extend the mapping...
00670     real val = real(-1000 - int(m.size()));
00671     m[strval] = val;
00672 
00673     if(!mapfiles[fieldnum])
00674     {
00675         string fname = getMapFilePath(fieldnum);
00676         force_mkdir_for_file(fname);
00677         mapfiles[fieldnum] = fopen(fname.c_str(),"a");
00678         if(!mapfiles[fieldnum])
00679             PLERROR("In TextFilesVMatrix::getMapping - Could not open map file %s\n for appending\n",fname.c_str());
00680     }
00681 
00682     fprintf(mapfiles[fieldnum],"\n\"%s\" %f", strval.c_str(), val);
00683     return val;
00684 }
00685 
00686 TVec<string> TextFilesVMatrix::splitIntoFields(const string& raw_row) const
00687 {
00688     return split_quoted_delimiter(removeblanks(raw_row), delimiter,quote_delimiter);
00689 }
00690 
00691 TVec<string> TextFilesVMatrix::getTextFields(int i) const
00692 {
00693     string rowi = getTextRow(i);
00694     TVec<string> fields =  splitIntoFields(rowi);
00695     if(fields.size() != fieldspec.size())
00696         PLERROR("In TextFilesVMatrix::getTextFields - In getting fields of row %d, wrong number of fields: %d (should be %d):\n%s\n",i,fields.size(),fieldspec.size(),rowi.c_str());
00697     for(int k=0; k<fields.size(); k++)
00698         fields[k] = removeblanks(fields[k]);
00699     return fields;
00700 }
00701 
00702 real TextFilesVMatrix::getPostalEncoding(const string& strval, bool display_warning) const
00703 {
00704     if(strval=="")
00705         return MISSING_VALUE;
00706 
00707     char first_char = strval[0];
00708     int second_digit = strval[1];
00709     real val = 0;
00710     if(first_char=='A')
00711         val = 30 + second_digit;
00712     else if(first_char=='B')
00713         val = 40 + second_digit;
00714     else if(first_char=='C')
00715         val = 50 + second_digit;
00716     else if(first_char=='E')
00717         val = 60 + second_digit;
00718     else if(first_char=='G')
00719         val = 0 + second_digit;
00720     else if(first_char=='H')
00721         val = 10 + second_digit;
00722     else if(first_char=='J')
00723         val = 20 + second_digit;
00724     else if(first_char=='K')
00725         val = 70 + second_digit;
00726     else if(first_char=='L')
00727         val = 80 + second_digit;
00728     else if(first_char=='M')
00729         val = 90 + second_digit;
00730     else if(first_char=='N')
00731         val = 100 + second_digit;
00732     else if(first_char=='P')
00733         val = 110 + second_digit;
00734     else if(first_char=='R')
00735         val = 120 + second_digit;
00736     else if(first_char=='S')
00737         val = 130 + second_digit;
00738     else if(first_char=='T')
00739         val = 140 + second_digit;
00740     else if(first_char=='V')
00741         val = 150 + second_digit;
00742     else if(first_char=='W')
00743         val = 160 + second_digit;
00744     else if(first_char=='X')
00745         val = 170 + second_digit;
00746     else if(first_char=='Y')
00747         val = 180 + second_digit;
00748     else if(first_char=='0' || first_char=='1' || first_char=='2' || first_char=='3' ||
00749             first_char=='4' || first_char=='5' || first_char=='6' || first_char=='7' ||
00750             first_char=='8' || first_char=='9') {
00751         // That would be a C.P.
00752         int first_digit = strval[0];
00753         val = 260 + first_digit * 10 + second_digit;
00754     }
00755     else {
00756         //http://en.wikipedia.org/wiki/Canadian_postal_code
00757         //No postal code includes the letters D, F, I, O, Q, or U,
00758         //as the OCR equipment used in automated sorting could easily
00759         //confuse them with other letters and digits, 
00760         if (display_warning) {
00761             string errmsg;
00762             if(first_char=='D' ||first_char=='F' ||first_char=='I' ||
00763                first_char=='O' ||first_char=='Q' ||first_char=='U')
00764                 errmsg = "Postal code don't use letters D, F, I, O, Q, or U: ";
00765             else
00766                 errmsg = "Currently only some postal codes are supported: ";
00767 
00768             errmsg += "can't process " + strval + ", value will be set to 0.";
00769             PLWARNING(errmsg.c_str());
00770         }
00771         val = 0;
00772     }
00773 
00774     return val;
00775 }
00776 
00777 void TextFilesVMatrix::transformStringToValue(int k, string strval, Vec dest) const
00778 {
00779     strval = removeblanks(strval);
00780     string fieldname = fieldspec[k].first;
00781     string fieldtype = fieldspec[k].second;
00782     real val;
00783 
00784     if(dest.length() != colrange[k].second)
00785         PLERROR("In TextFilesVMatrix::transformStringToValue, destination vec for field %d should be of length %d, not %d",k,colrange[k].second, dest.length());
00786 
00787 
00788     if(fieldtype=="skip")
00789     {
00790         // do nothing, simply skip it
00791         return;
00792     }
00793     
00794     if(strval=="")  // missing
00795         dest[0] = MISSING_VALUE;
00796     else if(fieldtype=="auto")
00797     {
00798         if(pl_isnumber(strval,&val))
00799             dest[0] = real(val);
00800         else
00801             dest[0] = getMapping(k, strval);
00802     }
00803     else if(fieldtype=="auto-num")
00804     {//We suppose the decimal point is '.'
00805         string s=strval;
00806         if(strval[0]=='$')
00807             s.erase(0,1);
00808         if(strval[strval.size()-1]=='$')
00809             s.erase(s.end());
00810         
00811         for(unsigned int pos=0; pos<strval.size(); pos++)
00812             if(s[pos]==',')
00813                 s.erase(pos--,1);
00814         if(pl_isnumber(s,&val))
00815             dest[0] = real(val);
00816         else
00817             PLERROR("In TextFilesVMatrix::transformStringToValue -"
00818                     " expedted [$]number[$] as the value for field %d(%s)."
00819                     " Got %s", k, fieldname.c_str(), strval.c_str());
00820     }
00821     else if(fieldtype=="char")
00822     {
00823         dest[0] = getMapping(k, strval);
00824     }
00825     else if(fieldtype=="num")
00826     {
00827         if(pl_isnumber(strval,&val))
00828             dest[0] = real(val);
00829         else
00830             PLERROR("In TextFilesVMatrix::transformStringToValue - expedted a number as the value for field %d(%s). Got %s",k,fieldname.c_str(),strval.c_str());
00831                 
00832     }
00833     else if(fieldtype=="date")
00834     {
00835         dest[0] = date_to_float(PDate(strval));
00836     }
00837     else if(fieldtype=="jdate")
00838     {
00839         dest[0] = PDate(strval).toJulianDay();
00840     }
00841     else if(fieldtype=="sas_date")
00842     {
00843         if(strval == "0")  // missing
00844             dest[0] = MISSING_VALUE;
00845         else if(pl_isnumber(strval,&val)) {
00846             dest[0] = val;
00847             if (val <= 0) {
00848                 PLERROR("In TextFilesVMatrix::transformStringToValue - "
00849                         "I didn't know a sas_date could be negative");
00850             }
00851         }
00852         else
00853             PLERROR("In TextFilesVMatrix::transformStringToValue - "
00854                     "Error while parsing a sas_date");
00855     }
00856     else if(fieldtype=="YYYYMM")
00857     {
00858         if(!pl_isnumber(strval) || toint(strval)<197000)
00859             dest[0] = MISSING_VALUE;
00860         else
00861             dest[0] = PDate(strval+"01").toJulianDay();
00862     }
00863     else if(fieldtype=="postal")
00864     {
00865         dest[0] = getPostalEncoding(strval);
00866     }
00867     else if(fieldtype=="dollar" || fieldtype=="dollar-comma")
00868     {
00869         char char_torm = ' ';
00870         if(fieldtype=="dollar-comma")
00871             char_torm = ',';
00872         if(strval[0]=='$')
00873         {
00874             string s = "";
00875             for(unsigned int pos=1; pos<strval.size(); pos++)
00876                 if(strval[pos]!=char_torm)
00877                     s += strval[pos];
00878 
00879             if(pl_isnumber(s,&val))
00880                 dest[0] = real(val);
00881             else
00882                 PLERROR("In TextFilesVMatrix::transformStringToValue - Goat as value '%s' while parsing field %d (%s) with fieldtype %s",strval.c_str(),k,fieldname.c_str(),fieldtype.c_str());
00883         }
00884         else
00885             PLERROR("In TextFilesVMatrix::transformStringToValue - Got as value '%s' while expecting a value beggining with '$' while parsing field %d (%s) with fieldtype %s",strval.c_str(),k,fieldname.c_str(),fieldtype.c_str());
00886     }
00887     else if(fieldtype=="bell_range") {
00888         if (strval == "Negative Value") {
00889             // We put an arbitrary negative value since we don't have more info.
00890             dest[0] = -100;
00891         } else {
00892             // A range of the kind "A: $0- 250".
00893             string s = "";
00894             unsigned int pos;
00895             unsigned int end;
00896             for (pos=0; pos<strval.size() && (strval[pos] == ' ' || !pl_isnumber(strval.substr(pos,1))); pos++) {}
00897             for (end=pos; end<strval.size() && strval[end] != ' ' && pl_isnumber(strval.substr(end,1)); end++) {
00898                 s += strval[end];
00899             }
00900             real number_1,number_2;
00901             if (!pl_isnumber(s,&number_1) || is_missing(number_1)) {
00902                 PLERROR(("TextFilesVMatrix::transformStringToValue: " + strval +
00903                          " is not a well formatted Bell range").c_str());
00904             }
00905             s = "";
00906             for (pos=end; pos<strval.size() && (strval[pos] == ' ' || !pl_isnumber(strval.substr(pos,1))); pos++) {}
00907             for (end=pos; end<strval.size() && strval[end] != ' ' && pl_isnumber(strval.substr(end,1)); end++) {
00908                 s += strval[end];
00909             }
00910             if (!pl_isnumber(s,&number_2) || is_missing(number_2)) {
00911                 PLERROR(("TextFilesVMatrix::transformStringToValue: " + strval +
00912                          " is not a well formatted Bell range").c_str());
00913             }
00914             dest[0] = (number_1 + number_2) / (real) 2;
00915         }
00916     }
00917     else if(fieldtype=="num-comma")
00918     {
00919         string s="";
00920         for(uint i=0;i<strval.length();i++)
00921         {
00922             if(strval[i]!=',')
00923                 s=s+strval[i];
00924         }
00925         if(pl_isnumber(s,&val))
00926             dest[0] = real(val);
00927         else
00928             PLERROR("In TextFilesVMatrix::transformStringToValue - expedted a number as the value for field %d(%s). Got %s",k,fieldname.c_str(),strval.c_str());
00929                 
00930     }
00931     else
00932     {
00933         PLERROR("TextFilesVMatrix::TextFilesVMatrix::transformStringToValue, Invalid field type specification for field %s: %s",fieldname.c_str(), fieldtype.c_str());
00934     }
00935 }
00936 
00937 void TextFilesVMatrix::getNewRow(int i, const Vec& v) const
00938 {
00939     TVec<string> fields = getTextFields(i);
00940     int n = fields.size();
00941 
00942     for(int k=0; k<n; k++)
00943     {
00944         string fieldname = fieldspec[k].first;
00945         string fieldtype = fieldspec[k].second;
00946         string strval = fields[k];
00947         Vec dest = v.subVec(colrange[k].first, colrange[k].second);
00948 
00949         try
00950         { transformStringToValue(k, strval, dest); }
00951         catch(const PLearnError& e)
00952         {
00953             PLERROR("In TextFilesVMatrix, while parsing field %d (%s) of row %d: \n%s",
00954                     k,fieldname.c_str(),i,e.message().c_str());
00955         }
00956     }
00957 }
00958 
00959 void TextFilesVMatrix::declareOptions(OptionList& ol)
00960 {
00961     declareOption(ol, "metadatapath", &TextFilesVMatrix::metadatapath, OptionBase::buildoption,
00962                   "Path of the metadata directory (in which to store the index, ...)\n"
00963                   "DEPRECATED: use metadatadir instead.\n");
00964 
00965     declareOption(ol, "txtfilenames", &TextFilesVMatrix::txtfilenames, OptionBase::buildoption,
00966                   "A list of paths to raw text files containing the records");
00967 
00968     declareOption(ol, "delimiter", &TextFilesVMatrix::delimiter, OptionBase::buildoption,
00969                   "Delimiter to use to split the fields.  Common delimiters are:\n"
00970                   "- \"\\t\" : used for SAS files (the default)\n"
00971                   "- \",\"  : used for CSV files\n"
00972                   "- \";\"  : used for a variant of CSV files\n"
00973                   "If more then 1 delimiter, we will select one based on the"
00974                   " first two line\n");
00975 
00976     declareOption(ol, "quote_delimiter", &TextFilesVMatrix::quote_delimiter, OptionBase::buildoption,
00977         "The escape character to indicate the delimiter is not considered.\n"
00978         "For instance, '\"' is frequently used.");
00979 
00980     declareOption(ol, "skipheader", &TextFilesVMatrix::skipheader, OptionBase::buildoption,
00981                   "An (optional) list of integers, one for each of the txtfilenames,\n"
00982                   "indicating the number of header lines at the top of the file to be skipped.");
00983 
00984     declareOption(ol, "fieldspec", &TextFilesVMatrix::fieldspec, OptionBase::buildoption,
00985                   "Specification of field names and types (type indicates how the text field is to be mapped to one or more reals)\n"
00986                   "Currently supported types: \n"
00987                   "- skip       : Ignore the content of the field, won't be inserted in the resulting VMat\n"
00988                   "- auto       : If a numeric value, keep it as is, if not, look it up in the mapping (possibly inserting a new mapping if it's not there) \n"
00989                   "- auto-num   : take any float value with the decimal separator as the dot. If there is a $\n"
00990                   "               at the start or end it is removed. If there are commas they are removed.\n"
00991                   "- num        : numeric value, keep as is\n"
00992                   "- num-comma  : numeric value where thousands are separeted by comma\n"
00993                   "- char       : look it up in the mapping (possibly inserting a new mapping if it's not there)\n"
00994                   "- date       : date of the form 25DEC2003 or 25-dec-2003 or 2003/12/25 or 20031225, will be mapped to float date format 1031225\n"
00995                   "- jdate      : date of the form 25DEC2003 or 25-dec-2003 or 2003/12/25 or 20031225, will be mapped to *julian* date format\n"
00996                   "- sas_date   : date used by SAS = number of days since Jan. 1st, 1960 (with 0 = missing)\n"
00997                   "- YYYYMM     : date of the form YYYYMM (e.g: 200312), will be mapped to the julian date of the corresponding month. Everthing "\
00998                   "               other than a number or lower than 197000 is considered as nan\n"
00999                   "- postal     : canadian postal code \n"
01000                   "- dollar     : strangely formatted field with dollar amount. Format is sth like '$12 003'\n"
01001                   "- dollar-comma : strangely formatted field with dollar amount. Format is sth like '$12,003'\n"
01002                   "- bell_range : a range like \"A: $0- 250\", replaced by the average of the two bounds;\n"
01003                   "               if the \"Negative Value\" string is found, it is replaced by -100\n"
01004         );
01005 
01006     declareOption(ol, "auto_extend_map", &TextFilesVMatrix::auto_extend_map, OptionBase::buildoption,
01007                   "If true, new strings for fields of type AUTO will automatically appended to the mapping (in the metadata/mappings/fieldname.map file)");
01008 
01009     declareOption(ol, "auto_build_map", &TextFilesVMatrix::auto_build_map, OptionBase::buildoption,
01010                   "If true, all mappings will be automatically computed at build time if they do not exist yet\n");
01011 
01012     declareOption(ol, "build_vmatrix_stringmap", &TextFilesVMatrix::build_vmatrix_stringmap,
01013                   OptionBase::buildoption,
01014                   "If true, standard vmatrix stringmap will be built from the txtmat specific stringmap");
01015 
01016     declareOption(ol, "reorder_fieldspec_from_headers", 
01017                   &TextFilesVMatrix::reorder_fieldspec_from_headers,
01018                   OptionBase::buildoption,
01019                   "If true, will reorder the fieldspec in the order given "
01020                   "by the field names taken from txtfilenames.");
01021 
01022     declareOption(ol, "partial_match", 
01023                   &TextFilesVMatrix::partial_match,
01024                   OptionBase::buildoption,
01025                   "If true, will repeatedly expand all fieldspec name ending "
01026                   "with * to the full name from header."
01027                   "The expansion is equivalent to the regex 'field_spec_name*'."
01028                   "The option reorder_fieldspec_from_headers must be true");
01029 
01030     declareOption(ol, "default_spec", 
01031                   &TextFilesVMatrix::default_spec,
01032                   OptionBase::buildoption,
01033                   "If there is no fieldspec for a fieldname, we will use this"
01034                   "value. reorder_fieldspec_from_headers must be true.");
01035 
01036     // Now call the parent class' declareOptions
01037     inherited::declareOptions(ol);
01038 }
01039 
01040 void TextFilesVMatrix::readAndCheckOptionName(PStream& in, const string& optionname, char buf[])
01041 {
01042     in.skipBlanksAndComments();
01043     in.readUntil(buf, sizeof(buf), "= ");
01044     string option = removeblanks(buf);
01045     in.skipBlanksAndComments();
01046     char eq = in.get();
01047     if(option!=optionname || eq!='=')
01048         PLERROR("In TextFilesVMatrix::readAndCheckOptionName - "
01049                 "Bad syntax in .txtmat file.\n"
01050                 "Expected option %s = ...\n"
01051                 "Read %s %c\n", optionname.c_str(), option.c_str(), eq);
01052 }
01053 
01054 
01055 // ### Nothing to add here, simply calls build_
01056 void TextFilesVMatrix::build()
01057 {
01058     inherited::build();
01059     build_();
01060 }
01061 
01062 TextFilesVMatrix::~TextFilesVMatrix()
01063 {
01064     if(idxfile)
01065         fclose(idxfile);
01066     for(int k=0; k<txtfiles.length(); k++)
01067         fclose(txtfiles[k]);
01068 
01069     for(int k=0; k<mapfiles.size(); k++)
01070     {
01071         if(mapfiles[k])
01072             fclose(mapfiles[k]);
01073     }
01074 }
01075 
01076 void TextFilesVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
01077 {
01078     inherited::makeDeepCopyFromShallowCopy(copies);
01079     idxfile=0;
01080     txtfiles.resize(0);
01081     //the map should be already build.
01082     auto_build_map=false;
01083     build();
01084 }
01085 
01086 } // end of namespace PLearn
01087 
01088 
01089 /*
01090   Local Variables:
01091   mode:c++
01092   c-basic-offset:4
01093   c-file-style:"stroustrup"
01094   c-file-offsets:((innamespace . 0)(inline-open . 0))
01095   indent-tabs-mode:nil
01096   fill-column:79
01097   End:
01098 */
01099 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines