PLearn 0.1
DictionaryVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DictionaryVMatrix.cc
00004 //
00005 // Copyright (C) 2004 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: DictionaryVMatrix.cc 8617 2008-03-03 17:45:54Z nouiz $
00037  ******************************************************* */
00038 
00039 // Authors: Hugo Larochelle
00040 
00044 #include "DictionaryVMatrix.h"
00045 //#include "DiskVMatrix.h"
00046 #include "plearn/io/openFile.h"
00047 #include "plearn/io/fileutils.h"
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 
00053 DictionaryVMatrix::DictionaryVMatrix()
00054     :python(), delimiters(" \t"), code(""), remove_rows_with_oov(false)
00055     /* ### Initialise all fields to their default value */
00056 {
00057     data=0;
00058     // build_();
00059 }
00060 
00061 PLEARN_IMPLEMENT_OBJECT(DictionaryVMatrix,
00062                         "VMat of text files, encoded  with Dictionaries",
00063                         "The lines of the text files that are empty are ommited. If no Dictionary\n"
00064                         "objects are given by the user, then new Dictionary objects\n"
00065                         "are created and updated from the text files.\n"
00066                         "A Python script can be provided to preprocess each\n"
00067                         "row of all files. The script must define a function called\n"
00068                         "process_string_row(string_row), where string_row is a list of\n"
00069                         "strings corresponding to the symbolic fields of a row in\n"
00070                         "the input files. This function must return a list of processed\n"
00071                         "strings, which will consist of the actual data contained by\n"
00072                         "the VMatrix. Note that process_string_row can return a list\n"
00073                         "that has more or less strings then the input files has fields.\n"
00074                         "The length of the returned list will determine the width of\n"
00075                         "the VMatrix. Here is an example of a Python code that\n"
00076                         "puts the first field to lower case and does nothing to the second:\n"
00077                         "\n"
00078                         "\"def process_string_row(string_row):\\n"
00079                         "         ret = string_row[:]\\n"
00080                         "         ret[0] = string_row[0].lower()\\n"
00081                         "         ret[1] = string_row[1]\\n "
00082                         "         return ret \""
00083     );
00084 
00085 void DictionaryVMatrix::getNewRow(int i, const Vec& v) const
00086 {
00087     v << data(i);
00088 }
00090 real DictionaryVMatrix::getStringVal(int col, const string & str) const
00091 {
00092     int ret = dictionaries[col]->getId(str);
00093     if(ret == -1) return MISSING_VALUE;
00094     else return ret;
00095 }
00096 
00097 string DictionaryVMatrix::getValString(int col, real val) const
00098 {
00099     if(is_missing(val))return tostring(val);
00100     return dictionaries[col]->getSymbol((int)val);
00101 }
00102 
00103 PP<Dictionary>  DictionaryVMatrix::getDictionary(int col) const
00104 {
00105     if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getDictionary() : invalid col %d, width()=%d", col, width_);
00106     return  dictionaries[col];
00107 }
00108 
00109 
00110 void DictionaryVMatrix::getValues(int row, int col, Vec& values) const
00111 {
00112     if(row < 0 || row >= length_) PLERROR("In DictionaryVMatrix::getValues() : invalid row %d, length()=%d", row, length_);
00113     if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getValues() : invalid col %d, width()=%d", col, width_);
00114     TVec<string> options(option_fields[col].length());
00115     for(int i=0; i<options.length(); i++)
00116     {
00117         options[i] = dictionaries[option_fields[col][i]]->getSymbol((int)data(row,option_fields[col][i]));
00118     }
00119     dictionaries[col]->getValues(options, values);
00120 }
00121 
00122 void DictionaryVMatrix::getValues(const Vec& input, int col, Vec& values) const
00123 {
00124     if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getValues() : invalid col %d, width()=%d", col, width_);
00125     TVec<string> options(option_fields[col].length());
00126     for(int i=0; i<options.length(); i++)
00127     {
00128         options[i] = dictionaries[option_fields[col][i]]->getSymbol((int)input[option_fields[col][i]]);
00129     }
00130     dictionaries[col]->getValues(options, values);
00131 }
00132 
00133 
00134 void DictionaryVMatrix::declareOptions(OptionList& ol)
00135 {
00136     declareOption(ol, "file_names", &DictionaryVMatrix::file_names, OptionBase::buildoption,
00137                   "The text files from which we create the VMat");
00138     declareOption(ol, "dictionaries", &DictionaryVMatrix::dictionaries, OptionBase::buildoption,
00139                   "Vector of dictionaries\n");
00140     declareOption(ol, "option_fields", &DictionaryVMatrix::option_fields, OptionBase::buildoption,
00141                   "Vector of the fields corresponding to the options of the Dictionary, for every Dictionary\n");
00142     declareOption(ol, "data", &DictionaryVMatrix::data, OptionBase::buildoption,
00143                   "Encoded Matrix\n");
00144     declareOption(ol, "delimiters", &DictionaryVMatrix::delimiters, OptionBase::buildoption,
00145                   "Delimiters for file fields (or attributs)\n");
00146     declareOption(ol, "code", &DictionaryVMatrix::code, OptionBase::buildoption,
00147                   "Snippet of python code that processes the text in the input files\n");
00148     declareOption(ol, "minimum_frequencies", &DictionaryVMatrix::minimum_frequencies, OptionBase::buildoption,
00149                   "Minimum frequency for a token to be added in a Dictionary, for the different fields\n");
00150     declareOption(ol, "symbols_to_ignore", &DictionaryVMatrix::symbols_to_ignore, OptionBase::buildoption,
00151                   "Symbols that are ignored, i.e. considered as \"<oov>\" a priori when reading\n"
00152                   "the text files. In NLP, this is usually called a stop word list.\n");
00153     declareOption(ol, "remove_rows_with_oov", &DictionaryVMatrix::remove_rows_with_oov, OptionBase::buildoption,
00154                   "Indication that a row that has a OOV symbol field should be ignored, i.e. removed\n"
00155                   "of the VMatrix.\n");
00156     declareOption(ol, "data", &DictionaryVMatrix::data, OptionBase::learntoption,
00157                   "Matrix containing the concatenated and encoded text files\n");
00158 
00159 
00160     // Now call the parent class' declareOptions
00161     inherited::declareOptions(ol);
00162 }
00163 
00164 void DictionaryVMatrix::build_()
00165 {
00166     //if(data.length() != 0)
00167     //{
00168     //    width_ = data.width();
00169     //    length_ = data.length();
00170     //    if(option_fields.length()==0) option_fields.resize(width_);
00171     //    return;
00172     //}
00173     
00174     string line = "";
00175     vector<string> tokens;
00176     TVec<string> tokens_vec;
00177     TVec< hash_map<string,int> > frequencies;
00178     int it=0;
00179     int nlines = 0;
00180     bool row_has_oov = false;
00181 
00182     if (!python && code != "")
00183     {
00184         python = new PythonCodeSnippet(code);
00185         PLASSERT( python );
00186         python->build();
00187     }
00188 
00189     if(data.length()!=0) data.clear();
00190 
00191     length_ = 0;
00192 
00193     // Prepocessing of the data...
00194     for(int k=0; k<file_names.length(); k++)
00195     {
00196         PPath input_file = file_names[k];
00197         PStream input_stream = openFile(input_file, PStream::raw_ascii);
00198         updateMtime(input_file);
00199         input_stream.skipBlanks();
00200         while (!input_stream.eof()){
00201             input_stream.getline(line);
00202             input_stream.skipBlanks();
00203             tokens = split(line, delimiters);
00204             if(python)
00205             {
00206                 tokens_vec.resize(int(tokens.size()));
00207                 for(int i=0; i<tokens_vec.length(); i++)
00208                     tokens_vec[i] = tokens[i];
00209                 tokens_vec = python->invoke("process_string_row",tokens_vec).as<TVec<string> >();
00210                 tokens.resize(tokens_vec.length());
00211                 for(int i=0; i<tokens_vec.length(); i++)
00212                     tokens[i] = tokens_vec[i];
00213             }
00214 
00215             // Set n_attributes
00216             if(it==0)
00217             {
00218                 it++;
00219                 n_attributes = int(tokens.size());
00220                 if(minimum_frequencies.length() != 0 && minimum_frequencies.length() != n_attributes)
00221                     PLERROR("In DictionaryVMatrix::build_(): number of attributes (%d) and size of minimum_frequencies (%d) is different", n_attributes, minimum_frequencies.length());
00222 
00223                 if(minimum_frequencies.length() == 0)
00224                 {
00225                     minimum_frequencies.resize(n_attributes);
00226                     minimum_frequencies.fill(-1);
00227                 }
00228 
00229                 // If no dictionaries are specified, then create some
00230                 if(dictionaries.length() == 0)
00231                 {
00232                     dictionaries.resize(n_attributes);
00233                     for(int i=0; i<n_attributes; i++)
00234                     {
00235                         dictionaries[i] = new Dictionary();
00236                         dictionaries[i]->update_mode = UPDATE;
00237                         dictionaries[i]->build();
00238                     }
00239 
00240                 }
00241                 if(dictionaries.length() != n_attributes)
00242                     PLERROR("In DictionaryVMatrix::build_(): number of attributes (%d) and number of dictionaries (%d) is different", n_attributes, dictionaries.length());
00243                 if(option_fields.length()==0) option_fields.resize(n_attributes);
00244                 frequencies.resize(n_attributes);
00245             }
00246 
00247             if(symbols_to_ignore.length() != 0)
00248                 for(int i=0; i<int(tokens.size()); i++)
00249                     if(symbols_to_ignore[i].find(tokens[i]) >= 0)
00250                         tokens[i] = dictionaries[i]->oov_symbol;
00251 
00252             row_has_oov = false;
00253             for(int i=0; i<int(tokens.size()); i++)
00254                 if(tokens[i] == dictionaries[i]->oov_symbol)
00255                     row_has_oov = true;
00256 
00257             // Count frequencies...
00258             for(int j=0; j<n_attributes; j++)
00259             {
00260                 if(tokens[j] != "nan") // Detect missing values
00261                 {
00262                     if(frequencies[j].find(tokens[j]) == frequencies[j].end())
00263                         frequencies[j][tokens[j]] = 1;
00264                     else
00265                         frequencies[j][tokens[j]]++;
00266                 }
00267             }
00268 
00269             if(!remove_rows_with_oov || !row_has_oov) length_++;
00270         }
00271     }
00272 
00273     it = 0;
00274     for(int k=0; k<file_names.length(); k++)
00275     {
00276         nlines = it;
00277         PPath input_file = file_names[k];
00278         PStream input_stream = openFile(input_file, PStream::raw_ascii);
00279         input_stream.skipBlanks();
00280         while (!input_stream.eof()){
00281             if(it==0) data.resize(length_,n_attributes);
00282             input_stream.getline(line);
00283             input_stream.skipBlanks();
00284             tokens = split(line, delimiters);
00285 
00286             if(python)
00287             {
00288                 tokens_vec.resize(int(tokens.size()));
00289                 for(int i=0; i<tokens_vec.length(); i++)
00290                     tokens_vec[i] = tokens[i];
00291                 tokens_vec = python->invoke("process_string_row",tokens_vec).as<TVec<string> >();
00292                 tokens.resize(tokens_vec.length());
00293                 for(int i=0; i<tokens_vec.length(); i++)
00294                     tokens[i] = tokens_vec[i];
00295             }
00296 
00297             if(symbols_to_ignore.length() != 0)
00298                 for(int i=0; i<int(tokens.size()); i++)
00299                     if(symbols_to_ignore[i].find(tokens[i]) >= 0)
00300                         tokens[i] = dictionaries[i]->oov_symbol;
00301 
00302             /*
00303             for(int i=0; i<to_lower_case.size(); i++)
00304                 tokens[to_lower_case[i]] = lowerstring(tokens[to_lower_case[i]]);
00305             */
00306 
00307             if((int)tokens.size() != n_attributes)
00308                 PLERROR("In DictionaryVMatrix::build_(): line %d (\"%s\") of file %s doesn't have %d attributes", it-nlines+1, line.c_str(), input_file.c_str(), n_attributes);
00309 
00310             // Insert symbols in dictionaries (if they can be updated)
00311             for(int j=0; j<n_attributes; j++)
00312             {
00313                 if(tokens[j] == "nan") // Detect missing values
00314                     data(it,j) = MISSING_VALUE;
00315                 else
00316                 {
00317                     if(frequencies[j][tokens[j]] >= minimum_frequencies[j])
00318                     {
00319                         TVec<string> options(option_fields[j].length());
00320                         for(int k_it=0; k_it<options.length(); k_it++)
00321                             options[k_it] = tokens[option_fields[j][k_it]];
00322                         data(it,j) = dictionaries[j]->getId(tokens[j],options);
00323                     }
00324                     else
00325                         data(it,j) = dictionaries[j]->getId(dictionaries[j]->oov_symbol);
00326                 }
00327             }
00328             row_has_oov = false;
00329             for(int j=0; j<n_attributes; j++)
00330                 if(data(it,j) == dictionaries[j]->getId(dictionaries[j]->oov_symbol))
00331                     row_has_oov = true;
00332 
00333             if(!remove_rows_with_oov || !row_has_oov) it++;
00334         }
00335     }
00336     width_ = n_attributes;
00337     data.resize(it,n_attributes);
00338     length_ = it;
00339 
00340     // Making sure that the dictionaries cannot be
00341     // updated anymore, since they should contain
00342     // all the needed information about the data
00343     for(int i=0; i<dictionaries.length(); i++)
00344         dictionaries[i]->setUpdateMode(NO_UPDATE);
00345 }
00346 
00347 // ### Nothing to add here, simply calls build_
00348 void DictionaryVMatrix::build()
00349 {
00350     inherited::build();
00351     build_();
00352 }
00353 
00354 void DictionaryVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00355 {
00356     inherited::makeDeepCopyFromShallowCopy(copies);
00357 
00358     deepCopyField(data, copies);
00359     deepCopyField(file_names, copies);
00360     deepCopyField(dictionaries, copies);
00361     deepCopyField(option_fields, copies);
00362     deepCopyField(minimum_frequencies, copies);
00363     deepCopyField(symbols_to_ignore, copies);
00364     deepCopyField(python, copies);
00365 }
00366 
00367 } // end of namespace PLearn
00368 
00369 
00370 /*
00371   Local Variables:
00372   mode:c++
00373   c-basic-offset:4
00374   c-file-style:"stroustrup"
00375   c-file-offsets:((innamespace . 0)(inline-open . 0))
00376   indent-tabs-mode:nil
00377   fill-column:79
00378   End:
00379 */
00380 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines