PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // DictionaryVMatrix.cc 00004 // 00005 // Copyright (C) 2004 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: DictionaryVMatrix.cc 8617 2008-03-03 17:45:54Z nouiz $ 00037 ******************************************************* */ 00038 00039 // Authors: Hugo Larochelle 00040 00044 #include "DictionaryVMatrix.h" 00045 //#include "DiskVMatrix.h" 00046 #include "plearn/io/openFile.h" 00047 #include "plearn/io/fileutils.h" 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 00053 DictionaryVMatrix::DictionaryVMatrix() 00054 :python(), delimiters(" \t"), code(""), remove_rows_with_oov(false) 00055 /* ### Initialise all fields to their default value */ 00056 { 00057 data=0; 00058 // build_(); 00059 } 00060 00061 PLEARN_IMPLEMENT_OBJECT(DictionaryVMatrix, 00062 "VMat of text files, encoded with Dictionaries", 00063 "The lines of the text files that are empty are ommited. If no Dictionary\n" 00064 "objects are given by the user, then new Dictionary objects\n" 00065 "are created and updated from the text files.\n" 00066 "A Python script can be provided to preprocess each\n" 00067 "row of all files. The script must define a function called\n" 00068 "process_string_row(string_row), where string_row is a list of\n" 00069 "strings corresponding to the symbolic fields of a row in\n" 00070 "the input files. This function must return a list of processed\n" 00071 "strings, which will consist of the actual data contained by\n" 00072 "the VMatrix. Note that process_string_row can return a list\n" 00073 "that has more or less strings then the input files has fields.\n" 00074 "The length of the returned list will determine the width of\n" 00075 "the VMatrix. Here is an example of a Python code that\n" 00076 "puts the first field to lower case and does nothing to the second:\n" 00077 "\n" 00078 "\"def process_string_row(string_row):\\n" 00079 " ret = string_row[:]\\n" 00080 " ret[0] = string_row[0].lower()\\n" 00081 " ret[1] = string_row[1]\\n " 00082 " return ret \"" 00083 ); 00084 00085 void DictionaryVMatrix::getNewRow(int i, const Vec& v) const 00086 { 00087 v << data(i); 00088 } 00090 real DictionaryVMatrix::getStringVal(int col, const string & str) const 00091 { 00092 int ret = dictionaries[col]->getId(str); 00093 if(ret == -1) return MISSING_VALUE; 00094 else return ret; 00095 } 00096 00097 string DictionaryVMatrix::getValString(int col, real val) const 00098 { 00099 if(is_missing(val))return tostring(val); 00100 return dictionaries[col]->getSymbol((int)val); 00101 } 00102 00103 PP<Dictionary> DictionaryVMatrix::getDictionary(int col) const 00104 { 00105 if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getDictionary() : invalid col %d, width()=%d", col, width_); 00106 return dictionaries[col]; 00107 } 00108 00109 00110 void DictionaryVMatrix::getValues(int row, int col, Vec& values) const 00111 { 00112 if(row < 0 || row >= length_) PLERROR("In DictionaryVMatrix::getValues() : invalid row %d, length()=%d", row, length_); 00113 if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getValues() : invalid col %d, width()=%d", col, width_); 00114 TVec<string> options(option_fields[col].length()); 00115 for(int i=0; i<options.length(); i++) 00116 { 00117 options[i] = dictionaries[option_fields[col][i]]->getSymbol((int)data(row,option_fields[col][i])); 00118 } 00119 dictionaries[col]->getValues(options, values); 00120 } 00121 00122 void DictionaryVMatrix::getValues(const Vec& input, int col, Vec& values) const 00123 { 00124 if(col < 0 || col >= width_) PLERROR("In DictionaryVMatrix::getValues() : invalid col %d, width()=%d", col, width_); 00125 TVec<string> options(option_fields[col].length()); 00126 for(int i=0; i<options.length(); i++) 00127 { 00128 options[i] = dictionaries[option_fields[col][i]]->getSymbol((int)input[option_fields[col][i]]); 00129 } 00130 dictionaries[col]->getValues(options, values); 00131 } 00132 00133 00134 void DictionaryVMatrix::declareOptions(OptionList& ol) 00135 { 00136 declareOption(ol, "file_names", &DictionaryVMatrix::file_names, OptionBase::buildoption, 00137 "The text files from which we create the VMat"); 00138 declareOption(ol, "dictionaries", &DictionaryVMatrix::dictionaries, OptionBase::buildoption, 00139 "Vector of dictionaries\n"); 00140 declareOption(ol, "option_fields", &DictionaryVMatrix::option_fields, OptionBase::buildoption, 00141 "Vector of the fields corresponding to the options of the Dictionary, for every Dictionary\n"); 00142 declareOption(ol, "data", &DictionaryVMatrix::data, OptionBase::buildoption, 00143 "Encoded Matrix\n"); 00144 declareOption(ol, "delimiters", &DictionaryVMatrix::delimiters, OptionBase::buildoption, 00145 "Delimiters for file fields (or attributs)\n"); 00146 declareOption(ol, "code", &DictionaryVMatrix::code, OptionBase::buildoption, 00147 "Snippet of python code that processes the text in the input files\n"); 00148 declareOption(ol, "minimum_frequencies", &DictionaryVMatrix::minimum_frequencies, OptionBase::buildoption, 00149 "Minimum frequency for a token to be added in a Dictionary, for the different fields\n"); 00150 declareOption(ol, "symbols_to_ignore", &DictionaryVMatrix::symbols_to_ignore, OptionBase::buildoption, 00151 "Symbols that are ignored, i.e. considered as \"<oov>\" a priori when reading\n" 00152 "the text files. In NLP, this is usually called a stop word list.\n"); 00153 declareOption(ol, "remove_rows_with_oov", &DictionaryVMatrix::remove_rows_with_oov, OptionBase::buildoption, 00154 "Indication that a row that has a OOV symbol field should be ignored, i.e. removed\n" 00155 "of the VMatrix.\n"); 00156 declareOption(ol, "data", &DictionaryVMatrix::data, OptionBase::learntoption, 00157 "Matrix containing the concatenated and encoded text files\n"); 00158 00159 00160 // Now call the parent class' declareOptions 00161 inherited::declareOptions(ol); 00162 } 00163 00164 void DictionaryVMatrix::build_() 00165 { 00166 //if(data.length() != 0) 00167 //{ 00168 // width_ = data.width(); 00169 // length_ = data.length(); 00170 // if(option_fields.length()==0) option_fields.resize(width_); 00171 // return; 00172 //} 00173 00174 string line = ""; 00175 vector<string> tokens; 00176 TVec<string> tokens_vec; 00177 TVec< hash_map<string,int> > frequencies; 00178 int it=0; 00179 int nlines = 0; 00180 bool row_has_oov = false; 00181 00182 if (!python && code != "") 00183 { 00184 python = new PythonCodeSnippet(code); 00185 PLASSERT( python ); 00186 python->build(); 00187 } 00188 00189 if(data.length()!=0) data.clear(); 00190 00191 length_ = 0; 00192 00193 // Prepocessing of the data... 00194 for(int k=0; k<file_names.length(); k++) 00195 { 00196 PPath input_file = file_names[k]; 00197 PStream input_stream = openFile(input_file, PStream::raw_ascii); 00198 updateMtime(input_file); 00199 input_stream.skipBlanks(); 00200 while (!input_stream.eof()){ 00201 input_stream.getline(line); 00202 input_stream.skipBlanks(); 00203 tokens = split(line, delimiters); 00204 if(python) 00205 { 00206 tokens_vec.resize(int(tokens.size())); 00207 for(int i=0; i<tokens_vec.length(); i++) 00208 tokens_vec[i] = tokens[i]; 00209 tokens_vec = python->invoke("process_string_row",tokens_vec).as<TVec<string> >(); 00210 tokens.resize(tokens_vec.length()); 00211 for(int i=0; i<tokens_vec.length(); i++) 00212 tokens[i] = tokens_vec[i]; 00213 } 00214 00215 // Set n_attributes 00216 if(it==0) 00217 { 00218 it++; 00219 n_attributes = int(tokens.size()); 00220 if(minimum_frequencies.length() != 0 && minimum_frequencies.length() != n_attributes) 00221 PLERROR("In DictionaryVMatrix::build_(): number of attributes (%d) and size of minimum_frequencies (%d) is different", n_attributes, minimum_frequencies.length()); 00222 00223 if(minimum_frequencies.length() == 0) 00224 { 00225 minimum_frequencies.resize(n_attributes); 00226 minimum_frequencies.fill(-1); 00227 } 00228 00229 // If no dictionaries are specified, then create some 00230 if(dictionaries.length() == 0) 00231 { 00232 dictionaries.resize(n_attributes); 00233 for(int i=0; i<n_attributes; i++) 00234 { 00235 dictionaries[i] = new Dictionary(); 00236 dictionaries[i]->update_mode = UPDATE; 00237 dictionaries[i]->build(); 00238 } 00239 00240 } 00241 if(dictionaries.length() != n_attributes) 00242 PLERROR("In DictionaryVMatrix::build_(): number of attributes (%d) and number of dictionaries (%d) is different", n_attributes, dictionaries.length()); 00243 if(option_fields.length()==0) option_fields.resize(n_attributes); 00244 frequencies.resize(n_attributes); 00245 } 00246 00247 if(symbols_to_ignore.length() != 0) 00248 for(int i=0; i<int(tokens.size()); i++) 00249 if(symbols_to_ignore[i].find(tokens[i]) >= 0) 00250 tokens[i] = dictionaries[i]->oov_symbol; 00251 00252 row_has_oov = false; 00253 for(int i=0; i<int(tokens.size()); i++) 00254 if(tokens[i] == dictionaries[i]->oov_symbol) 00255 row_has_oov = true; 00256 00257 // Count frequencies... 00258 for(int j=0; j<n_attributes; j++) 00259 { 00260 if(tokens[j] != "nan") // Detect missing values 00261 { 00262 if(frequencies[j].find(tokens[j]) == frequencies[j].end()) 00263 frequencies[j][tokens[j]] = 1; 00264 else 00265 frequencies[j][tokens[j]]++; 00266 } 00267 } 00268 00269 if(!remove_rows_with_oov || !row_has_oov) length_++; 00270 } 00271 } 00272 00273 it = 0; 00274 for(int k=0; k<file_names.length(); k++) 00275 { 00276 nlines = it; 00277 PPath input_file = file_names[k]; 00278 PStream input_stream = openFile(input_file, PStream::raw_ascii); 00279 input_stream.skipBlanks(); 00280 while (!input_stream.eof()){ 00281 if(it==0) data.resize(length_,n_attributes); 00282 input_stream.getline(line); 00283 input_stream.skipBlanks(); 00284 tokens = split(line, delimiters); 00285 00286 if(python) 00287 { 00288 tokens_vec.resize(int(tokens.size())); 00289 for(int i=0; i<tokens_vec.length(); i++) 00290 tokens_vec[i] = tokens[i]; 00291 tokens_vec = python->invoke("process_string_row",tokens_vec).as<TVec<string> >(); 00292 tokens.resize(tokens_vec.length()); 00293 for(int i=0; i<tokens_vec.length(); i++) 00294 tokens[i] = tokens_vec[i]; 00295 } 00296 00297 if(symbols_to_ignore.length() != 0) 00298 for(int i=0; i<int(tokens.size()); i++) 00299 if(symbols_to_ignore[i].find(tokens[i]) >= 0) 00300 tokens[i] = dictionaries[i]->oov_symbol; 00301 00302 /* 00303 for(int i=0; i<to_lower_case.size(); i++) 00304 tokens[to_lower_case[i]] = lowerstring(tokens[to_lower_case[i]]); 00305 */ 00306 00307 if((int)tokens.size() != n_attributes) 00308 PLERROR("In DictionaryVMatrix::build_(): line %d (\"%s\") of file %s doesn't have %d attributes", it-nlines+1, line.c_str(), input_file.c_str(), n_attributes); 00309 00310 // Insert symbols in dictionaries (if they can be updated) 00311 for(int j=0; j<n_attributes; j++) 00312 { 00313 if(tokens[j] == "nan") // Detect missing values 00314 data(it,j) = MISSING_VALUE; 00315 else 00316 { 00317 if(frequencies[j][tokens[j]] >= minimum_frequencies[j]) 00318 { 00319 TVec<string> options(option_fields[j].length()); 00320 for(int k_it=0; k_it<options.length(); k_it++) 00321 options[k_it] = tokens[option_fields[j][k_it]]; 00322 data(it,j) = dictionaries[j]->getId(tokens[j],options); 00323 } 00324 else 00325 data(it,j) = dictionaries[j]->getId(dictionaries[j]->oov_symbol); 00326 } 00327 } 00328 row_has_oov = false; 00329 for(int j=0; j<n_attributes; j++) 00330 if(data(it,j) == dictionaries[j]->getId(dictionaries[j]->oov_symbol)) 00331 row_has_oov = true; 00332 00333 if(!remove_rows_with_oov || !row_has_oov) it++; 00334 } 00335 } 00336 width_ = n_attributes; 00337 data.resize(it,n_attributes); 00338 length_ = it; 00339 00340 // Making sure that the dictionaries cannot be 00341 // updated anymore, since they should contain 00342 // all the needed information about the data 00343 for(int i=0; i<dictionaries.length(); i++) 00344 dictionaries[i]->setUpdateMode(NO_UPDATE); 00345 } 00346 00347 // ### Nothing to add here, simply calls build_ 00348 void DictionaryVMatrix::build() 00349 { 00350 inherited::build(); 00351 build_(); 00352 } 00353 00354 void DictionaryVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00355 { 00356 inherited::makeDeepCopyFromShallowCopy(copies); 00357 00358 deepCopyField(data, copies); 00359 deepCopyField(file_names, copies); 00360 deepCopyField(dictionaries, copies); 00361 deepCopyField(option_fields, copies); 00362 deepCopyField(minimum_frequencies, copies); 00363 deepCopyField(symbols_to_ignore, copies); 00364 deepCopyField(python, copies); 00365 } 00366 00367 } // end of namespace PLearn 00368 00369 00370 /* 00371 Local Variables: 00372 mode:c++ 00373 c-basic-offset:4 00374 c-file-style:"stroustrup" 00375 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00376 indent-tabs-mode:nil 00377 fill-column:79 00378 End: 00379 */ 00380 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :