PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2003 Olivier Delalleau 00008 // 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ******************************************************* 00039 * $Id: SelectColumnsVMatrix.cc 9773 2008-12-11 19:18:44Z nouiz $ 00040 ******************************************************* */ 00041 00042 #include "SelectColumnsVMatrix.h" 00043 #include <plearn/io/load_and_save.h> 00044 #define PL_LOG_MODULE_NAME "SelectColumnsVMatrix" 00045 #include <plearn/io/pl_log.h> 00046 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00053 PLEARN_IMPLEMENT_OBJECT(SelectColumnsVMatrix, 00054 "Selects variables from a source matrix according to given vector of indices.", 00055 "Alternatively, the variables can be given by their names. In this case, if a\n" 00056 "field does not exist and the 'extend_with_missing' option is set to 'true',\n" 00057 "the field will be added and filled with missing values.\n" 00058 ); 00059 00061 // SelectColumnsVMatrix // 00063 SelectColumnsVMatrix::SelectColumnsVMatrix() 00064 : extend_with_missing(false), 00065 fields_partial_match(false), 00066 fields_regex(false), 00067 inverse_fields_selection(false), 00068 warn_non_selected_field(false) 00069 {} 00070 00071 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source, 00072 TVec<string> the_fields, 00073 bool the_extend_with_missing, 00074 bool call_build_): 00075 inherited(the_source, call_build_), 00076 extend_with_missing(the_extend_with_missing), 00077 fields(the_fields), 00078 fields_partial_match(false), 00079 fields_regex(false), 00080 inverse_fields_selection(false), 00081 warn_non_selected_field(false) 00082 { 00083 if (call_build_) 00084 build_(); 00085 } 00086 00087 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source, 00088 TVec<int> the_indices, 00089 bool call_build_): 00090 inherited(the_source, call_build_), 00091 extend_with_missing(false), 00092 indices(the_indices), 00093 fields_partial_match(false), 00094 fields_regex(false), 00095 inverse_fields_selection(false), 00096 warn_non_selected_field(false) 00097 { 00098 if (call_build_) 00099 build_(); 00100 } 00101 00102 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source, Vec the_indices) 00103 : extend_with_missing(false), 00104 fields_partial_match(false), 00105 fields_regex(false), 00106 inverse_fields_selection(false), 00107 warn_non_selected_field(false) 00108 { 00109 source = the_source; 00110 indices.resize(the_indices.length()); 00111 // copy the real the_indices into the integer indices 00112 indices << the_indices; 00113 build_(); 00114 } 00115 00117 // get // 00119 real SelectColumnsVMatrix::get(int i, int j) const { 00120 #ifdef BOUNDCHECK 00121 if(i>=length_ ||j>=width_||i<0||j<0) 00122 PLERROR("In SelectColumnsVMatrix::getSubRow - " 00123 "requested index: (%d,%d) but width of %d and length of %d", 00124 i,j,width_,length_); 00125 #endif 00126 int col; 00127 col = sel_indices[j]; 00128 if (col == -1) 00129 return MISSING_VALUE; 00130 return source->get(i, col); 00131 } 00132 00134 // getSubRow // 00136 void SelectColumnsVMatrix::getSubRow(int i, int j, Vec v) const 00137 { 00138 #ifdef BOUNDCHECK 00139 if(i>=length_ ||j>=width_||i<0||j<0) 00140 PLERROR("In SelectColumnsVMatrix::getSubRow - " 00141 "requested index: (%d,%d) but width of %d and length of %d", 00142 i,j,width_,length_); 00143 #endif 00144 int col; 00145 for(int jj=0; jj<v.length(); jj++) { 00146 col = sel_indices[j+jj]; 00147 if (col == -1) 00148 v[jj] = MISSING_VALUE; 00149 else 00150 v[jj] = source->get(i, col); 00151 } 00152 } 00153 00155 // declareOptions // 00157 void SelectColumnsVMatrix::declareOptions(OptionList &ol) 00158 { 00159 declareOption(ol, "fields", &SelectColumnsVMatrix::fields, OptionBase::buildoption, 00160 "The names of the fields to extract (will override 'indices' if provided). Can\n" 00161 "be a range of the form Field_1-Field_n, if 'extend_with_missing' is false."); 00162 00163 declareOption(ol, "save_fields", &SelectColumnsVMatrix::save_fields, 00164 OptionBase::buildoption, 00165 "Optional file where the fieldnames of this VMatrix should be saved."); 00166 00167 declareOption(ol, "fields_partial_match", &SelectColumnsVMatrix::fields_partial_match, OptionBase::buildoption, 00168 "If set to 1, then a field will be kept iff it contains one of the strings from 'fields'."); 00169 00170 declareOption(ol, "fields_regex", &SelectColumnsVMatrix::fields_regex, OptionBase::buildoption, 00171 "If set to 1, then a field will be kept iff it match one of the strings from 'fields'. A match is when the two string is equal or when fields end with a *, we allow any end."); 00172 00173 declareOption(ol, "indices", &SelectColumnsVMatrix::indices, OptionBase::buildoption, 00174 "The array of column indices to extract."); 00175 00176 declareOption(ol, "extend_with_missing", &SelectColumnsVMatrix::extend_with_missing, OptionBase::buildoption, 00177 "If set to 1, then fields specified in the 'fields' option that do not exist\n" 00178 "in the source VMatrix will be filled with missing values."); 00179 00180 declareOption(ol, "inverse_fields_selection", 00181 &SelectColumnsVMatrix::inverse_fields_selection, 00182 OptionBase::buildoption, 00183 "If set to true, the selection is reversed. This provides an easy\n" 00184 "way to specify columns we do not want."); 00185 00186 declareOption(ol, "warn_non_selected_field", 00187 &SelectColumnsVMatrix::warn_non_selected_field, 00188 OptionBase::buildoption, 00189 "If set to true, we generate a PLWARNING with all fields " 00190 "that are not selected."); 00191 00192 inherited::declareOptions(ol); 00193 00194 redeclareOption(ol, "length", &SelectColumnsVMatrix::length_, 00195 OptionBase::nosave, 00196 "Taken from source"); 00197 00198 redeclareOption(ol, "width", &SelectColumnsVMatrix::width_, 00199 OptionBase::nosave, 00200 "Computed from parameter"); 00201 } 00202 00204 // makeDeepCopyFromShallowCopy // 00206 void SelectColumnsVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00207 { 00208 inherited::makeDeepCopyFromShallowCopy(copies); 00209 deepCopyField(sinput, copies); 00210 deepCopyField(indices, copies); 00211 deepCopyField(fields, copies); 00212 deepCopyField(sel_indices, copies); 00213 } 00214 00216 // build // 00218 void SelectColumnsVMatrix::build() 00219 { 00220 inherited::build(); 00221 build_(); 00222 } 00223 00225 // build_ // 00227 void SelectColumnsVMatrix::build_() 00228 { 00229 if (source) { 00230 updateMtime(source); 00231 if (fields.isNotEmpty()) { 00232 getIndicesFromFields(); 00233 } 00234 00235 //we inverse the selection 00236 if(inverse_fields_selection){ 00237 TVec<int> newindices; 00238 for (int i = 0;i<source.width();i++){ 00239 bool found=false; 00240 for(int j=0;j<indices.size();j++) 00241 if(indices[j]==i){ 00242 found = true; 00243 break; 00244 } 00245 if(!found) 00246 newindices.append(i); 00247 } 00248 sel_indices = newindices; 00249 } else 00250 sel_indices = indices; 00251 00252 if(warn_non_selected_field){ 00253 TVec<string> v; 00254 for(int i=0;i<source.width();i++) 00255 { 00256 bool found=false; 00257 for(int j=0;j<sel_indices.size();j++) 00258 if(sel_indices[j]==i){ 00259 found=true; 00260 break; 00261 } 00262 if(!found) 00263 v.append(source.fieldName(i)); 00264 } 00265 if(v.size()>0) 00266 PLWARNING("In SelectColumnsVMatrix::build_() - There are %d" 00267 " columns in the source matrix that we do not " 00268 "select: %s",v.size(),tostring(v).c_str()); 00269 } 00270 00271 // Copy matrix dimensions 00272 width_ = sel_indices.length(); 00273 length_ = source->length(); 00274 00275 if(!extend_with_missing) 00276 // The new width cannot exceed the old one. 00277 PLCHECK(source->width() >= width_); 00278 00279 //if we don't add new columns and the source columns type are emtpy, 00280 //they should be empty in this matrix if they are not already set. 00281 if(targetsize_<0 && !extend_with_missing && source->targetsize()==0) 00282 targetsize_=0; 00283 if(weightsize_<0 && !extend_with_missing && source->weightsize()==0) 00284 weightsize_=0; 00285 computeMissingSizeValue(false); 00286 00287 #if 0 00288 // Disabled for now, since it gives way too many false positives in 00289 // some cases. Todo: figure out a way to warn in a useful way when 00290 // a SelectColumnsVMatrix is given as a train or test set to a learner, 00291 // with inputsize and friends not set manually. 00292 00293 // Give warning if inputsize, and friends are not specified, since we 00294 // can't just copy the values from the underlying VMat. A smarter 00295 // version of this class could Figure out the right size for inputsize, 00296 // targetsize and weightsize, at least for some cases (for example when 00297 // the indices are nondecreasing). 00298 if (inputsize_ < 0 || targetsize_ < 0 || weightsize_ < 0) 00299 PLWARNING("In SelectColumnsVMatrix::build_ inputsize, targetsize or weightsize " 00300 "not set. You may want to set them yourself in the .plearn file."); 00301 #endif 00302 00303 // Copy the appropriate VMFields 00304 fieldinfos.resize(width()); 00305 if (source->getFieldInfos().size() > 0) { 00306 for (int i=0; i<width(); ++i) { 00307 int col = sel_indices[i]; 00308 if (col == -1) { 00309 if (extend_with_missing) 00310 // This must be because it is a field that did not exist in 00311 // the source VMat. 00312 fieldinfos[i] = VMField(fields[i]); 00313 else 00314 PLERROR("In SelectColumnsVMatrix::build_ - '-1' is not a valid value in indices"); 00315 } else { 00316 fieldinfos[i] = source->getFieldInfos(col); 00317 } 00318 } 00319 } 00320 00321 sinput.resize(width()); 00322 sinput.fill(MISSING_VALUE); 00323 00324 if(!save_fields.isEmpty()) 00325 PLearn::save(save_fields, fieldNames()); 00326 } 00327 } 00328 00329 void SelectColumnsVMatrix::getIndicesFromFields(){ 00330 // Find out the indices from the fields. 00331 indices.resize(0); 00332 if (!fields_partial_match && ! fields_regex) { 00333 TVec<string> missing_field; 00334 for (int i = 0; i < fields.length(); i++) { 00335 string the_field = fields[i]; 00336 int the_index = source->fieldIndex(the_field); // string only 00337 if (!extend_with_missing && the_index == -1) { 00338 // The_field does not exist AS A STRING in the vmat 00339 // It may be of the form FIELD1-FIELDN (a range of fields). 00340 size_t pos = the_field.find('-'); 00341 bool ok = false; 00342 if (pos != string::npos) { 00343 string field1 = the_field.substr(0, pos); 00344 string fieldn = the_field.substr(pos + 1); 00345 int the_index1 = source->getFieldIndex(field1); // either string or number 00346 int the_indexn = source->getFieldIndex(fieldn); // either string or number 00347 if (the_index1 >= 0 && the_indexn > the_index1) { 00348 // Indeed, this is a range. 00349 ok = true; 00350 for (int j = the_index1; j <= the_indexn; j++) 00351 indices.append(j); 00352 } 00353 } 00354 // OR it may be a number by itself only 00355 else if (pl_islong(the_field) && 00356 (the_index = source->getFieldIndex(the_field)) != -1) 00357 { 00358 ok = true; 00359 indices.append(the_index); 00360 } 00361 if (!ok && !inverse_fields_selection) 00362 PLERROR("In SelectColumnsVMatrix::build_ - Unknown field \"%s\" in source VMat;\n" 00363 " (you may want to use the 'extend_with_missing' option)", the_field.c_str()); 00364 else if(!ok) 00365 PLWARNING("In SelectColumnsVMatrix::build_ - Unknown field \"%s\" in source VMat;\n" 00366 " as we want to ignore this field, we ignore this error.", the_field.c_str()); 00367 00368 } else 00369 indices.append(the_index); 00370 if(extend_with_missing && the_index == -1) 00371 missing_field.append(the_field); 00372 } 00373 if(missing_field.size()>0){ 00374 PLWARNING("In SelectColumnsVMatrix::build_() - We are" 00375 " adding %d columns to the source matrix with missing values." 00376 " The columns names are: %s",missing_field.size(), 00377 tostring(missing_field).c_str()); 00378 } 00379 00380 } else if(fields_regex) { 00381 // We need to check whether or not we should add each field. 00382 TVec<string> source_fields = source->fieldNames(); 00383 for (int j = 0; j < fields.length(); j++){ 00384 string f = fields[j]; 00385 if(f[f.size()-1]=='*'){ 00386 f.resize(f.size()-1);//remove the last caracter (*) 00387 for (int i = 0; i < source_fields.length(); i++) 00388 if (string_begins_with(source_fields[i],f)) { 00389 // We want to add this field. 00390 indices.append(i); 00391 DBG_MODULE_LOG<<fields[j]<<" matched "<<source_fields[i]<<endl; 00392 } 00393 }else{ 00394 for (int i = 0; i < source_fields.length(); i++) 00395 if(source_fields[i]==f){ 00396 indices.append(i); 00397 //their should be only one fields with a given name 00398 break; 00399 } 00400 } 00401 } 00402 } else { 00403 // We need to check whether or not we should add each field. 00404 TVec<string> source_fields = source->fieldNames(); 00405 for (int i = 0; i < source_fields.length(); i++) 00406 for (int j = 0; j < fields.length(); j++) 00407 if (source_fields[i].find(fields[j]) != string::npos) { 00408 // We want to add this field. 00409 indices.append(i); 00410 DBG_MODULE_LOG<<fields[j]<<" matched "<<source_fields[i]<<endl; 00411 break; 00412 } 00413 } 00414 } 00415 00417 // getStringToRealMapping // 00419 const map<string,real>& SelectColumnsVMatrix::getStringToRealMapping(int col) const { 00420 int the_col; 00421 static map<string, real> empty_mapping; 00422 the_col = sel_indices[col]; 00423 if (the_col == -1) 00424 return empty_mapping; 00425 return source->getStringToRealMapping(the_col); 00426 } 00427 00429 // getStringVal // 00431 real SelectColumnsVMatrix::getStringVal(int col, const string & str) const { 00432 int the_col; 00433 the_col = sel_indices[col]; 00434 if (the_col == -1) 00435 return MISSING_VALUE; 00436 return source->getStringVal(the_col, str); 00437 } 00438 00440 // getRealToStringMapping // 00442 const map<real,string>& SelectColumnsVMatrix::getRealToStringMapping(int col) const { 00443 int the_col; 00444 static map<real, string> empty_mapping; 00445 the_col = sel_indices[col]; 00446 if (the_col == -1) 00447 return empty_mapping; 00448 return source->getRealToStringMapping(the_col); 00449 } 00450 00452 // getValString // 00454 string SelectColumnsVMatrix::getValString(int col, real val) const { 00455 int the_col; 00456 the_col = sel_indices[col]; 00457 if (the_col == -1) 00458 return ""; 00459 return source->getValString(the_col, val); 00460 } 00461 00462 PP<Dictionary> SelectColumnsVMatrix::getDictionary(int col) const 00463 { 00464 if(col>=width_) 00465 PLERROR("access out of bound. Width=%i accessed col=%i",width_,col); 00466 int the_col; 00467 the_col = sel_indices[col]; 00468 if (the_col == -1) 00469 return 0; 00470 return source->getDictionary(the_col); 00471 } 00472 00473 00474 void SelectColumnsVMatrix::getValues(int row, int col, Vec& values) const 00475 { 00476 if(col>=width_) 00477 PLERROR("access out of bound. Width=%i accessed col=%i",width_,col); 00478 int the_col; 00479 the_col = sel_indices[col]; 00480 if (the_col == -1) 00481 values.resize(0); 00482 else 00483 source->getValues(row,the_col,values); 00484 } 00485 00486 void SelectColumnsVMatrix::getValues(const Vec& input, int col, Vec& values) const 00487 { 00488 if(col>=width_) 00489 PLERROR("access out of bound. Width=%i accessed col=%i",width_,col); 00490 int the_col; 00491 the_col = sel_indices[col]; 00492 if (the_col == -1) 00493 values.resize(0); 00494 else 00495 { 00496 for(int i=0; i<sel_indices.length(); i++) 00497 sinput[sel_indices[i]] = input[i]; 00498 source->getValues(sinput, the_col, values); 00499 } 00500 } 00501 00502 } // end of namespace PLearn 00503 00504 00505 /* 00506 Local Variables: 00507 mode:c++ 00508 c-basic-offset:4 00509 c-file-style:"stroustrup" 00510 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00511 indent-tabs-mode:nil 00512 fill-column:79 00513 End: 00514 */ 00515 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :