PLearn 0.1
SelectColumnsVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003 Olivier Delalleau
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 //
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 //
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 //
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 //
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 //
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************
00039  * $Id: SelectColumnsVMatrix.cc 9773 2008-12-11 19:18:44Z nouiz $
00040  ******************************************************* */
00041 
00042 #include "SelectColumnsVMatrix.h"
00043 #include <plearn/io/load_and_save.h>
00044 #define PL_LOG_MODULE_NAME "SelectColumnsVMatrix"
00045 #include <plearn/io/pl_log.h>
00046 
00047 
00048 namespace PLearn {
00049 using namespace std;
00050 
00053 PLEARN_IMPLEMENT_OBJECT(SelectColumnsVMatrix,
00054                         "Selects variables from a source matrix according to given vector of indices.",
00055                         "Alternatively, the variables can be given by their names. In this case, if a\n"
00056                         "field does not exist and the 'extend_with_missing' option is set to 'true',\n"
00057                         "the field will be added and filled with missing values.\n"
00058     );
00059 
00061 // SelectColumnsVMatrix //
00063 SelectColumnsVMatrix::SelectColumnsVMatrix()
00064     : extend_with_missing(false),
00065       fields_partial_match(false),
00066       fields_regex(false),
00067       inverse_fields_selection(false),
00068       warn_non_selected_field(false)
00069 {}
00070 
00071 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source,
00072                                            TVec<string> the_fields,
00073                                            bool the_extend_with_missing,
00074                                            bool call_build_):
00075     inherited(the_source, call_build_),
00076     extend_with_missing(the_extend_with_missing),
00077     fields(the_fields),
00078     fields_partial_match(false),
00079     fields_regex(false),
00080     inverse_fields_selection(false),
00081     warn_non_selected_field(false)
00082 {
00083     if (call_build_)
00084         build_();
00085 }
00086 
00087 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source,
00088                                            TVec<int> the_indices,
00089                                            bool call_build_):
00090     inherited(the_source, call_build_),
00091     extend_with_missing(false),
00092     indices(the_indices),
00093     fields_partial_match(false),
00094     fields_regex(false),
00095     inverse_fields_selection(false),
00096     warn_non_selected_field(false)
00097 {
00098     if (call_build_)
00099         build_();
00100 }
00101 
00102 SelectColumnsVMatrix::SelectColumnsVMatrix(VMat the_source, Vec the_indices)
00103     : extend_with_missing(false),
00104       fields_partial_match(false),
00105       fields_regex(false),
00106       inverse_fields_selection(false),
00107       warn_non_selected_field(false)
00108 {
00109     source = the_source;
00110     indices.resize(the_indices.length());
00111     // copy the real the_indices into the integer indices
00112     indices << the_indices;
00113     build_();
00114 }
00115 
00117 // get //
00119 real SelectColumnsVMatrix::get(int i, int j) const {
00120 #ifdef BOUNDCHECK
00121     if(i>=length_ ||j>=width_||i<0||j<0)
00122         PLERROR("In SelectColumnsVMatrix::getSubRow - "
00123                 "requested index: (%d,%d) but width of %d and length of %d",
00124                 i,j,width_,length_);
00125 #endif
00126     int col;
00127     col = sel_indices[j];
00128     if (col == -1)
00129         return MISSING_VALUE;
00130     return source->get(i, col);
00131 }
00132 
00134 // getSubRow //
00136 void SelectColumnsVMatrix::getSubRow(int i, int j, Vec v) const
00137 {
00138 #ifdef BOUNDCHECK
00139     if(i>=length_ ||j>=width_||i<0||j<0)
00140         PLERROR("In SelectColumnsVMatrix::getSubRow - "
00141                 "requested index: (%d,%d) but width of %d and length of %d",
00142                 i,j,width_,length_);
00143 #endif
00144     int col;
00145     for(int jj=0; jj<v.length(); jj++) {
00146         col = sel_indices[j+jj];
00147         if (col == -1)
00148             v[jj] = MISSING_VALUE;
00149         else
00150             v[jj] = source->get(i, col);
00151     }
00152 }
00153 
00155 // declareOptions //
00157 void SelectColumnsVMatrix::declareOptions(OptionList &ol)
00158 {
00159     declareOption(ol, "fields", &SelectColumnsVMatrix::fields, OptionBase::buildoption,
00160                   "The names of the fields to extract (will override 'indices' if provided). Can\n"
00161                   "be a range of the form Field_1-Field_n, if 'extend_with_missing' is false.");
00162 
00163     declareOption(ol, "save_fields", &SelectColumnsVMatrix::save_fields,
00164                   OptionBase::buildoption,
00165         "Optional file where the fieldnames of this VMatrix should be saved.");
00166 
00167     declareOption(ol, "fields_partial_match", &SelectColumnsVMatrix::fields_partial_match, OptionBase::buildoption,
00168                   "If set to 1, then a field will be kept iff it contains one of the strings from 'fields'.");
00169 
00170     declareOption(ol, "fields_regex", &SelectColumnsVMatrix::fields_regex, OptionBase::buildoption,
00171                   "If set to 1, then a field will be kept iff it match one of the strings from 'fields'. A match is when the two string is equal or when fields end with a *, we allow any end.");
00172 
00173     declareOption(ol, "indices", &SelectColumnsVMatrix::indices, OptionBase::buildoption,
00174                   "The array of column indices to extract.");
00175 
00176     declareOption(ol, "extend_with_missing", &SelectColumnsVMatrix::extend_with_missing, OptionBase::buildoption,
00177                   "If set to 1, then fields specified in the 'fields' option that do not exist\n"
00178                   "in the source VMatrix will be filled with missing values.");
00179 
00180     declareOption(ol, "inverse_fields_selection",
00181                   &SelectColumnsVMatrix::inverse_fields_selection,
00182                   OptionBase::buildoption,
00183         "If set to true, the selection is reversed. This provides an easy\n"
00184         "way to specify columns we do not want.");
00185 
00186     declareOption(ol, "warn_non_selected_field",
00187                   &SelectColumnsVMatrix::warn_non_selected_field,
00188                   OptionBase::buildoption,
00189                   "If set to true, we generate a PLWARNING with all fields "
00190                   "that are not selected.");
00191 
00192     inherited::declareOptions(ol);
00193 
00194     redeclareOption(ol, "length", &SelectColumnsVMatrix::length_,
00195                     OptionBase::nosave,
00196                     "Taken from source");
00197 
00198     redeclareOption(ol, "width", &SelectColumnsVMatrix::width_,
00199                     OptionBase::nosave,
00200                     "Computed from parameter");
00201 }
00202 
00204 // makeDeepCopyFromShallowCopy //
00206 void SelectColumnsVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00207 {
00208     inherited::makeDeepCopyFromShallowCopy(copies);
00209     deepCopyField(sinput, copies);
00210     deepCopyField(indices, copies);
00211     deepCopyField(fields, copies);
00212     deepCopyField(sel_indices, copies);
00213 }
00214 
00216 // build //
00218 void SelectColumnsVMatrix::build()
00219 {
00220     inherited::build();
00221     build_();
00222 }
00223 
00225 // build_ //
00227 void SelectColumnsVMatrix::build_()
00228 {
00229     if (source) {
00230         updateMtime(source);
00231         if (fields.isNotEmpty()) {
00232             getIndicesFromFields();
00233         }
00234         
00235         //we inverse the selection
00236         if(inverse_fields_selection){
00237             TVec<int> newindices;
00238             for (int i = 0;i<source.width();i++){
00239                 bool found=false;
00240                 for(int j=0;j<indices.size();j++)
00241                     if(indices[j]==i){
00242                         found = true;
00243                         break;
00244                     }
00245                 if(!found)
00246                     newindices.append(i);
00247             }
00248             sel_indices = newindices;
00249         } else
00250             sel_indices = indices;
00251 
00252         if(warn_non_selected_field){
00253             TVec<string> v;
00254             for(int i=0;i<source.width();i++)
00255             {
00256                 bool found=false;
00257                 for(int j=0;j<sel_indices.size();j++)
00258                     if(sel_indices[j]==i){
00259                         found=true;
00260                         break;
00261                     }
00262                 if(!found)
00263                     v.append(source.fieldName(i));
00264             }
00265             if(v.size()>0)
00266                 PLWARNING("In SelectColumnsVMatrix::build_() - There are %d"
00267                           " columns in the source matrix that we do not "
00268                           "select: %s",v.size(),tostring(v).c_str());
00269         }
00270 
00271         // Copy matrix dimensions
00272         width_ = sel_indices.length();
00273         length_ = source->length();
00274 
00275         if(!extend_with_missing)
00276             // The new width cannot exceed the old one.
00277             PLCHECK(source->width() >= width_);
00278 
00279         //if we don't add new columns and the source columns type are emtpy,
00280         //they should be empty in this matrix if they are not already set.
00281         if(targetsize_<0 && !extend_with_missing && source->targetsize()==0)
00282             targetsize_=0;
00283         if(weightsize_<0 && !extend_with_missing && source->weightsize()==0)
00284             weightsize_=0;
00285         computeMissingSizeValue(false);
00286 
00287 #if 0
00288         // Disabled for now, since it gives way too many false positives in
00289         // some cases. Todo: figure out a way to warn in a useful way when
00290         // a SelectColumnsVMatrix is given as a train or test set to a learner,
00291         // with inputsize and friends not set manually.
00292 
00293         // Give warning if inputsize, and friends are not specified, since we
00294         // can't just copy the values from the underlying VMat. A smarter
00295         // version of this class could Figure out the right size for inputsize,
00296         // targetsize and weightsize, at least for some cases (for example when
00297         // the indices are nondecreasing).
00298         if (inputsize_ < 0 || targetsize_ < 0 || weightsize_ < 0)
00299             PLWARNING("In SelectColumnsVMatrix::build_ inputsize, targetsize or weightsize "
00300                       "not set. You may want to set them yourself in the .plearn file.");
00301 #endif
00302 
00303         // Copy the appropriate VMFields
00304         fieldinfos.resize(width());
00305         if (source->getFieldInfos().size() > 0) {
00306             for (int i=0; i<width(); ++i) {
00307                 int col = sel_indices[i];
00308                 if (col == -1) {
00309                     if (extend_with_missing)
00310                         // This must be because it is a field that did not exist in
00311                         // the source VMat.
00312                         fieldinfos[i] = VMField(fields[i]);
00313                     else
00314                         PLERROR("In SelectColumnsVMatrix::build_ - '-1' is not a valid value in indices");
00315                 } else {
00316                     fieldinfos[i] = source->getFieldInfos(col);
00317                 }
00318             }
00319         }
00320 
00321         sinput.resize(width());
00322         sinput.fill(MISSING_VALUE);
00323 
00324         if(!save_fields.isEmpty())
00325             PLearn::save(save_fields, fieldNames());
00326     }
00327 }
00328 
00329 void SelectColumnsVMatrix::getIndicesFromFields(){
00330             // Find out the indices from the fields.
00331     indices.resize(0);
00332     if (!fields_partial_match && ! fields_regex) {
00333         TVec<string> missing_field;
00334         for (int i = 0; i < fields.length(); i++) {
00335             string the_field = fields[i];
00336             int the_index = source->fieldIndex(the_field);  // string only
00337             if (!extend_with_missing && the_index == -1) {
00338                 // The_field does not exist AS A STRING in the vmat
00339                 // It may be of the form FIELD1-FIELDN (a range of fields).
00340                 size_t pos = the_field.find('-');
00341                 bool ok = false;
00342                 if (pos != string::npos) {
00343                     string field1 = the_field.substr(0, pos);
00344                     string fieldn = the_field.substr(pos + 1);
00345                     int the_index1 = source->getFieldIndex(field1);  // either string or number
00346                     int the_indexn = source->getFieldIndex(fieldn);  // either string or number
00347                     if (the_index1 >= 0 && the_indexn > the_index1) {
00348                         // Indeed, this is a range.
00349                         ok = true;
00350                         for (int j = the_index1; j <= the_indexn; j++)
00351                             indices.append(j);
00352                     }
00353                 }
00354                 // OR it may be a number by itself only
00355                 else if (pl_islong(the_field) &&
00356                          (the_index = source->getFieldIndex(the_field)) != -1)
00357                 {
00358                     ok = true;
00359                     indices.append(the_index);
00360                 }
00361                 if (!ok && !inverse_fields_selection)
00362                     PLERROR("In SelectColumnsVMatrix::build_ - Unknown field \"%s\" in source VMat;\n"
00363                             "    (you may want to use the 'extend_with_missing' option)", the_field.c_str());
00364                 else if(!ok)
00365                     PLWARNING("In SelectColumnsVMatrix::build_ - Unknown field \"%s\" in source VMat;\n"
00366                               " as we want to ignore this field, we ignore this error.", the_field.c_str());
00367                     
00368             } else
00369                 indices.append(the_index);
00370             if(extend_with_missing && the_index == -1)
00371                 missing_field.append(the_field);
00372         }
00373         if(missing_field.size()>0){
00374             PLWARNING("In SelectColumnsVMatrix::build_() - We are"
00375                       " adding %d columns to the source matrix with missing values."
00376                       " The columns names are: %s",missing_field.size(),
00377                       tostring(missing_field).c_str());
00378         }
00379 
00380     } else if(fields_regex) {
00381         // We need to check whether or not we should add each field.
00382         TVec<string> source_fields = source->fieldNames();
00383         for (int j = 0; j < fields.length(); j++){
00384             string f = fields[j];
00385             if(f[f.size()-1]=='*'){
00386                 f.resize(f.size()-1);//remove the last caracter (*)
00387                 for (int i = 0; i < source_fields.length(); i++)
00388                     if (string_begins_with(source_fields[i],f)) {
00389                         // We want to add this field.
00390                         indices.append(i);
00391                         DBG_MODULE_LOG<<fields[j]<<" matched "<<source_fields[i]<<endl;
00392                     }
00393             }else{
00394                 for (int i = 0; i < source_fields.length(); i++)
00395                     if(source_fields[i]==f){
00396                         indices.append(i);
00397                         //their should be only one fields with a given name
00398                         break;
00399                     }
00400             }
00401         }
00402     } else {
00403         // We need to check whether or not we should add each field.
00404         TVec<string> source_fields = source->fieldNames();
00405         for (int i = 0; i < source_fields.length(); i++)
00406             for (int j = 0; j < fields.length(); j++)
00407                 if (source_fields[i].find(fields[j]) != string::npos) {
00408                     // We want to add this field.
00409                     indices.append(i);
00410                     DBG_MODULE_LOG<<fields[j]<<" matched "<<source_fields[i]<<endl;
00411                     break;
00412                 }
00413     }
00414 }
00415 
00417 // getStringToRealMapping //
00419 const map<string,real>& SelectColumnsVMatrix::getStringToRealMapping(int col) const {
00420     int the_col;
00421     static map<string, real> empty_mapping;
00422     the_col = sel_indices[col];
00423     if (the_col == -1)
00424         return empty_mapping;
00425     return source->getStringToRealMapping(the_col);
00426 }
00427 
00429 // getStringVal //
00431 real SelectColumnsVMatrix::getStringVal(int col, const string & str) const {
00432     int the_col;
00433     the_col = sel_indices[col];
00434     if (the_col == -1)
00435         return MISSING_VALUE;
00436     return source->getStringVal(the_col, str);
00437 }
00438 
00440 // getRealToStringMapping //
00442 const map<real,string>& SelectColumnsVMatrix::getRealToStringMapping(int col) const {
00443     int the_col;
00444     static map<real, string> empty_mapping;
00445     the_col = sel_indices[col];
00446     if (the_col == -1)
00447         return empty_mapping;
00448     return source->getRealToStringMapping(the_col);
00449 }
00450 
00452 // getValString //
00454 string SelectColumnsVMatrix::getValString(int col, real val) const {
00455     int the_col;
00456     the_col = sel_indices[col];
00457     if (the_col == -1)
00458         return "";
00459     return source->getValString(the_col, val);
00460 }
00461 
00462 PP<Dictionary> SelectColumnsVMatrix::getDictionary(int col) const
00463 {
00464     if(col>=width_)
00465         PLERROR("access out of bound. Width=%i accessed col=%i",width_,col);
00466     int the_col;
00467     the_col = sel_indices[col];
00468     if (the_col == -1)
00469         return 0;
00470     return source->getDictionary(the_col);
00471 }
00472 
00473 
00474 void SelectColumnsVMatrix::getValues(int row, int col, Vec& values) const
00475 {
00476     if(col>=width_)
00477         PLERROR("access out of bound. Width=%i accessed col=%i",width_,col);
00478     int the_col;
00479     the_col = sel_indices[col];
00480     if (the_col == -1)
00481         values.resize(0);
00482     else
00483         source->getValues(row,the_col,values);
00484 }
00485 
00486 void SelectColumnsVMatrix::getValues(const Vec& input, int col, Vec& values) const
00487 {
00488     if(col>=width_)
00489         PLERROR("access out of bound. Width=%i accessed col=%i",width_,col);
00490     int the_col;
00491     the_col = sel_indices[col];
00492     if (the_col == -1)
00493         values.resize(0);
00494     else
00495     {
00496         for(int i=0; i<sel_indices.length(); i++)
00497             sinput[sel_indices[i]] = input[i];
00498         source->getValues(sinput, the_col, values);
00499     }
00500 }
00501 
00502 } // end of namespace PLearn
00503 
00504 
00505 /*
00506   Local Variables:
00507   mode:c++
00508   c-basic-offset:4
00509   c-file-style:"stroustrup"
00510   c-file-offsets:((innamespace . 0)(inline-open . 0))
00511   indent-tabs-mode:nil
00512   fill-column:79
00513   End:
00514 */
00515 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines