PLearn 0.1
MissingIndicatorVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003 Olivier Delalleau
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************************    
00039    * $Id: MissingIndicatorVMatrix.cc 3658 2005-07-06 20:30:15  Godbout $
00040    ******************************************************************* */
00041 
00042 
00043 #include "MissingIndicatorVMatrix.h"
00044 #include <plearn/math/TMat_maths.h>
00045 #include <plearn/io/load_and_save.h>
00046 
00047 namespace PLearn {
00048 using namespace std;
00049 
00052 PLEARN_IMPLEMENT_OBJECT(
00053   MissingIndicatorVMatrix,
00054   "VMatrix class to add a missing indicator for each variable.",
00055   "For each variable with a missing value in the referenced train set, an indicator is added.\n"
00056   "It is set to 1 if the value of the corresponding variable`in the underlying dataset is missing.\n"
00057   "It is set to 0 otherwise.\n"
00058   );
00059 
00060 MissingIndicatorVMatrix::MissingIndicatorVMatrix()
00061 : number_of_train_samples_to_use(0.0)
00062 {
00063 }
00064 
00065 MissingIndicatorVMatrix::MissingIndicatorVMatrix(VMat the_source, VMat the_train_set, real the_number_of_train_samples_to_use)
00066 {
00067   source = the_source;
00068   train_set = the_train_set;
00069   number_of_train_samples_to_use = the_number_of_train_samples_to_use;
00070 }
00071 
00072 MissingIndicatorVMatrix::~MissingIndicatorVMatrix()
00073 {
00074 }
00075 
00076 void MissingIndicatorVMatrix::declareOptions(OptionList &ol)
00077 {
00078   declareOption(ol, "source", &MissingIndicatorVMatrix::source, OptionBase::buildoption, 
00079                 "The source VMatrix with missing values.\n");
00080 
00081   declareOption(ol, "train_set", &MissingIndicatorVMatrix::train_set, OptionBase::buildoption, 
00082                 "A referenced train set.\n"
00083                 "A missing indicator is added for variables with missing values in this data set.\n"
00084                 "It is used in combination with the option number_of_train_samples_to_use\n");
00085 
00086   declareOption(ol, "number_of_train_samples_to_use", &MissingIndicatorVMatrix::number_of_train_samples_to_use, OptionBase::buildoption, 
00087                 "The number of samples from the train set that will be examined to see\n"
00088                 "if an indicator should be added for each variable\n");
00089 
00090   declareOption(ol, "fields", &MissingIndicatorVMatrix::fields, OptionBase::buildoption,
00091                 "The names of the fields to extract if the train_set is not provided.");
00092 
00093   declareOption(ol, "save_fields_with_missing",
00094                 &MissingIndicatorVMatrix::save_fields_with_missing,
00095                 OptionBase::buildoption,
00096                 "The file name where we save the name of the fields that we"
00097                 " add an indicator. It can be reused with the options fields"
00098                 " to redo the same processing.");
00099 
00100   inherited::declareOptions(ol);
00101 }
00102 
00103 void MissingIndicatorVMatrix::build()
00104 {
00105   inherited::build();
00106   build_();
00107 }
00108 
00109 void MissingIndicatorVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00110 {
00111   deepCopyField(source, copies);
00112   deepCopyField(train_set, copies);
00113   deepCopyField(number_of_train_samples_to_use, copies);
00114   deepCopyField(source_rel_pos, copies);
00115   deepCopyField(fields, copies);
00116 
00117   inherited::makeDeepCopyFromShallowCopy(copies);
00118 }
00119 
00120 void MissingIndicatorVMatrix::getExample(int i, Vec& input, Vec& target, real& weight)
00121 {
00122     source->getExample(i, source_input, target, weight);
00123     for (int source_col = 0, new_col = 0; source_col < source_inputsize;
00124          source_col++)
00125     {
00126       input[new_col] = source_input[source_col];
00127       new_col += 1;
00128       if (source_rel_pos[source_col] < 0)
00129       {
00130           if (is_missing(source_input[source_col])) input[new_col] = 1.0;
00131           else input[new_col] = 0.0;
00132           new_col += 1;
00133       }
00134     }
00135 }
00136 
00137 real MissingIndicatorVMatrix::get(int i, int j) const
00138 {
00139   if (source_rel_pos[j] < 0)
00140   {
00141     if (is_missing(source->get(i, source_rel_pos[j - 1]))) return 1.0;
00142     else return 0.0;
00143   }
00144   return source->get(i, source_rel_pos[j]);
00145 }
00146 
00147 void MissingIndicatorVMatrix::getColumn(int i, Vec v) const
00148 {
00149   if (source_rel_pos[i] < 0) source->getColumn(source_rel_pos[i - 1], v);
00150   else source->getColumn(source_rel_pos[i], v);
00151   if (source_rel_pos[i] >= 0) return;
00152   for (int source_row = 0; source_row < v->length(); source_row++)
00153   {
00154     if (is_missing(v[source_row])) v[source_row] = 1.0;
00155     else v[source_row] = 0.0;
00156   } 
00157 }
00158 
00159 void MissingIndicatorVMatrix::build_()
00160 {
00161     if (!source) PLERROR("In MissingIndicatorVMatrix:: source vmat must be supplied");
00162     if(source->inputsize()<=0)
00163       PLERROR("In MissingIndicatorVMatrix::build_() source must have an inputsize <= 0."
00164               " inputsize = %d", source->inputsize());
00165     if(!train_set && !fields){
00166       train_set = source;
00167     }
00168     updateMtime(source);
00169     if(train_set)
00170       updateMtime(train_set);
00171     buildNewRecordFormat(); 
00172 }
00173 
00174 void MissingIndicatorVMatrix::buildNewRecordFormat()
00175 {
00176     source_inputsize = source->inputsize();
00177     TVec<int> train_var_missing(source_inputsize);
00178     train_var_missing.clear();
00179     int source_width = source->width();
00180 
00181     if(train_set){
00182       int train_length = train_set->length();
00183       if (number_of_train_samples_to_use > 0.0){
00184         if (number_of_train_samples_to_use < 1.0) train_length = (int) (number_of_train_samples_to_use * (real) train_length);
00185         else train_length = (int) number_of_train_samples_to_use;
00186       }
00187       if (train_length > train_set->length()) train_length = train_set->length();
00188 
00189       int train_width = train_set->width();
00190       int train_inputsize = train_set->inputsize();
00191 
00192       if(train_length < 1) 
00193         PLERROR("In MissingIndicatorVMatrix::length of the number of train"
00194                 " samples to use must be at least 1, got: %i", train_length);
00195       if(train_inputsize < 1) 
00196         PLERROR("In MissingIndicatorVMatrix::inputsize of the train vmat must"
00197                 " be supplied, got : %i", train_inputsize);
00198       if (train_width != source_width) 
00199         PLERROR("In MissingIndicatorVMatrix::train set and source width must"
00200                 " agree, got : %i, %i", train_width, source_width);
00201       if (train_set->targetsize() != source->targetsize())
00202         PLERROR("In MissingIndicatorVMatrix::train set and source targetsize"
00203                 " must agree, got : %i, %i", train_set->targetsize(),
00204                 source->targetsize());
00205       if (train_set->weightsize() != source->weightsize()) 
00206         PLERROR("In MissingIndicatorVMatrix::train set and source weightsize"
00207                 " must agree, got : %i, %i", train_set->weightsize(),
00208                 source->weightsize());
00209       if (train_inputsize != source_inputsize)
00210         PLERROR("In MissingIndicatorVMatrix::train set and source inputsize"
00211                 " must agree, got : %i, %i", train_inputsize, source_inputsize);
00212 
00213       Vec train_input(train_width);
00214 
00215       for (int train_row = 0; train_row < train_length; train_row++)
00216         {
00217           train_set->getRow(train_row, train_input);
00218           for (int train_col = 0; train_col < train_inputsize; train_col++)
00219             {
00220               if (is_missing(train_input[train_col])) 
00221                 train_var_missing[train_col] = 1;
00222             }
00223         }
00224     }else if(fields.size()>0){
00225       TVec<string> source_field_names = source->fieldNames();
00226       for(int i=0;i<fields.size();i++)
00227         {
00228           int index=source->getFieldIndex(fields[i]);
00229           if(index<0)
00230             PLERROR("In MissingIndicatorVMatrix::buildNewRecordFormat() - The fields '%s' is not in the source",
00231                     fields[i].c_str());
00232           else
00233             train_var_missing[index]=1;
00234         }
00235       
00236     }else
00237        PLERROR("In MissingIndicatorVMatrix, the train_set or fields option should be provided.");
00238 
00239     int added_colomns = sum(train_var_missing);
00240     width_ = source_width + added_colomns;
00241 
00242     TVec<string> source_field_names(source_width);
00243     source_rel_pos.resize(width_);
00244     TVec<string> new_field_names(width_);
00245     source_field_names = source->fieldNames();
00246     int new_col = 0;
00247     TVec<string> miss;
00248     for (int source_col = 0; source_col < source_inputsize; source_col++)
00249     {
00250       new_field_names[new_col] = source_field_names[source_col];
00251       source_rel_pos[new_col] = source_col;
00252       new_col += 1;
00253       if (train_var_missing[source_col] > 0)
00254       {
00255           new_field_names[new_col] = source_field_names[source_col] + "_MI";
00256           source_rel_pos[new_col] = -1;
00257           new_col += 1;
00258           miss->append(source->fieldName(source_col));
00259       }
00260     }
00261     PLCHECK(miss->size()==added_colomns);
00262     for (int source_col = source_inputsize; source_col < source_width; source_col++)
00263     {
00264       new_field_names[new_col] = source_field_names[source_col];
00265       source_rel_pos[new_col] = source_col;
00266       new_col += 1;
00267     }
00268     length_ = source->length();
00269     defineSizes(source_inputsize + added_colomns, source->targetsize(), source->weightsize(), source->extrasize());
00270 
00271     source_input.resize(source_inputsize);
00272     declareFieldNames(new_field_names);
00273 
00274     if(!save_fields_with_missing.empty())
00275       PLearn::save(save_fields_with_missing,miss);
00276       
00277 }
00278 
00279 } // end of namespcae PLearn
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines