PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2003 Olivier Delalleau 00008 // 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ******************************************************************* 00039 * $Id: MissingIndicatorVMatrix.cc 3658 2005-07-06 20:30:15 Godbout $ 00040 ******************************************************************* */ 00041 00042 00043 #include "MissingIndicatorVMatrix.h" 00044 #include <plearn/math/TMat_maths.h> 00045 #include <plearn/io/load_and_save.h> 00046 00047 namespace PLearn { 00048 using namespace std; 00049 00052 PLEARN_IMPLEMENT_OBJECT( 00053 MissingIndicatorVMatrix, 00054 "VMatrix class to add a missing indicator for each variable.", 00055 "For each variable with a missing value in the referenced train set, an indicator is added.\n" 00056 "It is set to 1 if the value of the corresponding variable`in the underlying dataset is missing.\n" 00057 "It is set to 0 otherwise.\n" 00058 ); 00059 00060 MissingIndicatorVMatrix::MissingIndicatorVMatrix() 00061 : number_of_train_samples_to_use(0.0) 00062 { 00063 } 00064 00065 MissingIndicatorVMatrix::MissingIndicatorVMatrix(VMat the_source, VMat the_train_set, real the_number_of_train_samples_to_use) 00066 { 00067 source = the_source; 00068 train_set = the_train_set; 00069 number_of_train_samples_to_use = the_number_of_train_samples_to_use; 00070 } 00071 00072 MissingIndicatorVMatrix::~MissingIndicatorVMatrix() 00073 { 00074 } 00075 00076 void MissingIndicatorVMatrix::declareOptions(OptionList &ol) 00077 { 00078 declareOption(ol, "source", &MissingIndicatorVMatrix::source, OptionBase::buildoption, 00079 "The source VMatrix with missing values.\n"); 00080 00081 declareOption(ol, "train_set", &MissingIndicatorVMatrix::train_set, OptionBase::buildoption, 00082 "A referenced train set.\n" 00083 "A missing indicator is added for variables with missing values in this data set.\n" 00084 "It is used in combination with the option number_of_train_samples_to_use\n"); 00085 00086 declareOption(ol, "number_of_train_samples_to_use", &MissingIndicatorVMatrix::number_of_train_samples_to_use, OptionBase::buildoption, 00087 "The number of samples from the train set that will be examined to see\n" 00088 "if an indicator should be added for each variable\n"); 00089 00090 declareOption(ol, "fields", &MissingIndicatorVMatrix::fields, OptionBase::buildoption, 00091 "The names of the fields to extract if the train_set is not provided."); 00092 00093 declareOption(ol, "save_fields_with_missing", 00094 &MissingIndicatorVMatrix::save_fields_with_missing, 00095 OptionBase::buildoption, 00096 "The file name where we save the name of the fields that we" 00097 " add an indicator. It can be reused with the options fields" 00098 " to redo the same processing."); 00099 00100 inherited::declareOptions(ol); 00101 } 00102 00103 void MissingIndicatorVMatrix::build() 00104 { 00105 inherited::build(); 00106 build_(); 00107 } 00108 00109 void MissingIndicatorVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00110 { 00111 deepCopyField(source, copies); 00112 deepCopyField(train_set, copies); 00113 deepCopyField(number_of_train_samples_to_use, copies); 00114 deepCopyField(source_rel_pos, copies); 00115 deepCopyField(fields, copies); 00116 00117 inherited::makeDeepCopyFromShallowCopy(copies); 00118 } 00119 00120 void MissingIndicatorVMatrix::getExample(int i, Vec& input, Vec& target, real& weight) 00121 { 00122 source->getExample(i, source_input, target, weight); 00123 for (int source_col = 0, new_col = 0; source_col < source_inputsize; 00124 source_col++) 00125 { 00126 input[new_col] = source_input[source_col]; 00127 new_col += 1; 00128 if (source_rel_pos[source_col] < 0) 00129 { 00130 if (is_missing(source_input[source_col])) input[new_col] = 1.0; 00131 else input[new_col] = 0.0; 00132 new_col += 1; 00133 } 00134 } 00135 } 00136 00137 real MissingIndicatorVMatrix::get(int i, int j) const 00138 { 00139 if (source_rel_pos[j] < 0) 00140 { 00141 if (is_missing(source->get(i, source_rel_pos[j - 1]))) return 1.0; 00142 else return 0.0; 00143 } 00144 return source->get(i, source_rel_pos[j]); 00145 } 00146 00147 void MissingIndicatorVMatrix::getColumn(int i, Vec v) const 00148 { 00149 if (source_rel_pos[i] < 0) source->getColumn(source_rel_pos[i - 1], v); 00150 else source->getColumn(source_rel_pos[i], v); 00151 if (source_rel_pos[i] >= 0) return; 00152 for (int source_row = 0; source_row < v->length(); source_row++) 00153 { 00154 if (is_missing(v[source_row])) v[source_row] = 1.0; 00155 else v[source_row] = 0.0; 00156 } 00157 } 00158 00159 void MissingIndicatorVMatrix::build_() 00160 { 00161 if (!source) PLERROR("In MissingIndicatorVMatrix:: source vmat must be supplied"); 00162 if(source->inputsize()<=0) 00163 PLERROR("In MissingIndicatorVMatrix::build_() source must have an inputsize <= 0." 00164 " inputsize = %d", source->inputsize()); 00165 if(!train_set && !fields){ 00166 train_set = source; 00167 } 00168 updateMtime(source); 00169 if(train_set) 00170 updateMtime(train_set); 00171 buildNewRecordFormat(); 00172 } 00173 00174 void MissingIndicatorVMatrix::buildNewRecordFormat() 00175 { 00176 source_inputsize = source->inputsize(); 00177 TVec<int> train_var_missing(source_inputsize); 00178 train_var_missing.clear(); 00179 int source_width = source->width(); 00180 00181 if(train_set){ 00182 int train_length = train_set->length(); 00183 if (number_of_train_samples_to_use > 0.0){ 00184 if (number_of_train_samples_to_use < 1.0) train_length = (int) (number_of_train_samples_to_use * (real) train_length); 00185 else train_length = (int) number_of_train_samples_to_use; 00186 } 00187 if (train_length > train_set->length()) train_length = train_set->length(); 00188 00189 int train_width = train_set->width(); 00190 int train_inputsize = train_set->inputsize(); 00191 00192 if(train_length < 1) 00193 PLERROR("In MissingIndicatorVMatrix::length of the number of train" 00194 " samples to use must be at least 1, got: %i", train_length); 00195 if(train_inputsize < 1) 00196 PLERROR("In MissingIndicatorVMatrix::inputsize of the train vmat must" 00197 " be supplied, got : %i", train_inputsize); 00198 if (train_width != source_width) 00199 PLERROR("In MissingIndicatorVMatrix::train set and source width must" 00200 " agree, got : %i, %i", train_width, source_width); 00201 if (train_set->targetsize() != source->targetsize()) 00202 PLERROR("In MissingIndicatorVMatrix::train set and source targetsize" 00203 " must agree, got : %i, %i", train_set->targetsize(), 00204 source->targetsize()); 00205 if (train_set->weightsize() != source->weightsize()) 00206 PLERROR("In MissingIndicatorVMatrix::train set and source weightsize" 00207 " must agree, got : %i, %i", train_set->weightsize(), 00208 source->weightsize()); 00209 if (train_inputsize != source_inputsize) 00210 PLERROR("In MissingIndicatorVMatrix::train set and source inputsize" 00211 " must agree, got : %i, %i", train_inputsize, source_inputsize); 00212 00213 Vec train_input(train_width); 00214 00215 for (int train_row = 0; train_row < train_length; train_row++) 00216 { 00217 train_set->getRow(train_row, train_input); 00218 for (int train_col = 0; train_col < train_inputsize; train_col++) 00219 { 00220 if (is_missing(train_input[train_col])) 00221 train_var_missing[train_col] = 1; 00222 } 00223 } 00224 }else if(fields.size()>0){ 00225 TVec<string> source_field_names = source->fieldNames(); 00226 for(int i=0;i<fields.size();i++) 00227 { 00228 int index=source->getFieldIndex(fields[i]); 00229 if(index<0) 00230 PLERROR("In MissingIndicatorVMatrix::buildNewRecordFormat() - The fields '%s' is not in the source", 00231 fields[i].c_str()); 00232 else 00233 train_var_missing[index]=1; 00234 } 00235 00236 }else 00237 PLERROR("In MissingIndicatorVMatrix, the train_set or fields option should be provided."); 00238 00239 int added_colomns = sum(train_var_missing); 00240 width_ = source_width + added_colomns; 00241 00242 TVec<string> source_field_names(source_width); 00243 source_rel_pos.resize(width_); 00244 TVec<string> new_field_names(width_); 00245 source_field_names = source->fieldNames(); 00246 int new_col = 0; 00247 TVec<string> miss; 00248 for (int source_col = 0; source_col < source_inputsize; source_col++) 00249 { 00250 new_field_names[new_col] = source_field_names[source_col]; 00251 source_rel_pos[new_col] = source_col; 00252 new_col += 1; 00253 if (train_var_missing[source_col] > 0) 00254 { 00255 new_field_names[new_col] = source_field_names[source_col] + "_MI"; 00256 source_rel_pos[new_col] = -1; 00257 new_col += 1; 00258 miss->append(source->fieldName(source_col)); 00259 } 00260 } 00261 PLCHECK(miss->size()==added_colomns); 00262 for (int source_col = source_inputsize; source_col < source_width; source_col++) 00263 { 00264 new_field_names[new_col] = source_field_names[source_col]; 00265 source_rel_pos[new_col] = source_col; 00266 new_col += 1; 00267 } 00268 length_ = source->length(); 00269 defineSizes(source_inputsize + added_colomns, source->targetsize(), source->weightsize(), source->extrasize()); 00270 00271 source_input.resize(source_inputsize); 00272 declareFieldNames(new_field_names); 00273 00274 if(!save_fields_with_missing.empty()) 00275 PLearn::save(save_fields_with_missing,miss); 00276 00277 } 00278 00279 } // end of namespcae PLearn