PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2003, 2006 Olivier Delalleau 00008 // 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ********************************************************************* 00039 * $Id: VariableDeletionVMatrix.cc 3658 2005-07-06 20:30:15 Godbout $ 00040 ********************************************************************* */ 00041 00042 #include "VariableDeletionVMatrix.h" 00043 #include <plearn/vmat/SubVMatrix.h> 00044 #include <plearn/vmat/VMat_computeStats.h> 00045 #include <plearn/io/fileutils.h> 00046 #include <plearn/io/load_and_save.h> 00047 #define PL_LOG_MODULE_NAME "VariableDeletionVMatrix" 00048 #include <plearn/io/pl_log.h> 00049 00050 namespace PLearn { 00051 using namespace std; 00052 00053 PLEARN_IMPLEMENT_OBJECT( 00054 VariableDeletionVMatrix, 00055 "Deletes its source's inputs with most missing / constant values.", 00056 "The columns that are removed are those that:\n" 00057 "- contain a proportion of missing values higher than a threshold\n" 00058 " equal to 1 - min_non_missing_threshold, or\n" 00059 "- contain a single value whose proportion in the source dataset is\n" 00060 " higher than (or equal to) max_constant_threshold (if >0).\n" 00061 "\n" 00062 "Note that this selection is performed only on input variables (the\n" 00063 "inputsize is modified accordingly), while the target, weight and extra\n" 00064 "columns are always preserved.\n" 00065 ); 00066 00068 // VariableDeletionVMatrix // 00070 VariableDeletionVMatrix::VariableDeletionVMatrix(): 00071 min_non_missing_threshold(0), 00072 max_constant_threshold(0), 00073 number_of_train_samples(0), 00074 warn_removed_var(false), 00075 info_var_with_missing(false), 00076 deletion_threshold(-1), 00077 remove_columns_with_constant_value(-1) 00078 {} 00079 00080 VariableDeletionVMatrix::VariableDeletionVMatrix( 00081 VMat the_source, real the_min_non_missing_threshold, 00082 bool the_remove_columns_with_constant_value, 00083 int the_number_of_train_samples, 00084 bool call_build_): 00085 inherited(the_source, TVec<int>(), call_build_), 00086 min_non_missing_threshold(the_min_non_missing_threshold), 00087 number_of_train_samples(the_number_of_train_samples), 00088 remove_columns_with_constant_value(the_remove_columns_with_constant_value) 00089 { 00090 if (call_build_) 00091 build_(); 00092 PLDEPRECATED("In VariableDeletionVMatrix::VariableDeletionVMatrix - You " 00093 "are using a deprecated constructor"); 00094 // Note: this constructor should take as argument the new non-deprecated 00095 // options. 00096 } 00097 00099 // declareOptions // 00101 void VariableDeletionVMatrix::declareOptions(OptionList &ol) 00102 { 00103 00104 declareOption(ol, "min_non_missing_threshold", 00105 &VariableDeletionVMatrix::min_non_missing_threshold, 00106 OptionBase::buildoption, 00107 "Minimum proportion of non-missing values for a variable to be kept\n" 00108 "(a 1 means only variables with no missing values are kept).\n" 00109 "if >0, we will always remove columns with all missing value."); 00110 00111 declareOption(ol, "max_constant_threshold", 00112 &VariableDeletionVMatrix::max_constant_threshold, 00113 OptionBase::buildoption, 00114 "Maximum proportion of a unique value for a variable to be kept\n" 00115 "(contrary to 'min_non_missing_threshold', a variable will be\n" 00116 "removed if this proportion is attained, so that a 1 means only\n" 00117 "constant variables are removed, while 0 is a special value meaning\n" 00118 "that none is removed).\n" 00119 "Note also that this proportion is computed over the non-missing\n" 00120 "values only."); 00121 00122 declareOption(ol, "number_of_train_samples", 00123 &VariableDeletionVMatrix::number_of_train_samples, 00124 OptionBase::buildoption, 00125 "If equal to zero, all the underlying dataset samples are used to\n" 00126 "compute the percentages and constant values.\n" 00127 "If it is a fraction between 0 and 1, only this proportion of the\n" 00128 "samples will be used.\n" 00129 "If greater than or equal to 1, the integer portion will be\n" 00130 "interpreted as the number of samples to use."); 00131 00132 declareOption(ol, "warn_removed_var", 00133 &VariableDeletionVMatrix::warn_removed_var, 00134 OptionBase::buildoption, 00135 "If true, will print a warning about variable that are removed"); 00136 00137 declareOption(ol, "info_var_with_missing", 00138 &VariableDeletionVMatrix::info_var_with_missing, 00139 OptionBase::buildoption, 00140 "If true, will print the variable that have some missing" 00141 " that we keep."); 00142 00143 declareOption(ol, "save_deleted_columns", 00144 &VariableDeletionVMatrix::save_deleted_columns, 00145 OptionBase::buildoption, 00146 "If not empty will save the deleted culumns in this file." 00147 "If present, will verify that it have the same content then" 00148 " the calculated data."); 00149 00150 declareOption(ol, "complete_dataset", 00151 &VariableDeletionVMatrix::complete_dataset, 00152 OptionBase::learntoption, 00153 "DEPRECATED (use 'source' instead) - The data set with all variables\n" 00154 "to select the columns from."); 00155 00156 declareOption(ol, "train_set", &VariableDeletionVMatrix::train_set, OptionBase::buildoption, 00157 "The train set in which to compute the percentage of missing values.\n" 00158 "If null, will use the source to compute the percentage of missing values."); 00159 00160 declareOption(ol, "deletion_threshold", 00161 &VariableDeletionVMatrix::deletion_threshold, 00162 OptionBase::learntoption, 00163 "DEPRECATED (use 'min_non_missing_threshold' instead) - The\n" 00164 "percentage of non-missing values for a variable above which\n" 00165 "the variable will be selected."); 00166 00167 declareOption(ol, "remove_columns_with_constant_value", 00168 &VariableDeletionVMatrix::remove_columns_with_constant_value, 00169 OptionBase::learntoption, 00170 "DEPRECATED (use 'max_constant_threshold' instead) - If set to 1,\n" 00171 "the columns with constant non-missing values will be removed."); 00172 00173 inherited::declareOptions(ol); 00174 00175 // Hide unused parent class' options. 00176 00177 redeclareOption(ol, "fields", &VariableDeletionVMatrix::fields, 00178 OptionBase::nosave, 00179 "Not used in VariableDeletionVMatrix."); 00180 00181 redeclareOption(ol, "fields_partial_match", 00182 &VariableDeletionVMatrix::fields_partial_match, 00183 OptionBase::nosave, 00184 "Not used in VariableDeletionVMatrix."); 00185 00186 redeclareOption(ol, "indices", &VariableDeletionVMatrix::indices, 00187 OptionBase::nosave, 00188 "Not used in VariableDeletionVMatrix."); 00189 00190 redeclareOption(ol, "extend_with_missing", 00191 &VariableDeletionVMatrix::extend_with_missing, 00192 OptionBase::nosave, 00193 "Not used in VariableDeletionVMatrix."); 00194 00195 redeclareOption(ol, "inputsize", &VariableDeletionVMatrix::inputsize_, 00196 OptionBase::nosave, 00197 "Not used in VariableDeletionVMatrix."); 00198 00199 redeclareOption(ol, "targetsize", &VariableDeletionVMatrix::targetsize_, 00200 OptionBase::nosave, 00201 "Not used in VariableDeletionVMatrix."); 00202 00203 redeclareOption(ol, "weightsize", &VariableDeletionVMatrix::weightsize_, 00204 OptionBase::nosave, 00205 "Not used in VariableDeletionVMatrix."); 00206 00207 redeclareOption(ol, "extrasize", &VariableDeletionVMatrix::extrasize_, 00208 OptionBase::nosave, 00209 "Not used in VariableDeletionVMatrix."); 00210 } 00211 00213 // build // 00215 void VariableDeletionVMatrix::build() 00216 { 00217 //must be done even if we will call it later to have the 00218 //source metadatadir set correctly. 00219 bool saved_warn_non_selected_field=warn_non_selected_field; 00220 warn_non_selected_field=false; 00221 inherited::build(); 00222 warn_non_selected_field=saved_warn_non_selected_field; 00223 build_(); 00224 } 00225 00227 // makeDeepCopyFromShallowCopy // 00229 void VariableDeletionVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00230 { 00231 inherited::makeDeepCopyFromShallowCopy(copies); 00232 deepCopyField(complete_dataset, copies); 00233 deepCopyField(train_set, copies); 00234 } 00235 00237 // build_ // 00239 void VariableDeletionVMatrix::build_() 00240 { 00241 if (complete_dataset) { 00242 PLDEPRECATED("In VariableDeletionVMatrix::build_ - The option " 00243 "'complete_dataset' is deprecated, use source instead"); 00244 if (source && source != complete_dataset) 00245 PLERROR("In VariableDeletionVMatrix::build_ - A source was also " 00246 "specified, but it is different from 'complete_dataset'"); 00247 source = complete_dataset; 00248 } 00249 if (!is_equal(deletion_threshold, -1)) { 00250 PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the " 00251 "deprecated option 'deletion_threshold', you should " 00252 "instead use 'min_non_missing_threshold'"); 00253 min_non_missing_threshold = deletion_threshold; 00254 } 00255 if (remove_columns_with_constant_value != -1) { 00256 PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the " 00257 "deprecated option 'remove_columns_with_constant_value', " 00258 "you should instead use 'max_constant_threshold'"); 00259 max_constant_threshold = remove_columns_with_constant_value == 0 ? 0:1; 00260 } 00261 if(!source) 00262 PLERROR("In VariableDeletionVMatrix::build_ - The source VMat do not exist!"); 00263 00264 updateMtime(source); 00265 updateMtime(train_set); 00266 00267 int is = source->inputsize(); 00268 if (is < 0) 00269 PLERROR("In VariableDeletionVMatrix::build_ - The source VMat must " 00270 "have an inputsize defined"); 00271 00272 VMat the_train_source = train_set ? train_set : source; 00273 PLCHECK( the_train_source.width() == source->width() ); 00274 00275 if (number_of_train_samples > 0 && 00276 number_of_train_samples < the_train_source->length()) 00277 the_train_source = new SubVMatrix(the_train_source, 0, 0, 00278 number_of_train_samples, 00279 the_train_source->width()); 00280 TVec<StatsCollector> stats; 00281 if(min_non_missing_threshold > 0 || max_constant_threshold > 0){ 00282 int maxnvalues = -1; 00283 if(is_equal(max_constant_threshold,1)) 00284 //We don't need all the value, if (min==max && non_missing_value>0) it is constant value. 00285 maxnvalues = 0; 00286 if(!the_train_source->hasMetaDataDir() && hasMetaDataDir()) 00287 the_train_source->setMetaDataDir(getMetaDataDir()+"/source"); 00288 stats = the_train_source-> 00289 getPrecomputedStatsFromFile("stats_variableDeletionVMatrix_"+ 00290 tostring(maxnvalues)+".psave", 00291 maxnvalues, true); 00292 PLCHECK( stats.length() == source->width() ); 00293 } 00294 00295 indices.resize(0); 00296 00297 // First remove columns that have too many missing values. 00298 if (min_non_missing_threshold > 0){ 00299 int min_non_missing = 00300 int(round(min_non_missing_threshold * the_train_source->length())); 00301 TVec<int> have_missing; 00302 for (int i = 0; i < is; i++){ 00303 if (stats[i].nnonmissing() >= min_non_missing 00304 && stats[i].nnonmissing() > 0) 00305 indices.append(i); 00306 else if (warn_removed_var) 00307 PLWARNING("In VariableDeletionVMatrix::build_() var '%s'" 00308 " have too many missing (%d/%d). We remove it.", 00309 source->fieldName(i).c_str(), 00310 int(stats[i].nmissing()), 00311 int(stats[i].n())); 00312 if (info_var_with_missing && stats[i].nmissing() > 0){ 00313 have_missing.append(i); 00314 } 00315 } 00316 if(have_missing.length()>0){ 00317 string s="INFO: In build_() variable with missing value (var,nb_missing/nb_value): "; 00318 for(int k=0;k<have_missing.length();k++){ 00319 int i = have_missing[k]; 00320 s+=" ("+source->fieldName(i) 00321 +","+tostring(stats[i].nmissing()) 00322 +"/"+ tostring(stats[i].n())+")"; 00323 00324 } 00325 MODULE_LOG<<s<<endl; 00326 } 00327 00328 } else 00329 for (int i = 0; i < is; i++) 00330 indices.append(i); 00331 // Then remove columns that are too constant. 00332 TVec<int> final_indices; 00333 if (is_equal(max_constant_threshold,1)){ 00334 TVec<int> const_indices; 00335 for (int k = 0; k < indices.length(); k++) { 00336 int i = indices[k]; 00337 StatsCollector stat = stats[i]; 00338 if(!(stat.min()==stat.max() && stat.nnonmissing()>0)) 00339 final_indices.append(i); 00340 else if (warn_removed_var) 00341 const_indices.append(i); 00342 } 00343 if(warn_removed_var && const_indices.length()>0){ 00344 string s = " WARNING: In VariableDeletionVMatrix::build_() - The following tuple (variable, constant value) indicate variable that are removed because they are constant: \n"; 00345 for(int k=0;k<const_indices.length();k++){ 00346 int i = const_indices[k]; 00347 StatsCollector stat = stats[i]; 00348 s+=" ("+source->fieldName(i)+","+tostring(stat.min())+")"; 00349 } 00350 NORMAL_LOG<<s<<endl; 00351 } 00352 indices.resize(final_indices.length()); 00353 indices << final_indices; 00354 }else if (max_constant_threshold > 0){ 00355 for (int k = 0; k < indices.length(); k++) { 00356 int i = indices[k]; 00357 int max_constant_absolute = 00358 int(round(max_constant_threshold * stats[i].nnonmissing())); 00359 map<real, StatsCollectorCounts>* counts = stats[i].getCounts(); 00360 map<real, StatsCollectorCounts>::const_iterator it; 00361 bool ok = true; 00362 int n; 00363 for (it = counts->begin(); ok && it != counts->end(); it++) { 00364 n = int(round(it->second.n)); 00365 if (n >= max_constant_absolute) 00366 ok = false; 00367 } 00368 if (ok) 00369 final_indices.append(i); 00370 else if (warn_removed_var) 00371 PLWARNING("In VariableDeletionVMatrix::build_() var '%s'" 00372 " is too constant. Value %f happen %d/%f", 00373 source->fieldName(i).c_str(), it->first, 00374 n, stats[i].nnonmissing()); 00375 } 00376 indices.resize(final_indices.length()); 00377 indices << final_indices; 00378 } 00379 // Define sizes. 00380 inputsize_ = indices.length(); 00381 targetsize_ = source->targetsize(); 00382 weightsize_ = source->weightsize(); 00383 extrasize_ = source->extrasize(); 00384 00385 // Add target, weight and extra columns. 00386 for (int i = 0; i < source->targetsize(); i++) 00387 indices.append(is + i); 00388 if (source->targetsize() > 0) 00389 is += source->targetsize(); 00390 for (int i = 0; i < source->weightsize(); i++) 00391 indices.append(is + i); 00392 if (source->weightsize() > 0) 00393 is += source->weightsize(); 00394 for (int i = 0; i < source->extrasize(); i++) 00395 indices.append(is + i); 00396 00397 // We have modified the selected columns, so the parent class must be 00398 // re-built. 00399 inherited::build(); 00400 00401 if(!save_deleted_columns.empty()){ 00402 if(isfile(save_deleted_columns)){ 00403 TVec<int> indices2; 00404 PLearn::load(save_deleted_columns, indices2); 00405 if(indices!=indices2){ 00406 PLWARNING("In VariableDeletionVMatrix::build_() - the calculated" 00407 " indices(%d) differ from the saved indices(%d) in file '%s'." 00408 " We overwrite it.", 00409 indices2.length(), indices.length(), save_deleted_columns.c_str()); 00410 PLearn::save(save_deleted_columns,indices); 00411 } 00412 }else{ 00413 PLearn::save(save_deleted_columns,indices); 00414 } 00415 } 00416 } 00417 00418 } // end of namespace PLearn 00419 00420 00421 /* 00422 Local Variables: 00423 mode:c++ 00424 c-basic-offset:4 00425 c-file-style:"stroustrup" 00426 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00427 indent-tabs-mode:nil 00428 fill-column:79 00429 End: 00430 */ 00431 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :