PLearn 0.1
VariableDeletionVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003, 2006 Olivier Delalleau
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 //
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 //
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 //
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 //
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 //
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *********************************************************************
00039  * $Id: VariableDeletionVMatrix.cc 3658 2005-07-06 20:30:15  Godbout $
00040  ********************************************************************* */
00041 
00042 #include "VariableDeletionVMatrix.h"
00043 #include <plearn/vmat/SubVMatrix.h>
00044 #include <plearn/vmat/VMat_computeStats.h>
00045 #include <plearn/io/fileutils.h>
00046 #include <plearn/io/load_and_save.h>
00047 #define PL_LOG_MODULE_NAME "VariableDeletionVMatrix"
00048 #include <plearn/io/pl_log.h>
00049 
00050 namespace PLearn {
00051 using namespace std;
00052 
00053 PLEARN_IMPLEMENT_OBJECT(
00054     VariableDeletionVMatrix,
00055     "Deletes its source's inputs with most missing / constant values.",
00056     "The columns that are removed are those that:\n"
00057     "- contain a proportion of missing values higher than a threshold\n"
00058     "  equal to 1 - min_non_missing_threshold, or\n"
00059     "- contain a single value whose proportion in the source dataset is\n"
00060     "  higher than (or equal to) max_constant_threshold (if >0).\n"
00061     "\n"
00062     "Note that this selection is performed only on input variables (the\n"
00063     "inputsize is modified accordingly), while the target, weight and extra\n"
00064     "columns are always preserved.\n"
00065 );
00066 
00068 // VariableDeletionVMatrix //
00070 VariableDeletionVMatrix::VariableDeletionVMatrix():
00071     min_non_missing_threshold(0),
00072     max_constant_threshold(0),
00073     number_of_train_samples(0),
00074     warn_removed_var(false),
00075     info_var_with_missing(false),
00076     deletion_threshold(-1),
00077     remove_columns_with_constant_value(-1)
00078 {}
00079 
00080 VariableDeletionVMatrix::VariableDeletionVMatrix(
00081         VMat the_source, real the_min_non_missing_threshold, 
00082         bool the_remove_columns_with_constant_value,
00083         int  the_number_of_train_samples,
00084         bool call_build_):
00085     inherited(the_source, TVec<int>(), call_build_),
00086     min_non_missing_threshold(the_min_non_missing_threshold),
00087     number_of_train_samples(the_number_of_train_samples),
00088     remove_columns_with_constant_value(the_remove_columns_with_constant_value)
00089 {
00090     if (call_build_)
00091         build_();
00092     PLDEPRECATED("In VariableDeletionVMatrix::VariableDeletionVMatrix - You "
00093                  "are using a deprecated constructor");
00094     // Note: this constructor should take as argument the new non-deprecated
00095     // options.
00096 }
00097 
00099 // declareOptions //
00101 void VariableDeletionVMatrix::declareOptions(OptionList &ol)
00102 {
00103 
00104     declareOption(ol, "min_non_missing_threshold",
00105                   &VariableDeletionVMatrix::min_non_missing_threshold,
00106                   OptionBase::buildoption,
00107         "Minimum proportion of non-missing values for a variable to be kept\n"
00108         "(a 1 means only variables with no missing values are kept).\n"
00109         "if >0, we will always remove columns with all missing value.");
00110 
00111     declareOption(ol, "max_constant_threshold",
00112                   &VariableDeletionVMatrix::max_constant_threshold,
00113                   OptionBase::buildoption,
00114         "Maximum proportion of a unique value for a variable to be kept\n"
00115         "(contrary to 'min_non_missing_threshold', a variable will be\n"
00116         "removed if this proportion is attained, so that a 1 means only\n"
00117         "constant variables are removed, while 0 is a special value meaning\n"
00118         "that none is removed).\n"
00119         "Note also that this proportion is computed over the non-missing\n"
00120         "values only.");
00121 
00122     declareOption(ol, "number_of_train_samples",
00123                   &VariableDeletionVMatrix::number_of_train_samples,
00124                   OptionBase::buildoption,
00125         "If equal to zero, all the underlying dataset samples are used to\n"
00126         "compute the percentages and constant values.\n"
00127         "If it is a fraction between 0 and 1, only this proportion of the\n"
00128         "samples will be used.\n"
00129         "If greater than or equal to 1, the integer portion will be\n"
00130         "interpreted as the number of samples to use.");
00131 
00132     declareOption(ol, "warn_removed_var",
00133                   &VariableDeletionVMatrix::warn_removed_var,
00134                   OptionBase::buildoption,
00135                   "If true, will print a warning about variable that are removed");
00136 
00137     declareOption(ol, "info_var_with_missing",
00138                   &VariableDeletionVMatrix::info_var_with_missing,
00139                   OptionBase::buildoption,
00140                   "If true, will print the variable that have some missing"
00141                   " that we keep.");
00142 
00143     declareOption(ol, "save_deleted_columns",
00144                   &VariableDeletionVMatrix::save_deleted_columns,
00145                   OptionBase::buildoption,
00146                   "If not empty will save the deleted culumns in this file."
00147                   "If present, will verify that it have the same content then"
00148                   " the calculated data.");
00149 
00150     declareOption(ol, "complete_dataset",
00151                   &VariableDeletionVMatrix::complete_dataset,
00152                   OptionBase::learntoption,
00153         "DEPRECATED (use 'source' instead) - The data set with all variables\n"
00154         "to select the columns from.");
00155 
00156     declareOption(ol, "train_set", &VariableDeletionVMatrix::train_set, OptionBase::buildoption,
00157                   "The train set in which to compute the percentage of missing values.\n"
00158                   "If null, will use the source to compute the percentage of missing values.");
00159 
00160     declareOption(ol, "deletion_threshold",
00161                   &VariableDeletionVMatrix::deletion_threshold,
00162                   OptionBase::learntoption,
00163         "DEPRECATED (use 'min_non_missing_threshold' instead) - The\n"
00164         "percentage of non-missing values for a variable above which\n"
00165         "the variable will be selected.");
00166 
00167     declareOption(ol, "remove_columns_with_constant_value",
00168                   &VariableDeletionVMatrix::remove_columns_with_constant_value,
00169                   OptionBase::learntoption,
00170         "DEPRECATED (use 'max_constant_threshold' instead) - If set to 1,\n"
00171         "the columns with constant non-missing values will be removed.");
00172 
00173     inherited::declareOptions(ol);
00174 
00175     // Hide unused parent class' options.
00176 
00177     redeclareOption(ol, "fields", &VariableDeletionVMatrix::fields,
00178                                   OptionBase::nosave,
00179         "Not used in VariableDeletionVMatrix.");
00180 
00181     redeclareOption(ol, "fields_partial_match",
00182                     &VariableDeletionVMatrix::fields_partial_match,
00183                     OptionBase::nosave,
00184         "Not used in VariableDeletionVMatrix.");
00185 
00186     redeclareOption(ol, "indices", &VariableDeletionVMatrix::indices,
00187                                    OptionBase::nosave,
00188         "Not used in VariableDeletionVMatrix.");
00189 
00190     redeclareOption(ol, "extend_with_missing",
00191                     &VariableDeletionVMatrix::extend_with_missing,
00192                     OptionBase::nosave,
00193         "Not used in VariableDeletionVMatrix.");
00194 
00195     redeclareOption(ol, "inputsize", &VariableDeletionVMatrix::inputsize_,
00196                                      OptionBase::nosave,
00197         "Not used in VariableDeletionVMatrix.");
00198 
00199     redeclareOption(ol, "targetsize", &VariableDeletionVMatrix::targetsize_,
00200                                      OptionBase::nosave,
00201         "Not used in VariableDeletionVMatrix.");
00202                 
00203     redeclareOption(ol, "weightsize", &VariableDeletionVMatrix::weightsize_,
00204                                      OptionBase::nosave,
00205         "Not used in VariableDeletionVMatrix.");
00206     
00207     redeclareOption(ol, "extrasize", &VariableDeletionVMatrix::extrasize_,
00208                                      OptionBase::nosave,
00209         "Not used in VariableDeletionVMatrix.");
00210 }
00211 
00213 // build //
00215 void VariableDeletionVMatrix::build()
00216 {
00217     //must be done even if we will call it later to have the
00218     //source metadatadir set correctly.
00219     bool saved_warn_non_selected_field=warn_non_selected_field;
00220     warn_non_selected_field=false;
00221     inherited::build();
00222     warn_non_selected_field=saved_warn_non_selected_field;
00223     build_();
00224 }
00225 
00227 // makeDeepCopyFromShallowCopy //
00229 void VariableDeletionVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00230 {
00231     inherited::makeDeepCopyFromShallowCopy(copies);
00232     deepCopyField(complete_dataset, copies);
00233     deepCopyField(train_set, copies);
00234 }
00235 
00237 // build_ //
00239 void VariableDeletionVMatrix::build_()
00240 {
00241     if (complete_dataset) {
00242         PLDEPRECATED("In VariableDeletionVMatrix::build_ - The option "
00243                      "'complete_dataset' is deprecated, use source instead");
00244         if (source && source != complete_dataset)
00245             PLERROR("In VariableDeletionVMatrix::build_ - A source was also "
00246                     "specified, but it is different from 'complete_dataset'");
00247         source = complete_dataset;
00248     }
00249     if (!is_equal(deletion_threshold, -1)) {
00250         PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the "
00251                      "deprecated option 'deletion_threshold', you should "
00252                      "instead use 'min_non_missing_threshold'");
00253         min_non_missing_threshold = deletion_threshold;
00254     }
00255     if (remove_columns_with_constant_value != -1) {
00256         PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the "
00257                      "deprecated option 'remove_columns_with_constant_value', "
00258                      "you should instead use 'max_constant_threshold'");
00259         max_constant_threshold = remove_columns_with_constant_value == 0 ? 0:1;
00260     }
00261     if(!source)
00262         PLERROR("In VariableDeletionVMatrix::build_ - The source VMat do not exist!");
00263 
00264     updateMtime(source);
00265     updateMtime(train_set);
00266 
00267     int is = source->inputsize();
00268     if (is < 0)
00269         PLERROR("In VariableDeletionVMatrix::build_ - The source VMat must "
00270                 "have an inputsize defined");
00271 
00272     VMat the_train_source = train_set ? train_set : source;
00273     PLCHECK( the_train_source.width() == source->width() );
00274 
00275     if (number_of_train_samples > 0 &&
00276         number_of_train_samples < the_train_source->length())
00277         the_train_source = new SubVMatrix(the_train_source, 0, 0,
00278                                           number_of_train_samples,
00279                                           the_train_source->width());
00280     TVec<StatsCollector> stats;
00281     if(min_non_missing_threshold > 0 || max_constant_threshold > 0){
00282         int maxnvalues = -1;
00283         if(is_equal(max_constant_threshold,1))
00284 //We don't need all the value, if (min==max && non_missing_value>0) it is constant value.
00285             maxnvalues = 0;
00286         if(!the_train_source->hasMetaDataDir() && hasMetaDataDir())
00287             the_train_source->setMetaDataDir(getMetaDataDir()+"/source");
00288         stats = the_train_source->
00289             getPrecomputedStatsFromFile("stats_variableDeletionVMatrix_"+
00290                                         tostring(maxnvalues)+".psave",
00291                                         maxnvalues, true);
00292         PLCHECK( stats.length() == source->width() );
00293     }
00294 
00295     indices.resize(0);
00296 
00297     // First remove columns that have too many missing values.
00298     if (min_non_missing_threshold > 0){
00299         int min_non_missing =
00300             int(round(min_non_missing_threshold * the_train_source->length()));
00301         TVec<int> have_missing;
00302         for (int i = 0; i < is; i++){
00303             if (stats[i].nnonmissing() >= min_non_missing 
00304                 && stats[i].nnonmissing() > 0)
00305                 indices.append(i);
00306             else if (warn_removed_var)
00307                 PLWARNING("In VariableDeletionVMatrix::build_() var '%s'"
00308                           " have too many missing (%d/%d). We remove it.",
00309                           source->fieldName(i).c_str(),
00310                           int(stats[i].nmissing()),
00311                           int(stats[i].n()));
00312             if (info_var_with_missing && stats[i].nmissing() > 0){
00313                 have_missing.append(i);
00314             }
00315         }
00316         if(have_missing.length()>0){
00317             string s="INFO: In build_() variable with missing value (var,nb_missing/nb_value): ";
00318             for(int k=0;k<have_missing.length();k++){
00319                 int i = have_missing[k];
00320                 s+=" ("+source->fieldName(i)
00321                     +","+tostring(stats[i].nmissing())
00322                     +"/"+ tostring(stats[i].n())+")";
00323 
00324             }
00325             MODULE_LOG<<s<<endl;
00326         }
00327                 
00328     } else
00329         for (int i = 0; i < is; i++)
00330             indices.append(i);
00331     // Then remove columns that are too constant.
00332     TVec<int> final_indices;
00333     if (is_equal(max_constant_threshold,1)){
00334         TVec<int> const_indices;
00335         for (int k = 0; k < indices.length(); k++) {
00336             int i = indices[k];
00337             StatsCollector stat = stats[i];
00338             if(!(stat.min()==stat.max() && stat.nnonmissing()>0))
00339                 final_indices.append(i);
00340             else if (warn_removed_var)
00341                 const_indices.append(i);
00342         }
00343         if(warn_removed_var && const_indices.length()>0){
00344             string s = " WARNING: In VariableDeletionVMatrix::build_() - The following tuple (variable, constant value) indicate variable that are removed because they are constant: \n";
00345             for(int k=0;k<const_indices.length();k++){
00346                 int i = const_indices[k];
00347                 StatsCollector stat = stats[i];
00348                 s+=" ("+source->fieldName(i)+","+tostring(stat.min())+")";
00349             }
00350             NORMAL_LOG<<s<<endl;
00351         }
00352         indices.resize(final_indices.length());
00353         indices << final_indices; 
00354     }else if (max_constant_threshold > 0){
00355         for (int k = 0; k < indices.length(); k++) {
00356             int i = indices[k];
00357             int max_constant_absolute =
00358                 int(round(max_constant_threshold * stats[i].nnonmissing()));
00359             map<real, StatsCollectorCounts>* counts = stats[i].getCounts();
00360             map<real, StatsCollectorCounts>::const_iterator it;
00361             bool ok = true;
00362             int n;
00363             for (it = counts->begin(); ok && it != counts->end(); it++) {
00364                 n = int(round(it->second.n));
00365                 if (n >= max_constant_absolute)
00366                     ok = false;
00367             }
00368             if (ok)
00369                 final_indices.append(i);
00370             else if (warn_removed_var)
00371                 PLWARNING("In VariableDeletionVMatrix::build_() var '%s'"
00372                           " is too constant. Value %f happen %d/%f",
00373                           source->fieldName(i).c_str(), it->first, 
00374                           n, stats[i].nnonmissing());
00375         }
00376         indices.resize(final_indices.length());
00377         indices << final_indices;
00378     }
00379     // Define sizes.
00380     inputsize_  = indices.length();
00381     targetsize_ = source->targetsize();
00382     weightsize_ = source->weightsize();
00383     extrasize_  = source->extrasize();
00384 
00385     // Add target, weight and extra columns.
00386     for (int i = 0; i < source->targetsize(); i++)
00387         indices.append(is + i);
00388     if (source->targetsize() > 0)
00389         is += source->targetsize();
00390     for (int i = 0; i < source->weightsize(); i++)
00391         indices.append(is + i);
00392     if (source->weightsize() > 0)
00393         is += source->weightsize();
00394     for (int i = 0; i < source->extrasize(); i++)
00395         indices.append(is + i);
00396 
00397     // We have modified the selected columns, so the parent class must be
00398     // re-built.
00399     inherited::build();
00400 
00401     if(!save_deleted_columns.empty()){
00402         if(isfile(save_deleted_columns)){
00403             TVec<int> indices2;
00404             PLearn::load(save_deleted_columns, indices2);
00405             if(indices!=indices2){
00406                 PLWARNING("In VariableDeletionVMatrix::build_() - the calculated"
00407                           " indices(%d) differ from the saved indices(%d) in file '%s'."
00408                           " We overwrite it.",
00409                         indices2.length(), indices.length(), save_deleted_columns.c_str());
00410                 PLearn::save(save_deleted_columns,indices);
00411             }
00412         }else{
00413             PLearn::save(save_deleted_columns,indices);
00414         }
00415     }
00416 }
00417 
00418 } // end of namespace PLearn
00419 
00420 
00421 /*
00422   Local Variables:
00423   mode:c++
00424   c-basic-offset:4
00425   c-file-style:"stroustrup"
00426   c-file-offsets:((innamespace . 0)(inline-open . 0))
00427   indent-tabs-mode:nil
00428   fill-column:79
00429   End:
00430 */
00431 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines