PLearn 0.1
|
#include <VariableDeletionVMatrix.h>
Public Member Functions | |
VariableDeletionVMatrix () | |
VariableDeletionVMatrix (VMat the_source, real the_min_non_missing_threshold, bool the_remove_columns_with_constant_value, int the_number_of_train_samples, bool call_build_=true) | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual VariableDeletionVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
SelectColumnsVMatrix. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
real | min_non_missing_threshold |
real | max_constant_threshold |
int | number_of_train_samples |
bool | warn_removed_var |
bool | info_var_with_missing |
VMat | train_set |
string | save_deleted_columns |
VMat | complete_dataset |
real | deletion_threshold |
int | remove_columns_with_constant_value |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Private Types | |
typedef SelectColumnsVMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
void | buildIndices () |
Definition at line 52 of file VariableDeletionVMatrix.h.
typedef SelectColumnsVMatrix PLearn::VariableDeletionVMatrix::inherited [private] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 54 of file VariableDeletionVMatrix.h.
PLearn::VariableDeletionVMatrix::VariableDeletionVMatrix | ( | ) |
Definition at line 70 of file VariableDeletionVMatrix.cc.
: min_non_missing_threshold(0), max_constant_threshold(0), number_of_train_samples(0), warn_removed_var(false), info_var_with_missing(false), deletion_threshold(-1), remove_columns_with_constant_value(-1) {}
PLearn::VariableDeletionVMatrix::VariableDeletionVMatrix | ( | VMat | the_source, |
real | the_min_non_missing_threshold, | ||
bool | the_remove_columns_with_constant_value, | ||
int | the_number_of_train_samples, | ||
bool | call_build_ = true |
||
) |
Definition at line 80 of file VariableDeletionVMatrix.cc.
References build_(), and PLDEPRECATED.
: inherited(the_source, TVec<int>(), call_build_), min_non_missing_threshold(the_min_non_missing_threshold), number_of_train_samples(the_number_of_train_samples), remove_columns_with_constant_value(the_remove_columns_with_constant_value) { if (call_build_) build_(); PLDEPRECATED("In VariableDeletionVMatrix::VariableDeletionVMatrix - You " "are using a deprecated constructor"); // Note: this constructor should take as argument the new non-deprecated // options. }
string PLearn::VariableDeletionVMatrix::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
OptionList & PLearn::VariableDeletionVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
RemoteMethodMap & PLearn::VariableDeletionVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
Object * PLearn::VariableDeletionVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
StaticInitializer VariableDeletionVMatrix::_static_initializer_ & PLearn::VariableDeletionVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
void PLearn::VariableDeletionVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 215 of file VariableDeletionVMatrix.cc.
References PLearn::SelectColumnsVMatrix::build(), build_(), and PLearn::SelectColumnsVMatrix::warn_non_selected_field.
Referenced by PLearn::Experimentation::setSourceDataset().
{ //must be done even if we will call it later to have the //source metadatadir set correctly. bool saved_warn_non_selected_field=warn_non_selected_field; warn_non_selected_field=false; inherited::build(); warn_non_selected_field=saved_warn_non_selected_field; build_(); }
void PLearn::VariableDeletionVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 239 of file VariableDeletionVMatrix.cc.
References PLearn::TVec< T >::append(), PLearn::SelectColumnsVMatrix::build(), complete_dataset, deletion_threshold, PLearn::endl(), PLearn::VMatrix::extrasize_, PLearn::VMat::fieldName(), PLearn::VMatrix::getMetaDataDir(), PLearn::VMatrix::getPrecomputedStatsFromFile(), PLearn::VMatrix::hasMetaDataDir(), i, PLearn::SelectColumnsVMatrix::indices, info_var_with_missing, PLearn::VMatrix::inputsize_, PLearn::is_equal(), PLearn::isfile(), PLearn::VMat::length(), PLearn::TVec< T >::length(), PLearn::VMatrix::length(), PLearn::Object::load(), PLearn::StatsCollector::max(), max_constant_threshold, PLearn::StatsCollector::min(), min_non_missing_threshold, n, PLearn::StatsCollector::nnonmissing(), NORMAL_LOG, number_of_train_samples, PLCHECK, PLDEPRECATED, PLERROR, PLWARNING, remove_columns_with_constant_value, PLearn::TVec< T >::resize(), PLearn::VMatrix::save(), save_deleted_columns, PLearn::SourceVMatrix::source, PLearn::VMatrix::targetsize_, PLearn::tostring(), train_set, PLearn::VMatrix::updateMtime(), warn_removed_var, PLearn::VMatrix::weightsize_, and PLearn::VMat::width().
Referenced by build(), and VariableDeletionVMatrix().
{ if (complete_dataset) { PLDEPRECATED("In VariableDeletionVMatrix::build_ - The option " "'complete_dataset' is deprecated, use source instead"); if (source && source != complete_dataset) PLERROR("In VariableDeletionVMatrix::build_ - A source was also " "specified, but it is different from 'complete_dataset'"); source = complete_dataset; } if (!is_equal(deletion_threshold, -1)) { PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the " "deprecated option 'deletion_threshold', you should " "instead use 'min_non_missing_threshold'"); min_non_missing_threshold = deletion_threshold; } if (remove_columns_with_constant_value != -1) { PLDEPRECATED("In VariableDeletionVMatrix::build_ - You are using the " "deprecated option 'remove_columns_with_constant_value', " "you should instead use 'max_constant_threshold'"); max_constant_threshold = remove_columns_with_constant_value == 0 ? 0:1; } if(!source) PLERROR("In VariableDeletionVMatrix::build_ - The source VMat do not exist!"); updateMtime(source); updateMtime(train_set); int is = source->inputsize(); if (is < 0) PLERROR("In VariableDeletionVMatrix::build_ - The source VMat must " "have an inputsize defined"); VMat the_train_source = train_set ? train_set : source; PLCHECK( the_train_source.width() == source->width() ); if (number_of_train_samples > 0 && number_of_train_samples < the_train_source->length()) the_train_source = new SubVMatrix(the_train_source, 0, 0, number_of_train_samples, the_train_source->width()); TVec<StatsCollector> stats; if(min_non_missing_threshold > 0 || max_constant_threshold > 0){ int maxnvalues = -1; if(is_equal(max_constant_threshold,1)) //We don't need all the value, if (min==max && non_missing_value>0) it is constant value. maxnvalues = 0; if(!the_train_source->hasMetaDataDir() && hasMetaDataDir()) the_train_source->setMetaDataDir(getMetaDataDir()+"/source"); stats = the_train_source-> getPrecomputedStatsFromFile("stats_variableDeletionVMatrix_"+ tostring(maxnvalues)+".psave", maxnvalues, true); PLCHECK( stats.length() == source->width() ); } indices.resize(0); // First remove columns that have too many missing values. if (min_non_missing_threshold > 0){ int min_non_missing = int(round(min_non_missing_threshold * the_train_source->length())); TVec<int> have_missing; for (int i = 0; i < is; i++){ if (stats[i].nnonmissing() >= min_non_missing && stats[i].nnonmissing() > 0) indices.append(i); else if (warn_removed_var) PLWARNING("In VariableDeletionVMatrix::build_() var '%s'" " have too many missing (%d/%d). We remove it.", source->fieldName(i).c_str(), int(stats[i].nmissing()), int(stats[i].n())); if (info_var_with_missing && stats[i].nmissing() > 0){ have_missing.append(i); } } if(have_missing.length()>0){ string s="INFO: In build_() variable with missing value (var,nb_missing/nb_value): "; for(int k=0;k<have_missing.length();k++){ int i = have_missing[k]; s+=" ("+source->fieldName(i) +","+tostring(stats[i].nmissing()) +"/"+ tostring(stats[i].n())+")"; } MODULE_LOG<<s<<endl; } } else for (int i = 0; i < is; i++) indices.append(i); // Then remove columns that are too constant. TVec<int> final_indices; if (is_equal(max_constant_threshold,1)){ TVec<int> const_indices; for (int k = 0; k < indices.length(); k++) { int i = indices[k]; StatsCollector stat = stats[i]; if(!(stat.min()==stat.max() && stat.nnonmissing()>0)) final_indices.append(i); else if (warn_removed_var) const_indices.append(i); } if(warn_removed_var && const_indices.length()>0){ string s = " WARNING: In VariableDeletionVMatrix::build_() - The following tuple (variable, constant value) indicate variable that are removed because they are constant: \n"; for(int k=0;k<const_indices.length();k++){ int i = const_indices[k]; StatsCollector stat = stats[i]; s+=" ("+source->fieldName(i)+","+tostring(stat.min())+")"; } NORMAL_LOG<<s<<endl; } indices.resize(final_indices.length()); indices << final_indices; }else if (max_constant_threshold > 0){ for (int k = 0; k < indices.length(); k++) { int i = indices[k]; int max_constant_absolute = int(round(max_constant_threshold * stats[i].nnonmissing())); map<real, StatsCollectorCounts>* counts = stats[i].getCounts(); map<real, StatsCollectorCounts>::const_iterator it; bool ok = true; int n; for (it = counts->begin(); ok && it != counts->end(); it++) { n = int(round(it->second.n)); if (n >= max_constant_absolute) ok = false; } if (ok) final_indices.append(i); else if (warn_removed_var) PLWARNING("In VariableDeletionVMatrix::build_() var '%s'" " is too constant. Value %f happen %d/%f", source->fieldName(i).c_str(), it->first, n, stats[i].nnonmissing()); } indices.resize(final_indices.length()); indices << final_indices; } // Define sizes. inputsize_ = indices.length(); targetsize_ = source->targetsize(); weightsize_ = source->weightsize(); extrasize_ = source->extrasize(); // Add target, weight and extra columns. for (int i = 0; i < source->targetsize(); i++) indices.append(is + i); if (source->targetsize() > 0) is += source->targetsize(); for (int i = 0; i < source->weightsize(); i++) indices.append(is + i); if (source->weightsize() > 0) is += source->weightsize(); for (int i = 0; i < source->extrasize(); i++) indices.append(is + i); // We have modified the selected columns, so the parent class must be // re-built. inherited::build(); if(!save_deleted_columns.empty()){ if(isfile(save_deleted_columns)){ TVec<int> indices2; PLearn::load(save_deleted_columns, indices2); if(indices!=indices2){ PLWARNING("In VariableDeletionVMatrix::build_() - the calculated" " indices(%d) differ from the saved indices(%d) in file '%s'." " We overwrite it.", indices2.length(), indices.length(), save_deleted_columns.c_str()); PLearn::save(save_deleted_columns,indices); } }else{ PLearn::save(save_deleted_columns,indices); } } }
void PLearn::VariableDeletionVMatrix::buildIndices | ( | ) | [private] |
string PLearn::VariableDeletionVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
void PLearn::VariableDeletionVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 101 of file VariableDeletionVMatrix.cc.
References PLearn::OptionBase::buildoption, complete_dataset, PLearn::declareOption(), PLearn::SelectColumnsVMatrix::declareOptions(), deletion_threshold, PLearn::SelectColumnsVMatrix::extend_with_missing, PLearn::VMatrix::extrasize_, PLearn::SelectColumnsVMatrix::fields, PLearn::SelectColumnsVMatrix::fields_partial_match, PLearn::SelectColumnsVMatrix::indices, info_var_with_missing, PLearn::VMatrix::inputsize_, PLearn::OptionBase::learntoption, max_constant_threshold, min_non_missing_threshold, PLearn::OptionBase::nosave, number_of_train_samples, PLearn::redeclareOption(), remove_columns_with_constant_value, save_deleted_columns, PLearn::VMatrix::targetsize_, train_set, warn_removed_var, and PLearn::VMatrix::weightsize_.
{ declareOption(ol, "min_non_missing_threshold", &VariableDeletionVMatrix::min_non_missing_threshold, OptionBase::buildoption, "Minimum proportion of non-missing values for a variable to be kept\n" "(a 1 means only variables with no missing values are kept).\n" "if >0, we will always remove columns with all missing value."); declareOption(ol, "max_constant_threshold", &VariableDeletionVMatrix::max_constant_threshold, OptionBase::buildoption, "Maximum proportion of a unique value for a variable to be kept\n" "(contrary to 'min_non_missing_threshold', a variable will be\n" "removed if this proportion is attained, so that a 1 means only\n" "constant variables are removed, while 0 is a special value meaning\n" "that none is removed).\n" "Note also that this proportion is computed over the non-missing\n" "values only."); declareOption(ol, "number_of_train_samples", &VariableDeletionVMatrix::number_of_train_samples, OptionBase::buildoption, "If equal to zero, all the underlying dataset samples are used to\n" "compute the percentages and constant values.\n" "If it is a fraction between 0 and 1, only this proportion of the\n" "samples will be used.\n" "If greater than or equal to 1, the integer portion will be\n" "interpreted as the number of samples to use."); declareOption(ol, "warn_removed_var", &VariableDeletionVMatrix::warn_removed_var, OptionBase::buildoption, "If true, will print a warning about variable that are removed"); declareOption(ol, "info_var_with_missing", &VariableDeletionVMatrix::info_var_with_missing, OptionBase::buildoption, "If true, will print the variable that have some missing" " that we keep."); declareOption(ol, "save_deleted_columns", &VariableDeletionVMatrix::save_deleted_columns, OptionBase::buildoption, "If not empty will save the deleted culumns in this file." "If present, will verify that it have the same content then" " the calculated data."); declareOption(ol, "complete_dataset", &VariableDeletionVMatrix::complete_dataset, OptionBase::learntoption, "DEPRECATED (use 'source' instead) - The data set with all variables\n" "to select the columns from."); declareOption(ol, "train_set", &VariableDeletionVMatrix::train_set, OptionBase::buildoption, "The train set in which to compute the percentage of missing values.\n" "If null, will use the source to compute the percentage of missing values."); declareOption(ol, "deletion_threshold", &VariableDeletionVMatrix::deletion_threshold, OptionBase::learntoption, "DEPRECATED (use 'min_non_missing_threshold' instead) - The\n" "percentage of non-missing values for a variable above which\n" "the variable will be selected."); declareOption(ol, "remove_columns_with_constant_value", &VariableDeletionVMatrix::remove_columns_with_constant_value, OptionBase::learntoption, "DEPRECATED (use 'max_constant_threshold' instead) - If set to 1,\n" "the columns with constant non-missing values will be removed."); inherited::declareOptions(ol); // Hide unused parent class' options. redeclareOption(ol, "fields", &VariableDeletionVMatrix::fields, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "fields_partial_match", &VariableDeletionVMatrix::fields_partial_match, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "indices", &VariableDeletionVMatrix::indices, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "extend_with_missing", &VariableDeletionVMatrix::extend_with_missing, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "inputsize", &VariableDeletionVMatrix::inputsize_, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "targetsize", &VariableDeletionVMatrix::targetsize_, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "weightsize", &VariableDeletionVMatrix::weightsize_, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); redeclareOption(ol, "extrasize", &VariableDeletionVMatrix::extrasize_, OptionBase::nosave, "Not used in VariableDeletionVMatrix."); }
static const PPath& PLearn::VariableDeletionVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 93 of file VariableDeletionVMatrix.h.
VariableDeletionVMatrix * PLearn::VariableDeletionVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
OptionList & PLearn::VariableDeletionVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
OptionMap & PLearn::VariableDeletionVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
RemoteMethodMap & PLearn::VariableDeletionVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 65 of file VariableDeletionVMatrix.cc.
void PLearn::VariableDeletionVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 229 of file VariableDeletionVMatrix.cc.
References complete_dataset, PLearn::deepCopyField(), PLearn::SelectColumnsVMatrix::makeDeepCopyFromShallowCopy(), and train_set.
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(complete_dataset, copies); deepCopyField(train_set, copies); }
Reimplemented from PLearn::SelectColumnsVMatrix.
Definition at line 93 of file VariableDeletionVMatrix.h.
Definition at line 68 of file VariableDeletionVMatrix.h.
Referenced by build_(), declareOptions(), makeDeepCopyFromShallowCopy(), and PLearn::Experimentation::setSourceDataset().
Definition at line 69 of file VariableDeletionVMatrix.h.
Referenced by build_(), declareOptions(), and PLearn::Experimentation::setSourceDataset().
Definition at line 62 of file VariableDeletionVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 59 of file VariableDeletionVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 58 of file VariableDeletionVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 60 of file VariableDeletionVMatrix.h.
Referenced by build_(), declareOptions(), and PLearn::Experimentation::setSourceDataset().
Definition at line 70 of file VariableDeletionVMatrix.h.
Referenced by build_(), declareOptions(), and PLearn::Experimentation::setSourceDataset().
Definition at line 65 of file VariableDeletionVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 64 of file VariableDeletionVMatrix.h.
Referenced by build_(), declareOptions(), makeDeepCopyFromShallowCopy(), and PLearn::Experimentation::setSourceDataset().
Definition at line 61 of file VariableDeletionVMatrix.h.
Referenced by build_(), and declareOptions().