PLearn 0.1
|
#include <MissingIndicatorVMatrix.h>
Public Member Functions | |
MissingIndicatorVMatrix () | |
MissingIndicatorVMatrix (VMat the_source, VMat the_train_set, real the_number_of_train_samples_to_use) | |
virtual | ~MissingIndicatorVMatrix () |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Does the necessary operations to transform a shallow copy (this) into a deep copy by deep-copying all the members that need to be. | |
virtual void | getExample (int i, Vec &input, Vec &target, real &weight) |
Default version calls getSubRow based on inputsize_ targetsize_ weightsize_ But exotic subclasses may construct, input, target and weight however they please. | |
virtual real | get (int i, int j) const |
This method must be implemented in all subclasses. | |
virtual void | getColumn (int i, Vec v) const |
Copies column i into v (which must have appropriate length equal to the VMat's length). | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual MissingIndicatorVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static void | declareOptions (OptionList &ol) |
Declare this class' options. | |
static string | _classname_ () |
MissingIndicatorVMatrix. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
VMat | source |
The source VMatrix with missing values. | |
VMat | train_set |
A referenced train set. | |
real | number_of_train_samples_to_use |
The number of samples from the train set that will be examined to see if an indicator should be added for each variable. | |
TVec< string > | fields |
string | save_fields_with_missing |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Private Types | |
typedef VMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
void | buildNewRecordFormat () |
Private Attributes | |
int | source_inputsize |
Vec | source_input |
TVec< int > | source_rel_pos |
Definition at line 53 of file MissingIndicatorVMatrix.h.
typedef VMatrix PLearn::MissingIndicatorVMatrix::inherited [private] |
Reimplemented from PLearn::VMatrix.
Definition at line 55 of file MissingIndicatorVMatrix.h.
PLearn::MissingIndicatorVMatrix::MissingIndicatorVMatrix | ( | ) |
Definition at line 60 of file MissingIndicatorVMatrix.cc.
: number_of_train_samples_to_use(0.0) { }
PLearn::MissingIndicatorVMatrix::MissingIndicatorVMatrix | ( | VMat | the_source, |
VMat | the_train_set, | ||
real | the_number_of_train_samples_to_use | ||
) |
Definition at line 65 of file MissingIndicatorVMatrix.cc.
References number_of_train_samples_to_use, source, and train_set.
{ source = the_source; train_set = the_train_set; number_of_train_samples_to_use = the_number_of_train_samples_to_use; }
PLearn::MissingIndicatorVMatrix::~MissingIndicatorVMatrix | ( | ) | [virtual] |
Definition at line 72 of file MissingIndicatorVMatrix.cc.
{ }
string PLearn::MissingIndicatorVMatrix::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
OptionList & PLearn::MissingIndicatorVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
RemoteMethodMap & PLearn::MissingIndicatorVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
Object * PLearn::MissingIndicatorVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
StaticInitializer MissingIndicatorVMatrix::_static_initializer_ & PLearn::MissingIndicatorVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
void PLearn::MissingIndicatorVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::VMatrix.
Definition at line 103 of file MissingIndicatorVMatrix.cc.
References PLearn::VMatrix::build(), and build_().
Referenced by PLearn::Preprocessing::manageTrainTestUnknownSets().
{ inherited::build(); build_(); }
void PLearn::MissingIndicatorVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VMatrix.
Definition at line 159 of file MissingIndicatorVMatrix.cc.
References buildNewRecordFormat(), fields, PLERROR, source, train_set, and PLearn::VMatrix::updateMtime().
Referenced by build().
{ if (!source) PLERROR("In MissingIndicatorVMatrix:: source vmat must be supplied"); if(source->inputsize()<=0) PLERROR("In MissingIndicatorVMatrix::build_() source must have an inputsize <= 0." " inputsize = %d", source->inputsize()); if(!train_set && !fields){ train_set = source; } updateMtime(source); if(train_set) updateMtime(train_set); buildNewRecordFormat(); }
void PLearn::MissingIndicatorVMatrix::buildNewRecordFormat | ( | ) | [private] |
Definition at line 174 of file MissingIndicatorVMatrix.cc.
References PLearn::TVec< T >::append(), PLearn::TVec< T >::clear(), PLearn::VMatrix::declareFieldNames(), PLearn::VMatrix::defineSizes(), PLearn::VMat::fieldName(), fields, PLearn::VMat::getFieldIndex(), i, PLearn::is_missing(), PLearn::VMat::length(), PLearn::VMatrix::length_, number_of_train_samples_to_use, PLCHECK, PLERROR, PLearn::TVec< T >::resize(), PLearn::save(), save_fields_with_missing, PLearn::TVec< T >::size(), source, source_input, source_inputsize, source_rel_pos, PLearn::sum(), train_set, PLearn::VMat::width(), and PLearn::VMatrix::width_.
Referenced by build_().
{ source_inputsize = source->inputsize(); TVec<int> train_var_missing(source_inputsize); train_var_missing.clear(); int source_width = source->width(); if(train_set){ int train_length = train_set->length(); if (number_of_train_samples_to_use > 0.0){ if (number_of_train_samples_to_use < 1.0) train_length = (int) (number_of_train_samples_to_use * (real) train_length); else train_length = (int) number_of_train_samples_to_use; } if (train_length > train_set->length()) train_length = train_set->length(); int train_width = train_set->width(); int train_inputsize = train_set->inputsize(); if(train_length < 1) PLERROR("In MissingIndicatorVMatrix::length of the number of train" " samples to use must be at least 1, got: %i", train_length); if(train_inputsize < 1) PLERROR("In MissingIndicatorVMatrix::inputsize of the train vmat must" " be supplied, got : %i", train_inputsize); if (train_width != source_width) PLERROR("In MissingIndicatorVMatrix::train set and source width must" " agree, got : %i, %i", train_width, source_width); if (train_set->targetsize() != source->targetsize()) PLERROR("In MissingIndicatorVMatrix::train set and source targetsize" " must agree, got : %i, %i", train_set->targetsize(), source->targetsize()); if (train_set->weightsize() != source->weightsize()) PLERROR("In MissingIndicatorVMatrix::train set and source weightsize" " must agree, got : %i, %i", train_set->weightsize(), source->weightsize()); if (train_inputsize != source_inputsize) PLERROR("In MissingIndicatorVMatrix::train set and source inputsize" " must agree, got : %i, %i", train_inputsize, source_inputsize); Vec train_input(train_width); for (int train_row = 0; train_row < train_length; train_row++) { train_set->getRow(train_row, train_input); for (int train_col = 0; train_col < train_inputsize; train_col++) { if (is_missing(train_input[train_col])) train_var_missing[train_col] = 1; } } }else if(fields.size()>0){ TVec<string> source_field_names = source->fieldNames(); for(int i=0;i<fields.size();i++) { int index=source->getFieldIndex(fields[i]); if(index<0) PLERROR("In MissingIndicatorVMatrix::buildNewRecordFormat() - The fields '%s' is not in the source", fields[i].c_str()); else train_var_missing[index]=1; } }else PLERROR("In MissingIndicatorVMatrix, the train_set or fields option should be provided."); int added_colomns = sum(train_var_missing); width_ = source_width + added_colomns; TVec<string> source_field_names(source_width); source_rel_pos.resize(width_); TVec<string> new_field_names(width_); source_field_names = source->fieldNames(); int new_col = 0; TVec<string> miss; for (int source_col = 0; source_col < source_inputsize; source_col++) { new_field_names[new_col] = source_field_names[source_col]; source_rel_pos[new_col] = source_col; new_col += 1; if (train_var_missing[source_col] > 0) { new_field_names[new_col] = source_field_names[source_col] + "_MI"; source_rel_pos[new_col] = -1; new_col += 1; miss->append(source->fieldName(source_col)); } } PLCHECK(miss->size()==added_colomns); for (int source_col = source_inputsize; source_col < source_width; source_col++) { new_field_names[new_col] = source_field_names[source_col]; source_rel_pos[new_col] = source_col; new_col += 1; } length_ = source->length(); defineSizes(source_inputsize + added_colomns, source->targetsize(), source->weightsize(), source->extrasize()); source_input.resize(source_inputsize); declareFieldNames(new_field_names); if(!save_fields_with_missing.empty()) PLearn::save(save_fields_with_missing,miss); }
string PLearn::MissingIndicatorVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
void PLearn::MissingIndicatorVMatrix::declareOptions | ( | OptionList & | ol | ) | [static] |
Declare this class' options.
Reimplemented from PLearn::VMatrix.
Definition at line 76 of file MissingIndicatorVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::VMatrix::declareOptions(), fields, number_of_train_samples_to_use, save_fields_with_missing, source, and train_set.
{ declareOption(ol, "source", &MissingIndicatorVMatrix::source, OptionBase::buildoption, "The source VMatrix with missing values.\n"); declareOption(ol, "train_set", &MissingIndicatorVMatrix::train_set, OptionBase::buildoption, "A referenced train set.\n" "A missing indicator is added for variables with missing values in this data set.\n" "It is used in combination with the option number_of_train_samples_to_use\n"); declareOption(ol, "number_of_train_samples_to_use", &MissingIndicatorVMatrix::number_of_train_samples_to_use, OptionBase::buildoption, "The number of samples from the train set that will be examined to see\n" "if an indicator should be added for each variable\n"); declareOption(ol, "fields", &MissingIndicatorVMatrix::fields, OptionBase::buildoption, "The names of the fields to extract if the train_set is not provided."); declareOption(ol, "save_fields_with_missing", &MissingIndicatorVMatrix::save_fields_with_missing, OptionBase::buildoption, "The file name where we save the name of the fields that we" " add an indicator. It can be reused with the options fields" " to redo the same processing."); inherited::declareOptions(ol); }
static const PPath& PLearn::MissingIndicatorVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::VMatrix.
Definition at line 97 of file MissingIndicatorVMatrix.h.
MissingIndicatorVMatrix * PLearn::MissingIndicatorVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::VMatrix.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
This method must be implemented in all subclasses.
Returns element (i,j).
Implements PLearn::VMatrix.
Definition at line 137 of file MissingIndicatorVMatrix.cc.
References PLearn::is_missing(), source, and source_rel_pos.
{ if (source_rel_pos[j] < 0) { if (is_missing(source->get(i, source_rel_pos[j - 1]))) return 1.0; else return 0.0; } return source->get(i, source_rel_pos[j]); }
Copies column i into v (which must have appropriate length equal to the VMat's length).
Reimplemented from PLearn::VMatrix.
Definition at line 147 of file MissingIndicatorVMatrix.cc.
References PLearn::VMat::getColumn(), PLearn::is_missing(), PLearn::TVec< T >::length(), source, and source_rel_pos.
{ if (source_rel_pos[i] < 0) source->getColumn(source_rel_pos[i - 1], v); else source->getColumn(source_rel_pos[i], v); if (source_rel_pos[i] >= 0) return; for (int source_row = 0; source_row < v->length(); source_row++) { if (is_missing(v[source_row])) v[source_row] = 1.0; else v[source_row] = 0.0; } }
void PLearn::MissingIndicatorVMatrix::getExample | ( | int | i, |
Vec & | input, | ||
Vec & | target, | ||
real & | weight | ||
) | [virtual] |
Default version calls getSubRow based on inputsize_ targetsize_ weightsize_ But exotic subclasses may construct, input, target and weight however they please.
If not a weighted matrix, weight should be set to default value 1.
Reimplemented from PLearn::VMatrix.
Definition at line 120 of file MissingIndicatorVMatrix.cc.
References PLearn::VMat::getExample(), PLearn::is_missing(), source, source_input, source_inputsize, and source_rel_pos.
{ source->getExample(i, source_input, target, weight); for (int source_col = 0, new_col = 0; source_col < source_inputsize; source_col++) { input[new_col] = source_input[source_col]; new_col += 1; if (source_rel_pos[source_col] < 0) { if (is_missing(source_input[source_col])) input[new_col] = 1.0; else input[new_col] = 0.0; new_col += 1; } } }
OptionList & PLearn::MissingIndicatorVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
OptionMap & PLearn::MissingIndicatorVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
RemoteMethodMap & PLearn::MissingIndicatorVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 58 of file MissingIndicatorVMatrix.cc.
void PLearn::MissingIndicatorVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Does the necessary operations to transform a shallow copy (this) into a deep copy by deep-copying all the members that need to be.
This needs to be overridden by every class that adds "complex" data members to the class, such as Vec
, Mat
, PP<Something>
, etc. Typical implementation:
void CLASS_OF_THIS::makeDeepCopyFromShallowCopy(CopiesMap& copies) { inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(complex_data_member1, copies); deepCopyField(complex_data_member2, copies); ... }
copies | A map used by the deep-copy mechanism to keep track of already-copied objects. |
Reimplemented from PLearn::VMatrix.
Definition at line 109 of file MissingIndicatorVMatrix.cc.
References PLearn::deepCopyField(), fields, PLearn::VMatrix::makeDeepCopyFromShallowCopy(), number_of_train_samples_to_use, source, source_rel_pos, and train_set.
{ deepCopyField(source, copies); deepCopyField(train_set, copies); deepCopyField(number_of_train_samples_to_use, copies); deepCopyField(source_rel_pos, copies); deepCopyField(fields, copies); inherited::makeDeepCopyFromShallowCopy(copies); }
Reimplemented from PLearn::VMatrix.
Definition at line 97 of file MissingIndicatorVMatrix.h.
Definition at line 72 of file MissingIndicatorVMatrix.h.
Referenced by build_(), buildNewRecordFormat(), declareOptions(), and makeDeepCopyFromShallowCopy().
The number of samples from the train set that will be examined to see if an indicator should be added for each variable.
Definition at line 69 of file MissingIndicatorVMatrix.h.
Referenced by buildNewRecordFormat(), declareOptions(), makeDeepCopyFromShallowCopy(), PLearn::Preprocessing::manageTrainTestUnknownSets(), and MissingIndicatorVMatrix().
Definition at line 73 of file MissingIndicatorVMatrix.h.
Referenced by buildNewRecordFormat(), and declareOptions().
The source VMatrix with missing values.
Definition at line 60 of file MissingIndicatorVMatrix.h.
Referenced by build_(), buildNewRecordFormat(), declareOptions(), get(), getColumn(), getExample(), makeDeepCopyFromShallowCopy(), PLearn::Preprocessing::manageTrainTestUnknownSets(), and MissingIndicatorVMatrix().
Definition at line 91 of file MissingIndicatorVMatrix.h.
Referenced by buildNewRecordFormat(), and getExample().
Definition at line 90 of file MissingIndicatorVMatrix.h.
Referenced by buildNewRecordFormat(), and getExample().
Definition at line 92 of file MissingIndicatorVMatrix.h.
Referenced by buildNewRecordFormat(), get(), getColumn(), getExample(), and makeDeepCopyFromShallowCopy().
A referenced train set.
A missing indicator is added for variables with missing values in this data set. It is used in combination with the option number_of_train_samples_to_use.
Definition at line 65 of file MissingIndicatorVMatrix.h.
Referenced by build_(), buildNewRecordFormat(), declareOptions(), makeDeepCopyFromShallowCopy(), PLearn::Preprocessing::manageTrainTestUnknownSets(), and MissingIndicatorVMatrix().