PLearn 0.1
|
#include <ProcessDatasetVMatrix.h>
Public Member Functions | |
ProcessDatasetVMatrix () | |
Default constructor. | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual ProcessDatasetVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
int | max_mbytes_disk |
int | max_mbytes_memory |
string | duplicate |
string | input_normalization |
string | precompute |
string | target_normalization |
VMat | source |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Private Types | |
typedef ForwardVMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Definition at line 51 of file ProcessDatasetVMatrix.h.
typedef ForwardVMatrix PLearn::ProcessDatasetVMatrix::inherited [private] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 56 of file ProcessDatasetVMatrix.h.
PLearn::ProcessDatasetVMatrix::ProcessDatasetVMatrix | ( | ) |
Default constructor.
Definition at line 58 of file ProcessDatasetVMatrix.cc.
: max_mbytes_disk (1000), max_mbytes_memory (30), duplicate ("none"), input_normalization ("none"), precompute ("auto"), target_normalization("none") {}
string PLearn::ProcessDatasetVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
OptionList & PLearn::ProcessDatasetVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
RemoteMethodMap & PLearn::ProcessDatasetVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
Object * PLearn::ProcessDatasetVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
StaticInitializer ProcessDatasetVMatrix::_static_initializer_ & PLearn::ProcessDatasetVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
void PLearn::ProcessDatasetVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 123 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::build(), and build_().
{ inherited::build(); build_(); }
void PLearn::ProcessDatasetVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 132 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::build(), duplicate, input_normalization, PLearn::isfile(), PLearn::VMat::length(), max_mbytes_disk, max_mbytes_memory, n, PLERROR, precompute, source, target_normalization, PLearn::VMatrix::updateMtime(), PLearn::ForwardVMatrix::vm, w, and PLearn::VMat::width().
Referenced by build().
{ if (!source) { // Empty VMatrix. vm = 0; inherited::build(); return; } if (source->inputsize() < 0 || source->targetsize() < 0 || source->weightsize() < 0) PLERROR("In ProcessDatasetVMatrix::build_ - The source VMat's sizes must be defined"); updateMtime(source); vm = source; PPath filename_base = string("processed_dataset") + "-input_normalization=" + input_normalization + "-target_normalization=" + target_normalization + "-duplicate=" + duplicate; bool target_normalization_is_performed = false; if (input_normalization == "none") { } else if (input_normalization == "standard") { int n_inputs = vm->inputsize(); if (target_normalization == "standard") { target_normalization_is_performed = true; n_inputs += vm->targetsize(); } vm = new ShiftAndRescaleVMatrix(vm, n_inputs); } else PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the " "'input_normalization' option: %s", input_normalization.c_str()); if (!target_normalization_is_performed) { if (target_normalization == "none") { } else if (target_normalization == "standard") { VMat noninput_part = new SubVMatrix(vm, 0, vm->inputsize(), vm->length(), vm->targetsize() + vm->weightsize()); VMat input_part = new SubVMatrix(vm, 0, 0, vm->length(), vm->inputsize()); noninput_part = new ShiftAndRescaleVMatrix(noninput_part, vm->targetsize()); VMat result = new ConcatColumnsVMatrix(input_part, noninput_part); result->defineSizes(vm->inputsize(), vm->targetsize(), vm->weightsize()); vm = result; } } if (duplicate == "all") { } else if (duplicate == "no_same_input") { if (!source->hasMetaDataDir()) PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have " "a metadata directory in order to compute duplicated samples"); PPath meta = source->getMetaDataDir(); // TODO Continue here! } int n = source->length(); int w = source->width(); string precomp = precompute; if (precompute == "auto") { // Need to find out whether to precompute in memory or on disk. real memory_used = n / real(1024) * w / real(1024) * sizeof(real); // pout << "Memory used: " << memory_used << " Mbs" << endl; if (memory_used <= max_mbytes_memory) precomp = "memory"; else if (memory_used <= max_mbytes_disk) precomp = "disk"; else precomp = "none"; } if (precomp == "none") { } else if (precomp == "memory") { vm = new MemoryVMatrix(vm); } else if (precomp == "disk") { if (!source->hasMetaDataDir()) PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have " "a metadata directory in order to precompute on disk"); PPath metadata = source->getMetaDataDir(); PPath filename = metadata / (filename_base + ".pmat"); bool need_recompute = true; VMat old_vm; if (isfile(filename)) { old_vm = new FileVMatrix(filename); if (old_vm->length() == n && old_vm->width() == w) need_recompute = false; } if (need_recompute) { vm->savePMAT(filename); vm = new FileVMatrix(filename); } else vm = old_vm; } else PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the " "'precompute' option: '%s'", precompute.c_str()); inherited::build(); }
string PLearn::ProcessDatasetVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
void PLearn::ProcessDatasetVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 75 of file ProcessDatasetVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::ForwardVMatrix::declareOptions(), duplicate, input_normalization, max_mbytes_disk, max_mbytes_memory, PLearn::OptionBase::nosave, precompute, PLearn::redeclareOption(), source, target_normalization, and PLearn::ForwardVMatrix::vm.
{ declareOption(ol, "source", &ProcessDatasetVMatrix::source, OptionBase::buildoption, "The underlying VMatrix."); declareOption(ol, "input_normalization", &ProcessDatasetVMatrix::input_normalization, OptionBase::buildoption, "Kind of normalization performed on the input features:\n" " - 'none' : no normalization\n" " - 'standard' : rescaled to have mean = 0 and standard deviation = 1\n"); declareOption(ol, "target_normalization", &ProcessDatasetVMatrix::target_normalization, OptionBase::buildoption, "Kind of normalization performed on the target features (see 'input_normalization')"); declareOption(ol, "duplicate", &ProcessDatasetVMatrix::duplicate, OptionBase::buildoption, "What to do with duplicated / conflicting samples:\n" " - 'all' : nothing (all samples are kept)\n" " - 'no_same_input' : samples must have a unique input (first one is kept)\n" " - 'no_same_input_and_target' : only exact duplicates are removed (first one is kept)\n" "Note: duplicated samples will only be removed after normalization is performed."); declareOption(ol, "precompute", &ProcessDatasetVMatrix::precompute, OptionBase::buildoption, "How to precompute the dataset:\n" " - 'none' : it is not precomputed\n" " - 'memory' : it is precomputed in memory\n" " - 'disk' : it is precomputed in the underlying VMat metadatadir\n" " - 'auto' : it is precomputed in memory if it takes less than 'max_mbytes' Mb,\n" " it is precomputed in the underlying VMat metadatadir if it takes\n" " less than 'max_mbytes_disk' Mb, otherwise it is not precomputed.\n"); declareOption(ol, "max_mbytes_memory", &ProcessDatasetVMatrix::max_mbytes_memory, OptionBase::buildoption, "Maximum number of megabytes allowed in memory when 'precompute' is set to 'auto'"); declareOption(ol, "max_mbytes_disk", &ProcessDatasetVMatrix::max_mbytes_disk, OptionBase::buildoption, "Maximum number of megabytes allowed on disk when 'precompute' is set to 'auto'"); // Now call the parent class' declareOptions inherited::declareOptions(ol); // Hide unused options. redeclareOption(ol, "vm", &ProcessDatasetVMatrix::vm, OptionBase::nosave, "Defined at build time."); }
static const PPath& PLearn::ProcessDatasetVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 108 of file ProcessDatasetVMatrix.h.
ProcessDatasetVMatrix * PLearn::ProcessDatasetVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
OptionList & PLearn::ProcessDatasetVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
OptionMap & PLearn::ProcessDatasetVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
RemoteMethodMap & PLearn::ProcessDatasetVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
void PLearn::ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 230 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::makeDeepCopyFromShallowCopy(), and PLERROR.
{ inherited::makeDeepCopyFromShallowCopy(copies); // ### Call deepCopyField on all "pointer-like" fields // ### that you wish to be deepCopied rather than // ### shallow-copied. // ### ex: // deepCopyField(trainvec, copies); // ### Remove this line when you have fully implemented this method. PLERROR("ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); }
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 108 of file ProcessDatasetVMatrix.h.
Definition at line 72 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 70 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 71 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 74 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 76 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 75 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().