|
PLearn 0.1
|
#include <ProcessDatasetVMatrix.h>


Public Member Functions | |
| ProcessDatasetVMatrix () | |
| Default constructor. | |
| virtual void | build () |
| Simply calls inherited::build() then build_(). | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual ProcessDatasetVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
| static string | _classname_ () |
| Declares name and deepCopy methods. | |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Public Attributes | |
| int | max_mbytes_disk |
| int | max_mbytes_memory |
| string | duplicate |
| string | input_normalization |
| string | precompute |
| string | target_normalization |
| VMat | source |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declares this class' options. | |
Private Types | |
| typedef ForwardVMatrix | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
Definition at line 51 of file ProcessDatasetVMatrix.h.
typedef ForwardVMatrix PLearn::ProcessDatasetVMatrix::inherited [private] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 56 of file ProcessDatasetVMatrix.h.
| PLearn::ProcessDatasetVMatrix::ProcessDatasetVMatrix | ( | ) |
Default constructor.
Definition at line 58 of file ProcessDatasetVMatrix.cc.
:
max_mbytes_disk (1000),
max_mbytes_memory (30),
duplicate ("none"),
input_normalization ("none"),
precompute ("auto"),
target_normalization("none")
{}
| string PLearn::ProcessDatasetVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| OptionList & PLearn::ProcessDatasetVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| RemoteMethodMap & PLearn::ProcessDatasetVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| Object * PLearn::ProcessDatasetVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| StaticInitializer ProcessDatasetVMatrix::_static_initializer_ & PLearn::ProcessDatasetVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| void PLearn::ProcessDatasetVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 123 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::build(), and build_().
{
inherited::build();
build_();
}

| void PLearn::ProcessDatasetVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 132 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::build(), duplicate, input_normalization, PLearn::isfile(), PLearn::VMat::length(), max_mbytes_disk, max_mbytes_memory, n, PLERROR, precompute, source, target_normalization, PLearn::VMatrix::updateMtime(), PLearn::ForwardVMatrix::vm, w, and PLearn::VMat::width().
Referenced by build().
{
if (!source) {
// Empty VMatrix.
vm = 0;
inherited::build();
return;
}
if (source->inputsize() < 0 || source->targetsize() < 0 || source->weightsize() < 0)
PLERROR("In ProcessDatasetVMatrix::build_ - The source VMat's sizes must be defined");
updateMtime(source);
vm = source;
PPath filename_base = string("processed_dataset")
+ "-input_normalization=" + input_normalization
+ "-target_normalization=" + target_normalization
+ "-duplicate=" + duplicate;
bool target_normalization_is_performed = false;
if (input_normalization == "none") {
} else if (input_normalization == "standard") {
int n_inputs = vm->inputsize();
if (target_normalization == "standard") {
target_normalization_is_performed = true;
n_inputs += vm->targetsize();
}
vm = new ShiftAndRescaleVMatrix(vm, n_inputs);
} else
PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the "
"'input_normalization' option: %s", input_normalization.c_str());
if (!target_normalization_is_performed) {
if (target_normalization == "none") {
} else if (target_normalization == "standard") {
VMat noninput_part = new SubVMatrix(vm, 0, vm->inputsize(), vm->length(),
vm->targetsize() + vm->weightsize());
VMat input_part = new SubVMatrix(vm, 0, 0, vm->length(), vm->inputsize());
noninput_part = new ShiftAndRescaleVMatrix(noninput_part, vm->targetsize());
VMat result = new ConcatColumnsVMatrix(input_part, noninput_part);
result->defineSizes(vm->inputsize(), vm->targetsize(), vm->weightsize());
vm = result;
}
}
if (duplicate == "all") {
} else if (duplicate == "no_same_input") {
if (!source->hasMetaDataDir())
PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have "
"a metadata directory in order to compute duplicated samples");
PPath meta = source->getMetaDataDir();
// TODO Continue here!
}
int n = source->length();
int w = source->width();
string precomp = precompute;
if (precompute == "auto") {
// Need to find out whether to precompute in memory or on disk.
real memory_used = n / real(1024) * w / real(1024) * sizeof(real);
// pout << "Memory used: " << memory_used << " Mbs" << endl;
if (memory_used <= max_mbytes_memory)
precomp = "memory";
else if (memory_used <= max_mbytes_disk)
precomp = "disk";
else
precomp = "none";
}
if (precomp == "none") {
} else if (precomp == "memory") {
vm = new MemoryVMatrix(vm);
} else if (precomp == "disk") {
if (!source->hasMetaDataDir())
PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have "
"a metadata directory in order to precompute on disk");
PPath metadata = source->getMetaDataDir();
PPath filename = metadata / (filename_base + ".pmat");
bool need_recompute = true;
VMat old_vm;
if (isfile(filename)) {
old_vm = new FileVMatrix(filename);
if (old_vm->length() == n && old_vm->width() == w)
need_recompute = false;
}
if (need_recompute) {
vm->savePMAT(filename);
vm = new FileVMatrix(filename);
} else
vm = old_vm;
} else
PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the "
"'precompute' option: '%s'", precompute.c_str());
inherited::build();
}


| string PLearn::ProcessDatasetVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| void PLearn::ProcessDatasetVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 75 of file ProcessDatasetVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::ForwardVMatrix::declareOptions(), duplicate, input_normalization, max_mbytes_disk, max_mbytes_memory, PLearn::OptionBase::nosave, precompute, PLearn::redeclareOption(), source, target_normalization, and PLearn::ForwardVMatrix::vm.
{
declareOption(ol, "source", &ProcessDatasetVMatrix::source, OptionBase::buildoption,
"The underlying VMatrix.");
declareOption(ol, "input_normalization", &ProcessDatasetVMatrix::input_normalization, OptionBase::buildoption,
"Kind of normalization performed on the input features:\n"
" - 'none' : no normalization\n"
" - 'standard' : rescaled to have mean = 0 and standard deviation = 1\n");
declareOption(ol, "target_normalization", &ProcessDatasetVMatrix::target_normalization, OptionBase::buildoption,
"Kind of normalization performed on the target features (see 'input_normalization')");
declareOption(ol, "duplicate", &ProcessDatasetVMatrix::duplicate, OptionBase::buildoption,
"What to do with duplicated / conflicting samples:\n"
" - 'all' : nothing (all samples are kept)\n"
" - 'no_same_input' : samples must have a unique input (first one is kept)\n"
" - 'no_same_input_and_target' : only exact duplicates are removed (first one is kept)\n"
"Note: duplicated samples will only be removed after normalization is performed.");
declareOption(ol, "precompute", &ProcessDatasetVMatrix::precompute, OptionBase::buildoption,
"How to precompute the dataset:\n"
" - 'none' : it is not precomputed\n"
" - 'memory' : it is precomputed in memory\n"
" - 'disk' : it is precomputed in the underlying VMat metadatadir\n"
" - 'auto' : it is precomputed in memory if it takes less than 'max_mbytes' Mb,\n"
" it is precomputed in the underlying VMat metadatadir if it takes\n"
" less than 'max_mbytes_disk' Mb, otherwise it is not precomputed.\n");
declareOption(ol, "max_mbytes_memory", &ProcessDatasetVMatrix::max_mbytes_memory, OptionBase::buildoption,
"Maximum number of megabytes allowed in memory when 'precompute' is set to 'auto'");
declareOption(ol, "max_mbytes_disk", &ProcessDatasetVMatrix::max_mbytes_disk, OptionBase::buildoption,
"Maximum number of megabytes allowed on disk when 'precompute' is set to 'auto'");
// Now call the parent class' declareOptions
inherited::declareOptions(ol);
// Hide unused options.
redeclareOption(ol, "vm", &ProcessDatasetVMatrix::vm, OptionBase::nosave,
"Defined at build time.");
}

| static const PPath& PLearn::ProcessDatasetVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 108 of file ProcessDatasetVMatrix.h.
| ProcessDatasetVMatrix * PLearn::ProcessDatasetVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| OptionList & PLearn::ProcessDatasetVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| OptionMap & PLearn::ProcessDatasetVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| RemoteMethodMap & PLearn::ProcessDatasetVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 70 of file ProcessDatasetVMatrix.cc.
| void PLearn::ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::ForwardVMatrix.
Definition at line 230 of file ProcessDatasetVMatrix.cc.
References PLearn::ForwardVMatrix::makeDeepCopyFromShallowCopy(), and PLERROR.
{
inherited::makeDeepCopyFromShallowCopy(copies);
// ### Call deepCopyField on all "pointer-like" fields
// ### that you wish to be deepCopied rather than
// ### shallow-copied.
// ### ex:
// deepCopyField(trainvec, copies);
// ### Remove this line when you have fully implemented this method.
PLERROR("ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
}

Reimplemented from PLearn::ForwardVMatrix.
Definition at line 108 of file ProcessDatasetVMatrix.h.
Definition at line 72 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 70 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 71 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 74 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 76 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 75 of file ProcessDatasetVMatrix.h.
Referenced by build_(), and declareOptions().
1.7.4