PLearn 0.1
|
#include <RemoveDuplicateVMatrix.h>
Public Member Functions | |
RemoveDuplicateVMatrix () | |
Default constructor. | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual RemoveDuplicateVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
real | epsilon |
int | max_source_length |
bool | only_input |
int | verbosity |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Private Types | |
typedef SelectRowsVMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Definition at line 52 of file RemoveDuplicateVMatrix.h.
typedef SelectRowsVMatrix PLearn::RemoveDuplicateVMatrix::inherited [private] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 57 of file RemoveDuplicateVMatrix.h.
PLearn::RemoveDuplicateVMatrix::RemoveDuplicateVMatrix | ( | ) |
Default constructor.
Definition at line 54 of file RemoveDuplicateVMatrix.cc.
: epsilon(1e-6), max_source_length(10000), only_input(false), verbosity(1) { // ... // ### You may or may not want to call build_() to finish building the object // build_(); }
string PLearn::RemoveDuplicateVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
OptionList & PLearn::RemoveDuplicateVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
RemoteMethodMap & PLearn::RemoveDuplicateVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
Object * PLearn::RemoveDuplicateVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
StaticInitializer RemoveDuplicateVMatrix::_static_initializer_ & PLearn::RemoveDuplicateVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
void PLearn::RemoveDuplicateVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 109 of file RemoveDuplicateVMatrix.cc.
References PLearn::SelectRowsVMatrix::build(), and build_().
{ // ### Nothing to add here, simply calls build_ inherited::build(); build_(); }
void PLearn::RemoveDuplicateVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 119 of file RemoveDuplicateVMatrix.cc.
References PLearn::TVec< T >::append(), PLearn::SelectRowsVMatrix::build(), PLearn::Kernel::build(), PLearn::Kernel::computeGramMatrix(), PLearn::TVec< T >::data(), PLearn::endl(), epsilon, PLearn::DistanceKernel::evaluate(), PLearn::fast_exact_is_equal(), PLearn::TVec< T >::fill(), PLearn::VMat::getSubRow(), i, PLearn::SelectRowsVMatrix::indices, PLearn::SelectRowsVMatrix::indices_vmat, PLearn::iterate(), j, PLearn::VMatrix::length(), PLearn::TVec< T >::length(), PLearn::VMat::length(), max_source_length, n, only_input, PLWARNING, PLearn::pout, PLearn::DistanceKernel::pow_distance, PLearn::Kernel::report_progress, PLearn::TVec< T >::resize(), PLearn::TMat< T >::resize(), PLearn::DistanceKernel::setDataForKernelMatrix(), PLearn::SourceVMatrix::source, PLearn::VMatrix::updateMtime(), verbosity, w, and PLearn::VMat::width().
Referenced by build().
{ updateMtime(indices_vmat); updateMtime(source); if (source) { DistanceKernel dk; dk.pow_distance = true; if (verbosity >= 1) dk.report_progress = true; else dk.report_progress = false; dk.build(); int old_is = source->inputsize(); int old_ts = source->targetsize(); int old_ws = source->weightsize(); if (!only_input) source->defineSizes(source->width(), 0, 0); dk.setDataForKernelMatrix(source); int n = source.length(); bool compute_gram = (n <= max_source_length); Mat distances; if (compute_gram) { if (n > 10000 && verbosity >= 2) PLWARNING("In RemoveDuplicateVMatrix::build_ - Computing a large Gram " "matrix (%d x %d), there may not be enough memory available", n, n); distances.resize(n, n); dk.computeGramMatrix(distances); } if (!only_input) source->defineSizes(old_is, old_ts, old_ws); TVec<bool> removed(n); removed.fill(false); Vec row_i, row_j; if (fast_exact_is_equal(epsilon, 0) || !compute_gram) { int w = only_input ? source->inputsize() : source->width(); row_i.resize(w); row_j.resize(w); } real delta = epsilon > 0 ? epsilon : 1e-4; int count = 0; PP<ProgressBar> pb; bool report_progress = (!compute_gram && verbosity >= 1); int iterate = 0; if (report_progress) pb = new ProgressBar("Looking for duplicated entries", (n * (n - 1)) / 2); for (int i = 0; i < n; i++) { if (!removed[i]) { if (!compute_gram) source->getSubRow(i, 0, row_i); for (int j = i + 1; j < n; j++) { if (!removed[j]) { bool equal; if (compute_gram) equal = (distances(i,j) < delta); else { source->getSubRow(j, 0, row_j); equal = (dk.evaluate(row_i, row_j) < delta); } if (equal && fast_exact_is_equal(epsilon, 0)) { // More accurate check. if (compute_gram) { source->getSubRow(i, 0, row_i); source->getSubRow(j, 0, row_j); } real* data_i = row_i->data(); real* data_j = row_j->data(); int w = row_i->length(); for (int k = 0; k < w; k++, data_i++, data_j++) if (!fast_exact_is_equal(*data_i, *data_j)) equal = false; } if (equal) { if (verbosity >= 5) pout << "Removed sample " << j << " (duplicated with sample " << i << ")" << endl; removed[j] = true; count++; } } } } iterate += (n - i); if (report_progress) pb->update(iterate); } indices.resize(0); for (int i = 0; i < n; i++) if (!removed[i]) indices.append(i); inherited::build(); if (verbosity >= 2){ if (count > 0) pout << "Removed a total of " << count << " duplicated samples (new length: " << length() << ")" << endl; else pout << "No duplicated samples found." << endl; } } }
string PLearn::RemoveDuplicateVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
void PLearn::RemoveDuplicateVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 73 of file RemoveDuplicateVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::SelectRowsVMatrix::declareOptions(), epsilon, PLearn::SelectRowsVMatrix::indices, PLearn::SelectRowsVMatrix::indices_vmat, max_source_length, PLearn::OptionBase::nosave, only_input, PLearn::redeclareOption(), and verbosity.
{ // ### Declare all of this object's options here // ### For the "flags" of each option, you should typically specify // ### one of OptionBase::buildoption, OptionBase::learntoption or // ### OptionBase::tuningoption. Another possible flag to be combined with // ### is OptionBase::nosave declareOption(ol, "epsilon", &RemoveDuplicateVMatrix::epsilon, OptionBase::buildoption, "Two points will be considered equal iff their square distance is < epsilon.\n" "If epsilon is set to 0, a more accurate check is performed and only samples\n" "which are perfectly equal are removed.\n"); declareOption(ol, "only_input", &RemoveDuplicateVMatrix::only_input, OptionBase::buildoption, "If set to 1, only the input part will be considered when computing the inter-points\n" "distance. If set to 0, the whole row of the matrix is considered.\n"); declareOption(ol, "max_source_length", &RemoveDuplicateVMatrix::max_source_length, OptionBase::buildoption, "If the source's length is higher than this value, the whole Gram matrix will\n" "not be stored in memory (which will be slightly slower).\n"); declareOption(ol, "verbosity", &RemoveDuplicateVMatrix::verbosity, OptionBase::buildoption, "Controls the amount of output."); // Now call the parent class' declareOptions inherited::declareOptions(ol); redeclareOption(ol, "indices", &RemoveDuplicateVMatrix::indices, OptionBase::nosave, "The indices will be computed at build time."); redeclareOption(ol, "indices_vmat", &RemoveDuplicateVMatrix::indices_vmat, OptionBase::nosave, "Unused."); }
static const PPath& PLearn::RemoveDuplicateVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 106 of file RemoveDuplicateVMatrix.h.
RemoveDuplicateVMatrix * PLearn::RemoveDuplicateVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
OptionList & PLearn::RemoveDuplicateVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
OptionMap & PLearn::RemoveDuplicateVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
RemoteMethodMap & PLearn::RemoveDuplicateVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
void PLearn::RemoveDuplicateVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 223 of file RemoveDuplicateVMatrix.cc.
References PLearn::SelectRowsVMatrix::makeDeepCopyFromShallowCopy(), and PLERROR.
{ inherited::makeDeepCopyFromShallowCopy(copies); // ### Call deepCopyField on all "pointer-like" fields // ### that you wish to be deepCopied rather than // ### shallow-copied. // ### ex: // deepCopyField(trainvec, copies); // ### Remove this line when you have fully implemented this method. PLERROR("RemoveDuplicateVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); }
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 106 of file RemoveDuplicateVMatrix.h.
Definition at line 71 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 72 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 74 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().