|
PLearn 0.1
|
#include <RemoveDuplicateVMatrix.h>


Public Member Functions | |
| RemoveDuplicateVMatrix () | |
| Default constructor. | |
| virtual void | build () |
| Simply calls inherited::build() then build_(). | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual RemoveDuplicateVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
| static string | _classname_ () |
| Declares name and deepCopy methods. | |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Public Attributes | |
| real | epsilon |
| int | max_source_length |
| bool | only_input |
| int | verbosity |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declares this class' options. | |
Private Types | |
| typedef SelectRowsVMatrix | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
Definition at line 52 of file RemoveDuplicateVMatrix.h.
typedef SelectRowsVMatrix PLearn::RemoveDuplicateVMatrix::inherited [private] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 57 of file RemoveDuplicateVMatrix.h.
| PLearn::RemoveDuplicateVMatrix::RemoveDuplicateVMatrix | ( | ) |
Default constructor.
Definition at line 54 of file RemoveDuplicateVMatrix.cc.
: epsilon(1e-6), max_source_length(10000), only_input(false), verbosity(1) { // ... // ### You may or may not want to call build_() to finish building the object // build_(); }
| string PLearn::RemoveDuplicateVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| OptionList & PLearn::RemoveDuplicateVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| RemoteMethodMap & PLearn::RemoveDuplicateVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| Object * PLearn::RemoveDuplicateVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| StaticInitializer RemoveDuplicateVMatrix::_static_initializer_ & PLearn::RemoveDuplicateVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| void PLearn::RemoveDuplicateVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 109 of file RemoveDuplicateVMatrix.cc.
References PLearn::SelectRowsVMatrix::build(), and build_().
{
// ### Nothing to add here, simply calls build_
inherited::build();
build_();
}

| void PLearn::RemoveDuplicateVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 119 of file RemoveDuplicateVMatrix.cc.
References PLearn::TVec< T >::append(), PLearn::SelectRowsVMatrix::build(), PLearn::Kernel::build(), PLearn::Kernel::computeGramMatrix(), PLearn::TVec< T >::data(), PLearn::endl(), epsilon, PLearn::DistanceKernel::evaluate(), PLearn::fast_exact_is_equal(), PLearn::TVec< T >::fill(), PLearn::VMat::getSubRow(), i, PLearn::SelectRowsVMatrix::indices, PLearn::SelectRowsVMatrix::indices_vmat, PLearn::iterate(), j, PLearn::VMatrix::length(), PLearn::TVec< T >::length(), PLearn::VMat::length(), max_source_length, n, only_input, PLWARNING, PLearn::pout, PLearn::DistanceKernel::pow_distance, PLearn::Kernel::report_progress, PLearn::TVec< T >::resize(), PLearn::TMat< T >::resize(), PLearn::DistanceKernel::setDataForKernelMatrix(), PLearn::SourceVMatrix::source, PLearn::VMatrix::updateMtime(), verbosity, w, and PLearn::VMat::width().
Referenced by build().
{
updateMtime(indices_vmat);
updateMtime(source);
if (source) {
DistanceKernel dk;
dk.pow_distance = true;
if (verbosity >= 1)
dk.report_progress = true;
else
dk.report_progress = false;
dk.build();
int old_is = source->inputsize();
int old_ts = source->targetsize();
int old_ws = source->weightsize();
if (!only_input)
source->defineSizes(source->width(), 0, 0);
dk.setDataForKernelMatrix(source);
int n = source.length();
bool compute_gram = (n <= max_source_length);
Mat distances;
if (compute_gram) {
if (n > 10000 && verbosity >= 2)
PLWARNING("In RemoveDuplicateVMatrix::build_ - Computing a large Gram "
"matrix (%d x %d), there may not be enough memory available", n, n);
distances.resize(n, n);
dk.computeGramMatrix(distances);
}
if (!only_input)
source->defineSizes(old_is, old_ts, old_ws);
TVec<bool> removed(n);
removed.fill(false);
Vec row_i, row_j;
if (fast_exact_is_equal(epsilon, 0) || !compute_gram) {
int w = only_input ? source->inputsize() : source->width();
row_i.resize(w);
row_j.resize(w);
}
real delta = epsilon > 0 ? epsilon : 1e-4;
int count = 0;
PP<ProgressBar> pb;
bool report_progress = (!compute_gram && verbosity >= 1);
int iterate = 0;
if (report_progress)
pb = new ProgressBar("Looking for duplicated entries", (n * (n - 1)) / 2);
for (int i = 0; i < n; i++) {
if (!removed[i]) {
if (!compute_gram)
source->getSubRow(i, 0, row_i);
for (int j = i + 1; j < n; j++) {
if (!removed[j]) {
bool equal;
if (compute_gram)
equal = (distances(i,j) < delta);
else {
source->getSubRow(j, 0, row_j);
equal = (dk.evaluate(row_i, row_j) < delta);
}
if (equal && fast_exact_is_equal(epsilon, 0)) {
// More accurate check.
if (compute_gram) {
source->getSubRow(i, 0, row_i);
source->getSubRow(j, 0, row_j);
}
real* data_i = row_i->data();
real* data_j = row_j->data();
int w = row_i->length();
for (int k = 0; k < w; k++, data_i++, data_j++)
if (!fast_exact_is_equal(*data_i, *data_j))
equal = false;
}
if (equal) {
if (verbosity >= 5)
pout << "Removed sample " << j
<< " (duplicated with sample " << i << ")" << endl;
removed[j] = true;
count++;
}
}
}
}
iterate += (n - i);
if (report_progress)
pb->update(iterate);
}
indices.resize(0);
for (int i = 0; i < n; i++)
if (!removed[i])
indices.append(i);
inherited::build();
if (verbosity >= 2){
if (count > 0)
pout << "Removed a total of " << count << " duplicated samples (new length: "
<< length() << ")" << endl;
else
pout << "No duplicated samples found." << endl;
}
}
}


| string PLearn::RemoveDuplicateVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| void PLearn::RemoveDuplicateVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 73 of file RemoveDuplicateVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::SelectRowsVMatrix::declareOptions(), epsilon, PLearn::SelectRowsVMatrix::indices, PLearn::SelectRowsVMatrix::indices_vmat, max_source_length, PLearn::OptionBase::nosave, only_input, PLearn::redeclareOption(), and verbosity.
{
// ### Declare all of this object's options here
// ### For the "flags" of each option, you should typically specify
// ### one of OptionBase::buildoption, OptionBase::learntoption or
// ### OptionBase::tuningoption. Another possible flag to be combined with
// ### is OptionBase::nosave
declareOption(ol, "epsilon", &RemoveDuplicateVMatrix::epsilon, OptionBase::buildoption,
"Two points will be considered equal iff their square distance is < epsilon.\n"
"If epsilon is set to 0, a more accurate check is performed and only samples\n"
"which are perfectly equal are removed.\n");
declareOption(ol, "only_input", &RemoveDuplicateVMatrix::only_input, OptionBase::buildoption,
"If set to 1, only the input part will be considered when computing the inter-points\n"
"distance. If set to 0, the whole row of the matrix is considered.\n");
declareOption(ol, "max_source_length", &RemoveDuplicateVMatrix::max_source_length, OptionBase::buildoption,
"If the source's length is higher than this value, the whole Gram matrix will\n"
"not be stored in memory (which will be slightly slower).\n");
declareOption(ol, "verbosity", &RemoveDuplicateVMatrix::verbosity, OptionBase::buildoption,
"Controls the amount of output.");
// Now call the parent class' declareOptions
inherited::declareOptions(ol);
redeclareOption(ol, "indices", &RemoveDuplicateVMatrix::indices, OptionBase::nosave,
"The indices will be computed at build time.");
redeclareOption(ol, "indices_vmat", &RemoveDuplicateVMatrix::indices_vmat, OptionBase::nosave,
"Unused.");
}

| static const PPath& PLearn::RemoveDuplicateVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 106 of file RemoveDuplicateVMatrix.h.
| RemoveDuplicateVMatrix * PLearn::RemoveDuplicateVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| OptionList & PLearn::RemoveDuplicateVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| OptionMap & PLearn::RemoveDuplicateVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| RemoteMethodMap & PLearn::RemoveDuplicateVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 68 of file RemoveDuplicateVMatrix.cc.
| void PLearn::RemoveDuplicateVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 223 of file RemoveDuplicateVMatrix.cc.
References PLearn::SelectRowsVMatrix::makeDeepCopyFromShallowCopy(), and PLERROR.
{
inherited::makeDeepCopyFromShallowCopy(copies);
// ### Call deepCopyField on all "pointer-like" fields
// ### that you wish to be deepCopied rather than
// ### shallow-copied.
// ### ex:
// deepCopyField(trainvec, copies);
// ### Remove this line when you have fully implemented this method.
PLERROR("RemoveDuplicateVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
}

Reimplemented from PLearn::SelectRowsVMatrix.
Definition at line 106 of file RemoveDuplicateVMatrix.h.
Definition at line 71 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 72 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
Definition at line 74 of file RemoveDuplicateVMatrix.h.
Referenced by build_(), and declareOptions().
1.7.4