PLearn 0.1
|
#include <FilteredVMatrix.h>
Public Member Functions | |
FilteredVMatrix () | |
Default constructor. | |
FilteredVMatrix (VMat the_source, const string &program_string, const PPath &the_metadatadir="", bool the_report_progress=true, bool allow_repeat_rows_=false, const string &repeat_id_field_name_="", const string &repeat_count_field_name_="", bool call_build_=true) | |
Convenience constructor. | |
virtual void | setMetaDataDir (const PPath &the_metadatadir) |
Also sets the source's meta-data dir if it's not already set. | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual FilteredVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
bool | report_progress |
string | prg |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
void | openIndex () |
Generates the index file if it does not already exist. | |
void | computeFilteredIndices () |
Compute the filtered indices. | |
virtual void | getNewRow (int i, const Vec &v) const |
This is the only method requiring implementation. | |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Protected Attributes | |
VMatLanguage | program |
IntVecFile | indexes |
The i-th element is the index of the i-th selected item in the source VMatrix. | |
TVec< int > | mem_indices |
Indices stored in memory when no metadata directory is available. | |
bool | allow_repeat_rows |
string | repeat_id_field_name |
string | repeat_count_field_name |
bool | warn_no_metadatadir |
Private Types | |
typedef SourceVMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
Private Attributes | |
bool | build_complete |
A custom hack to make sure openIndex() is not called before the build is complete. |
Definition at line 54 of file FilteredVMatrix.h.
typedef SourceVMatrix PLearn::FilteredVMatrix::inherited [private] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.h.
PLearn::FilteredVMatrix::FilteredVMatrix | ( | ) |
Default constructor.
Definition at line 61 of file FilteredVMatrix.cc.
: build_complete(false), allow_repeat_rows(false), repeat_id_field_name(""), repeat_count_field_name(""), warn_no_metadatadir(false), report_progress(true) {}
PLearn::FilteredVMatrix::FilteredVMatrix | ( | VMat | the_source, |
const string & | program_string, | ||
const PPath & | the_metadatadir = "" , |
||
bool | the_report_progress = true , |
||
bool | allow_repeat_rows_ = false , |
||
const string & | repeat_id_field_name_ = "" , |
||
const string & | repeat_count_field_name_ = "" , |
||
bool | call_build_ = true |
||
) |
Convenience constructor.
Definition at line 70 of file FilteredVMatrix.cc.
References build_(), and PLearn::VMatrix::metadatadir.
: inherited(the_source, call_build_), allow_repeat_rows(allow_repeat_rows_), repeat_id_field_name(repeat_id_field_name_), repeat_count_field_name(repeat_count_field_name_), report_progress(the_report_progress), prg(program_string) { // Note that although VMatrix::build_ would be tempted to call // setMetaDataDir when inherited(the_source, true) is called above (if // call_build_ is true), the metadatadir is only set below, so it will not // happen. metadatadir = the_metadatadir; if (call_build_) build_(); }
string PLearn::FilteredVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
OptionList & PLearn::FilteredVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
RemoteMethodMap & PLearn::FilteredVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
Object * PLearn::FilteredVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
StaticInitializer FilteredVMatrix::_static_initializer_ & PLearn::FilteredVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
void PLearn::FilteredVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::SourceVMatrix.
Definition at line 323 of file FilteredVMatrix.cc.
References PLearn::SourceVMatrix::build(), build_(), and build_complete.
{ build_complete = false; inherited::build(); build_(); }
void PLearn::FilteredVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 278 of file FilteredVMatrix.cc.
References PLearn::TVec< T >::append(), build_complete, PLearn::VMatLanguage::compileString(), computeFilteredIndices(), PLearn::VMatrix::getMetaDataDir(), PLearn::VMatrix::hasMetaDataDir(), PLearn::VMatrix::length_, PLWARNING, prg, program, repeat_count_field_name, repeat_id_field_name, PLearn::VMatrix::setFieldInfos(), setMetaDataDir(), PLearn::SourceVMatrix::setMetaInfoFromSource(), PLearn::VMatLanguage::setSource(), PLearn::SourceVMatrix::source, PLearn::VMatrix::updateMtime(), warn_no_metadatadir, and PLearn::VMatrix::width_.
Referenced by build(), and FilteredVMatrix().
{ if (source) { updateMtime(source); vector<string> fieldnames; program.setSource(source); program.compileString(prg,fieldnames); build_complete = true; if (hasMetaDataDir()) // Calling setMetaDataDir() will compute indices and save them // in the give metadata directory. setMetaDataDir(getMetaDataDir()); else { // Compute selected indices in memory only. if(warn_no_metadatadir) PLWARNING("In FilteredVMatrix::build_() - there is no " "metadatadir: recomputing filtered indices"); computeFilteredIndices(); } Array<VMField> finfos= source->getFieldInfos().copy(); if(!repeat_id_field_name.empty()) { finfos.append(VMField(repeat_id_field_name)); if(0 < width_) ++width_; } if(!repeat_count_field_name.empty()) { finfos.append(VMField(repeat_count_field_name)); if(0 < width_) ++width_; } setFieldInfos(finfos); setMetaInfoFromSource(); } else length_ = width_ = 0; }
string PLearn::FilteredVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
void PLearn::FilteredVMatrix::computeFilteredIndices | ( | ) | [protected] |
Compute the filtered indices.
Definition at line 96 of file FilteredVMatrix.cc.
References allow_repeat_rows, PLearn::TVec< T >::append(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), PLearn::VMat::length(), PLearn::VMatrix::length_, mem_indices, program, report_progress, PLearn::TVec< T >::resize(), PLearn::VMatLanguage::run(), PLearn::SourceVMatrix::source, and x.
Referenced by build_(), and openIndex().
{ int l = source.length(); Vec result(1); PP<ProgressBar> pb; if (report_progress) pb = new ProgressBar("Filtering source vmat", l); mem_indices.resize(0); for(int i=0; i<l; i++) { if (report_progress) pb->update(i); program.run(i,result); if(!allow_repeat_rows) { if(!fast_exact_is_equal(result[0], 0)) mem_indices.append(i); } else for(int x = int(round(result[0])); x > 0; --x) mem_indices.append(i); } length_ = mem_indices.length(); }
void PLearn::FilteredVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 221 of file FilteredVMatrix.cc.
References allow_repeat_rows, PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::SourceVMatrix::declareOptions(), PLearn::VMatrix::extrasize_, PLearn::VMatrix::inputsize_, PLearn::VMatrix::length_, PLearn::OptionBase::nosave, prg, PLearn::redeclareOption(), repeat_count_field_name, repeat_id_field_name, report_progress, PLearn::VMatrix::targetsize_, warn_no_metadatadir, PLearn::VMatrix::weightsize_, and PLearn::VMatrix::width_.
{ declareOption(ol, "prg", &FilteredVMatrix::prg, OptionBase::buildoption, "The VPL code that should produce a single scalar, indicating whether \n" "we should keep the line (if the produced scalar is non zero) or throw it away (if it's zero)"); declareOption(ol, "report_progress", &FilteredVMatrix::report_progress, OptionBase::buildoption, "Whether to report the filtering progress or not."); declareOption(ol, "allow_repeat_rows", &FilteredVMatrix::allow_repeat_rows, OptionBase::buildoption, "When true, the result of the program indicates the number of times this row should be repeated.\n" "Simple filtering when false."); declareOption(ol, "repeat_id_field_name", &FilteredVMatrix::repeat_id_field_name, OptionBase::buildoption, "Field name for the repetition id (0, 1, ..., n-1). No field is added if empty."); declareOption(ol, "repeat_count_field_name", &FilteredVMatrix::repeat_count_field_name, OptionBase::buildoption, "Field name for the number of repetitions (n). No field is added if empty."); declareOption(ol, "warn_no_metadatadir", &FilteredVMatrix::warn_no_metadatadir, OptionBase::buildoption, "If True, generate a warning when there is no metadatadir.\n" "This can be useful as if there is no metadatadir we will recompute\n" "filtered indices every time."); // Now call the parent class' declareOptions inherited::declareOptions(ol); declareOption(ol, "length", &FilteredVMatrix::length_, OptionBase::nosave, "The number of example. Computed each time from source."); redeclareOption(ol, "inputsize", &FilteredVMatrix::inputsize_, OptionBase::nosave, "Taken from source in FilteredVMatrix."); redeclareOption(ol, "targetsize", &FilteredVMatrix::targetsize_, OptionBase::nosave, "Taken from source in FilteredVMatrix."); redeclareOption(ol, "weightsize", &FilteredVMatrix::weightsize_, OptionBase::nosave, "Taken from source in FilteredVMatrix."); redeclareOption(ol, "extrasize", &FilteredVMatrix::extrasize_, OptionBase::nosave, "Taken from source in FilteredVMatrix."); redeclareOption(ol, "width", &FilteredVMatrix::width_, OptionBase::nosave, "Taken from source in FilteredVMatrix."); }
static const PPath& PLearn::FilteredVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 132 of file FilteredVMatrix.h.
FilteredVMatrix * PLearn::FilteredVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
This is the only method requiring implementation.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 179 of file FilteredVMatrix.cc.
References i, indexes, PLearn::TVec< T >::isEmpty(), j, PLearn::VMatrix::length(), PLearn::IntVecFile::length(), mem_indices, PLERROR, repeat_count_field_name, repeat_id_field_name, PLearn::SourceVMatrix::source, PLearn::TVec< T >::subVec(), and PLearn::VMat::width().
{ if (mem_indices.isEmpty() && indexes.length() == -1) PLERROR("In FilteredVMatrix::getNewRow - Filtered indices are not\n" "set yet."); int j= source->width(); int idx = mem_indices.isEmpty() ? indexes[i] : mem_indices[i]; source->getRow(idx, v.subVec(0, j)); if(!repeat_id_field_name.empty()) { int k= 1; while(k <= i && (mem_indices.isEmpty() ? indexes[i]==indexes[i-k] : mem_indices[i] == mem_indices[i-k])) ++k; v[j++]= static_cast<real>(k-1); } if(!repeat_count_field_name.empty()) { int k0= 1; while(k0 <= i && (mem_indices.isEmpty() ? indexes[i]==indexes[i-k0] : mem_indices[i] == mem_indices[i-k0])) ++k0; --k0; int k1= 1; while(k1+i < length() && (mem_indices.isEmpty() ? indexes[i]==indexes[i+k1] : mem_indices[i] == mem_indices[i+k1])) ++k1; v[j++]= static_cast<real>(k0+k1); } }
OptionList & PLearn::FilteredVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
OptionMap & PLearn::FilteredVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
RemoteMethodMap & PLearn::FilteredVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
void PLearn::FilteredVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 333 of file FilteredVMatrix.cc.
References PLearn::deepCopyField(), PLearn::SourceVMatrix::makeDeepCopyFromShallowCopy(), and mem_indices.
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(mem_indices, copies); }
void PLearn::FilteredVMatrix::openIndex | ( | ) | [protected] |
Generates the index file if it does not already exist.
If it exists and is up to date, simply opens it.
Definition at line 124 of file FilteredVMatrix.cc.
References PLearn::PPath::absolute(), PLearn::IntVecFile::append(), PLearn::IntVecFile::close(), computeFilteredIndices(), PLearn::force_mkdir(), PLearn::VMatrix::getMetaDataDir(), PLearn::VMatrix::hasMetaDataDir(), i, indexes, PLearn::isfile(), PLearn::VMatrix::isUpToDate(), PLearn::TVec< T >::length(), PLearn::IntVecFile::length(), PLearn::VMatrix::length_, PLearn::VMatrix::lockMetaDataDir(), mem_indices, PLearn::IntVecFile::open(), PLASSERT, PLERROR, PLearn::rm(), and PLearn::VMatrix::unlockMetaDataDir().
Referenced by setMetaDataDir().
{ PLASSERT(hasMetaDataDir()); PPath idxfname = getMetaDataDir() / "filtered.idx"; if(!force_mkdir(getMetaDataDir())) PLERROR("In FilteredVMatrix::openIndex - Could not create " "directory %s", getMetaDataDir().absolute().c_str()); lockMetaDataDir(); try{ if(isUpToDate(idxfname)) indexes.open(idxfname.absolute()); else // let's (re)create the index { computeFilteredIndices(); rm(idxfname); // force remove it indexes.open(idxfname.absolute(), true); for (int i = 0; i < mem_indices.length(); i++) indexes.append(mem_indices[i]); indexes.close(); indexes.open(idxfname.absolute()); mem_indices = TVec<int>(); // Free memory. } }catch(const PLearnError& e){ unlockMetaDataDir(); //we erase the file if we are creating it // as it can be partilly saved. if(!isUpToDate(idxfname) && isfile(idxfname)) rm(idxfname); throw e; } unlockMetaDataDir(); length_ = indexes.length(); }
void PLearn::FilteredVMatrix::setMetaDataDir | ( | const PPath & | the_metadatadir | ) | [virtual] |
Also sets the source's meta-data dir if it's not already set.
If there are fields that have no corresponding .smap .notes or .binning info files but the source has those files for a field with the same name, then those of the source will be set also for this vmatrix.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 163 of file FilteredVMatrix.cc.
References build_complete, openIndex(), PLearn::SourceVMatrix::setMetaDataDir(), and PLearn::SourceVMatrix::setMetaInfoFromSource().
Referenced by build_().
{ inherited::setMetaDataDir(the_metadatadir); if (build_complete) { // Only call openIndex() if the build has been completed, // otherwise the filtering program will not be ready yet. openIndex(); // We call 'setMetaInfoFromSource' only after the index file has been // correctly read. setMetaInfoFromSource(); } }
Reimplemented from PLearn::SourceVMatrix.
Definition at line 132 of file FilteredVMatrix.h.
bool PLearn::FilteredVMatrix::allow_repeat_rows [protected] |
Definition at line 75 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), and declareOptions().
bool PLearn::FilteredVMatrix::build_complete [private] |
A custom hack to make sure openIndex() is not called before the build is complete.
Definition at line 62 of file FilteredVMatrix.h.
Referenced by build(), build_(), and setMetaDataDir().
IntVecFile PLearn::FilteredVMatrix::indexes [protected] |
The i-th element is the index of the i-th selected item in the source VMatrix.
Definition at line 70 of file FilteredVMatrix.h.
Referenced by getNewRow(), and openIndex().
TVec<int> PLearn::FilteredVMatrix::mem_indices [protected] |
Indices stored in memory when no metadata directory is available.
Definition at line 73 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), getNewRow(), makeDeepCopyFromShallowCopy(), and openIndex().
string PLearn::FilteredVMatrix::prg |
Definition at line 93 of file FilteredVMatrix.h.
Referenced by build_(), and declareOptions().
VMatLanguage PLearn::FilteredVMatrix::program [protected] |
Definition at line 66 of file FilteredVMatrix.h.
Referenced by build_(), and computeFilteredIndices().
string PLearn::FilteredVMatrix::repeat_count_field_name [protected] |
Definition at line 77 of file FilteredVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
string PLearn::FilteredVMatrix::repeat_id_field_name [protected] |
Definition at line 76 of file FilteredVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
Definition at line 92 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), and declareOptions().
bool PLearn::FilteredVMatrix::warn_no_metadatadir [protected] |
Definition at line 78 of file FilteredVMatrix.h.
Referenced by build_(), and declareOptions().