|
PLearn 0.1
|
#include <FilteredVMatrix.h>


Public Member Functions | |
| FilteredVMatrix () | |
| Default constructor. | |
| FilteredVMatrix (VMat the_source, const string &program_string, const PPath &the_metadatadir="", bool the_report_progress=true, bool allow_repeat_rows_=false, const string &repeat_id_field_name_="", const string &repeat_count_field_name_="", bool call_build_=true) | |
| Convenience constructor. | |
| virtual void | setMetaDataDir (const PPath &the_metadatadir) |
| Also sets the source's meta-data dir if it's not already set. | |
| virtual void | build () |
| Simply calls inherited::build() then build_(). | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual FilteredVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
| static string | _classname_ () |
| Declares name and deepCopy methods. | |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Public Attributes | |
| bool | report_progress |
| string | prg |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
| void | openIndex () |
| Generates the index file if it does not already exist. | |
| void | computeFilteredIndices () |
| Compute the filtered indices. | |
| virtual void | getNewRow (int i, const Vec &v) const |
| This is the only method requiring implementation. | |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declares this class' options. | |
Protected Attributes | |
| VMatLanguage | program |
| IntVecFile | indexes |
| The i-th element is the index of the i-th selected item in the source VMatrix. | |
| TVec< int > | mem_indices |
| Indices stored in memory when no metadata directory is available. | |
| bool | allow_repeat_rows |
| string | repeat_id_field_name |
| string | repeat_count_field_name |
| bool | warn_no_metadatadir |
Private Types | |
| typedef SourceVMatrix | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
Private Attributes | |
| bool | build_complete |
| A custom hack to make sure openIndex() is not called before the build is complete. | |
Definition at line 54 of file FilteredVMatrix.h.
typedef SourceVMatrix PLearn::FilteredVMatrix::inherited [private] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.h.
| PLearn::FilteredVMatrix::FilteredVMatrix | ( | ) |
Default constructor.
Definition at line 61 of file FilteredVMatrix.cc.
:
build_complete(false),
allow_repeat_rows(false),
repeat_id_field_name(""),
repeat_count_field_name(""),
warn_no_metadatadir(false),
report_progress(true)
{}
| PLearn::FilteredVMatrix::FilteredVMatrix | ( | VMat | the_source, |
| const string & | program_string, | ||
| const PPath & | the_metadatadir = "", |
||
| bool | the_report_progress = true, |
||
| bool | allow_repeat_rows_ = false, |
||
| const string & | repeat_id_field_name_ = "", |
||
| const string & | repeat_count_field_name_ = "", |
||
| bool | call_build_ = true |
||
| ) |
Convenience constructor.
Definition at line 70 of file FilteredVMatrix.cc.
References build_(), and PLearn::VMatrix::metadatadir.
:
inherited(the_source, call_build_),
allow_repeat_rows(allow_repeat_rows_),
repeat_id_field_name(repeat_id_field_name_),
repeat_count_field_name(repeat_count_field_name_),
report_progress(the_report_progress),
prg(program_string)
{
// Note that although VMatrix::build_ would be tempted to call
// setMetaDataDir when inherited(the_source, true) is called above (if
// call_build_ is true), the metadatadir is only set below, so it will not
// happen.
metadatadir = the_metadatadir;
if (call_build_)
build_();
}

| string PLearn::FilteredVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| OptionList & PLearn::FilteredVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| RemoteMethodMap & PLearn::FilteredVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| Object * PLearn::FilteredVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| StaticInitializer FilteredVMatrix::_static_initializer_ & PLearn::FilteredVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| void PLearn::FilteredVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::SourceVMatrix.
Definition at line 323 of file FilteredVMatrix.cc.
References PLearn::SourceVMatrix::build(), build_(), and build_complete.
{
build_complete = false;
inherited::build();
build_();
}

| void PLearn::FilteredVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 278 of file FilteredVMatrix.cc.
References PLearn::TVec< T >::append(), build_complete, PLearn::VMatLanguage::compileString(), computeFilteredIndices(), PLearn::VMatrix::getMetaDataDir(), PLearn::VMatrix::hasMetaDataDir(), PLearn::VMatrix::length_, PLWARNING, prg, program, repeat_count_field_name, repeat_id_field_name, PLearn::VMatrix::setFieldInfos(), setMetaDataDir(), PLearn::SourceVMatrix::setMetaInfoFromSource(), PLearn::VMatLanguage::setSource(), PLearn::SourceVMatrix::source, PLearn::VMatrix::updateMtime(), warn_no_metadatadir, and PLearn::VMatrix::width_.
Referenced by build(), and FilteredVMatrix().
{
if (source) {
updateMtime(source);
vector<string> fieldnames;
program.setSource(source);
program.compileString(prg,fieldnames);
build_complete = true;
if (hasMetaDataDir())
// Calling setMetaDataDir() will compute indices and save them
// in the give metadata directory.
setMetaDataDir(getMetaDataDir());
else {
// Compute selected indices in memory only.
if(warn_no_metadatadir)
PLWARNING("In FilteredVMatrix::build_() - there is no "
"metadatadir: recomputing filtered indices");
computeFilteredIndices();
}
Array<VMField> finfos= source->getFieldInfos().copy();
if(!repeat_id_field_name.empty())
{
finfos.append(VMField(repeat_id_field_name));
if(0 < width_)
++width_;
}
if(!repeat_count_field_name.empty())
{
finfos.append(VMField(repeat_count_field_name));
if(0 < width_)
++width_;
}
setFieldInfos(finfos);
setMetaInfoFromSource();
} else
length_ = width_ = 0;
}


| string PLearn::FilteredVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| void PLearn::FilteredVMatrix::computeFilteredIndices | ( | ) | [protected] |
Compute the filtered indices.
Definition at line 96 of file FilteredVMatrix.cc.
References allow_repeat_rows, PLearn::TVec< T >::append(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), PLearn::VMat::length(), PLearn::VMatrix::length_, mem_indices, program, report_progress, PLearn::TVec< T >::resize(), PLearn::VMatLanguage::run(), PLearn::SourceVMatrix::source, and x.
Referenced by build_(), and openIndex().
{
int l = source.length();
Vec result(1);
PP<ProgressBar> pb;
if (report_progress)
pb = new ProgressBar("Filtering source vmat", l);
mem_indices.resize(0);
for(int i=0; i<l; i++)
{
if (report_progress)
pb->update(i);
program.run(i,result);
if(!allow_repeat_rows)
{
if(!fast_exact_is_equal(result[0], 0))
mem_indices.append(i);
}
else
for(int x = int(round(result[0])); x > 0; --x)
mem_indices.append(i);
}
length_ = mem_indices.length();
}


| void PLearn::FilteredVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 221 of file FilteredVMatrix.cc.
References allow_repeat_rows, PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::SourceVMatrix::declareOptions(), PLearn::VMatrix::extrasize_, PLearn::VMatrix::inputsize_, PLearn::VMatrix::length_, PLearn::OptionBase::nosave, prg, PLearn::redeclareOption(), repeat_count_field_name, repeat_id_field_name, report_progress, PLearn::VMatrix::targetsize_, warn_no_metadatadir, PLearn::VMatrix::weightsize_, and PLearn::VMatrix::width_.
{
declareOption(ol, "prg", &FilteredVMatrix::prg, OptionBase::buildoption,
"The VPL code that should produce a single scalar, indicating whether \n"
"we should keep the line (if the produced scalar is non zero) or throw it away (if it's zero)");
declareOption(ol, "report_progress", &FilteredVMatrix::report_progress, OptionBase::buildoption,
"Whether to report the filtering progress or not.");
declareOption(ol, "allow_repeat_rows", &FilteredVMatrix::allow_repeat_rows, OptionBase::buildoption,
"When true, the result of the program indicates the number of times this row should be repeated.\n"
"Simple filtering when false.");
declareOption(ol, "repeat_id_field_name", &FilteredVMatrix::repeat_id_field_name, OptionBase::buildoption,
"Field name for the repetition id (0, 1, ..., n-1). No field is added if empty.");
declareOption(ol, "repeat_count_field_name", &FilteredVMatrix::repeat_count_field_name, OptionBase::buildoption,
"Field name for the number of repetitions (n). No field is added if empty.");
declareOption(ol, "warn_no_metadatadir",
&FilteredVMatrix::warn_no_metadatadir,
OptionBase::buildoption,
"If True, generate a warning when there is no metadatadir.\n"
"This can be useful as if there is no metadatadir we will recompute\n"
"filtered indices every time.");
// Now call the parent class' declareOptions
inherited::declareOptions(ol);
declareOption(ol, "length", &FilteredVMatrix::length_,
OptionBase::nosave,
"The number of example. Computed each time from source.");
redeclareOption(ol, "inputsize", &FilteredVMatrix::inputsize_,
OptionBase::nosave,
"Taken from source in FilteredVMatrix.");
redeclareOption(ol, "targetsize", &FilteredVMatrix::targetsize_,
OptionBase::nosave,
"Taken from source in FilteredVMatrix.");
redeclareOption(ol, "weightsize", &FilteredVMatrix::weightsize_,
OptionBase::nosave,
"Taken from source in FilteredVMatrix.");
redeclareOption(ol, "extrasize", &FilteredVMatrix::extrasize_,
OptionBase::nosave,
"Taken from source in FilteredVMatrix.");
redeclareOption(ol, "width", &FilteredVMatrix::width_,
OptionBase::nosave,
"Taken from source in FilteredVMatrix.");
}

| static const PPath& PLearn::FilteredVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 132 of file FilteredVMatrix.h.
| FilteredVMatrix * PLearn::FilteredVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
This is the only method requiring implementation.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 179 of file FilteredVMatrix.cc.
References i, indexes, PLearn::TVec< T >::isEmpty(), j, PLearn::VMatrix::length(), PLearn::IntVecFile::length(), mem_indices, PLERROR, repeat_count_field_name, repeat_id_field_name, PLearn::SourceVMatrix::source, PLearn::TVec< T >::subVec(), and PLearn::VMat::width().
{
if (mem_indices.isEmpty() && indexes.length() == -1)
PLERROR("In FilteredVMatrix::getNewRow - Filtered indices are not\n"
"set yet.");
int j= source->width();
int idx = mem_indices.isEmpty() ? indexes[i] : mem_indices[i];
source->getRow(idx, v.subVec(0, j));
if(!repeat_id_field_name.empty())
{
int k= 1;
while(k <= i && (mem_indices.isEmpty()
? indexes[i]==indexes[i-k]
: mem_indices[i] == mem_indices[i-k]))
++k;
v[j++]= static_cast<real>(k-1);
}
if(!repeat_count_field_name.empty())
{
int k0= 1;
while(k0 <= i && (mem_indices.isEmpty()
? indexes[i]==indexes[i-k0]
: mem_indices[i] == mem_indices[i-k0]))
++k0;
--k0;
int k1= 1;
while(k1+i < length() && (mem_indices.isEmpty()
? indexes[i]==indexes[i+k1]
: mem_indices[i] == mem_indices[i+k1]))
++k1;
v[j++]= static_cast<real>(k0+k1);
}
}

| OptionList & PLearn::FilteredVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| OptionMap & PLearn::FilteredVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| RemoteMethodMap & PLearn::FilteredVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::SourceVMatrix.
Definition at line 56 of file FilteredVMatrix.cc.
| void PLearn::FilteredVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 333 of file FilteredVMatrix.cc.
References PLearn::deepCopyField(), PLearn::SourceVMatrix::makeDeepCopyFromShallowCopy(), and mem_indices.
{
inherited::makeDeepCopyFromShallowCopy(copies);
deepCopyField(mem_indices, copies);
}

| void PLearn::FilteredVMatrix::openIndex | ( | ) | [protected] |
Generates the index file if it does not already exist.
If it exists and is up to date, simply opens it.
Definition at line 124 of file FilteredVMatrix.cc.
References PLearn::PPath::absolute(), PLearn::IntVecFile::append(), PLearn::IntVecFile::close(), computeFilteredIndices(), PLearn::force_mkdir(), PLearn::VMatrix::getMetaDataDir(), PLearn::VMatrix::hasMetaDataDir(), i, indexes, PLearn::isfile(), PLearn::VMatrix::isUpToDate(), PLearn::TVec< T >::length(), PLearn::IntVecFile::length(), PLearn::VMatrix::length_, PLearn::VMatrix::lockMetaDataDir(), mem_indices, PLearn::IntVecFile::open(), PLASSERT, PLERROR, PLearn::rm(), and PLearn::VMatrix::unlockMetaDataDir().
Referenced by setMetaDataDir().
{
PLASSERT(hasMetaDataDir());
PPath idxfname = getMetaDataDir() / "filtered.idx";
if(!force_mkdir(getMetaDataDir()))
PLERROR("In FilteredVMatrix::openIndex - Could not create "
"directory %s", getMetaDataDir().absolute().c_str());
lockMetaDataDir();
try{
if(isUpToDate(idxfname))
indexes.open(idxfname.absolute());
else // let's (re)create the index
{
computeFilteredIndices();
rm(idxfname); // force remove it
indexes.open(idxfname.absolute(), true);
for (int i = 0; i < mem_indices.length(); i++)
indexes.append(mem_indices[i]);
indexes.close();
indexes.open(idxfname.absolute());
mem_indices = TVec<int>(); // Free memory.
}
}catch(const PLearnError& e){
unlockMetaDataDir();
//we erase the file if we are creating it
// as it can be partilly saved.
if(!isUpToDate(idxfname) && isfile(idxfname))
rm(idxfname);
throw e;
}
unlockMetaDataDir();
length_ = indexes.length();
}


| void PLearn::FilteredVMatrix::setMetaDataDir | ( | const PPath & | the_metadatadir | ) | [virtual] |
Also sets the source's meta-data dir if it's not already set.
If there are fields that have no corresponding .smap .notes or .binning info files but the source has those files for a field with the same name, then those of the source will be set also for this vmatrix.
Reimplemented from PLearn::SourceVMatrix.
Definition at line 163 of file FilteredVMatrix.cc.
References build_complete, openIndex(), PLearn::SourceVMatrix::setMetaDataDir(), and PLearn::SourceVMatrix::setMetaInfoFromSource().
Referenced by build_().
{
inherited::setMetaDataDir(the_metadatadir);
if (build_complete) {
// Only call openIndex() if the build has been completed,
// otherwise the filtering program will not be ready yet.
openIndex();
// We call 'setMetaInfoFromSource' only after the index file has been
// correctly read.
setMetaInfoFromSource();
}
}


Reimplemented from PLearn::SourceVMatrix.
Definition at line 132 of file FilteredVMatrix.h.
bool PLearn::FilteredVMatrix::allow_repeat_rows [protected] |
Definition at line 75 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), and declareOptions().
bool PLearn::FilteredVMatrix::build_complete [private] |
A custom hack to make sure openIndex() is not called before the build is complete.
Definition at line 62 of file FilteredVMatrix.h.
Referenced by build(), build_(), and setMetaDataDir().
IntVecFile PLearn::FilteredVMatrix::indexes [protected] |
The i-th element is the index of the i-th selected item in the source VMatrix.
Definition at line 70 of file FilteredVMatrix.h.
Referenced by getNewRow(), and openIndex().
TVec<int> PLearn::FilteredVMatrix::mem_indices [protected] |
Indices stored in memory when no metadata directory is available.
Definition at line 73 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), getNewRow(), makeDeepCopyFromShallowCopy(), and openIndex().
| string PLearn::FilteredVMatrix::prg |
Definition at line 93 of file FilteredVMatrix.h.
Referenced by build_(), and declareOptions().
VMatLanguage PLearn::FilteredVMatrix::program [protected] |
Definition at line 66 of file FilteredVMatrix.h.
Referenced by build_(), and computeFilteredIndices().
string PLearn::FilteredVMatrix::repeat_count_field_name [protected] |
Definition at line 77 of file FilteredVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
string PLearn::FilteredVMatrix::repeat_id_field_name [protected] |
Definition at line 76 of file FilteredVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
Definition at line 92 of file FilteredVMatrix.h.
Referenced by computeFilteredIndices(), and declareOptions().
bool PLearn::FilteredVMatrix::warn_no_metadatadir [protected] |
Definition at line 78 of file FilteredVMatrix.h.
Referenced by build_(), and declareOptions().
1.7.4