|
PLearn 0.1
|
this class is a wrapper for a .vmat VMatrix. More...
#include <VVMatrix.h>


Public Member Functions | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual VVMatrix * | deepCopy (CopiesMap &copies) const |
| virtual void | build () |
| Simply calls inherited::build() then build_(). | |
| const string & | getCode () |
| bool | isPrecomputedAndUpToDate () |
| returns true if the .vmat is precomputed *and* if that precomputed data is more recent than the .vmat file | |
| string | getPrecomputedDataName () |
| VVMatrix (const string &filename_) | |
| VVMatrix () | |
| virtual real | get (int i, int j) const |
| This method must be implemented in all subclasses. | |
| virtual void | getSubRow (int i, int j, Vec v) const |
| It is suggested that this method be implemented in subclasses to speed up accesses (default version repeatedly calls get(i,j) which may have a significant overhead). | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
Static Public Member Functions | |
| static string | _classname_ () |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
| static time_t | getDateOfVMat (const string &filename) |
| get "real" date of .vmat according to its dependencies | |
| static VMat | buildFilteredVMatFromVPL (VMat source, const string &code, const string &ivfname, time_t date_of_code) |
| returns the source VMat after filtering it with 'code' . | |
Public Attributes | |
| string | the_filename |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declare this class' options. | |
| static VMat | createPreproVMat (const string &filename) |
| loads a .vmat file and return the VMat object built from it | |
| static void | generateVMatIndex (VMat source, const string &meta_data_dir, const string &filename, time_t date_of_code, const string &in, size_t idx_prefilter, size_t cidx_prefilter, size_t idx_postfilter, size_t cidx_postfilter, size_t idx_process, size_t cidx_process, size_t idx_shuffle, size_t cidx_shuffle, size_t idx_join, size_t cidx_join) |
| static void | processJoinSection (const vector< string > &code, VMat &tmpsource) |
| static vector< vector< string > > | extractSourceMatrix (const string &str, const string &filename) |
| static void | generateFilterIndexFile (VMat source, const string &code, const string &ivfname) |
Protected Attributes | |
| string | code |
| VMat | the_mat |
Private Types | |
| typedef VMatrix | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
this class is a wrapper for a .vmat VMatrix.
Definition at line 48 of file VVMatrix.h.
typedef VMatrix PLearn::VVMatrix::inherited [private] |
Reimplemented from PLearn::VMatrix.
Definition at line 53 of file VVMatrix.h.
| PLearn::VVMatrix::VVMatrix | ( | const string & | filename_ | ) | [inline] |
Definition at line 110 of file VVMatrix.h.
:the_filename(filename_){build_();}
| PLearn::VVMatrix::VVMatrix | ( | ) | [inline] |
Definition at line 111 of file VVMatrix.h.
{};
| string PLearn::VVMatrix::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
| OptionList & PLearn::VVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
| RemoteMethodMap & PLearn::VVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
| Object * PLearn::VVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 67 of file VVMatrix.cc.
| StaticInitializer VVMatrix::_static_initializer_ & PLearn::VVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
| void PLearn::VVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::VMatrix.
Definition at line 603 of file VVMatrix.cc.
{
inherited::build();
build_();
}
| void PLearn::VVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VMatrix.
Definition at line 609 of file VVMatrix.cc.
References PLearn::force_mkdir(), j, PLearn::newObject(), PLERROR, PLearn::readFileAndMacroProcess(), and PLearn::removeblanks().
{
if (the_filename != "") {
setMetaDataDir( PPath(the_filename+".metadata").absolute() );
force_mkdir( getMetaDataDir() );
time_t latest=0;
code = readFileAndMacroProcess(the_filename,latest);
if(removeblanks(code)[0]=='<') // old xml-like format
the_mat = createPreproVMat(the_filename);
else // New standard PLearn object description format
{
the_mat = dynamic_cast<VMatrix*>(newObject(code));
if(the_mat.isNull())
PLERROR("Object described in %s is not a VMatrix subclass",the_filename.c_str());
the_mat->setMetaDataDir(getMetaDataDir());
}
updateMtime(the_mat->getMtime());
updateMtime(latest);
length_ = the_mat.length();
width_ = the_mat.width();
// Copy the sizes.
copySizesFrom(the_mat);
// Copy the string mappings.
copyStringMappingsFrom(the_mat);
// Copy the parent field names
fieldinfos.resize(width_);
if (the_mat->getFieldInfos().size() > 0)
for(int j=0; j<width_; j++)
fieldinfos[j] = the_mat->getFieldInfos()[j];
}
}

| VMat PLearn::VVMatrix::buildFilteredVMatFromVPL | ( | VMat | source, |
| const string & | code, | ||
| const string & | ivfname, | ||
| time_t | date_of_code | ||
| ) | [static] |
returns the source VMat after filtering it with 'code' .
ivfname is the name of the index file and date_of_code is the code last modification date. If existant, index file is used as is, provided its date is > the latest date of all files (code + all its dependencies)
Definition at line 103 of file VVMatrix.cc.
References PLearn::getNonBlankLines(), PLearn::isfile(), and PLearn::mtime().
{
if(getNonBlankLines(code).empty())
return source;
if(!isfile(ivfname) || mtime(ivfname) < date_of_code)
generateFilterIndexFile(source, code, ivfname);
IntVecFile ivf(ivfname,true);
return new SelectRowsFileIndexVMatrix(source, ivfname);
}

| string PLearn::VVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 67 of file VVMatrix.cc.
| VMat PLearn::VVMatrix::createPreproVMat | ( | const string & | filename | ) | [static, protected] |
loads a .vmat file and return the VMat object built from it
Definition at line 343 of file VVMatrix.cc.
References PLearn::endl(), PLearn::extract_directory(), PLearn::force_mkdir(), PLearn::force_rmdir(), PLearn::getDataSet(), PLearn::hconcat(), i, in, PLearn::isdir(), PLearn::isfile(), PLearn::PP< T >::isNull(), j, PLearn::max(), PLearn::mtime(), PLearn::mvforce(), NEW_SYNTAX_CHAR, PLearn::newObject(), PLearn::VMatLanguage::output_preproc, PLearn::pathexists(), PLERROR, PLWARNING, PLearn::readFileAndMacroProcess(), PLearn::remove_trailing_slash(), PLearn::removeblanks(), PLearn::rm(), PLearn::TVec< T >::size(), slash, slash_char, PLearn::split(), PLearn::toint(), and PLearn::vconcat().
{
time_t date_of_code = 0;
string in=readFileAndMacroProcess(filename,date_of_code);
size_t idx_source = in.find("<SOURCES>");
size_t idx_prefilter = in.find("<PREFILTER>");
size_t idx_postfilter = in.find("<POSTFILTER>");
size_t idx_process = in.find("<PROCESSING>");
size_t idx_shuffle = in.find("<SHUFFLE>");
size_t idx_join = in.find("<JOIN>");
size_t idx_precompute = in.find("<PRECOMPUTE>");
size_t idx_sizes = in.find("<SIZES>");
size_t cidx_source = 0, cidx_prefilter = 0, cidx_postfilter = 0,
cidx_process = 0, cidx_shuffle = 0, cidx_precompute = 0,
cidx_join = 0, cidx_sizes = 0;
string precomp;
VMat source;
bool olddebugval=VMatLanguage::output_preproc;
VMatLanguage::output_preproc = in.find("<DEBUG>")!=string::npos;
if( VMatLanguage::output_preproc )
cerr<<"DEBUG is on (remove <DEBUG> in "+filename+" to turn it off)"<<endl;
PPath meta_data_dir = remove_trailing_slash(filename)+".metadata";
force_mkdir(meta_data_dir);
date_of_code=max(date_of_code,getDateOfVMat(filename));
// remove pollution : all stuff that has possibly been interrupted during computation
rm ( meta_data_dir / "incomplete.*" );
bool sizes_spec = false;
int inputsize = -1;
int targetsize = -1;
int weightsize = -1;
// Check if sizes are specified.
if(idx_sizes != string::npos)
{
sizes_spec = true;
idx_sizes += 7;
cidx_sizes = in.find("</SIZES>");
if(cidx_sizes == string::npos)
PLERROR("Cannot find closing tag </SIZES>. File is %s",filename.c_str());
vector<string> sizes = split(in.substr(idx_sizes, cidx_sizes - idx_sizes),"\n");
if (sizes.size() != 3) {
PLERROR("You should specify exactly 3 sizes between <SIZES> and </SIZES> (file is %s)", filename.c_str());
}
inputsize = toint(sizes[0]);
targetsize = toint(sizes[1]);
weightsize = toint(sizes[2]);
}
if(isfile(meta_data_dir / "precomputed.pmat"))
{
// Precomputed version exist in pmat format.
// If it seems old, we display a warning (one may still want to use it
// if he knows the changes made to the vmat code do not alter the data).
if(mtime(meta_data_dir / "precomputed.pmat") < date_of_code) {
PLWARNING("In VVMatrix::createPreproVMat - The precomputed data (in %s) is older than current code, you may want to recompute again", meta_data_dir.c_str());
}
source = new FileVMatrix(meta_data_dir / "precomputed.pmat");
source->setMetaDataDir(meta_data_dir);
source->updateMtime(date_of_code);
source->defineSizes(inputsize, targetsize, weightsize);
return source;
}
if( pathexists(meta_data_dir / "precomputed.dmat") )
{
// precomputed version exist in DiskVMatrix format,
// so we use it *if it is up to date*
if ( mtime(meta_data_dir/"precomputed.dmat/indexfile") > date_of_code )
{
source = new DiskVMatrix(meta_data_dir/"precomputed.dmat");
source->setMetaDataDir(meta_data_dir);
source->updateMtime(date_of_code);
source->defineSizes(inputsize, targetsize, weightsize);
return source;
}
}
// if index_section is true, then this dataset needs a file containing the index of the rows to keep
bool index_section = idx_prefilter!=string::npos || idx_postfilter!=string::npos || idx_shuffle!=string::npos ;
// if true, index file lacks or is out of date
bool must_regen_index = index_section &&
(!isfile(meta_data_dir/"source.indexes") || date_of_code > mtime(meta_data_dir/"source.indexes"));
// erase obsolete source.index if necessary
if(isfile(meta_data_dir/"source.indexes") && !index_section)
rm (meta_data_dir/"source.indexes");
if(idx_source!=string::npos)
{
// go over tag
idx_source+=9;
cidx_source=in.find("</SOURCES>");
if(cidx_source==string::npos)
PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str());
// 'sec' is the text content of the <SOURCES> section
string sec=in.substr(idx_source,cidx_source-idx_source);
if (sec[1] == NEW_SYNTAX_CHAR) {
// We are using the new syntax
sec[1] = ' '; // remove the special character indicating the new syntax
source = dynamic_cast<VMatrix*>(newObject(sec));
if(source.isNull()) {
PLERROR("In VVMatrix::createPreproVMat '%s' is not a valid VMatrix subclass",sec.c_str());
}
} else {
vector<vector<string> > mstr = extractSourceMatrix(sec,filename);
Array<VMat> vmrows(int(mstr.size()));
// we need to build a VMat that is the concatenation of the datasets contained in 'mstr'
for(unsigned int i=0;i<mstr.size();i++)
{
Array<VMat> ar( (int)mstr[i].size() );
for(unsigned int j=0;j<mstr[i].size();j++)
{
vector<string> vecstr;
vecstr=split(mstr[i][j],":");
ar[j]=getDataSet(vecstr[0]);
// handle different cases of dataset specification
// legal formats are:
// 1- simple dataset filename
// 2- intVecFile specification : filename:intVecFile_Filename
// 3- range specifiaction : filename:start:length
// 4- range specifiaction + intVecFile: filename:intVecFile_Filename:start:length
switch(vecstr.size())
{
case 1: // only dataset name so we do nothing
break;
case 2: // we have an intVecFile specification
// prefix with the path to the current vmat
if(vecstr[1][0]!=slash_char)
vecstr[1]=extract_directory(filename)+vecstr[1];
ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]);
break;
case 3: // submatRows range specification
ar[j]=ar[j].subMatRows(toint(vecstr[1]),toint(vecstr[2]));
break;
case 4: // intVecFile + submatRows
if(vecstr[1][0]!=slash_char)
vecstr[1]=extract_directory(filename)+vecstr[1];
ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]);
ar[j]=ar[j].subMatRows(toint(vecstr[2]),toint(vecstr[3]));
break;
default:
PLERROR("Strange number of semicolumns... Format of source must be something.vmat[:indexfile.index][:start_row:length]. File is %s",filename.c_str());
break;
}
}
// if we have more than one filename in this row, we use hconcat to consolidate 'ar'.
vmrows[i] = ar.size()==1?ar[0]:hconcat(ar);
if(vmrows[i].length()==-1)
PLERROR("Trying to hconcat matrix with different lengths! File is %s",filename.c_str());
}
source = vconcat(vmrows);
if(mstr.size()==0)
PLERROR("No source matrix found in <SOURCES> section! File is %s",filename.c_str());
}
}
else PLERROR("Need at least a <SOURCES> section ! File is %s",filename.c_str());
if(must_regen_index)
generateVMatIndex(source, meta_data_dir, filename, date_of_code, in, idx_prefilter, cidx_prefilter,
idx_postfilter, cidx_postfilter, idx_process, cidx_process,
idx_shuffle, cidx_shuffle, idx_join, cidx_join);
if(idx_join!=string::npos)
{
cidx_join=in.find("</JOIN>");
if(cidx_join==string::npos)
PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str());
vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n");
processJoinSection(code,source);
}
if(idx_process!=string::npos)
{
cidx_process=in.find("</PROCESSING>");
if(cidx_process==string::npos)
PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str());
string code=in.substr(idx_process+12,cidx_process-idx_process-12);
source = new PreprocessingVMatrix(source,code);
}
// if source.index exists at this point, we need to apply it
if(isfile(meta_data_dir/"source.indexes"))
source = new SelectRowsFileIndexVMatrix(source,meta_data_dir/"source.indexes");
// next lines handle precomputation of matrix
if(idx_precompute!=string::npos)
{
idx_precompute+=12;
cidx_precompute=in.find("</PRECOMPUTE>");
if(cidx_precompute==string::npos)
PLERROR("Cannot find closing tag </PRECOMPUTE>. File is %s",filename.c_str());
precomp=removeblanks(in.substr(idx_precompute,cidx_precompute-idx_precompute));
if(precomp == "dmat")
{
cout << "Rendering DMAT : " << meta_data_dir/"precomputed.dmat" << endl;
source->saveDMAT(meta_data_dir/"incomplete.precomputed.dmat");
int cnt=0;
if (isdir(meta_data_dir/"precomputed.dmat")) {
while(cnt++ < 5 &&
!force_rmdir(meta_data_dir/"precomputed.dmat"))
{
cerr<<"Could not rm -rf '"+meta_data_dir+
slash+"precomputed.dmat/'. Maybe 'Stale NFS file handle' curse again. Retrying in 2 sec."
<<endl;
sleep(2);
}
}
mvforce(meta_data_dir/"incomplete.precomputed.dmat",
meta_data_dir/"precomputed.dmat");
if(pathexists(meta_data_dir/"incomplete.precomputed.dmat.metadata"))
{
rm(meta_data_dir/"precomputed.dmat.metadata");
mvforce(meta_data_dir/"incomplete.precomputed.dmat.metadata",
meta_data_dir/"precomputed.dmat.metadata");
}
source=new DiskVMatrix(meta_data_dir/"precomputed.dmat");
}
else if(precomp == "pmat")
{
cout<<"Rendering PMAT : "<<meta_data_dir+slash+"precomputed.pmat"<<endl;
source->savePMAT(meta_data_dir+slash+"incomplete.precomputed.pmat");
mvforce(meta_data_dir/"incomplete.precomputed.pmat",
meta_data_dir/"precomputed.pmat");
if(pathexists(meta_data_dir/"incomplete.precomputed.pmat.metadata"))
mvforce(meta_data_dir/"incomplete.precomputed.pmat.metadata",
meta_data_dir/"precomputed.pmat.metadata");
// Save the string mappings.
source->setMetaDataDir(meta_data_dir);
source->saveAllStringMappings();
source = new FileVMatrix(meta_data_dir/"precomputed.pmat");
source->setMetaDataDir(meta_data_dir);
source->setMtime(date_of_code);
source->defineSizes(inputsize, targetsize, weightsize);
// TODO The 3 lines above are duplicated throughout the code, this is ugly.
}
else if(precomp!="")
PLERROR("Unsupported precomputing format : %s. File is %s", precomp.c_str(),filename.c_str());
}
VMatLanguage::output_preproc=olddebugval;
source->setMetaDataDir(meta_data_dir);
source->updateMtime(date_of_code);
if (sizes_spec) {
source->defineSizes(inputsize, targetsize, weightsize);
}
return source;
}

| void PLearn::VVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declare this class' options.
Reimplemented from PLearn::VMatrix.
Definition at line 72 of file VVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), and the_filename.
{
declareOption(ol, "filename", &VVMatrix::the_filename, OptionBase::buildoption, "Path to the .vmat file");
inherited::declareOptions(ol);
}

| static const PPath& PLearn::VVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::VMatrix.
Definition at line 67 of file VVMatrix.cc.
| vector< vector< string > > PLearn::VVMatrix::extractSourceMatrix | ( | const string & | str, |
| const string & | filename | ||
| ) | [static, protected] |
Definition at line 114 of file VVMatrix.cc.
References PLearn::extract_directory(), PLearn::getNonBlankLines(), i, j, NEW_SYNTAX_CHAR, PLearn::pathexists(), PLearn::removeblanks(), slash_char, and PLearn::split().
{
vector<vector<string> > mstr;
if (str[0] == NEW_SYNTAX_CHAR) {
// We are using the new syntax : we don't care about this for now,
// too bad for the getDate method !
return mstr;
}
vector<string> lines=getNonBlankLines(str);
for(unsigned int i=0;i<lines.size();i++)
mstr.push_back(split(lines[i],"|"));
for(unsigned int i=0;i<mstr.size();i++)
for(unsigned int j=0;j<mstr[i].size();j++)
{
string srcstr = removeblanks(mstr[i][j]);
if(srcstr[0]!=slash_char)
{
string potential_dir = extract_directory(filename);
size_t p = srcstr.find(":");
string potential_path = potential_dir + srcstr.substr(0,p);
if(pathexists(potential_path))
srcstr=potential_dir+srcstr;
}
mstr[i][j] = srcstr;
}
return mstr;
}

| void PLearn::VVMatrix::generateFilterIndexFile | ( | VMat | source, |
| const string & | code, | ||
| const string & | ivfname | ||
| ) | [static, protected] |
Definition at line 81 of file VVMatrix.cc.
References PLearn::IntVecFile::append(), PLearn::IntVecFile::close(), PLearn::VMatLanguage::compileString(), PLearn::fast_exact_is_equal(), PLearn::VMat::fieldName(), i, PLearn::VMat::length(), PLearn::rm(), PLearn::VMatLanguage::run(), and PLearn::VMat::width().
{
rm(ivfname);
IntVecFile ivf(ivfname,true);
VMatLanguage filt(source);
vector<string> fn;
for(int i=0;i<source.width();i++)
fn.push_back(source->fieldName(i));
filt.compileString(code,fn);
Vec bla(1);
Vec src(source.width());
ProgressBar pb(cerr,"Filtering",source.length());
for(int i=0;i<source.length();i++)
{
filt.run(i,bla);
if(!fast_exact_is_equal(bla[0], 0))
ivf.append(i);
pb(i);
}
ivf.close();
}

| void PLearn::VVMatrix::generateVMatIndex | ( | VMat | source, |
| const string & | meta_data_dir, | ||
| const string & | filename, | ||
| time_t | date_of_code, | ||
| const string & | in, | ||
| size_t | idx_prefilter, | ||
| size_t | cidx_prefilter, | ||
| size_t | idx_postfilter, | ||
| size_t | cidx_postfilter, | ||
| size_t | idx_process, | ||
| size_t | cidx_process, | ||
| size_t | idx_shuffle, | ||
| size_t | cidx_shuffle, | ||
| size_t | idx_join, | ||
| size_t | cidx_join | ||
| ) | [static, protected] |
Definition at line 218 of file VVMatrix.cc.
References PLearn::IntVecFile::append(), PLearn::endl(), PLearn::IntVecFile::getVec(), i, PLearn::isfile(), PLearn::VMat::length(), PLearn::IntVecFile::length(), PLearn::manual_seed(), PLearn::mv(), PLERROR, PLearn::removeblanks(), PLearn::rm(), PLearn::seed(), PLearn::shuffleElements(), slash, PLearn::split(), and PLearn::toint().
{
VMat tmpsource = source;
rm(meta_data_dir+slash+"source.indexes");
if(idx_prefilter!=string::npos)
{
idx_prefilter+=11;
cidx_prefilter=in.find("</PREFILTER>");
if(cidx_prefilter==string::npos)
PLERROR("Cannot find closing tag </PREFILTER>. File is %s",filename.c_str());
string code=in.substr(idx_prefilter,cidx_prefilter-idx_prefilter);
cout<<"Prefiltering.. "<<endl;
tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.prefilter.indexes",date_of_code);
}
if(idx_join!=string::npos)
{
cidx_join=in.find("</JOIN>");
if(cidx_join==string::npos)
PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str());
vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n");
processJoinSection(code,tmpsource);
}
if(idx_process!=string::npos)
{
cidx_process=in.find("</PROCESSING>");
if(cidx_process==string::npos)
PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str());
string code=in.substr(idx_process+12,cidx_process-idx_process-12);
tmpsource = new PreprocessingVMatrix(tmpsource,code);
}
if(idx_postfilter!=string::npos)
{
idx_postfilter+=12;
cidx_postfilter=in.find("</POSTFILTER>");
if(cidx_postfilter==string::npos)
PLERROR("Cannot find closing tag </POSTFILTER>. File is %s",filename.c_str());
string code=in.substr(idx_postfilter,cidx_postfilter-idx_postfilter);
cout<<"Postfiltering.. "<<endl;
tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.postfilter.indexes",date_of_code);
}
// combines pre and postfilter index file in a single one
if(isfile(meta_data_dir+slash+"incomplete.source.prefilter.indexes"))
if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes"))
{
IntVecFile ivf(meta_data_dir+slash+"source.indexes",true);
IntVecFile preivf(meta_data_dir+slash+"incomplete.source.prefilter.indexes");
IntVecFile postivf(meta_data_dir+slash+"incomplete.source.postfilter.indexes");
for(int i=0;i<postivf.length();i++)
ivf.append(preivf[postivf[i]]);
}
else
mv(meta_data_dir+slash+"incomplete.source.prefilter.indexes",
meta_data_dir+slash+"source.indexes");
else
if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes"))
mv(meta_data_dir+slash+"incomplete.source.postfilter.indexes",
meta_data_dir+slash+"source.indexes");
if(idx_shuffle!=string::npos)
{
idx_shuffle+=9;
cidx_shuffle=in.find("</SHUFFLE>");
if(cidx_shuffle==string::npos)
PLERROR("Cannot find closing tag </SHUFFLE>. File is %s",filename.c_str());
string seedstr=removeblanks(in.substr(idx_shuffle,cidx_shuffle-idx_shuffle));
if(seedstr=="")
manual_seed(clock());
else
{
int seed = toint(seedstr);
manual_seed(seed);
}
// if a source.indexes file exists, shuffle it, otherwise, create it
TVec<int> idx;
if(isfile(meta_data_dir+slash+"source.indexes"))
{
IntVecFile ivf(meta_data_dir+slash+"source.indexes");
idx=ivf.getVec();
}
else idx=TVec<int>(0,tmpsource->length()-1,1);
shuffleElements(idx);
rm(meta_data_dir+slash+"source.indexes");
IntVecFile newivf(meta_data_dir+slash+"source.indexes",true);
newivf.append(idx);
}
// remove intermediate files
rm(meta_data_dir+slash+"incomplete.source.prefilter.indexes");
rm(meta_data_dir+slash+"incomplete.source.postfilter.indexes");
}

This method must be implemented in all subclasses.
Returns element (i,j).
Implements PLearn::VMatrix.
Definition at line 125 of file VVMatrix.h.
{return the_mat->get(i,j);}
| const string& PLearn::VVMatrix::getCode | ( | ) | [inline] |
Definition at line 94 of file VVMatrix.h.
{return code;}
| time_t PLearn::VVMatrix::getDateOfVMat | ( | const string & | filename | ) | [static] |
get "real" date of .vmat according to its dependencies
Definition at line 143 of file VVMatrix.cc.
References PLearn::extract_directory(), PLearn::getDataSetDate(), PLearn::getDateOfCode(), i, in, j, PLearn::max(), PLearn::mtime(), PLERROR, PLearn::readFileAndMacroProcess(), slash_char, PLearn::split(), and PLearn::strlen().
{
time_t latest = 0;
string in=readFileAndMacroProcess(filename, latest);
size_t idx_source = in.find("<SOURCES>");
size_t cidx_source;
latest=max(getDateOfCode(filename),latest);
if(idx_source!=string::npos)
{
idx_source += strlen("<SOURCES>"); // skip beyond
cidx_source=in.find("</SOURCES>");
if(cidx_source==string::npos)
PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str());
string sec=in.substr(idx_source,cidx_source-idx_source);
vector<vector<string> > mstr = extractSourceMatrix(sec,filename);
// TODO Make it work with the new syntax
for(unsigned int i=0;i<mstr.size();i++)
for(unsigned int j=0;j<mstr[i].size();j++)
{
vector<string> vecstr;
vecstr=split(mstr[i][j],":");
if(vecstr.size()==2 || vecstr.size()==4)
{
// also check for date of possible IntVecFile
if(vecstr[1][0]!=slash_char)
vecstr[1]=extract_directory(filename)+vecstr[1];
latest=max(mtime(vecstr[1]),latest);
}
latest=max(getDataSetDate(vecstr[0]),latest);
}
}
return latest;
}

| OptionList & PLearn::VVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 67 of file VVMatrix.cc.
| OptionMap & PLearn::VVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 67 of file VVMatrix.cc.
| string PLearn::VVMatrix::getPrecomputedDataName | ( | ) |
Definition at line 332 of file VVMatrix.cc.
References PLearn::isfile(), PLearn::pathexists(), PLearn::remove_trailing_slash(), and slash.
Referenced by PLearn::vmatmain().
{
string meta_data_dir=remove_trailing_slash(the_filename)+".metadata";
if(isfile(meta_data_dir+slash+"precomputed.pmat"))
return meta_data_dir+slash+"precomputed.pmat";
if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash))
return meta_data_dir+slash+"precomputed.dmat";
return "** no precomputed data found **";
}


| RemoteMethodMap & PLearn::VVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 67 of file VVMatrix.cc.
It is suggested that this method be implemented in subclasses to speed up accesses (default version repeatedly calls get(i,j) which may have a significant overhead).
Fills v with the subrow i lying between columns j (inclusive) and j+v.length() (exclusive).
Reimplemented from PLearn::VMatrix.
Definition at line 126 of file VVMatrix.h.
{the_mat->getSubRow(i,j,v);}
| bool PLearn::VVMatrix::isPrecomputedAndUpToDate | ( | ) |
returns true if the .vmat is precomputed *and* if that precomputed data is more recent than the .vmat file
Definition at line 320 of file VVMatrix.cc.
References PLearn::isfile(), PLearn::mtime(), PLearn::pathexists(), PLearn::remove_trailing_slash(), and slash.
Referenced by PLearn::vmatmain().
{
string meta_data_dir=remove_trailing_slash(the_filename)+".metadata";
if(isfile(meta_data_dir+slash+"precomputed.pmat") &&
mtime(meta_data_dir+slash+"precomputed.pmat") > getMtime())
return true;
if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash) &&
mtime(meta_data_dir+slash+"precomputed.dmat"+slash+"indexfile") > getMtime())
return true;
return false;
}


| void PLearn::VVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::VMatrix.
Definition at line 670 of file VVMatrix.cc.
References PLearn::deepCopyField().
{
inherited::makeDeepCopyFromShallowCopy(copies);
deepCopyField(the_mat, copies);
}

| void PLearn::VVMatrix::processJoinSection | ( | const vector< string > & | code, |
| VMat & | tmpsource | ||
| ) | [static, protected] |
Definition at line 179 of file VVMatrix.cc.
References PLearn::JoinVMatrix::addStatField(), PLearn::getDataSet(), i, PLearn::isBlank(), j, PLERROR, PLearn::removeblanks(), and PLearn::split().
{
unsigned int i=0;
while(i<code.size())
{
while(isBlank(code[i]))i++;
VMat slave=getDataSet(code[i++]);
vector<string> mess=split(code[i++],"[]");
if(mess.size()!=3)
PLERROR("In JOIN section, field correspondance syntax is : [master1,master2] -> [slave1,slave2]");
vector<string> mfields=split(mess[0],",");
vector<string> sfields=split(mess[2],",");
TVec<int> midx(int(mfields.size())), sidx(int(sfields.size()));
for(unsigned int j=0;j<mfields.size();j++)
midx[j]=tmpsource->fieldIndex(mfields[j]);
for(unsigned int j=0;j<sfields.size();j++)
sidx[j]=slave->fieldIndex(sfields[j]);
JoinVMatrix * jvm= new JoinVMatrix(tmpsource,slave,midx,sidx);
// browse field declarations of the <JOIN> section to declare new fields in the resulting JoinVMatrix
while(i<code.size() && code[i].find(":")!=string::npos)
{
vector<string> crunch=split(code[i],":");
if(crunch.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname");
vector<string> st=split(removeblanks(crunch[0]),"()");
if(st.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname");
jvm->addStatField(removeblanks(st[0]),removeblanks(st[1]),removeblanks(crunch[1]));
i++;
}
while(i<code.size() && isBlank(code[i]))i++;
tmpsource=jvm;
}
}

Reimplemented from PLearn::VMatrix.
Definition at line 90 of file VVMatrix.h.
string PLearn::VVMatrix::code [protected] |
Definition at line 57 of file VVMatrix.h.
Definition at line 86 of file VVMatrix.h.
Referenced by declareOptions().
VMat PLearn::VVMatrix::the_mat [protected] |
Definition at line 58 of file VVMatrix.h.
1.7.4