PLearn 0.1
|
#include <VMatLanguage.h>
Public Member Functions | |
VMatLanguage () | |
VMatLanguage (VMat vmsrc) | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual VMatLanguage * | deepCopy (CopiesMap &copies) const |
virtual void | build () |
Post-constructor. | |
virtual void | run (const Vec &srcvec, const Vec &result, int rowindex=-1) const |
Executes the program on the srcvec, copy resulting stack to result rowindex is only there for instruction 'rowindex' that pushes it on the stack. | |
void | run (int rowindex, const Vec &result) const |
Gets the row with the given rowindex from the vmsource VMat and applies program to it. | |
void | setSource (VMat the_source) |
void | setSourceFieldNames (TVec< string > the_srcfieldnames) |
TVec< string > | getOutputFieldNames () const |
int | inputsize () const |
int | outputsize () const |
void | compileStream (PStream &in, vector< string > &fieldnames) |
takes a string, filename, or PStream and generate the bytecode from it On exit, fieldnames will contain all the fieldnames of the resulting matrix If the program accesses fields by name, you must either have called setSource OR setSourceFieldNames prior to calling one of the compile... | |
void | compileString (const string &code, vector< string > &fieldnames) |
void | compileFile (const PPath &filename, vector< string > &fieldnames) |
void | compileString (const string &code, TVec< string > &fieldnames) |
operator bool () const | |
void | clear () |
Make it an empty program by clearing outputfieldnames, program, mappings. | |
int | pstackSize () const |
Vec | getMemory () const |
Return the contents of the memory buffer. | |
void | setMemory (const Vec &new_mem) const |
Set the contents of the memory buffer. | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Does the necessary operations to transform a shallow copy (this) into a deep copy by deep-copying all the members that need to be. | |
Static Public Member Functions | |
static string | _classname_ () |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
static void | declareOptions (OptionList &ol) |
Declare options (data fields) for the class. | |
static void | getOutputFieldNamesFromString (const string &code, vector< string > &fieldnames) |
static void | getOutputFieldNamesFromString (const string &code, TVec< string > &fieldnames) |
static void | getOutputFieldNamesFromStream (PStream &in, vector< string > &fieldnames) |
static void | staticPreprocess (PStream &in, map< string, string > &defines, string &processed_sourcecode, vector< string > &fieldnames) |
Public Attributes | |
string | sourcecode |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
static bool | output_preproc = false |
Private Types | |
typedef Object | inherited |
Private Member Functions | |
void | generateCode (const string &processed_sourcecode) |
void | generateCode (PStream &processed_sourcecode) |
void | preprocess (PStream &in, map< string, string > &defines, string &processed_sourcecode, vector< string > &fieldnames) |
void | build_ () |
Object-specific post-constructor. | |
Static Private Member Functions | |
static void | build_opcodes_map () |
builds the opcodes map if it does not already exist | |
Private Attributes | |
VMat | vmsource |
TVec< string > | srcfieldnames |
TVec< string > | outputfieldnames |
TVec< int > | program |
TVec< RealMapping > | mappings |
Vec | pstack |
Vec | myvec |
Vec | mem |
Static Private Attributes | |
static map< string, int > | opcodes |
Definition at line 62 of file VMatLanguage.h.
typedef Object PLearn::VMatLanguage::inherited [private] |
Reimplemented from PLearn::Object.
Definition at line 64 of file VMatLanguage.h.
PLearn::VMatLanguage::VMatLanguage | ( | ) | [inline] |
Definition at line 93 of file VMatLanguage.h.
PLearn::VMatLanguage::VMatLanguage | ( | VMat | vmsrc | ) |
Definition at line 197 of file VMatLanguage.cc.
string PLearn::VMatLanguage::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
OptionList & PLearn::VMatLanguage::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
RemoteMethodMap & PLearn::VMatLanguage::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
Object * PLearn::VMatLanguage::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
StaticInitializer VMatLanguage::_static_initializer_ & PLearn::VMatLanguage::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
void PLearn::VMatLanguage::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::Object.
Definition at line 231 of file VMatLanguage.cc.
{ inherited::build(); build_(); }
void PLearn::VMatLanguage::build_ | ( | ) | [private] |
Object-specific post-constructor.
This method should be redefined in subclasses and do the actual building of the object according to previously set option fields. Constructors can just set option fields, and then call build_. This method is NOT virtual, and will typically be called only from three places: a constructor, the public virtual build()
method, and possibly the public virtual read method (which calls its parent's read). build_()
can assume that its parent's build_()
has already been called.
Reimplemented from PLearn::Object.
Definition at line 238 of file VMatLanguage.cc.
{ build_opcodes_map(); }
void PLearn::VMatLanguage::build_opcodes_map | ( | ) | [static, private] |
builds the opcodes map if it does not already exist
builds the map if it does not already exist
Definition at line 844 of file VMatLanguage.cc.
{ if(opcodes.empty()) { opcodes["__insertconstant"] = 0; // followed by a floating point number (4 bytes, just like int) opcodes["__getfieldval"] = 1; // followed by field# opcodes["__applymapping"] = 2; // followed by mapping# opcodes["pop"] = 3; opcodes["dup"] = 4; opcodes["exch"] = 5; opcodes["onehot"] = 6; opcodes["+"] = 7; opcodes["-"] = 8; opcodes["*"] = 9; opcodes["/"] = 10; opcodes["=="] = 11; opcodes["!="] = 12; opcodes[">"] = 13; opcodes[">="] = 14; opcodes["<"] = 15; opcodes["<="] = 16; opcodes["and"] = 17; opcodes["or"] = 18; opcodes["not"] = 19; opcodes["ifelse"] = 20; opcodes["fabs"] = 21; opcodes["rint"] = 22; opcodes["floor"] = 23; opcodes["ceil"] = 24; opcodes["log"] = 25; opcodes["exp"] = 26; opcodes["rowindex"] = 27; // pushes the rownum on the stack opcodes["isnan"] = 28; opcodes["year"] = 29; // from format CYYMMDD -> YYYY opcodes["month"] = 30; // CYYMMDD -> MM opcodes["day"] = 31; // CYYMMDD -> DD opcodes["daydiff"] = 32; // nb. days opcodes["monthdiff"] = 33; // continuous: nb. days / (365.25/12) opcodes["yeardiff"] = 34; // continuous: nb. days / 365.25 opcodes["year_month_day"] = 35; // CYYMMDD -> YYYY MM DD opcodes["todate"] = 36; // YYYY MM DD -> CYYMMDD opcodes["dayofweek"] = 37; // from CYYMMDD -> [0..6] (0=monday; 6=sunday) opcodes["today"] = 38; // today's date CYYMMDD opcodes["date2julian"] = 39; // CYYMMDD -> nb. days opcodes["julian2date"] = 40; // nb. days -> CYYMMDD opcodes["min"] = 41; // a b -> (a<b? a : b) opcodes["max"] = 42; // a b -> (a<b? b : a) opcodes["sqrt"] = 43; opcodes["^"] = 44; opcodes["mod"] = 45; opcodes["vecscalmul"] = 46; // x1 ... xn alpha n --> (x1*alpha) ... (xn*alpha) opcodes["__getfieldsrange"] = 47; // %N:%M pushes field %N up to %M. M must be >= N. opcodes["select"] = 48; // v0 v1 v2 v3 ... vn-1 n i --> vi opcodes["length"] = 49; // the length of the currently processed column. opcodes["sign"] = 50; // a --> sign(a) (0 -1 or +1) opcodes["get"] = 51; // pos --> value_of_stack_at_pos (if pos is negative then it's relative to stacke end ex: -1 get will get the previous element of stack) opcodes["memput"] = 52; // a mempos --> ( a is saved in memory position mempos) opcodes["memget"] = 53; // mempos --> val ( gets val from memory in position mempos) opcodes["neg"] = 54; // a --> -a opcodes["missing"] = 55; // a missing value opcodes["sumabs"] = 56; // v0 v1 v2 ... vn --> sum_i |vi| (no pop, and starts from the beginning of the stack) opcodes["weeknumber"] = 57; // CYYMMDD -> week number in the year between 0 and 52 incl. (ISO 8601 minus 1) opcodes["dayofyear"] = 58; // CYYMMDD -> number of days since january 1 of year CYY opcodes["nextincal"] = 59; // cal# JDate -> next jdate on or after given jdate in global calendar cal# opcodes["previncal"] = 60; // cal# JDate -> previous jdate on or before given jdate in global calendar cal# opcodes["gausshot"] = 61; // index nclasses sigma --> smooth one-hot opcodes["sigmoid"] = 62; // a -> sigmoid(a) opcodes["cos"] = 63; // a -> cos(a) opcodes["varproduct"] = 64; // a0 a1 ... an n+1 b0 b1 ... bm m+1 ... num_vars -> res0 ... (product of encoded variables) opcodes["thermometer"] = 65; // index nclasses -> thermometer encoding opcodes["erf"] = 66; } }
string PLearn::VMatLanguage::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
void PLearn::VMatLanguage::clear | ( | ) |
Make it an empty program by clearing outputfieldnames, program, mappings.
Definition at line 281 of file VMatLanguage.cc.
Referenced by PLearn::VPLProcessor::initializeInputPrograms(), PLearn::VPLPreprocessedLearner2::initializeInputPrograms(), and PLearn::VPLPreprocessedLearner2::initializeOutputPrograms().
{ outputfieldnames.resize(0); program.resize(0); mappings.resize(0); }
void PLearn::VMatLanguage::compileFile | ( | const PPath & | filename, |
vector< string > & | fieldnames | ||
) |
Definition at line 654 of file VMatLanguage.cc.
References in, PLearn::openFile(), and PLearn::PStream::raw_ascii.
{ PStream in = openFile(filename, PStream::raw_ascii, "r"); compileStream(in,fieldnames); }
void PLearn::VMatLanguage::compileStream | ( | PStream & | in, |
vector< string > & | fieldnames | ||
) |
takes a string, filename, or PStream and generate the bytecode from it On exit, fieldnames will contain all the fieldnames of the resulting matrix If the program accesses fields by name, you must either have called setSource OR setSourceFieldNames prior to calling one of the compile...
methods.
Definition at line 660 of file VMatLanguage.cc.
References PLearn::endl(), i, PLearn::perr, PLERROR, and PLearn::tostring().
{ map<string,string> defines; string processed_sourcecode; program.resize(0); mappings.resize(0); // first, warn user if a fieldname appears twice or more in the source matrix string fname; for(int i=0;i<srcfieldnames.length();i++) { fname = srcfieldnames[i]; if (!fname.empty()) { fname = string("@") + fname; if(defines.find(fname) != defines.end()) PLERROR("fieldname '%s' is duplicate in processed matrix", fname.c_str()); defines[fname]=string("%")+tostring(i); } } // the filednames parameter is an output vector in which we put the fieldnames of the final VMat fieldnames.clear(); preprocess(in, defines, processed_sourcecode, fieldnames); outputfieldnames.resize(int(fieldnames.size())); for(unsigned int k=0; k < fieldnames.size(); k++) outputfieldnames[k] = fieldnames[k]; if(output_preproc) { perr<<"Preprocessed code:"<<endl<<processed_sourcecode<<endl; perr<<"FieldNames : "<<endl<<fieldnames<<endl; } generateCode(processed_sourcecode); }
void PLearn::VMatLanguage::compileString | ( | const string & | code, |
vector< string > & | fieldnames | ||
) |
Definition at line 639 of file VMatLanguage.cc.
References in, PLearn::openString(), and PLearn::PStream::raw_ascii.
Referenced by PLearn::RandomSamplesVMatrix::build_(), PLearn::ProcessingVMatrix::build_(), PLearn::FilteredVMatrix::build_(), PLearn::VVMatrix::generateFilterIndexFile(), PLearn::VPLProcessor::initializeInputPrograms(), PLearn::VPLPreprocessedLearner2::initializeInputPrograms(), PLearn::VPLPreprocessedLearner2::initializeOutputPrograms(), PLearn::VPLCombinedLearner::initializeOutputPrograms(), and PLearn::vmatmain().
{ PStream in = openString(code, PStream::raw_ascii); compileStream(in,fieldnames); }
void PLearn::VMatLanguage::compileString | ( | const string & | code, |
TVec< string > & | fieldnames | ||
) |
Definition at line 645 of file VMatLanguage.cc.
References i, and PLearn::TVec< T >::resize().
{ vector<string> names; compileString(code, names); fieldnames.resize(int(names.size())); for(unsigned int i=0; i< names.size(); i++) fieldnames[i] = names[i]; }
void PLearn::VMatLanguage::declareOptions | ( | OptionList & | ol | ) | [static] |
Declare options (data fields) for the class.
Redefine this in subclasses: call declareOption
(...) for each option, and then call inherited::declareOptions(options)
. Please call the inherited
method AT THE END to get the options listed in a consistent order (from most recently defined to least recently defined).
static void MyDerivedClass::declareOptions(OptionList& ol) { declareOption(ol, "inputsize", &MyObject::inputsize_, OptionBase::buildoption, "The size of the input; it must be provided"); declareOption(ol, "weights", &MyObject::weights, OptionBase::learntoption, "The learned model weights"); inherited::declareOptions(ol); }
ol | List of options that is progressively being constructed for the current class. |
Reimplemented from PLearn::Object.
Definition at line 244 of file VMatLanguage.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::OptionBase::learntoption, mappings, outputfieldnames, program, sourcecode, srcfieldnames, and vmsource.
{ declareOption(ol, "sourcecode", &VMatLanguage::sourcecode, OptionBase::buildoption, "The VPL sourcecode of the program."); declareOption(ol, "outputfieldnames", &VMatLanguage::outputfieldnames, OptionBase::learntoption, "The output fieldnames produced by the program"); declareOption(ol, "vmsource", &VMatLanguage::vmsource, OptionBase::learntoption, "The VMat that was set by setSource"); declareOption(ol, "srcfieldnames", &VMatLanguage::srcfieldnames, OptionBase::learntoption, "The fieldnames that were set by setSourceFieldNames"); declareOption(ol, "program", &VMatLanguage::program, OptionBase::learntoption, "The opcodes of the compiled program"); declareOption(ol, "mappings", &VMatLanguage::mappings, OptionBase::learntoption, "The mappings of the compiled program"); inherited::declareOptions(ol); }
static const PPath& PLearn::VMatLanguage::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::Object.
Definition at line 96 of file VMatLanguage.h.
{ return outputfieldnames; }
VMatLanguage * PLearn::VMatLanguage::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
void PLearn::VMatLanguage::generateCode | ( | const string & | processed_sourcecode | ) | [private] |
Definition at line 631 of file VMatLanguage.cc.
References in, PLearn::openString(), and PLearn::PStream::raw_ascii.
{ PStream in = openString(processed_sourcecode, PStream::raw_ascii); generateCode(in); }
void PLearn::VMatLanguage::generateCode | ( | PStream & | processed_sourcecode | ) | [private] |
Definition at line 575 of file VMatLanguage.cc.
References a, b, PLearn::PStream::eof(), PLearn::peekAfterSkipBlanks(), PLearn::pl_isnumber(), PLERROR, PLearn::split(), PLearn::tofloat(), and PLearn::toint().
{ char car; string token; map<string,int>::iterator pos; car = peekAfterSkipBlanks(processed_sourcecode); while(!processed_sourcecode.eof()) { if (car=='{') { int mapnum = mappings.size(); mappings.resize(mapnum+1); mappings[mapnum].read(processed_sourcecode); program.append(opcodes["__applymapping"]); program.append(mapnum); } else { processed_sourcecode>>token; if( pl_isnumber(token)) // assume we have a float { float zefloat=tofloat(token); program.append(opcodes["__insertconstant"]); program.append(*(int*)&zefloat); } else if (token[0]=='%') { vector<string> parts=split(token,":"); if (parts.size()==1) // expecting e.g. %10 for column 10 { program.append(opcodes["__getfieldval"]); int val=toint(token.substr(1)); program.append(val); } else if (parts.size()==2) // expecting e.g. %10-%20 for columns 10 to 20 inclusive { program.append(opcodes["__getfieldsrange"]); int a=toint(parts[0].substr(1)); int b=toint(parts[1].substr(1)); program.append(a); program.append(b); } } else { pos=opcodes.find(token); if(pos!=opcodes.end()) program.append(pos->second); else PLERROR("Undefined keyword : '%s'",token.c_str()); } } car=peekAfterSkipBlanks(processed_sourcecode); } }
Vec PLearn::VMatLanguage::getMemory | ( | ) | const [inline] |
Return the contents of the memory buffer.
Definition at line 149 of file VMatLanguage.h.
{ return mem; }
OptionList & PLearn::VMatLanguage::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
OptionMap & PLearn::VMatLanguage::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
TVec<string> PLearn::VMatLanguage::getOutputFieldNames | ( | ) | const [inline] |
Definition at line 112 of file VMatLanguage.h.
{ return outputfieldnames; }
void PLearn::VMatLanguage::getOutputFieldNamesFromStream | ( | PStream & | in, |
vector< string > & | fieldnames | ||
) | [static] |
Definition at line 714 of file VMatLanguage.cc.
{ map<string,string> defines; string processed_sourcecode; fieldnames.clear(); staticPreprocess(in, defines, processed_sourcecode, fieldnames); }
void PLearn::VMatLanguage::getOutputFieldNamesFromString | ( | const string & | code, |
TVec< string > & | fieldnames | ||
) | [static] |
Definition at line 705 of file VMatLanguage.cc.
References i, and PLearn::TVec< T >::resize().
{ vector<string> names; getOutputFieldNamesFromString(code, names); fieldnames.resize(int(names.size())); for(unsigned int i=0; i< names.size(); i++) fieldnames[i] = names[i]; }
void PLearn::VMatLanguage::getOutputFieldNamesFromString | ( | const string & | code, |
vector< string > & | fieldnames | ||
) | [static] |
Definition at line 699 of file VMatLanguage.cc.
References in, PLearn::openString(), and PLearn::PStream::raw_ascii.
Referenced by PLearn::VPLPreprocessedLearner2::build_(), and PLearn::VPLCombinedLearner::initializeCostNames().
{ PStream in = openString(code, PStream::raw_ascii); getOutputFieldNamesFromStream(in, fieldnames); }
RemoteMethodMap & PLearn::VMatLanguage::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 192 of file VMatLanguage.cc.
int PLearn::VMatLanguage::inputsize | ( | ) | const [inline] |
Definition at line 115 of file VMatLanguage.h.
References PLearn::TVec< T >::length().
Referenced by PLearn::VPLCombinedLearner::computeOutput(), and PLearn::VPLCombinedLearner::computeOutputAndCosts().
{ return srcfieldnames.length(); }
void PLearn::VMatLanguage::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Does the necessary operations to transform a shallow copy (this) into a deep copy by deep-copying all the members that need to be.
This needs to be overridden by every class that adds "complex" data members to the class, such as Vec
, Mat
, PP<Something>
, etc. Typical implementation:
void CLASS_OF_THIS::makeDeepCopyFromShallowCopy(CopiesMap& copies) { inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(complex_data_member1, copies); deepCopyField(complex_data_member2, copies); ... }
copies | A map used by the deep-copy mechanism to keep track of already-copied objects. |
Reimplemented from PLearn::Object.
Definition at line 1403 of file VMatLanguage.cc.
References PLearn::deepCopyField().
Referenced by PLearn::VPLProcessor::makeDeepCopyFromShallowCopy(), PLearn::VPLPreprocessedLearner2::makeDeepCopyFromShallowCopy(), and PLearn::VPLCombinedLearner::makeDeepCopyFromShallowCopy().
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(vmsource, copies); deepCopyField(srcfieldnames, copies); deepCopyField(outputfieldnames, copies); deepCopyField(program, copies); deepCopyField(mappings, copies); deepCopyField(pstack, copies); deepCopyField(myvec, copies); deepCopyField(mem, copies); }
PLearn::VMatLanguage::operator bool | ( | ) | const [inline] |
Definition at line 139 of file VMatLanguage.h.
{ return program.length()>0; }
int PLearn::VMatLanguage::outputsize | ( | ) | const [inline] |
Definition at line 118 of file VMatLanguage.h.
{ return outputfieldnames.length(); }
void PLearn::VMatLanguage::preprocess | ( | PStream & | in, |
map< string, string > & | defines, | ||
string & | processed_sourcecode, | ||
vector< string > & | fieldnames | ||
) | [private] |
Definition at line 291 of file VMatLanguage.cc.
References a, b, PLearn::PStream::eof(), PLearn::PStream::get(), PLearn::getDataSetDate(), PLearn::PStream::getline(), PLearn::PStream::good(), i, PLearn::is_missing(), PLearn::isBlank(), PLearn::join(), PLearn::mtime(), PLearn::openFile(), PLearn::openString(), PLearn::pl_isnumber(), PLCHECK, PLERROR, PLWARNING, PLearn::PStream::raw_ascii, PLearn::removeblanks(), PLearn::skipRestOfLine(), PLearn::PStream::smartReadUntilNext(), PLearn::split(), PLearn::toint(), and PLearn::tostring().
{ string token; size_t spos; map<string, string>::iterator pos; while (in.good()) { in >> token; pos = defines.find(token); // Are we sitting on a mapping declaration? if (token[0] == '{') { // Skip mapping to avoid brackets conflicts with fieldcopy macro // syntax processed_sourcecode += token; // If the token is only a part of the mapping... if (token.find("}") == string::npos) { char car; // Just eat till the end of the mapping while ((car = in.get()) != '}' && !in.eof()) processed_sourcecode += car; processed_sourcecode += "}"; } } // Did we find a fieldName declaration? format is either :myField or // :myField:a:b else if (token[0] == ':') { if (isBlank(token.substr(1))) PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'"); vector<string> parts = split(token, ":"); if (parts.size() == 3) { int a = toint(parts[1]); int b = 0; // Let the chance for the second interval boundary to be a "DEFINE". // This is used with onehot and @myfield.ranges10.nbins // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins if (pl_isnumber(parts[2])) b = toint(parts[2]); else { if (defines.find(parts[2]) != defines.end()) b = toint(defines[parts[2]]); else PLERROR("found a undefined non-numeric boundary in multifield declaration : '%s'", parts[2].c_str()); } for (int i = a; i <= b; i++) fieldnames.push_back(parts[0] + tostring(i)); } else if (parts.size() == 1) fieldnames.push_back(token.substr(1)); else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10"); } // Did we find a fieldcopy macro? else if (token[0] == '[') { if (token[token.size() - 1] != ']') { // First read until the brackets are closed. string end_of_token; in.smartReadUntilNext("]", end_of_token, false, false); in.get(); // Read the ']' character. token += end_of_token + ']'; } vector<string> parts = split(token.substr(1, token.size()-2), ":"); // fieldcopy macro type is [start:end] or [start:end:vpl_code] // fields can be refered to as %number or @name if (parts.size() == 2 || parts.size() == 3) { string astr=parts[0].substr(1); string bstr=parts[1].substr(1); bool code_to_perform = (parts.size() == 3); string performed_code; if (code_to_perform) performed_code = parts[2]; int a=-1; int b=-1; if (parts[0][0] == '@') { for (int i = 0; i < srcfieldnames.length(); i++) if (srcfieldnames[i] == astr) { a = i; break; } } else if (parts[0][0] == '%') a = toint(parts[0].substr(1)); else if (parts[0] == "END") // Keyword indicating we go till the end. a = srcfieldnames.length() - 1; else PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", token.c_str()); if (parts[1][0] == '@') { for (int i = 0; i < srcfieldnames.length(); i++) if (srcfieldnames[i] == bstr) { b = i; break; } } else if (parts[1][0] == '%') b = toint(parts[1].substr(1)); else if (parts[1] == "END") // Keyword indicating we go till the end. b = srcfieldnames.length() - 1; else if (parts[1].substr(0,3) == "END") { // Keyword indicating we go till the end. int v = toint(parts[1].substr(3)); PLCHECK(v<0); b = srcfieldnames.length() - 1 + v; } else PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", token.c_str()); if (a > b) PLERROR("In copyfield macro '%s', you have specified a " "start field (%d) that is after the end field " "(%d). E.g.: [%%10:%%5]", token.c_str(), a, b); if (a == -1) PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str()); if (b == -1) PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str()); for (int i = a; i <= b; i++) { processed_sourcecode+=string("%")+tostring(i)+ " "; if (code_to_perform) processed_sourcecode += performed_code + " "; if (i >= srcfieldnames.length()) PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there " "are only %d fields available", i, srcfieldnames.length()); fieldnames.push_back(srcfieldnames[i]); } } else if (parts.size() == 1) // fieldcopy macro type is [field] { bool ignore_if_missing = false; if (parts[0][parts[0].size()-1] == '?') { ignore_if_missing = true; // Remove ending '?' character. parts[0] = parts[0].substr(0, parts[0].size()-1); } string astr = parts[0].substr(1); int a = -1; if (parts[0][0] == '@') { for (int i = 0;i < srcfieldnames.length(); i++) if (srcfieldnames[i] == astr) { a = i; break; } } else if (parts[0][0] == '%') a = toint(parts[0].substr(1)); else if (parts[0] == "END") // Keyword indicating we go till the end. a = srcfieldnames.length() - 1; else PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", token.c_str()); if (a == -1) { if (!ignore_if_missing) PLERROR("In copyfield macro, unknown field: '%s'", astr.c_str()); } else { processed_sourcecode += string("%") + tostring(a) + " "; if (a >= srcfieldnames.length()) PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there " "are only %d fields available", a, srcfieldnames.length()); fieldnames.push_back(srcfieldnames[a]); } } else PLERROR("Strange fieldcopy format. e.g : [%%0:%%5]. Found parts '%s'",join(parts," ").c_str()); } // did we find a comment? else if (token[0] == '#') skipRestOfLine(in); // include declaration else if (token=="INCLUDE") { in >> token; // Try to be intelligent and find out if the file belongs directly // to another .?mat (the case of a stats file for example) and warn // if the file is out of date // Mhhh.. is this still pertinent? This "stats" and "bins" thing is // semi-standard I think size_t idx_meta = token.find(".metadata"); size_t idx_stats = token.find("stats."); size_t idx_bins = token.find("bins."); if (idx_meta != string::npos && (idx_stats != string::npos || idx_bins != string::npos)) { string file = token.substr(0, idx_meta); if (getDataSetDate(file) > mtime(token)) PLWARNING("File %s seems out of date with parent matrix %s", token.c_str(), file.c_str()); } PStream incfile = openFile(token, PStream::raw_ascii, "r"); // process recursively this included file // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level preprocess(incfile, defines, processed_sourcecode, fieldnames); } // define declaration else if (token == "DEFINE") { in >> token; string str_buf; in.getline(str_buf); defines[token.c_str()] = str_buf; } else if (pos != defines.end()) { // the token is a macro (define) so we process it recursively until // it's stable (necessary since the define macro can use defines // recursively) string oldstr = pos->second; string newstr; bool unstable = true; while (unstable) { PStream strm = openString(oldstr, PStream::raw_ascii); newstr = ""; preprocess(strm, defines, newstr, fieldnames); if (removeblanks(oldstr) == removeblanks(newstr)) unstable = false; oldstr = newstr; } processed_sourcecode += newstr + " "; } // Did we find a reference to a string value of a VMatrix that has // overloaded getStringVal(..) e.g.:StrTableVMatrix? In VPL, you can // push on the stack the value of a string according to the string map // of a particular column e.g. : to push value of string "WBush" from // field MostSuspectAmericanPresidents, write // @MostSuspectsAmericanPresidents."WBush" else if ((token[0]=='@' || token[0]=='%') && token[token.length()-1]=='"' && (spos=token.find(".\""))!=string::npos) { string colname = token.substr(1, spos - 1); const string str = token.substr(spos + 2, token.length() - spos - 3); // Do we have a named field reference? if (token[0]=='@') { pos = defines.find(string("@") + colname); if (pos == defines.end()) PLERROR("unknown field : '%s'",colname.c_str()); colname = pos->second.substr(1); } const int colnum = toint(colname); const real r = vmsource->getStringVal(colnum, str); if (is_missing(r)) PLERROR("String '%s' is not a known string for the field '%s'", str.c_str(), token.c_str()); processed_sourcecode+=tostring(r)+" "; } else processed_sourcecode += token + " "; } }
int PLearn::VMatLanguage::pstackSize | ( | ) | const [inline] |
Definition at line 145 of file VMatLanguage.h.
{return pstack.size();}
void PLearn::VMatLanguage::run | ( | const Vec & | srcvec, |
const Vec & | result, | ||
int | rowindex = -1 |
||
) | const [virtual] |
Executes the program on the srcvec, copy resulting stack to result rowindex is only there for instruction 'rowindex' that pushes it on the stack.
Definition at line 918 of file VMatLanguage.cc.
References a, b, PLearn::TVec< T >::begin(), c, PLearn::Calendar::calendarTimeOnOrAfter(), PLearn::Calendar::calendarTimeOnOrBefore(), d, PLearn::TVec< T >::data(), PLearn::date_to_float(), PLearn::PDate::day, PLearn::TVec< T >::end(), PLearn::exp(), PLearn::fast_exact_is_equal(), PLearn::float_to_date(), PLearn::Calendar::getGlobalCalendar(), i, PLearn::is_missing(), j, PLearn::TVec< T >::length(), MISSING_VALUE, PLearn::PDate::month, n, N, PLearn::pl_erf(), pl_log, PLASSERT_MSG, PLERROR, PLearn::TVec< T >::pop(), PLearn::pow(), PLearn::TVec< T >::push(), PLearn::TVec< T >::resize(), PLearn::sigmoid(), PLearn::TVec< T >::size(), PLearn::sqrt(), PLearn::sumabs(), PLearn::PDate::today(), PLearn::PDate::toJulianDay(), PLearn::tostring(), and PLearn::PDate::year.
Referenced by PLearn::RandomSamplesVMatrix::build_(), PLearn::VPLPreprocessedLearner2::computeConfidenceFromOutput(), PLearn::FilteredVMatrix::computeFilteredIndices(), PLearn::VPLProcessor::computeOutput(), PLearn::VPLPreprocessedLearner2::computeOutput(), PLearn::VPLCombinedLearner::computeOutput(), PLearn::VPLPreprocessedLearner2::computeOutputAndCosts(), PLearn::VPLCombinedLearner::computeOutputAndCosts(), PLearn::VVMatrix::generateFilterIndexFile(), PLearn::ProcessingVMatrix::getNewRow(), and PLearn::vmatmain().
{ if (program.length() == 0 && sourcecode != "") { TVec<string> outnames; const_cast<VMatLanguage*>(this)->compileString(sourcecode, outnames); } if (srcvec.length()!=srcfieldnames.length()) PLERROR("In VMatLanguage::run, srcvec should have length %d, not %d.",srcfieldnames.length(),srcvec.length()); pstack.resize(0); TVec<int>::iterator pptr = program.begin(); const TVec<int>::iterator pptrend = program.end(); real* pfieldvalues = srcvec.data(); real a,b,c; while (pptr != pptrend) { const int op = *pptr++; switch(op) { case 0: // insertconstant pstack.push(*((float*)pptr++)); break; case 1: // getfieldval // XXX Question: why is the next PLERROR line commented? Is if made // obsolete by another bound check earlier in the code? Or is it // temporarily disabled? If the former, please *delete* the line, // together with this comment. If the latter, please reenable the // line or replace this comment by one explaining what needs to be // done before the line can be reenabled. (Bound checking in // general is a good idea...) // //if(*pptr > fieldvalues.width()) PLERROR("Tried to acces an out of bound field in VPL code"); pstack.push(pfieldvalues[*pptr++]); break; case 2: // applymapping pstack.push(mappings[*pptr++].map(pstack.pop())); break; case 3: // pop pstack.pop(); break; case 4: // dup pstack.push(pstack.top()); break; case 5: // exch b = pstack.pop(); a = pstack.pop(); pstack.push(b); pstack.push(a); break; case 6: // onehot { const int nclasses = int(pstack.pop()); const int index = int(pstack.pop()); for (int i = 0; i < nclasses; i++) pstack.push(i == index ? 1 : 0); break; } case 7: // + b = pstack.pop(); a = pstack.pop(); pstack.push(a+b); break; case 8: // - b = pstack.pop(); a = pstack.pop(); pstack.push(a-b); break; case 9: // * b = pstack.pop(); a = pstack.pop(); pstack.push(a*b); break; case 10: // / b = pstack.pop(); a = pstack.pop(); pstack.push(a/b); break; case 11: // == b = pstack.pop(); a = pstack.pop(); pstack.push(fast_exact_is_equal((float)a, (float)b) ?1 :0); break; case 12: // != b = pstack.pop(); a = pstack.pop(); pstack.push(!fast_exact_is_equal((float)a, (float)b) ?1 :0); break; case 13: // > b = pstack.pop(); a = pstack.pop(); pstack.push(((float)a>(float)b) ?1 :0); break; case 14: // >= b = pstack.pop(); a = pstack.pop(); pstack.push(((float)a>=(float)b) ?1 :0); break; case 15: // < b = pstack.pop(); a = pstack.pop(); pstack.push(((float)a<(float)b) ?1 :0); break; case 16: // <= b = pstack.pop(); a = pstack.pop(); pstack.push(((float)a<=(float)b) ?1 :0); break; case 17: // and b = pstack.pop(); a = pstack.pop(); pstack.push((!fast_exact_is_equal(a, 0) && !fast_exact_is_equal(b, 0)) ?1 :0); break; case 18: // or b = pstack.pop(); a = pstack.pop(); pstack.push((!fast_exact_is_equal(a, 0) || !fast_exact_is_equal(b, 0)) ?1 :0); break; case 19: // not pstack.push(fast_exact_is_equal(pstack.pop(), 0) ?1 :0); break; case 20: // ifelse c = pstack.pop(); b = pstack.pop(); a = pstack.pop(); pstack.push(!fast_exact_is_equal(a, 0)?b:c); break; case 21: // fabs pstack.push(fabs(pstack.pop())); break; case 22: // rint pstack.push(rint(pstack.pop())); break; case 23: // floor pstack.push(floor(pstack.pop())); break; case 24: // ceil pstack.push(ceil(pstack.pop())); break; case 25: // log pstack.push(pl_log(pstack.pop())); break; case 26: // exp pstack.push(exp(pstack.pop())); break; case 27: // rowindex pstack.push(real(rowindex)); break; case 28: // isnan pstack.push(isnan(pstack.pop())?1:0); break; case 29: //year pstack.push(float_to_date(pstack.pop()).year); break; case 30: //month pstack.push(float_to_date(pstack.pop()).month); break; case 31: //day pstack.push(float_to_date(pstack.pop()).day); break; case 32: //daydiff b= pstack.pop(); a= pstack.pop(); if (!isnan(a) && !isnan(b)) { pstack.push(float_to_date(a)-float_to_date(b)); } else { pstack.push(MISSING_VALUE); } break; case 33: //monthdiff b= pstack.pop(); a= pstack.pop(); pstack.push((float_to_date(a)-float_to_date(b))*(12.0/365.25)); break; case 34: //yeardiff b= pstack.pop(); a= pstack.pop(); if (is_missing(a) || is_missing(b)) pstack.push(MISSING_VALUE); else pstack.push((float_to_date(a)-float_to_date(b))/365.25); break; case 35: //year_month_day { PDate d(float_to_date(pstack.pop())); pstack.push(d.year); pstack.push(d.month); pstack.push(d.day); break; } case 36: //todate c = pstack.pop(); b = pstack.pop(); a = pstack.pop(); pstack.push(date_to_float(PDate((int)a, (int)b, (int)c))); break; case 37: //dayofweek pstack.push(float_to_date(pstack.pop()).dayOfWeek()); break; case 38: //today pstack.push(date_to_float(PDate::today())); break; case 39: //date2julian pstack.push(float_to_date(pstack.pop()).toJulianDay()); break; case 40: //julian2date pstack.push(date_to_float(PDate((int)pstack.pop()))); break; case 41: //min a= pstack.pop(); b= pstack.pop(); pstack.push(a<b? a : b); break; case 42: //max a= pstack.pop(); b= pstack.pop(); pstack.push(a<b? b : a); break; case 43: // sqrt pstack.push(sqrt(pstack.pop())); break; case 44: // ^ b= pstack.pop(); a= pstack.pop(); pstack.push(pow(a,b)); break; case 45: // mod a= pstack.pop(); b= pstack.pop(); pstack.push((int)b % (int)a); break; case 46: // vecscalmul { a = pstack.pop(); // n b = pstack.pop(); // alpha int start = int(pstack.length()-a); for (int i=0;i<a;i++) pstack[start+i] *= b; break; } case 47: // __getfieldsrange %M:%N pushes fields %N to %M inclusively on the stack { int M = *pptr++; int N = *pptr++; for (int i=M;i<=N;i++) pstack.push(pfieldvalues[i]); break; } case 48: // select: v0 v1 v2 v3 ... vn-1 n i --> vi { int i = (int)pstack.pop(); int n = (int)pstack.pop(); a = MISSING_VALUE; while(--n>=0) { if(n==i) a = pstack.pop(); else pstack.pop(); } pstack.push(a); break; } case 49: // length { pstack.push(srcvec.length()); break; } case 50: // sign { a = pstack.pop(); if(a>0) pstack.push(1.); else if(a<0) pstack.push(-1.); else pstack.push(0.); break; } case 51: // get { const int i = int(pstack.pop()); if (i >= 0) pstack.push(pstack[i]); else pstack.push(pstack.length() + i); break; } case 52: // memput { const int i = int(pstack.pop()); a = pstack.pop(); if (mem.size()<i+1) mem.resize(i+1); mem[i] = a; break; } case 53: // memget { const int i = int(pstack.pop()); pstack.push(mem[i]); break; } case 54: // neg pstack.push(-pstack.pop()); break; case 55: // missing pstack.push(MISSING_VALUE); break; case 56: // sumabs { real sumabs = 0; for (int i = 0; i < pstack.length(); i++) sumabs += fabs(pstack[i]); pstack.push(sumabs); break; } case 57: // weeknumber pstack.push(float_to_date(pstack.pop()).weekNumber()); break; case 58: // dayofyear pstack.push(float_to_date(pstack.pop()).dayOfYear()); break; case 59: // nextincal { const string cal_name = tostring(pstack.pop()); PDate d = float_to_date(pstack.pop()); JTime date = d.toJulianDay(); const Calendar* cal = Calendar::getGlobalCalendar(cal_name); if (cal) { JTime next = cal->calendarTimeOnOrAfter(date); if(next<0) PLERROR("VMatLanguage :: attempting 'nextincal' for date '%s' on " "calendar '%s' but no next-date found", d.info().c_str(), cal_name.c_str()); else pstack.push(date_to_float(PDate((int)next))); } else PLERROR("Global calendar '%s' does not exist", cal_name.c_str()); break; } case 60: // previncal { const string cal_name = tostring(pstack.pop()); PDate d = float_to_date(pstack.pop()); JTime date = d.toJulianDay(); const Calendar* cal = Calendar::getGlobalCalendar(cal_name); if (cal) { JTime next = cal->calendarTimeOnOrBefore(date); if(next<0) PLERROR("VMatLanguage :: attempting 'previncal' for date '%s' on " "calendar '%s' but no previous-date found", d.info().c_str(), cal_name.c_str()); else pstack.push(date_to_float(PDate((int)next))); } else PLERROR("Global calendar '%s' does not exist", cal_name.c_str()); break; } case 61: // gausshot { const real sigma = pstack.pop(); const int nclasses = int(pstack.pop()); const int index = int(pstack.pop()); for (int i = 0; i < nclasses; i++) { const real diff_index = i - index; pstack.push(exp(- diff_index*diff_index / sigma)); } break; } case 62: // sigmoid { pstack.push(sigmoid(pstack.pop())); break; } case 63: // cos { pstack.push(cos(pstack.pop())); break; } case 64: // varproduct { const real num_vars_real = pstack.pop(); if (num_vars_real <= 0) PLERROR("VMatLanguage: varproduct: num_vars must be a strictly positive number."); const int num_vars = (int)num_vars_real; if (num_vars != num_vars_real) PLERROR("VMatLanguage: varproduct: num_vars must be an integer."); TVec<Vec> vars(num_vars); int result_size = 1; // Store all the encoded variables on the stack into "vars" for (int i = num_vars - 1; i >= 0; i--) { real var_size_real = pstack.pop(); if (var_size_real <= 0) PLERROR("VMatLanguage: varproduct: var_size must be a strictly positive number."); int var_size = (int)var_size_real; if (var_size != var_size_real) PLERROR("VMatLanguage: varproduct: var_size must be an integer."); result_size *= var_size; vars[i].resize(var_size); for (int j = var_size - 1; j >= 0 ; j--) vars[i][j] = pstack.pop(); } Vec res(result_size, 1); int length_of_run = 1; int gap_between_runs = 1; // Accumulate variable product into "res" variable. for (int var_index = num_vars - 1; var_index >= 0; var_index--) { const int current_var_size = vars[var_index].size(); gap_between_runs *= current_var_size; int start_dest_index = 0; for (int var_data_index = 0; var_data_index < current_var_size; var_data_index++) { int dest_index = start_dest_index; while (dest_index < result_size) { int start_of_run = dest_index; while (dest_index - start_of_run < length_of_run) { res[dest_index++] *= vars[var_index][var_data_index]; } dest_index += gap_between_runs - 1; } start_dest_index += length_of_run; } length_of_run = gap_between_runs; } // Put the result onto the stack for (int i = 0; i < result_size; i++) pstack.push(res[i]); break; } case 65: // thermometer { const int nclasses = int(pstack.pop()); const int index = int(pstack.pop()); for (int i = 0; i < nclasses; i++) pstack.push(i > index ? 1 : 0); break; } case 66: // erf pstack.push(pl_erf(pstack.pop())); break; default: PLASSERT_MSG(false, "BUG IN VMatLanguage::run while running program: unexpected opcode: " + tostring(op)); } } if (pstack.length() > result.length()) PLERROR("Parsing VMatLanguage: left with %d too many items on the stack!", pstack.length()-result.length()); if (pstack.length() < result.length()) PLERROR("Parsing VMatLanguage: left with %d missing items on the stack!", result.length()-pstack.length()); pstack >> result; }
void PLearn::VMatLanguage::setMemory | ( | const Vec & | new_mem | ) | const |
Set the contents of the memory buffer.
Definition at line 1397 of file VMatLanguage.cc.
References PLearn::TVec< T >::size().
void PLearn::VMatLanguage::setSource | ( | VMat | the_source | ) |
Definition at line 262 of file VMatLanguage.cc.
References PLearn::TVec< T >::isEmpty().
Referenced by PLearn::ProcessingVMatrix::build_(), PLearn::RandomSamplesVMatrix::build_(), and PLearn::FilteredVMatrix::build_().
{ vmsource = the_source; // Set field names from the source VMat if it has field names, otherwise // set each field name to "". TVec<string> fnames = vmsource->fieldNames(); if (fnames.isEmpty()) fnames = TVec<string>(vmsource->width()); setSourceFieldNames(fnames); program.resize(0); }
void PLearn::VMatLanguage::setSourceFieldNames | ( | TVec< string > | the_srcfieldnames | ) |
Definition at line 274 of file VMatLanguage.cc.
References PLearn::TVec< T >::resize().
Referenced by PLearn::VPLProcessor::initializeInputPrograms(), PLearn::VPLPreprocessedLearner2::initializeInputPrograms(), PLearn::VPLPreprocessedLearner2::initializeOutputPrograms(), and PLearn::VPLCombinedLearner::initializeOutputPrograms().
{ srcfieldnames = the_srcfieldnames; myvec.resize(srcfieldnames.length()); }
void PLearn::VMatLanguage::staticPreprocess | ( | PStream & | in, |
map< string, string > & | defines, | ||
string & | processed_sourcecode, | ||
vector< string > & | fieldnames | ||
) | [static] |
Definition at line 723 of file VMatLanguage.cc.
References a, b, PLearn::PStream::eof(), PLearn::PStream::get(), PLearn::getDataSetDate(), PLearn::PStream::getline(), PLearn::PStream::good(), i, PLearn::isBlank(), PLearn::mtime(), PLearn::openFile(), PLearn::openString(), PLearn::pl_isnumber(), PLERROR, PLWARNING, PLearn::PStream::raw_ascii, PLearn::removeblanks(), PLearn::skipRestOfLine(), PLearn::split(), PLearn::toint(), and PLearn::tostring().
{ string token; map<string,string>::iterator pos; while(in.good()) { in >> token; pos=defines.find(token); // are we sitting on a mapping declaration? if(token[0]=='{') { //skip mapping to avoid brackets conflicts with fieldcopy macro syntax char car; processed_sourcecode+=token; // if the token is only a part of the mapping... if(token.find("}")==string::npos) { // just eat till the end of the mapping while((car=in.get())!='}' && !in.eof()) processed_sourcecode+=car; processed_sourcecode+="}"; } } // did we find a fieldName declaration? // format is either :myField or :myField:a:b else if(token[0]==':') { if(isBlank(token.substr(1))) PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'"); vector<string> parts=split(token,":"); if(parts.size()==3) { int a=toint(parts[1]); int b=0; // let the chance for the second interval boundary to be a "DEFINE" // this is used with onehot and @myfield.ranges10.nbins // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins if(pl_isnumber(parts[2])) b=toint(parts[2]); else { if(defines.find(parts[2])!=defines.end()) b=toint(defines[parts[2]]); else PLERROR("Found an undefined non-numeric boundary in multifield declaration : '%s'",parts[2].c_str()); } for(int i=a;i<=b;i++) fieldnames.push_back(parts[0]+tostring(i)); } else if (parts.size()==1) fieldnames.push_back(token.substr(1)); else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10"); } // Did we find a fieldcopy macro? else if(token[0]=='[') PLERROR("fieldcopy macro not supported in VMatLanguage::staticPreprocess"); // did we find a comment? else if(token[0]=='#') skipRestOfLine(in); // include declaration else if(token=="INCLUDE") { in >> token; // Try to be intelligent and find out if the file belongs directly to another .?mat (the case of a // stats file for example) and warn if the file is out of date // Mhhh.. is this still pertinent? This "stats" and "bins" thing is semi-standard I think size_t idx_meta = token.find(".metadata"); size_t idx_stats = token.find("stats."); size_t idx_bins = token.find("bins."); if(idx_meta!=string::npos && (idx_stats!=string::npos || idx_bins!=string::npos)) { string file=token.substr(0,idx_meta); if(getDataSetDate(file) > mtime(token)) PLWARNING("File %s seems out of date with parent matrix %s",token.c_str(),file.c_str()); } PStream incfile = openFile(token, PStream::raw_ascii, "r"); // process recursively this included file // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level staticPreprocess(incfile,defines, processed_sourcecode,fieldnames); } // define declaration else if(token=="DEFINE") { in >> token; string str_buf; in.getline(str_buf); defines[token.c_str()] = str_buf; } else if(pos!=defines.end()) { // the token is a macro (define) so we process it recursively until it's stable // (necessary since the define macro can use defines recursively) string oldstr=pos->second,newstr; bool unstable=true; while(unstable) { PStream strm = openString(oldstr, PStream::raw_ascii); newstr=""; staticPreprocess(strm,defines,newstr,fieldnames); if(removeblanks(oldstr)==removeblanks(newstr)) unstable=false; oldstr=newstr; } processed_sourcecode+=newstr + " "; } // did we find a reference to a string value of a VMatrix that has overloaded getStringVal(..) e.g.:StrTableVMatrix // In VPL, you can push on the stack the value of a string according to the string map of a particular column // e.g. : to push value of string "WBush" from field MostSuspectAmericanPresidents, write @MostSuspectsAmericanPresidents."WBush" else if ((token[0]=='@' || token[0]=='%') && token[token.length()-1]=='"' && token.find(".\"")!=string::npos) PLERROR("string values not supported in VMatLanguage::staticPreprocess"); else processed_sourcecode+=token + " "; } }
Reimplemented from PLearn::Object.
Definition at line 96 of file VMatLanguage.h.
TVec<RealMapping> PLearn::VMatLanguage::mappings [private] |
Definition at line 70 of file VMatLanguage.h.
Referenced by declareOptions().
Vec PLearn::VMatLanguage::mem [mutable, private] |
Definition at line 73 of file VMatLanguage.h.
Vec PLearn::VMatLanguage::myvec [mutable, private] |
Definition at line 72 of file VMatLanguage.h.
map< string, int > PLearn::VMatLanguage::opcodes [static, private] |
Definition at line 76 of file VMatLanguage.h.
bool PLearn::VMatLanguage::output_preproc = false [static] |
Definition at line 158 of file VMatLanguage.h.
Referenced by PLearn::VVMatrix::createPreproVMat().
TVec<string> PLearn::VMatLanguage::outputfieldnames [private] |
Definition at line 68 of file VMatLanguage.h.
Referenced by declareOptions().
TVec<int> PLearn::VMatLanguage::program [private] |
Definition at line 69 of file VMatLanguage.h.
Referenced by declareOptions().
Vec PLearn::VMatLanguage::pstack [mutable, private] |
Definition at line 71 of file VMatLanguage.h.
Definition at line 91 of file VMatLanguage.h.
Referenced by declareOptions().
TVec<string> PLearn::VMatLanguage::srcfieldnames [private] |
Definition at line 67 of file VMatLanguage.h.
Referenced by declareOptions().
VMat PLearn::VMatLanguage::vmsource [private] |
Definition at line 66 of file VMatLanguage.h.
Referenced by declareOptions().