PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 2002 Pascal Vincent and Julien Keable 00005 // 00006 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 00036 00037 00038 /* ******************************************************* 00039 * $Id: VMatLanguage.cc 10200 2009-05-12 17:17:48Z tihocan $ 00040 * This file is part of the PLearn library. 00041 ******************************************************* */ 00042 00043 #include "VMatLanguage.h" 00044 #include <plearn/base/PDate.h> 00045 #include <plearn/base/stringutils.h> 00046 #include <plearn/base/plerror.h> 00047 #include <plearn/db/getDataSet.h> 00048 #include <plearn/io/fileutils.h> 00049 #include <plearn/io/openFile.h> 00050 #include <plearn/misc/Calendar.h> 00051 #include <plearn/math/pl_erf.h> 00052 00053 namespace PLearn { 00054 using namespace std; 00055 00056 bool VMatLanguage::output_preproc=false; 00057 00058 00059 PLEARN_IMPLEMENT_OBJECT(VMatLanguage, 00060 "This class implements the VPL mini-language.", 00061 "VPL (VMat Processing Language) is a home brewed mini-language in postfix\n" 00062 "notation. As of today, it is used is the {PRE,POST}FILTERING and\n" 00063 "PROCESSING sections of a .vmat file. It supports INCLUDEs instructions\n" 00064 "and DEFINEs (dumb named string constants). It can handle reals as well\n" 00065 "as dates (format is: CYYMMDD, where C is 0 (1900-1999) or 1\n" 00066 "(2000-2099). For more info, you can look at PLearnCore/VMatLanguage.*.\n" 00067 "\n" 00068 "A VPL code snippet is always applied to the row of a VMatrix, and can\n" 00069 "only refer to data of that row (in the state it was before any\n" 00070 "processing.) The result of the execution will be a vector which is the\n" 00071 "execution stack at code termination, defining the row of same index in\n" 00072 "the resulting matrix.\n" 00073 "\n" 00074 "When you use VPL in a PROCESSING section, each field you declare must\n" 00075 "have its associated fieldname declaration. The compiler will ensure that\n" 00076 "the size of the result vector and the number of declared fieldnames\n" 00077 "match. This doesn't apply in the filtering sections, where you don't\n" 00078 "declare fieldnames, since the result is always a single value.\n" 00079 "\n" 00080 "To declare a fieldname, use a colon with the name immediately after. To\n" 00081 "batch-declare fieldnames, use eg. :myfield:1:10. This will declare\n" 00082 "fields myfield1 up to myfield10.\n" 00083 "\n" 00084 "There are two notations to refer to a field value: the @ symbol followed\n" 00085 "by the fieldname, or % followed by the field number (with valid indices.\n" 00086 "going from 0 to num_fields-1).\n" 00087 "\n" 00088 "To batch-copy fields, use the following syntax: [field1:fieldn], which copies\n" 00089 "all the fields from field1 up to and including fieldn (fields\n" 00090 "can be in @ or % notation, with the keyword 'END' denoting the last field).\n" 00091 "The fields can also be transformed with a VPL program using the syntax:\n" 00092 "[field1:fieldn:vpl_code], where vpl_code can be any VPL code, for example\n" 00093 "for a 0.5 thresholding: 0.5 < 0 1 ifelse. To copy a single field, use [field].\n" 00094 "There is also a special feature available only for single field copies: if you\n" 00095 "use the syntax [field?], then VPL will not produce an error if the field cannot\n" 00096 "be found. END-N (N an integer) will select a field counting from the END.\n" 00097 "\n" 00098 "Here's a real-life example of a VPL program:\n" 00099 "\n" 00100 " @lease_indicator 88 == 1 0 ifelse :lease_indicator\n" 00101 " @rate_class 1 - 7 onehot :rate_class:0:6\n" 00102 " @collision_deductible { 2->1; 4->2; 5->3; 6->4; 7->5;\n" 00103 " [8 8]->6; MISSING->0; OTHER->0 }\n" 00104 " 7 onehot :collision_deductible:0:6\n" 00105 " @roadstar_indicator 89 == 1 0 ifelse :roadstar_indicator\n" 00106 "\n" 00107 "In the following, the syntax\n" 00108 "\n" 00109 " a b c -> f(a,b,c)\n" 00110 "\n" 00111 "means that (a,b,c) in that order (i.e. 'a' bottommost and 'c' top-of-stack)\n" 00112 "are taken from the stack, and the result f(a,b,c) is pushed on the stack\n" 00113 "\n" 00114 "List of valid VPL operators:\n" 00115 "\n" 00116 " _ pop : pop last element from stack\n" 00117 " _ dup : duplicates last element on the stack\n" 00118 " _ exch : exchanges the two top-most elements on the stack\n" 00119 " _ onehot : index nclasses --> one-hot representation of index\n" 00120 " _ gausshot : index nclasses sigma --> smooth 'one-hot' representation of\n" 00121 " index using a gaussian of width sigma. Maximum value remains 1;\n" 00122 " useful if there is some locality structure in the classes\n" 00123 " _ thermometer : index nclasses --> thermometer representation of index\n" 00124 " _ + : a b --> a + b\n" 00125 " _ - : a b --> a - b\n" 00126 " _ * : a b --> a * b\n" 00127 " _ / : a b --> a / b\n" 00128 " _ neg : a --> -a\n" 00129 " _ == : a b --> a == b\n" 00130 " _ != : a b --> a != b\n" 00131 " _ > : a b --> a > b\n" 00132 " _ >= : a b --> a >= b\n" 00133 " _ < : a b --> a < b\n" 00134 " _ <= : a b --> a <= b\n" 00135 " _ and : a b --> a && b\n" 00136 " _ or : a b --> a || b\n" 00137 " _ not : a --> !a\n" 00138 " _ ifelse : a b c --> (a != 0? b : c)\n" 00139 " _ fabs : a --> fabs(a)\n" 00140 " _ rint : a --> rint(a) ; round to closest int\n" 00141 " _ floor : a --> floor(a)\n" 00142 " _ ceil : a --> ceil(a)\n" 00143 " _ log : a --> log(a) ; natural log\n" 00144 " _ exp : a --> exp(a) ; e^a\n" 00145 " _ sigmoid : a --> sigmoid(a); sigmoid\n" 00146 " _ cos : a --> cos(a) ; cosinus\n" 00147 " _ rowindex : pushes the row number in the VMat on the stack\n" 00148 " _ isnan : true if missing value\n" 00149 " _ missing : pushes a missing value\n" 00150 " _ year : CYYMMDD --> YYYY\n" 00151 " _ month : CYYMMDD --> MM\n" 00152 " _ day : CYYMMDD --> DD\n" 00153 " _ daydiff : CYYMMDD_a CYYMMDD_b --> (CYYMMDD_a - CYMMDD_b) in nb. of days\n" 00154 " _ monthdiff : continuous: nb. days / (365.25/12)\n" 00155 " _ yeardiff : continuous: nb. days / 365.25\n" 00156 " _ year_month_day : CYYMMDD --> YYYY MM DD\n" 00157 " _ todate : YYYY MM DD --> CYYMMDD\n" 00158 " _ dayofweek : from CYYMMDD --> [0..6] (0=monday 6=sunday)\n" 00159 " _ today : todays date CYYMMDD\n" 00160 " _ date2julian : CYYMMDD --> nb. days (JDate)\n" 00161 " _ julian2date : nb. days --> CYYMMDD\n" 00162 " _ weeknumber : CYYMMDD --> week number in the year between 0 and 52 incl.\n" 00163 " (ISO 8601 minus 1)\n" 00164 " _ dayofyear : CYYMMDD --> number of days since january 1 of year CYY \n" 00165 " _ nextincal : CYYMMDD cal# --> next CYYMMDD ON OR AFTER given jdate within\n" 00166 " global calendar 'cal#'; global calendar name\n" 00167 " should be a string repr. of the integer cal#\n" 00168 " If not found, return 0\n" 00169 " _ previncal : CYYMMDD cal# --> previous CYYMMDD ON OR BEFORE given jdatewithin\n" 00170 " global calendar 'cal#'; global calendar name\n" 00171 " should be a string repr. of the integer cal#\n" 00172 " If not found, return 0\n" 00173 " _ min : b a --> (a<b? a : b)\n" 00174 " _ max : b a --> (a<b? b : a)\n" 00175 " _ sqrt : a --> sqrt(a) ; square root\n" 00176 " _ ^ : a b --> pow(a,b) ; a^b\n" 00177 " _ mod : b a --> int(b) % int(a)\n" 00178 " _ vecscalmul : x1 ... xn alpha n --> (x1*alpha) ... (xn*alpha)\n" 00179 " _ select : v0 v1 v2 v3 ... vn-1 n i --> vi \n" 00180 " _ length : the length of the currently processed column.\n" 00181 " _ sign : a --> sign(a) (0 -1 or +1)\n" 00182 " _ get : pos --> value_of_stack_at_pos\n" 00183 " (if pos is negative then it's relative to stacke end\n" 00184 " ex: -1 get will get the previous element of stack)\n" 00185 " _ memput : a mempos --> (a is saved in memory position mempos)\n" 00186 " _ memget : mempos --> a (gets 'a' from memory in position mempos)\n" 00187 " _ sumabs : v0 v1 v2 ... vn --> sum_i |vi|\n" 00188 " (no pop, and starts from the beginning of the stack)\n" 00189 " _ varproduct : a0 a1 ... an n+1 b0 b1 ... bm m+1 ... num_vars -> res0 ..." 00190 " (product of encoded variables)\n" 00191 " _ erf : a --> erf(a) ; the error function erf\n" 00192 ); 00193 00195 // VMatLanguage // 00197 VMatLanguage::VMatLanguage(VMat vmsrc) 00198 { 00199 setSource(vmsrc); 00200 build_(); 00201 } 00202 00203 // returns oldest modification date of a file containing VPL code, searching recursively every 00204 // file placed after a INCLUDE token 00205 time_t getDateOfCode(const string& codefile) 00206 { 00207 time_t latest = mtime(codefile); 00208 string token; 00209 ifstream in(codefile.c_str()); 00210 if(in.bad()) 00211 PLERROR("Cannot open file : %s",codefile.c_str()); 00212 00213 in >> token; 00214 while(!in.eof()) 00215 { 00216 if(token=="INCLUDE") 00217 { 00218 in >> token; 00219 time_t t=getDateOfCode(token); 00220 if(t>latest) 00221 latest=t; 00222 } 00223 in >> token; 00224 } 00225 return latest; 00226 } 00227 00228 map<string, int> VMatLanguage::opcodes; 00229 00230 void 00231 VMatLanguage::build() 00232 { 00233 inherited::build(); 00234 build_(); 00235 } 00236 00237 void 00238 VMatLanguage::build_() 00239 { 00240 build_opcodes_map(); 00241 } 00242 00243 void 00244 VMatLanguage::declareOptions(OptionList &ol) 00245 { 00246 declareOption(ol, "sourcecode", &VMatLanguage::sourcecode, OptionBase::buildoption, 00247 "The VPL sourcecode of the program."); 00248 declareOption(ol, "outputfieldnames", &VMatLanguage::outputfieldnames, OptionBase::learntoption, 00249 "The output fieldnames produced by the program"); 00250 declareOption(ol, "vmsource", &VMatLanguage::vmsource, OptionBase::learntoption, 00251 "The VMat that was set by setSource"); 00252 declareOption(ol, "srcfieldnames", &VMatLanguage::srcfieldnames, OptionBase::learntoption, 00253 "The fieldnames that were set by setSourceFieldNames"); 00254 declareOption(ol, "program", &VMatLanguage::program, OptionBase::learntoption, 00255 "The opcodes of the compiled program"); 00256 declareOption(ol, "mappings", &VMatLanguage::mappings, OptionBase::learntoption, 00257 "The mappings of the compiled program"); 00258 00259 inherited::declareOptions(ol); 00260 } 00261 00262 void VMatLanguage::setSource(VMat the_source) 00263 { 00264 vmsource = the_source; 00265 // Set field names from the source VMat if it has field names, otherwise 00266 // set each field name to "". 00267 TVec<string> fnames = vmsource->fieldNames(); 00268 if (fnames.isEmpty()) 00269 fnames = TVec<string>(vmsource->width()); 00270 setSourceFieldNames(fnames); 00271 program.resize(0); 00272 } 00273 00274 void VMatLanguage::setSourceFieldNames(TVec<string> the_srcfieldnames) 00275 { 00276 srcfieldnames = the_srcfieldnames; 00277 myvec.resize(srcfieldnames.length()); 00278 } 00279 00281 void VMatLanguage::clear() 00282 { 00283 outputfieldnames.resize(0); 00284 program.resize(0); 00285 mappings.resize(0); 00286 } 00287 00289 // preprocess // 00291 void VMatLanguage::preprocess(PStream& in, map<string, string>& defines, 00292 string& processed_sourcecode, vector<string>& fieldnames) 00293 { 00294 string token; 00295 size_t spos; 00296 map<string, string>::iterator pos; 00297 while (in.good()) 00298 { 00299 in >> token; 00300 pos = defines.find(token); 00301 00302 // Are we sitting on a mapping declaration? 00303 if (token[0] == '{') 00304 { 00305 // Skip mapping to avoid brackets conflicts with fieldcopy macro 00306 // syntax 00307 processed_sourcecode += token; 00308 // If the token is only a part of the mapping... 00309 if (token.find("}") == string::npos) 00310 { 00311 char car; 00312 // Just eat till the end of the mapping 00313 while ((car = in.get()) != '}' && !in.eof()) 00314 processed_sourcecode += car; 00315 processed_sourcecode += "}"; 00316 } 00317 } 00318 // Did we find a fieldName declaration? format is either :myField or 00319 // :myField:a:b 00320 else if (token[0] == ':') 00321 { 00322 if (isBlank(token.substr(1))) 00323 PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'"); 00324 vector<string> parts = split(token, ":"); 00325 if (parts.size() == 3) 00326 { 00327 int a = toint(parts[1]); 00328 int b = 0; 00329 // Let the chance for the second interval boundary to be a "DEFINE". 00330 // This is used with onehot and @myfield.ranges10.nbins 00331 // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins 00332 if (pl_isnumber(parts[2])) 00333 b = toint(parts[2]); 00334 else 00335 { 00336 if (defines.find(parts[2]) != defines.end()) 00337 b = toint(defines[parts[2]]); 00338 else 00339 PLERROR("found a undefined non-numeric boundary in multifield declaration : '%s'", 00340 parts[2].c_str()); 00341 } 00342 00343 for (int i = a; i <= b; i++) 00344 fieldnames.push_back(parts[0] + tostring(i)); 00345 } 00346 else if (parts.size() == 1) 00347 fieldnames.push_back(token.substr(1)); 00348 else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10"); 00349 } 00350 // Did we find a fieldcopy macro? 00351 else if (token[0] == '[') 00352 { 00353 if (token[token.size() - 1] != ']') { 00354 // First read until the brackets are closed. 00355 string end_of_token; 00356 in.smartReadUntilNext("]", end_of_token, false, false); 00357 in.get(); // Read the ']' character. 00358 token += end_of_token + ']'; 00359 } 00360 vector<string> parts = split(token.substr(1, token.size()-2), ":"); 00361 00362 // fieldcopy macro type is [start:end] or [start:end:vpl_code] 00363 // fields can be refered to as %number or @name 00364 if (parts.size() == 2 || parts.size() == 3) 00365 { 00366 string astr=parts[0].substr(1); 00367 string bstr=parts[1].substr(1); 00368 bool code_to_perform = (parts.size() == 3); 00369 string performed_code; 00370 if (code_to_perform) 00371 performed_code = parts[2]; 00372 00373 int a=-1; 00374 int b=-1; 00375 00376 if (parts[0][0] == '@') 00377 { 00378 for (int i = 0; i < srcfieldnames.length(); i++) 00379 if (srcfieldnames[i] == astr) 00380 { 00381 a = i; 00382 break; 00383 } 00384 } 00385 else if (parts[0][0] == '%') 00386 a = toint(parts[0].substr(1)); 00387 else if (parts[0] == "END") 00388 // Keyword indicating we go till the end. 00389 a = srcfieldnames.length() - 1; 00390 else 00391 PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", 00392 token.c_str()); 00393 00394 if (parts[1][0] == '@') 00395 { 00396 for (int i = 0; i < srcfieldnames.length(); i++) 00397 if (srcfieldnames[i] == bstr) 00398 { 00399 b = i; 00400 break; 00401 } 00402 } 00403 else if (parts[1][0] == '%') 00404 b = toint(parts[1].substr(1)); 00405 else if (parts[1] == "END") 00406 // Keyword indicating we go till the end. 00407 b = srcfieldnames.length() - 1; 00408 00409 else if (parts[1].substr(0,3) == "END") 00410 { 00411 // Keyword indicating we go till the end. 00412 int v = toint(parts[1].substr(3)); 00413 PLCHECK(v<0); 00414 b = srcfieldnames.length() - 1 + v; 00415 } 00416 else 00417 PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", 00418 token.c_str()); 00419 00420 if (a > b) 00421 PLERROR("In copyfield macro '%s', you have specified a " 00422 "start field (%d) that is after the end field " 00423 "(%d). E.g.: [%%10:%%5]", token.c_str(), a, b); 00424 if (a == -1) 00425 PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str()); 00426 if (b == -1) 00427 PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str()); 00428 00429 for (int i = a; i <= b; i++) 00430 { 00431 processed_sourcecode+=string("%")+tostring(i)+ " "; 00432 if (code_to_perform) 00433 processed_sourcecode += performed_code + " "; 00434 if (i >= srcfieldnames.length()) 00435 PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there " 00436 "are only %d fields available", i, srcfieldnames.length()); 00437 fieldnames.push_back(srcfieldnames[i]); 00438 } 00439 } 00440 else if (parts.size() == 1) 00441 // fieldcopy macro type is [field] 00442 { 00443 bool ignore_if_missing = false; 00444 if (parts[0][parts[0].size()-1] == '?') { 00445 ignore_if_missing = true; 00446 // Remove ending '?' character. 00447 parts[0] = parts[0].substr(0, parts[0].size()-1); 00448 } 00449 00450 string astr = parts[0].substr(1); 00451 int a = -1; 00452 if (parts[0][0] == '@') 00453 { 00454 for (int i = 0;i < srcfieldnames.length(); i++) 00455 if (srcfieldnames[i] == astr) 00456 { 00457 a = i; 00458 break; 00459 } 00460 } 00461 else if (parts[0][0] == '%') 00462 a = toint(parts[0].substr(1)); 00463 else if (parts[0] == "END") 00464 // Keyword indicating we go till the end. 00465 a = srcfieldnames.length() - 1; 00466 else 00467 PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)", 00468 token.c_str()); 00469 00470 if (a == -1) { 00471 if (!ignore_if_missing) 00472 PLERROR("In copyfield macro, unknown field: '%s'", astr.c_str()); 00473 } 00474 else { 00475 processed_sourcecode += string("%") + tostring(a) + " "; 00476 if (a >= srcfieldnames.length()) 00477 PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there " 00478 "are only %d fields available", a, srcfieldnames.length()); 00479 fieldnames.push_back(srcfieldnames[a]); 00480 } 00481 } 00482 else PLERROR("Strange fieldcopy format. e.g : [%%0:%%5]. Found parts '%s'",join(parts," ").c_str()); 00483 } 00484 00485 // did we find a comment? 00486 else if (token[0] == '#') 00487 skipRestOfLine(in); 00488 00489 // include declaration 00490 else if (token=="INCLUDE") 00491 { 00492 in >> token; 00493 // Try to be intelligent and find out if the file belongs directly 00494 // to another .?mat (the case of a stats file for example) and warn 00495 // if the file is out of date 00496 00497 // Mhhh.. is this still pertinent? This "stats" and "bins" thing is 00498 // semi-standard I think 00499 size_t idx_meta = token.find(".metadata"); 00500 size_t idx_stats = token.find("stats."); 00501 size_t idx_bins = token.find("bins."); 00502 if (idx_meta != string::npos && (idx_stats != string::npos || idx_bins != string::npos)) 00503 { 00504 string file = token.substr(0, idx_meta); 00505 if (getDataSetDate(file) > mtime(token)) 00506 PLWARNING("File %s seems out of date with parent matrix %s", 00507 token.c_str(), file.c_str()); 00508 } 00509 00510 PStream incfile = openFile(token, PStream::raw_ascii, "r"); 00511 // process recursively this included file 00512 // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level 00513 preprocess(incfile, defines, processed_sourcecode, fieldnames); 00514 00515 } 00516 // define declaration 00517 else if (token == "DEFINE") 00518 { 00519 in >> token; 00520 string str_buf; 00521 in.getline(str_buf); 00522 defines[token.c_str()] = str_buf; 00523 } 00524 else if (pos != defines.end()) 00525 { 00526 // the token is a macro (define) so we process it recursively until 00527 // it's stable (necessary since the define macro can use defines 00528 // recursively) 00529 string oldstr = pos->second; 00530 string newstr; 00531 bool unstable = true; 00532 while (unstable) 00533 { 00534 PStream strm = openString(oldstr, PStream::raw_ascii); 00535 newstr = ""; 00536 preprocess(strm, defines, newstr, fieldnames); 00537 if (removeblanks(oldstr) == removeblanks(newstr)) 00538 unstable = false; 00539 oldstr = newstr; 00540 } 00541 processed_sourcecode += newstr + " "; 00542 } 00543 // Did we find a reference to a string value of a VMatrix that has 00544 // overloaded getStringVal(..) e.g.:StrTableVMatrix? In VPL, you can 00545 // push on the stack the value of a string according to the string map 00546 // of a particular column e.g. : to push value of string "WBush" from 00547 // field MostSuspectAmericanPresidents, write 00548 // @MostSuspectsAmericanPresidents."WBush" 00549 else if ((token[0]=='@' || token[0]=='%') && 00550 token[token.length()-1]=='"' && 00551 (spos=token.find(".\""))!=string::npos) 00552 { 00553 string colname = token.substr(1, spos - 1); 00554 const string str = token.substr(spos + 2, token.length() - spos - 3); 00555 // Do we have a named field reference? 00556 if (token[0]=='@') 00557 { 00558 pos = defines.find(string("@") + colname); 00559 if (pos == defines.end()) 00560 PLERROR("unknown field : '%s'",colname.c_str()); 00561 colname = pos->second.substr(1); 00562 } 00563 const int colnum = toint(colname); 00564 const real r = vmsource->getStringVal(colnum, str); 00565 if (is_missing(r)) 00566 PLERROR("String '%s' is not a known string for the field '%s'", 00567 str.c_str(), token.c_str()); 00568 processed_sourcecode+=tostring(r)+" "; 00569 } 00570 else 00571 processed_sourcecode += token + " "; 00572 } 00573 } 00574 00575 void VMatLanguage::generateCode(PStream& processed_sourcecode) 00576 { 00577 char car; 00578 string token; 00579 map<string,int>::iterator pos; 00580 car = peekAfterSkipBlanks(processed_sourcecode); 00581 while(!processed_sourcecode.eof()) 00582 { 00583 if (car=='{') 00584 { 00585 int mapnum = mappings.size(); 00586 mappings.resize(mapnum+1); 00587 mappings[mapnum].read(processed_sourcecode); 00588 program.append(opcodes["__applymapping"]); 00589 program.append(mapnum); 00590 } 00591 else 00592 { 00593 processed_sourcecode>>token; 00594 if( pl_isnumber(token)) 00595 // assume we have a float 00596 { 00597 float zefloat=tofloat(token); 00598 program.append(opcodes["__insertconstant"]); 00599 program.append(*(int*)&zefloat); 00600 } 00601 else if (token[0]=='%') 00602 { 00603 vector<string> parts=split(token,":"); 00604 if (parts.size()==1) // expecting e.g. %10 for column 10 00605 { 00606 program.append(opcodes["__getfieldval"]); 00607 int val=toint(token.substr(1)); 00608 program.append(val); 00609 } 00610 else if (parts.size()==2) // expecting e.g. %10-%20 for columns 10 to 20 inclusive 00611 { 00612 program.append(opcodes["__getfieldsrange"]); 00613 int a=toint(parts[0].substr(1)); 00614 int b=toint(parts[1].substr(1)); 00615 program.append(a); 00616 program.append(b); 00617 } 00618 } 00619 else 00620 { 00621 pos=opcodes.find(token); 00622 if(pos!=opcodes.end()) 00623 program.append(pos->second); 00624 else PLERROR("Undefined keyword : '%s'",token.c_str()); 00625 } 00626 } 00627 car=peekAfterSkipBlanks(processed_sourcecode); 00628 } 00629 } 00630 00631 void VMatLanguage::generateCode(const string& processed_sourcecode) 00632 { 00633 PStream in = openString(processed_sourcecode, PStream::raw_ascii); 00634 generateCode(in); 00635 } 00636 00637 //void declareField(int fieldindex, const string& fieldname, VMField::FieldType fieldtype=VMField::UnknownType) 00638 00639 void VMatLanguage::compileString(const string & code, vector<string>& fieldnames) 00640 { 00641 PStream in = openString(code, PStream::raw_ascii); 00642 compileStream(in,fieldnames); 00643 } 00644 00645 void VMatLanguage::compileString(const string & code, TVec<string>& fieldnames) 00646 { 00647 vector<string> names; 00648 compileString(code, names); 00649 fieldnames.resize(int(names.size())); 00650 for(unsigned int i=0; i< names.size(); i++) 00651 fieldnames[i] = names[i]; 00652 } 00653 00654 void VMatLanguage::compileFile(const PPath& filename, vector<string>& fieldnames) 00655 { 00656 PStream in = openFile(filename, PStream::raw_ascii, "r"); 00657 compileStream(in,fieldnames); 00658 } 00659 00660 void VMatLanguage::compileStream(PStream & in, vector<string>& fieldnames) 00661 { 00662 map<string,string> defines; 00663 string processed_sourcecode; 00664 00665 program.resize(0); 00666 mappings.resize(0); 00667 00668 // first, warn user if a fieldname appears twice or more in the source matrix 00669 string fname; 00670 for(int i=0;i<srcfieldnames.length();i++) 00671 { 00672 fname = srcfieldnames[i]; 00673 if (!fname.empty()) 00674 { 00675 fname = string("@") + fname; 00676 if(defines.find(fname) != defines.end()) 00677 PLERROR("fieldname '%s' is duplicate in processed matrix", fname.c_str()); 00678 defines[fname]=string("%")+tostring(i); 00679 } 00680 } 00681 00682 // the filednames parameter is an output vector in which we put the fieldnames of the final VMat 00683 fieldnames.clear(); 00684 preprocess(in, defines, processed_sourcecode, fieldnames); 00685 outputfieldnames.resize(int(fieldnames.size())); 00686 for(unsigned int k=0; k < fieldnames.size(); k++) 00687 outputfieldnames[k] = fieldnames[k]; 00688 00689 if(output_preproc) 00690 { 00691 perr<<"Preprocessed code:"<<endl<<processed_sourcecode<<endl; 00692 perr<<"FieldNames : "<<endl<<fieldnames<<endl; 00693 } 00694 generateCode(processed_sourcecode); 00695 } 00696 00697 00698 00699 void VMatLanguage::getOutputFieldNamesFromString(const string & code, vector<string>& fieldnames) 00700 { 00701 PStream in = openString(code, PStream::raw_ascii); 00702 getOutputFieldNamesFromStream(in, fieldnames); 00703 } 00704 00705 void VMatLanguage::getOutputFieldNamesFromString(const string & code, TVec<string>& fieldnames) 00706 { 00707 vector<string> names; 00708 getOutputFieldNamesFromString(code, names); 00709 fieldnames.resize(int(names.size())); 00710 for(unsigned int i=0; i< names.size(); i++) 00711 fieldnames[i] = names[i]; 00712 } 00713 00714 void VMatLanguage::getOutputFieldNamesFromStream(PStream &in, vector<string>& fieldnames) 00715 { 00716 00717 map<string,string> defines; 00718 string processed_sourcecode; 00719 fieldnames.clear(); 00720 staticPreprocess(in, defines, processed_sourcecode, fieldnames); 00721 } 00722 00723 void VMatLanguage::staticPreprocess(PStream& in, map<string, string>& defines, 00724 string& processed_sourcecode, vector<string>& fieldnames) 00725 { 00726 string token; 00727 map<string,string>::iterator pos; 00728 while(in.good()) 00729 { 00730 in >> token; 00731 pos=defines.find(token); 00732 00733 // are we sitting on a mapping declaration? 00734 if(token[0]=='{') 00735 { 00736 //skip mapping to avoid brackets conflicts with fieldcopy macro syntax 00737 char car; 00738 processed_sourcecode+=token; 00739 // if the token is only a part of the mapping... 00740 if(token.find("}")==string::npos) 00741 { 00742 // just eat till the end of the mapping 00743 while((car=in.get())!='}' && !in.eof()) 00744 processed_sourcecode+=car; 00745 processed_sourcecode+="}"; 00746 } 00747 } 00748 // did we find a fieldName declaration? 00749 // format is either :myField or :myField:a:b 00750 else if(token[0]==':') 00751 { 00752 if(isBlank(token.substr(1))) 00753 PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'"); 00754 vector<string> parts=split(token,":"); 00755 if(parts.size()==3) 00756 { 00757 int a=toint(parts[1]); 00758 int b=0; 00759 // let the chance for the second interval boundary to be a "DEFINE" 00760 // this is used with onehot and @myfield.ranges10.nbins 00761 // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins 00762 if(pl_isnumber(parts[2])) 00763 b=toint(parts[2]); 00764 else 00765 { 00766 if(defines.find(parts[2])!=defines.end()) 00767 b=toint(defines[parts[2]]); 00768 else 00769 PLERROR("Found an undefined non-numeric boundary in multifield declaration : '%s'",parts[2].c_str()); 00770 } 00771 00772 for(int i=a;i<=b;i++) 00773 fieldnames.push_back(parts[0]+tostring(i)); 00774 } 00775 else if (parts.size()==1) 00776 fieldnames.push_back(token.substr(1)); 00777 else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10"); 00778 } 00779 // Did we find a fieldcopy macro? 00780 else if(token[0]=='[') 00781 PLERROR("fieldcopy macro not supported in VMatLanguage::staticPreprocess"); 00782 // did we find a comment? 00783 else if(token[0]=='#') 00784 skipRestOfLine(in); 00785 // include declaration 00786 else if(token=="INCLUDE") 00787 { 00788 in >> token; 00789 // Try to be intelligent and find out if the file belongs directly to another .?mat (the case of a 00790 // stats file for example) and warn if the file is out of date 00791 00792 // Mhhh.. is this still pertinent? This "stats" and "bins" thing is semi-standard I think 00793 size_t idx_meta = token.find(".metadata"); 00794 size_t idx_stats = token.find("stats."); 00795 size_t idx_bins = token.find("bins."); 00796 if(idx_meta!=string::npos && (idx_stats!=string::npos || idx_bins!=string::npos)) 00797 { 00798 string file=token.substr(0,idx_meta); 00799 if(getDataSetDate(file) > mtime(token)) 00800 PLWARNING("File %s seems out of date with parent matrix %s",token.c_str(),file.c_str()); 00801 } 00802 00803 PStream incfile = openFile(token, PStream::raw_ascii, "r"); 00804 // process recursively this included file 00805 // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level 00806 staticPreprocess(incfile,defines, processed_sourcecode,fieldnames); 00807 00808 } 00809 // define declaration 00810 else if(token=="DEFINE") 00811 { 00812 in >> token; 00813 string str_buf; 00814 in.getline(str_buf); 00815 defines[token.c_str()] = str_buf; 00816 } 00817 else if(pos!=defines.end()) 00818 { 00819 // the token is a macro (define) so we process it recursively until it's stable 00820 // (necessary since the define macro can use defines recursively) 00821 string oldstr=pos->second,newstr; 00822 bool unstable=true; 00823 while(unstable) 00824 { 00825 PStream strm = openString(oldstr, PStream::raw_ascii); 00826 newstr=""; 00827 staticPreprocess(strm,defines,newstr,fieldnames); 00828 if(removeblanks(oldstr)==removeblanks(newstr)) 00829 unstable=false; 00830 oldstr=newstr; 00831 } 00832 processed_sourcecode+=newstr + " "; 00833 } 00834 // did we find a reference to a string value of a VMatrix that has overloaded getStringVal(..) e.g.:StrTableVMatrix 00835 // In VPL, you can push on the stack the value of a string according to the string map of a particular column 00836 // e.g. : to push value of string "WBush" from field MostSuspectAmericanPresidents, write @MostSuspectsAmericanPresidents."WBush" 00837 else if ((token[0]=='@' || token[0]=='%') && token[token.length()-1]=='"' && token.find(".\"")!=string::npos) 00838 PLERROR("string values not supported in VMatLanguage::staticPreprocess"); 00839 else processed_sourcecode+=token + " "; 00840 } 00841 } 00842 00844 void VMatLanguage::build_opcodes_map() 00845 { 00846 if(opcodes.empty()) 00847 { 00848 opcodes["__insertconstant"] = 0; // followed by a floating point number (4 bytes, just like int) 00849 opcodes["__getfieldval"] = 1; // followed by field# 00850 opcodes["__applymapping"] = 2; // followed by mapping# 00851 opcodes["pop"] = 3; 00852 opcodes["dup"] = 4; 00853 opcodes["exch"] = 5; 00854 opcodes["onehot"] = 6; 00855 opcodes["+"] = 7; 00856 opcodes["-"] = 8; 00857 opcodes["*"] = 9; 00858 opcodes["/"] = 10; 00859 opcodes["=="] = 11; 00860 opcodes["!="] = 12; 00861 opcodes[">"] = 13; 00862 opcodes[">="] = 14; 00863 opcodes["<"] = 15; 00864 opcodes["<="] = 16; 00865 opcodes["and"] = 17; 00866 opcodes["or"] = 18; 00867 opcodes["not"] = 19; 00868 opcodes["ifelse"] = 20; 00869 opcodes["fabs"] = 21; 00870 opcodes["rint"] = 22; 00871 opcodes["floor"] = 23; 00872 opcodes["ceil"] = 24; 00873 opcodes["log"] = 25; 00874 opcodes["exp"] = 26; 00875 opcodes["rowindex"] = 27; // pushes the rownum on the stack 00876 opcodes["isnan"] = 28; 00877 opcodes["year"] = 29; // from format CYYMMDD -> YYYY 00878 opcodes["month"] = 30; // CYYMMDD -> MM 00879 opcodes["day"] = 31; // CYYMMDD -> DD 00880 opcodes["daydiff"] = 32; // nb. days 00881 opcodes["monthdiff"] = 33; // continuous: nb. days / (365.25/12) 00882 opcodes["yeardiff"] = 34; // continuous: nb. days / 365.25 00883 opcodes["year_month_day"] = 35; // CYYMMDD -> YYYY MM DD 00884 opcodes["todate"] = 36; // YYYY MM DD -> CYYMMDD 00885 opcodes["dayofweek"] = 37; // from CYYMMDD -> [0..6] (0=monday; 6=sunday) 00886 opcodes["today"] = 38; // today's date CYYMMDD 00887 opcodes["date2julian"] = 39; // CYYMMDD -> nb. days 00888 opcodes["julian2date"] = 40; // nb. days -> CYYMMDD 00889 opcodes["min"] = 41; // a b -> (a<b? a : b) 00890 opcodes["max"] = 42; // a b -> (a<b? b : a) 00891 opcodes["sqrt"] = 43; 00892 opcodes["^"] = 44; 00893 opcodes["mod"] = 45; 00894 opcodes["vecscalmul"] = 46; // x1 ... xn alpha n --> (x1*alpha) ... (xn*alpha) 00895 opcodes["__getfieldsrange"] = 47; // %N:%M pushes field %N up to %M. M must be >= N. 00896 opcodes["select"] = 48; // v0 v1 v2 v3 ... vn-1 n i --> vi 00897 opcodes["length"] = 49; // the length of the currently processed column. 00898 opcodes["sign"] = 50; // a --> sign(a) (0 -1 or +1) 00899 opcodes["get"] = 51; // pos --> value_of_stack_at_pos (if pos is negative then it's relative to stacke end ex: -1 get will get the previous element of stack) 00900 opcodes["memput"] = 52; // a mempos --> ( a is saved in memory position mempos) 00901 opcodes["memget"] = 53; // mempos --> val ( gets val from memory in position mempos) 00902 opcodes["neg"] = 54; // a --> -a 00903 opcodes["missing"] = 55; // a missing value 00904 opcodes["sumabs"] = 56; // v0 v1 v2 ... vn --> sum_i |vi| (no pop, and starts from the beginning of the stack) 00905 opcodes["weeknumber"] = 57; // CYYMMDD -> week number in the year between 0 and 52 incl. (ISO 8601 minus 1) 00906 opcodes["dayofyear"] = 58; // CYYMMDD -> number of days since january 1 of year CYY 00907 opcodes["nextincal"] = 59; // cal# JDate -> next jdate on or after given jdate in global calendar cal# 00908 opcodes["previncal"] = 60; // cal# JDate -> previous jdate on or before given jdate in global calendar cal# 00909 opcodes["gausshot"] = 61; // index nclasses sigma --> smooth one-hot 00910 opcodes["sigmoid"] = 62; // a -> sigmoid(a) 00911 opcodes["cos"] = 63; // a -> cos(a) 00912 opcodes["varproduct"] = 64; // a0 a1 ... an n+1 b0 b1 ... bm m+1 ... num_vars -> res0 ... (product of encoded variables) 00913 opcodes["thermometer"] = 65; // index nclasses -> thermometer encoding 00914 opcodes["erf"] = 66; 00915 } 00916 } 00917 00918 void VMatLanguage::run(const Vec& srcvec, const Vec& result, int rowindex) const 00919 { 00920 if (program.length() == 0 && sourcecode != "") 00921 { 00922 TVec<string> outnames; 00923 const_cast<VMatLanguage*>(this)->compileString(sourcecode, outnames); 00924 } 00925 00926 00927 if (srcvec.length()!=srcfieldnames.length()) 00928 PLERROR("In VMatLanguage::run, srcvec should have length %d, not %d.",srcfieldnames.length(),srcvec.length()); 00929 00930 pstack.resize(0); 00931 TVec<int>::iterator pptr = program.begin(); 00932 const TVec<int>::iterator pptrend = program.end(); 00933 real* pfieldvalues = srcvec.data(); 00934 real a,b,c; 00935 00936 while (pptr != pptrend) 00937 { 00938 const int op = *pptr++; 00939 switch(op) 00940 { 00941 case 0: // insertconstant 00942 pstack.push(*((float*)pptr++)); 00943 break; 00944 case 1: // getfieldval 00945 // XXX Question: why is the next PLERROR line commented? Is if made 00946 // obsolete by another bound check earlier in the code? Or is it 00947 // temporarily disabled? If the former, please *delete* the line, 00948 // together with this comment. If the latter, please reenable the 00949 // line or replace this comment by one explaining what needs to be 00950 // done before the line can be reenabled. (Bound checking in 00951 // general is a good idea...) 00952 // 00953 //if(*pptr > fieldvalues.width()) PLERROR("Tried to acces an out of bound field in VPL code"); 00954 pstack.push(pfieldvalues[*pptr++]); 00955 break; 00956 case 2: // applymapping 00957 pstack.push(mappings[*pptr++].map(pstack.pop())); 00958 break; 00959 case 3: // pop 00960 pstack.pop(); 00961 break; 00962 case 4: // dup 00963 pstack.push(pstack.top()); 00964 break; 00965 case 5: // exch 00966 b = pstack.pop(); 00967 a = pstack.pop(); 00968 pstack.push(b); 00969 pstack.push(a); 00970 break; 00971 case 6: // onehot 00972 { 00973 const int nclasses = int(pstack.pop()); 00974 const int index = int(pstack.pop()); 00975 for (int i = 0; i < nclasses; i++) 00976 pstack.push(i == index ? 1 : 0); 00977 break; 00978 } 00979 case 7: // + 00980 b = pstack.pop(); 00981 a = pstack.pop(); 00982 pstack.push(a+b); 00983 break; 00984 case 8: // - 00985 b = pstack.pop(); 00986 a = pstack.pop(); 00987 pstack.push(a-b); 00988 break; 00989 case 9: // * 00990 b = pstack.pop(); 00991 a = pstack.pop(); 00992 pstack.push(a*b); 00993 break; 00994 case 10: // / 00995 b = pstack.pop(); 00996 a = pstack.pop(); 00997 pstack.push(a/b); 00998 break; 00999 case 11: // == 01000 b = pstack.pop(); 01001 a = pstack.pop(); 01002 pstack.push(fast_exact_is_equal((float)a, (float)b) ?1 :0); 01003 break; 01004 case 12: // != 01005 b = pstack.pop(); 01006 a = pstack.pop(); 01007 pstack.push(!fast_exact_is_equal((float)a, (float)b) ?1 :0); 01008 break; 01009 case 13: // > 01010 b = pstack.pop(); 01011 a = pstack.pop(); 01012 pstack.push(((float)a>(float)b) ?1 :0); 01013 break; 01014 case 14: // >= 01015 b = pstack.pop(); 01016 a = pstack.pop(); 01017 pstack.push(((float)a>=(float)b) ?1 :0); 01018 break; 01019 case 15: // < 01020 b = pstack.pop(); 01021 a = pstack.pop(); 01022 pstack.push(((float)a<(float)b) ?1 :0); 01023 break; 01024 case 16: // <= 01025 b = pstack.pop(); 01026 a = pstack.pop(); 01027 pstack.push(((float)a<=(float)b) ?1 :0); 01028 break; 01029 case 17: // and 01030 b = pstack.pop(); 01031 a = pstack.pop(); 01032 pstack.push((!fast_exact_is_equal(a, 0) && 01033 !fast_exact_is_equal(b, 0)) ?1 :0); 01034 break; 01035 case 18: // or 01036 b = pstack.pop(); 01037 a = pstack.pop(); 01038 pstack.push((!fast_exact_is_equal(a, 0) || 01039 !fast_exact_is_equal(b, 0)) ?1 :0); 01040 break; 01041 case 19: // not 01042 pstack.push(fast_exact_is_equal(pstack.pop(), 0) ?1 :0); 01043 break; 01044 case 20: // ifelse 01045 c = pstack.pop(); 01046 b = pstack.pop(); 01047 a = pstack.pop(); 01048 pstack.push(!fast_exact_is_equal(a, 0)?b:c); 01049 break; 01050 case 21: // fabs 01051 pstack.push(fabs(pstack.pop())); 01052 break; 01053 case 22: // rint 01054 pstack.push(rint(pstack.pop())); 01055 break; 01056 case 23: // floor 01057 pstack.push(floor(pstack.pop())); 01058 break; 01059 case 24: // ceil 01060 pstack.push(ceil(pstack.pop())); 01061 break; 01062 case 25: // log 01063 pstack.push(pl_log(pstack.pop())); 01064 break; 01065 case 26: // exp 01066 pstack.push(exp(pstack.pop())); 01067 break; 01068 case 27: // rowindex 01069 pstack.push(real(rowindex)); 01070 break; 01071 case 28: // isnan 01072 pstack.push(isnan(pstack.pop())?1:0); 01073 break; 01074 case 29: //year 01075 pstack.push(float_to_date(pstack.pop()).year); 01076 break; 01077 case 30: //month 01078 pstack.push(float_to_date(pstack.pop()).month); 01079 break; 01080 case 31: //day 01081 pstack.push(float_to_date(pstack.pop()).day); 01082 break; 01083 case 32: //daydiff 01084 b= pstack.pop(); 01085 a= pstack.pop(); 01086 if (!isnan(a) && !isnan(b)) { 01087 pstack.push(float_to_date(a)-float_to_date(b)); 01088 } 01089 else { 01090 pstack.push(MISSING_VALUE); 01091 } 01092 break; 01093 case 33: //monthdiff 01094 b= pstack.pop(); 01095 a= pstack.pop(); 01096 pstack.push((float_to_date(a)-float_to_date(b))*(12.0/365.25)); 01097 break; 01098 case 34: //yeardiff 01099 b= pstack.pop(); 01100 a= pstack.pop(); 01101 if (is_missing(a) || is_missing(b)) 01102 pstack.push(MISSING_VALUE); 01103 else 01104 pstack.push((float_to_date(a)-float_to_date(b))/365.25); 01105 break; 01106 case 35: //year_month_day 01107 { 01108 PDate d(float_to_date(pstack.pop())); 01109 pstack.push(d.year); 01110 pstack.push(d.month); 01111 pstack.push(d.day); 01112 break; 01113 } 01114 case 36: //todate 01115 c = pstack.pop(); 01116 b = pstack.pop(); 01117 a = pstack.pop(); 01118 pstack.push(date_to_float(PDate((int)a, (int)b, (int)c))); 01119 break; 01120 case 37: //dayofweek 01121 pstack.push(float_to_date(pstack.pop()).dayOfWeek()); 01122 break; 01123 case 38: //today 01124 pstack.push(date_to_float(PDate::today())); 01125 break; 01126 case 39: //date2julian 01127 pstack.push(float_to_date(pstack.pop()).toJulianDay()); 01128 break; 01129 case 40: //julian2date 01130 pstack.push(date_to_float(PDate((int)pstack.pop()))); 01131 break; 01132 case 41: //min 01133 a= pstack.pop(); 01134 b= pstack.pop(); 01135 pstack.push(a<b? a : b); 01136 break; 01137 case 42: //max 01138 a= pstack.pop(); 01139 b= pstack.pop(); 01140 pstack.push(a<b? b : a); 01141 break; 01142 case 43: // sqrt 01143 pstack.push(sqrt(pstack.pop())); 01144 break; 01145 case 44: // ^ 01146 b= pstack.pop(); 01147 a= pstack.pop(); 01148 pstack.push(pow(a,b)); 01149 break; 01150 case 45: // mod 01151 a= pstack.pop(); 01152 b= pstack.pop(); 01153 pstack.push((int)b % (int)a); 01154 break; 01155 case 46: // vecscalmul 01156 { 01157 a = pstack.pop(); // n 01158 b = pstack.pop(); // alpha 01159 int start = int(pstack.length()-a); 01160 for (int i=0;i<a;i++) 01161 pstack[start+i] *= b; 01162 break; 01163 } 01164 case 47: // __getfieldsrange %M:%N pushes fields %N to %M inclusively on the stack 01165 { 01166 int M = *pptr++; 01167 int N = *pptr++; 01168 for (int i=M;i<=N;i++) 01169 pstack.push(pfieldvalues[i]); 01170 break; 01171 } 01172 case 48: // select: v0 v1 v2 v3 ... vn-1 n i --> vi 01173 { 01174 int i = (int)pstack.pop(); 01175 int n = (int)pstack.pop(); 01176 a = MISSING_VALUE; 01177 while(--n>=0) 01178 { 01179 if(n==i) 01180 a = pstack.pop(); 01181 else 01182 pstack.pop(); 01183 } 01184 pstack.push(a); 01185 break; 01186 } 01187 case 49: // length 01188 { 01189 pstack.push(srcvec.length()); 01190 break; 01191 } 01192 case 50: // sign 01193 { 01194 a = pstack.pop(); 01195 if(a>0) 01196 pstack.push(1.); 01197 else if(a<0) 01198 pstack.push(-1.); 01199 else 01200 pstack.push(0.); 01201 break; 01202 } 01203 case 51: // get 01204 { 01205 const int i = int(pstack.pop()); 01206 if (i >= 0) 01207 pstack.push(pstack[i]); 01208 else 01209 pstack.push(pstack.length() + i); 01210 break; 01211 } 01212 case 52: // memput 01213 { 01214 const int i = int(pstack.pop()); 01215 a = pstack.pop(); 01216 if (mem.size()<i+1) 01217 mem.resize(i+1); 01218 mem[i] = a; 01219 break; 01220 } 01221 case 53: // memget 01222 { 01223 const int i = int(pstack.pop()); 01224 pstack.push(mem[i]); 01225 break; 01226 } 01227 case 54: // neg 01228 pstack.push(-pstack.pop()); 01229 break; 01230 case 55: // missing 01231 pstack.push(MISSING_VALUE); 01232 break; 01233 case 56: // sumabs 01234 { 01235 real sumabs = 0; 01236 for (int i = 0; i < pstack.length(); i++) 01237 sumabs += fabs(pstack[i]); 01238 pstack.push(sumabs); 01239 break; 01240 } 01241 case 57: // weeknumber 01242 pstack.push(float_to_date(pstack.pop()).weekNumber()); 01243 break; 01244 case 58: // dayofyear 01245 pstack.push(float_to_date(pstack.pop()).dayOfYear()); 01246 break; 01247 case 59: // nextincal 01248 { 01249 const string cal_name = tostring(pstack.pop()); 01250 PDate d = float_to_date(pstack.pop()); 01251 JTime date = d.toJulianDay(); 01252 const Calendar* cal = Calendar::getGlobalCalendar(cal_name); 01253 if (cal) 01254 { 01255 JTime next = cal->calendarTimeOnOrAfter(date); 01256 if(next<0) 01257 PLERROR("VMatLanguage :: attempting 'nextincal' for date '%s' on " 01258 "calendar '%s' but no next-date found", 01259 d.info().c_str(), cal_name.c_str()); 01260 else 01261 pstack.push(date_to_float(PDate((int)next))); 01262 } 01263 else 01264 PLERROR("Global calendar '%s' does not exist", cal_name.c_str()); 01265 break; 01266 } 01267 case 60: // previncal 01268 { 01269 const string cal_name = tostring(pstack.pop()); 01270 PDate d = float_to_date(pstack.pop()); 01271 JTime date = d.toJulianDay(); 01272 const Calendar* cal = Calendar::getGlobalCalendar(cal_name); 01273 if (cal) 01274 { 01275 JTime next = cal->calendarTimeOnOrBefore(date); 01276 if(next<0) 01277 PLERROR("VMatLanguage :: attempting 'previncal' for date '%s' on " 01278 "calendar '%s' but no previous-date found", 01279 d.info().c_str(), cal_name.c_str()); 01280 else 01281 pstack.push(date_to_float(PDate((int)next))); 01282 } 01283 else 01284 PLERROR("Global calendar '%s' does not exist", cal_name.c_str()); 01285 break; 01286 } 01287 case 61: // gausshot 01288 { 01289 const real sigma = pstack.pop(); 01290 const int nclasses = int(pstack.pop()); 01291 const int index = int(pstack.pop()); 01292 for (int i = 0; i < nclasses; i++) { 01293 const real diff_index = i - index; 01294 pstack.push(exp(- diff_index*diff_index / sigma)); 01295 } 01296 break; 01297 } 01298 case 62: // sigmoid 01299 { 01300 pstack.push(sigmoid(pstack.pop())); 01301 break; 01302 } 01303 case 63: // cos 01304 { 01305 pstack.push(cos(pstack.pop())); 01306 break; 01307 } 01308 case 64: // varproduct 01309 { 01310 const real num_vars_real = pstack.pop(); 01311 if (num_vars_real <= 0) 01312 PLERROR("VMatLanguage: varproduct: num_vars must be a strictly positive number."); 01313 const int num_vars = (int)num_vars_real; 01314 if (num_vars != num_vars_real) 01315 PLERROR("VMatLanguage: varproduct: num_vars must be an integer."); 01316 TVec<Vec> vars(num_vars); 01317 int result_size = 1; 01318 01319 // Store all the encoded variables on the stack into "vars" 01320 for (int i = num_vars - 1; i >= 0; i--) { 01321 real var_size_real = pstack.pop(); 01322 if (var_size_real <= 0) 01323 PLERROR("VMatLanguage: varproduct: var_size must be a strictly positive number."); 01324 int var_size = (int)var_size_real; 01325 if (var_size != var_size_real) 01326 PLERROR("VMatLanguage: varproduct: var_size must be an integer."); 01327 01328 result_size *= var_size; 01329 vars[i].resize(var_size); 01330 for (int j = var_size - 1; j >= 0 ; j--) 01331 vars[i][j] = pstack.pop(); 01332 } 01333 01334 Vec res(result_size, 1); 01335 int length_of_run = 1; 01336 int gap_between_runs = 1; 01337 01338 // Accumulate variable product into "res" variable. 01339 for (int var_index = num_vars - 1; var_index >= 0; var_index--) { 01340 const int current_var_size = vars[var_index].size(); 01341 gap_between_runs *= current_var_size; 01342 int start_dest_index = 0; 01343 01344 for (int var_data_index = 0; var_data_index < current_var_size; var_data_index++) { 01345 int dest_index = start_dest_index; 01346 while (dest_index < result_size) { 01347 int start_of_run = dest_index; 01348 while (dest_index - start_of_run < length_of_run) { 01349 res[dest_index++] *= vars[var_index][var_data_index]; 01350 } 01351 dest_index += gap_between_runs - 1; 01352 } 01353 start_dest_index += length_of_run; 01354 } 01355 length_of_run = gap_between_runs; 01356 } 01357 01358 // Put the result onto the stack 01359 for (int i = 0; i < result_size; i++) 01360 pstack.push(res[i]); 01361 break; 01362 } 01363 case 65: // thermometer 01364 { 01365 const int nclasses = int(pstack.pop()); 01366 const int index = int(pstack.pop()); 01367 for (int i = 0; i < nclasses; i++) 01368 pstack.push(i > index ? 1 : 0); 01369 01370 break; 01371 } 01372 case 66: // erf 01373 pstack.push(pl_erf(pstack.pop())); 01374 break; 01375 default: 01376 PLASSERT_MSG(false, "BUG IN VMatLanguage::run while running program: unexpected opcode: " + 01377 tostring(op)); 01378 } 01379 } 01380 01381 if (pstack.length() > result.length()) 01382 PLERROR("Parsing VMatLanguage: left with %d too many items on the stack!", 01383 pstack.length()-result.length()); 01384 if (pstack.length() < result.length()) 01385 PLERROR("Parsing VMatLanguage: left with %d missing items on the stack!", 01386 result.length()-pstack.length()); 01387 01388 pstack >> result; 01389 } 01390 01391 void VMatLanguage::run(int rowindex, const Vec& result) const 01392 { 01393 vmsource->getRow(rowindex,myvec); 01394 run(myvec, result, rowindex); 01395 } 01396 01397 void VMatLanguage::setMemory(const Vec& new_mem) const 01398 { 01399 mem.resize(new_mem.size()); 01400 mem << new_mem; 01401 } 01402 01403 void VMatLanguage::makeDeepCopyFromShallowCopy(CopiesMap& copies) 01404 { 01405 inherited::makeDeepCopyFromShallowCopy(copies); 01406 01407 deepCopyField(vmsource, copies); 01408 deepCopyField(srcfieldnames, copies); 01409 deepCopyField(outputfieldnames, copies); 01410 deepCopyField(program, copies); 01411 deepCopyField(mappings, copies); 01412 deepCopyField(pstack, copies); 01413 deepCopyField(myvec, copies); 01414 deepCopyField(mem, copies); 01415 } 01416 01417 01418 01419 void PreprocessingVMatrix::getNewRow(int i, const Vec& v) const 01420 { 01421 program->run(i,v); 01422 } 01423 01424 PLEARN_IMPLEMENT_OBJECT(PreprocessingVMatrix, "DEPRECATED: use ProcessingVMatrix instead", "NO HELP"); 01425 01426 01427 PreprocessingVMatrix::PreprocessingVMatrix(VMat the_source, const string& program_string) 01428 : source(the_source), program(new VMatLanguage(the_source)) 01429 { 01430 program->compileString(program_string,fieldnames); 01431 build(); 01432 } 01433 01434 void 01435 PreprocessingVMatrix::build() 01436 { 01437 inherited::build(); 01438 build_(); 01439 } 01440 01441 void 01442 PreprocessingVMatrix::build_() 01443 { 01444 if (source) { 01445 fieldinfos.resize((int)fieldnames.size()); 01446 for(unsigned int j=0; j<fieldnames.size(); j++) 01447 fieldinfos[j] = VMField(fieldnames[j]); 01448 01449 sourcevec.resize(source->width()); 01450 width_ = (int)fieldnames.size(); 01451 length_ = source.length(); 01452 } 01453 } 01454 01455 void 01456 PreprocessingVMatrix::declareOptions(OptionList &ol) 01457 { 01458 inherited::declareOptions(ol); 01459 declareOption(ol, "source", &PreprocessingVMatrix::source, OptionBase::buildoption, ""); 01460 declareOption(ol, "program", &PreprocessingVMatrix::program, OptionBase::buildoption, ""); 01461 declareOption(ol, "fieldnames", &PreprocessingVMatrix::fieldnames, OptionBase::buildoption, ""); 01462 } 01463 01464 01465 } // end of namespace PLearn 01466 01467 01468 /* 01469 Local Variables: 01470 mode:c++ 01471 c-basic-offset:4 01472 c-file-style:"stroustrup" 01473 c-file-offsets:((innamespace . 0)(inline-open . 0)) 01474 indent-tabs-mode:nil 01475 fill-column:79 01476 End: 01477 */ 01478 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :