PLearn 0.1
VMatLanguage.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 2002 Pascal Vincent and Julien Keable
00005 //
00006 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 
00036 
00037 
00038 /* *******************************************************
00039  * $Id: VMatLanguage.cc 10200 2009-05-12 17:17:48Z tihocan $
00040  * This file is part of the PLearn library.
00041  ******************************************************* */
00042 
00043 #include "VMatLanguage.h"
00044 #include <plearn/base/PDate.h>
00045 #include <plearn/base/stringutils.h>
00046 #include <plearn/base/plerror.h>
00047 #include <plearn/db/getDataSet.h>
00048 #include <plearn/io/fileutils.h>
00049 #include <plearn/io/openFile.h>
00050 #include <plearn/misc/Calendar.h>
00051 #include <plearn/math/pl_erf.h>
00052 
00053 namespace PLearn {
00054 using namespace std;
00055 
00056 bool VMatLanguage::output_preproc=false;
00057 
00058 
00059 PLEARN_IMPLEMENT_OBJECT(VMatLanguage,
00060                         "This class implements the VPL mini-language.",
00061                         "VPL (VMat Processing Language) is a home brewed mini-language in postfix\n"
00062                         "notation. As of today, it is used is the {PRE,POST}FILTERING and\n"
00063                         "PROCESSING sections of a .vmat file. It supports INCLUDEs instructions\n"
00064                         "and DEFINEs (dumb named string constants). It can handle reals as well\n"
00065                         "as dates (format is: CYYMMDD, where C is 0 (1900-1999) or 1\n"
00066                         "(2000-2099). For more info, you can look at PLearnCore/VMatLanguage.*.\n"
00067                         "\n"
00068                         "A VPL code snippet is always applied to the row of a VMatrix, and can\n"
00069                         "only refer to data of that row (in the state it was before any\n"
00070                         "processing.) The result of the execution will be a vector which is the\n"
00071                         "execution stack at code termination, defining the row of same index in\n"
00072                         "the resulting matrix.\n"
00073                         "\n"
00074                         "When you use VPL in a PROCESSING section, each field you declare must\n"
00075                         "have its associated fieldname declaration. The compiler will ensure that\n"
00076                         "the size of the result vector and the number of declared fieldnames\n"
00077                         "match. This doesn't apply in the filtering sections, where you don't\n"
00078                         "declare fieldnames, since the result is always a single value.\n"
00079                         "\n"
00080                         "To declare a fieldname, use a colon with the name immediately after. To\n"
00081                         "batch-declare fieldnames, use eg. :myfield:1:10. This will declare\n"
00082                         "fields myfield1 up to myfield10.\n"
00083                         "\n"
00084                         "There are two notations to refer to a field value: the @ symbol followed\n"
00085                         "by the fieldname, or % followed by the field number (with valid indices.\n"
00086                         "going from 0 to num_fields-1).\n"
00087                         "\n"
00088                         "To batch-copy fields, use the following syntax: [field1:fieldn], which copies\n"
00089                         "all the fields from field1 up to and including fieldn (fields\n"
00090                         "can be in @ or % notation, with the keyword 'END' denoting the last field).\n"
00091                         "The fields can also be transformed with a VPL program using the syntax:\n"
00092                         "[field1:fieldn:vpl_code], where vpl_code can be any VPL code, for example\n"
00093                         "for a 0.5 thresholding: 0.5 < 0 1 ifelse. To copy a single field, use [field].\n"
00094                         "There is also a special feature available only for single field copies: if you\n"
00095                         "use the syntax [field?], then VPL will not produce an error if the field cannot\n"
00096                         "be found. END-N (N an integer) will select a field counting from the END.\n"
00097                         "\n"
00098                         "Here's a real-life example of a VPL program:\n"
00099                         "\n"
00100                         "    @lease_indicator 88 == 1 0 ifelse :lease_indicator\n"
00101                         "    @rate_class 1 - 7 onehot :rate_class:0:6\n"
00102                         "    @collision_deductible { 2->1; 4->2; 5->3; 6->4; 7->5;\n"
00103                         "       [8 8]->6; MISSING->0; OTHER->0 }\n"
00104                         "      7 onehot :collision_deductible:0:6\n"
00105                         "    @roadstar_indicator 89 == 1 0 ifelse :roadstar_indicator\n"
00106                         "\n"
00107                         "In the following, the syntax\n"
00108                         "\n"
00109                         "    a b c -> f(a,b,c)\n"
00110                         "\n"
00111                         "means that (a,b,c) in that order (i.e. 'a' bottommost and 'c' top-of-stack)\n"
00112                         "are taken from the stack, and the result f(a,b,c) is pushed on the stack\n"
00113                         "\n"
00114                         "List of valid VPL operators:\n"
00115                         "\n"
00116                         " _ pop            : pop last element from stack\n"
00117                         " _ dup            : duplicates last element on the stack\n"
00118                         " _ exch           : exchanges the two top-most elements on the stack\n"
00119                         " _ onehot         : index nclasses --> one-hot representation of index\n"
00120                         " _ gausshot       : index nclasses sigma --> smooth 'one-hot' representation of\n"
00121                         "                    index using a gaussian of width sigma.  Maximum value remains 1;\n"
00122                         "                    useful if there is some locality structure in the classes\n"
00123                         " _ thermometer    : index nclasses --> thermometer representation of index\n"
00124                         " _ +              : a b   -->  a + b\n"
00125                         " _ -              : a b   -->  a - b\n"
00126                         " _ *              : a b   -->  a * b\n"
00127                         " _ /              : a b   -->  a / b\n"
00128                         " _ neg            : a     -->  -a\n"
00129                         " _ ==             : a b   -->  a == b\n"
00130                         " _ !=             : a b   -->  a != b\n"
00131                         " _ >              : a b   -->  a >  b\n"
00132                         " _ >=             : a b   -->  a >= b\n"
00133                         " _ <              : a b   -->  a <  b\n"
00134                         " _ <=             : a b   -->  a <= b\n"
00135                         " _ and            : a b   -->  a && b\n"
00136                         " _ or             : a b   -->  a || b\n"
00137                         " _ not            : a     -->  !a\n"
00138                         " _ ifelse         : a b c -->  (a != 0? b : c)\n"
00139                         " _ fabs           : a     -->  fabs(a)\n"
00140                         " _ rint           : a     -->  rint(a)   ; round to closest int\n"
00141                         " _ floor          : a     -->  floor(a)\n"
00142                         " _ ceil           : a     -->  ceil(a)\n"
00143                         " _ log            : a     -->  log(a)    ; natural log\n"
00144                         " _ exp            : a     -->  exp(a)    ; e^a\n"
00145                         " _ sigmoid        : a     -->  sigmoid(a); sigmoid\n"
00146                         " _ cos            : a     -->  cos(a)    ; cosinus\n"
00147                         " _ rowindex       : pushes the row number in the VMat on the stack\n"
00148                         " _ isnan          : true if missing value\n"
00149                         " _ missing        : pushes a missing value\n"
00150                         " _ year           : CYYMMDD --> YYYY\n"
00151                         " _ month          : CYYMMDD --> MM\n"
00152                         " _ day            : CYYMMDD --> DD\n"
00153                         " _ daydiff        : CYYMMDD_a CYYMMDD_b --> (CYYMMDD_a - CYMMDD_b) in nb. of days\n"
00154                         " _ monthdiff      : continuous: nb. days / (365.25/12)\n"
00155                         " _ yeardiff       : continuous: nb. days / 365.25\n"
00156                         " _ year_month_day : CYYMMDD      --> YYYY MM DD\n"
00157                         " _ todate         : YYYY MM DD   --> CYYMMDD\n"
00158                         " _ dayofweek      : from CYYMMDD --> [0..6] (0=monday  6=sunday)\n"
00159                         " _ today          : todays date CYYMMDD\n"
00160                         " _ date2julian    : CYYMMDD      --> nb. days (JDate)\n"
00161                         " _ julian2date    : nb. days     --> CYYMMDD\n"
00162                         " _ weeknumber     : CYYMMDD      --> week number in the year between 0 and 52 incl.\n"
00163                         "                                     (ISO 8601 minus 1)\n"
00164                         " _ dayofyear      : CYYMMDD      --> number of days since january 1 of year CYY \n"
00165                         " _ nextincal      : CYYMMDD cal# --> next CYYMMDD ON OR AFTER given jdate within\n"
00166                         "                                     global calendar 'cal#'; global calendar name\n"
00167                         "                                     should be a string repr. of the integer cal#\n"
00168                         "                                     If not found, return 0\n"
00169                         " _ previncal      : CYYMMDD cal# --> previous CYYMMDD ON OR BEFORE given jdatewithin\n"
00170                         "                                     global calendar 'cal#'; global calendar name\n"
00171                         "                                     should be a string repr. of the integer cal#\n"
00172                         "                                     If not found, return 0\n"
00173                         " _ min            : b a  -->  (a<b? a : b)\n"
00174                         " _ max            : b a  -->  (a<b? b : a)\n"
00175                         " _ sqrt           : a    -->  sqrt(a)    ; square root\n"
00176                         " _ ^              : a b  -->  pow(a,b)   ; a^b\n"
00177                         " _ mod            : b a  -->  int(b) % int(a)\n"
00178                         " _ vecscalmul     : x1 ... xn alpha n  -->  (x1*alpha) ... (xn*alpha)\n"
00179                         " _ select         : v0 v1 v2 v3 ... vn-1 n i  -->  vi  \n"
00180                         " _ length         : the length of the currently processed column.\n"
00181                         " _ sign           : a  -->  sign(a)  (0 -1 or +1)\n"
00182                         " _ get            : pos  -->  value_of_stack_at_pos\n"
00183                         "                    (if pos is negative then it's relative to stacke end\n"
00184                         "                    ex: -1 get will get the previous element of stack)\n"
00185                         " _ memput         : a mempos  -->    (a is saved in memory position mempos)\n"
00186                         " _ memget         : mempos    --> a  (gets 'a' from memory in position mempos)\n"
00187                         " _ sumabs         : v0 v1 v2 ... vn  -->  sum_i |vi|\n"
00188                         "                    (no pop, and starts from the beginning of the stack)\n"
00189                         " _ varproduct     : a0 a1 ... an n+1 b0 b1 ... bm m+1 ... num_vars -> res0 ..."
00190                         "                    (product of encoded variables)\n"
00191                         " _ erf            : a     -->  erf(a)    ; the error function erf\n"
00192     );
00193 
00195 // VMatLanguage //
00197 VMatLanguage::VMatLanguage(VMat vmsrc)
00198 {
00199     setSource(vmsrc);
00200     build_();
00201 }
00202 
00203 // returns oldest modification date of a file containing VPL code, searching recursively every
00204 // file placed after a INCLUDE token
00205 time_t getDateOfCode(const string& codefile)
00206 {
00207     time_t latest = mtime(codefile);
00208     string token;
00209     ifstream in(codefile.c_str());
00210     if(in.bad())
00211         PLERROR("Cannot open file : %s",codefile.c_str());
00212 
00213     in >> token;
00214     while(!in.eof())
00215     {
00216         if(token=="INCLUDE")
00217         {
00218             in >> token;
00219             time_t t=getDateOfCode(token);
00220             if(t>latest)
00221                 latest=t;
00222         }
00223         in >> token;
00224     }
00225     return latest;
00226 }
00227 
00228 map<string, int> VMatLanguage::opcodes;
00229 
00230 void
00231 VMatLanguage::build()
00232 {
00233     inherited::build();
00234     build_();
00235 }
00236 
00237 void
00238 VMatLanguage::build_()
00239 {
00240     build_opcodes_map();
00241 }
00242 
00243 void
00244 VMatLanguage::declareOptions(OptionList &ol)
00245 {
00246     declareOption(ol, "sourcecode", &VMatLanguage::sourcecode, OptionBase::buildoption,
00247                   "The VPL sourcecode of the program.");
00248     declareOption(ol, "outputfieldnames", &VMatLanguage::outputfieldnames, OptionBase::learntoption,
00249                   "The output fieldnames produced by the program");
00250     declareOption(ol, "vmsource", &VMatLanguage::vmsource, OptionBase::learntoption,
00251                   "The VMat that was set by setSource");
00252     declareOption(ol, "srcfieldnames", &VMatLanguage::srcfieldnames, OptionBase::learntoption,
00253                   "The fieldnames that were set by setSourceFieldNames");
00254     declareOption(ol, "program", &VMatLanguage::program, OptionBase::learntoption,
00255                   "The opcodes of the compiled program");
00256     declareOption(ol, "mappings", &VMatLanguage::mappings, OptionBase::learntoption,
00257                   "The mappings of the compiled program");
00258 
00259     inherited::declareOptions(ol);
00260 }
00261 
00262 void VMatLanguage::setSource(VMat the_source)
00263 {
00264     vmsource = the_source;
00265     // Set field names from the source VMat if it has field names, otherwise
00266     // set each field name to "".
00267     TVec<string> fnames = vmsource->fieldNames();
00268     if (fnames.isEmpty())
00269         fnames = TVec<string>(vmsource->width());
00270     setSourceFieldNames(fnames);
00271     program.resize(0);
00272 }
00273 
00274 void VMatLanguage::setSourceFieldNames(TVec<string> the_srcfieldnames)
00275 { 
00276     srcfieldnames = the_srcfieldnames; 
00277     myvec.resize(srcfieldnames.length());
00278 }
00279 
00281 void VMatLanguage::clear()
00282 {
00283     outputfieldnames.resize(0);
00284     program.resize(0);
00285     mappings.resize(0);
00286 }
00287 
00289 // preprocess //
00291 void VMatLanguage::preprocess(PStream& in, map<string, string>& defines,
00292                               string& processed_sourcecode, vector<string>& fieldnames)
00293 {
00294     string token;
00295     size_t spos;
00296     map<string, string>::iterator pos;
00297     while (in.good())
00298     {
00299         in >> token;
00300         pos = defines.find(token);
00301 
00302         // Are we sitting on a mapping declaration?
00303         if (token[0] == '{')
00304         {
00305             // Skip mapping to avoid brackets conflicts with fieldcopy macro
00306             // syntax
00307             processed_sourcecode += token;
00308             // If the token is only a part of the mapping...
00309             if (token.find("}") == string::npos)
00310             {
00311                 char car;
00312                 // Just eat till the end of the mapping
00313                 while ((car = in.get()) != '}' && !in.eof())
00314                     processed_sourcecode += car;
00315                 processed_sourcecode += "}";
00316             }
00317         }
00318         // Did we find a fieldName declaration? format is either :myField or
00319         // :myField:a:b
00320         else if (token[0] == ':')
00321         {
00322             if (isBlank(token.substr(1)))
00323                 PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'");
00324             vector<string> parts = split(token, ":");
00325             if (parts.size() == 3)
00326             {
00327                 int a = toint(parts[1]);
00328                 int b = 0;
00329                 // Let the chance for the second interval boundary to be a "DEFINE".
00330                 // This is used with onehot and @myfield.ranges10.nbins
00331                 // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins
00332                 if (pl_isnumber(parts[2]))
00333                     b = toint(parts[2]);
00334                 else
00335                 {
00336                     if (defines.find(parts[2]) != defines.end())
00337                         b = toint(defines[parts[2]]);
00338                     else
00339                         PLERROR("found a undefined non-numeric boundary in multifield declaration : '%s'",
00340                                 parts[2].c_str());
00341                 }
00342 
00343                 for (int i = a; i <= b; i++)
00344                     fieldnames.push_back(parts[0] + tostring(i));
00345             }
00346             else if (parts.size() == 1)
00347                 fieldnames.push_back(token.substr(1));
00348             else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10");
00349         }
00350         // Did we find a fieldcopy macro?
00351         else if (token[0] == '[')
00352         {
00353             if (token[token.size() - 1] != ']') {
00354                 // First read until the brackets are closed.
00355                 string end_of_token;
00356                 in.smartReadUntilNext("]", end_of_token, false, false);
00357                 in.get(); // Read the ']' character.
00358                 token += end_of_token + ']';
00359             }
00360             vector<string> parts = split(token.substr(1, token.size()-2), ":");
00361 
00362             // fieldcopy macro type is [start:end] or [start:end:vpl_code]
00363             // fields can be refered to as %number or @name
00364             if (parts.size() == 2 || parts.size() == 3)
00365             {
00366                 string astr=parts[0].substr(1);
00367                 string bstr=parts[1].substr(1);
00368                 bool code_to_perform = (parts.size() == 3);
00369                 string performed_code;
00370                 if (code_to_perform)
00371                     performed_code = parts[2];
00372 
00373                 int a=-1;
00374                 int b=-1;
00375 
00376                 if (parts[0][0] == '@')
00377                 {
00378                     for (int i = 0; i < srcfieldnames.length(); i++)
00379                         if (srcfieldnames[i] == astr)
00380                         {
00381                             a = i;
00382                             break;
00383                         }
00384                 }
00385                 else if (parts[0][0] == '%')
00386                     a = toint(parts[0].substr(1));
00387                 else if (parts[0] == "END")
00388                     // Keyword indicating we go till the end.
00389                     a = srcfieldnames.length() - 1;
00390                 else
00391                     PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)",
00392                         token.c_str());
00393 
00394                 if (parts[1][0] == '@')
00395                 {
00396                     for (int i = 0; i < srcfieldnames.length(); i++)
00397                         if (srcfieldnames[i] == bstr)
00398                         {
00399                             b = i;
00400                             break;
00401                         }
00402                 }
00403                 else if (parts[1][0] == '%')
00404                     b = toint(parts[1].substr(1));
00405                 else if (parts[1] == "END")
00406                     // Keyword indicating we go till the end.
00407                     b = srcfieldnames.length() - 1;
00408                 
00409                 else if (parts[1].substr(0,3) == "END")
00410                 {
00411                     // Keyword indicating we go till the end.
00412                     int v = toint(parts[1].substr(3));
00413                     PLCHECK(v<0);
00414                     b = srcfieldnames.length() - 1 + v;
00415                 }
00416                 else
00417                     PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)",
00418                             token.c_str());
00419 
00420                 if (a > b)
00421                     PLERROR("In copyfield macro '%s', you have specified a "
00422                             "start field (%d) that is after the end field "
00423                             "(%d). E.g.: [%%10:%%5]", token.c_str(), a, b);
00424                 if (a == -1)
00425                     PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str());
00426                 if (b == -1)
00427                     PLERROR("In copyfield macro, unknown field : '%s'", astr.c_str());
00428 
00429                 for (int i = a; i <= b; i++)
00430                 {
00431                     processed_sourcecode+=string("%")+tostring(i)+ " ";
00432                     if (code_to_perform)
00433                         processed_sourcecode += performed_code + " ";
00434                     if (i >= srcfieldnames.length())
00435                         PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there "
00436                                 "are only %d fields available", i, srcfieldnames.length());
00437                     fieldnames.push_back(srcfieldnames[i]);
00438                 }
00439             }
00440             else if (parts.size() == 1)
00441                 // fieldcopy macro type is [field]
00442             {
00443                 bool ignore_if_missing = false;
00444                 if (parts[0][parts[0].size()-1] == '?') {
00445                     ignore_if_missing = true;
00446                     // Remove ending '?' character.
00447                     parts[0] = parts[0].substr(0, parts[0].size()-1);
00448                 }
00449                 
00450                 string astr = parts[0].substr(1);
00451                 int a = -1;
00452                 if (parts[0][0] == '@')
00453                 {
00454                     for (int i = 0;i < srcfieldnames.length(); i++)
00455                         if (srcfieldnames[i] == astr)
00456                         {
00457                             a = i;
00458                             break;
00459                         }
00460                 }
00461                 else if (parts[0][0] == '%')
00462                     a = toint(parts[0].substr(1));
00463                 else if (parts[0] == "END")
00464                     // Keyword indicating we go till the end.
00465                     a = srcfieldnames.length() - 1;
00466                 else
00467                     PLERROR("fieldcopy macro syntax is : [start:end] EG: [@year:%%6]. 'end' must be after 'start'.. OR [field] to copy a single field (%s)",
00468                             token.c_str());
00469 
00470                 if (a == -1) {
00471                     if (!ignore_if_missing)
00472                         PLERROR("In copyfield macro, unknown field: '%s'", astr.c_str());
00473                 }
00474                 else {
00475                     processed_sourcecode += string("%") + tostring(a) + " ";
00476                     if (a >= srcfieldnames.length())
00477                         PLERROR("In VMatLanguage::preprocess - Asked field number %d, but there "
00478                                 "are only %d fields available", a, srcfieldnames.length());
00479                     fieldnames.push_back(srcfieldnames[a]);
00480                 }
00481             }
00482             else PLERROR("Strange fieldcopy format. e.g : [%%0:%%5]. Found parts '%s'",join(parts," ").c_str());
00483         }
00484 
00485         // did we find a comment?
00486         else if (token[0] == '#')
00487             skipRestOfLine(in);
00488 
00489         // include declaration
00490         else if (token=="INCLUDE")
00491         {
00492             in >> token;
00493             // Try to be intelligent and find out if the file belongs directly
00494             // to another .?mat (the case of a stats file for example) and warn
00495             // if the file is out of date
00496 
00497             // Mhhh.. is this still pertinent? This "stats" and "bins" thing is
00498             // semi-standard I think
00499             size_t idx_meta  =  token.find(".metadata");
00500             size_t idx_stats =  token.find("stats.");
00501             size_t idx_bins  =  token.find("bins.");
00502             if (idx_meta != string::npos && (idx_stats != string::npos || idx_bins != string::npos))
00503             {
00504                 string file = token.substr(0, idx_meta);
00505                 if (getDataSetDate(file) > mtime(token))
00506                     PLWARNING("File %s seems out of date with parent matrix %s",
00507                               token.c_str(), file.c_str());
00508             }
00509 
00510             PStream incfile = openFile(token, PStream::raw_ascii, "r");
00511             // process recursively this included file
00512             // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level
00513             preprocess(incfile, defines, processed_sourcecode, fieldnames);
00514         
00515         }
00516         // define declaration
00517         else if (token == "DEFINE")
00518         {
00519             in >> token;
00520             string str_buf;
00521             in.getline(str_buf);
00522             defines[token.c_str()] = str_buf;
00523         }
00524         else if (pos != defines.end())
00525         {
00526             // the token is a macro (define) so we process it recursively until
00527             // it's stable (necessary since the define macro can use defines
00528             // recursively)
00529             string oldstr = pos->second;
00530             string newstr;
00531             bool unstable = true;
00532             while (unstable)
00533             {
00534                 PStream strm = openString(oldstr, PStream::raw_ascii);
00535                 newstr = "";
00536                 preprocess(strm, defines, newstr, fieldnames);
00537                 if (removeblanks(oldstr) == removeblanks(newstr))
00538                     unstable = false;
00539                 oldstr = newstr;
00540             }
00541             processed_sourcecode += newstr + " ";
00542         }
00543         // Did we find a reference to a string value of a VMatrix that has
00544         // overloaded getStringVal(..) e.g.:StrTableVMatrix? In VPL, you can
00545         // push on the stack the value of a string according to the string map
00546         // of a particular column e.g. : to push value of string "WBush" from
00547         // field MostSuspectAmericanPresidents, write
00548         // @MostSuspectsAmericanPresidents."WBush"
00549         else if ((token[0]=='@' || token[0]=='%') &&
00550                  token[token.length()-1]=='"' &&
00551                  (spos=token.find(".\""))!=string::npos)
00552         {
00553             string colname = token.substr(1, spos - 1);
00554             const string str = token.substr(spos + 2, token.length() - spos - 3);
00555             // Do we have a named field reference?
00556             if (token[0]=='@')
00557             {
00558                 pos = defines.find(string("@") + colname);
00559                 if (pos == defines.end())
00560                     PLERROR("unknown field : '%s'",colname.c_str());
00561                 colname = pos->second.substr(1);
00562             }
00563             const int colnum = toint(colname);
00564             const real r = vmsource->getStringVal(colnum, str);
00565             if (is_missing(r))
00566                 PLERROR("String '%s' is not a known string for the field '%s'",
00567                         str.c_str(), token.c_str());
00568             processed_sourcecode+=tostring(r)+" ";
00569         }
00570         else
00571             processed_sourcecode += token + " ";
00572     }
00573 }
00574 
00575 void VMatLanguage::generateCode(PStream& processed_sourcecode)
00576 {
00577     char car;
00578     string token;
00579     map<string,int>::iterator pos;
00580     car = peekAfterSkipBlanks(processed_sourcecode);
00581     while(!processed_sourcecode.eof())
00582     {
00583         if (car=='{')
00584         {
00585             int mapnum = mappings.size();
00586             mappings.resize(mapnum+1);
00587             mappings[mapnum].read(processed_sourcecode);
00588             program.append(opcodes["__applymapping"]);
00589             program.append(mapnum);
00590         }
00591         else
00592         {
00593             processed_sourcecode>>token;
00594             if( pl_isnumber(token))
00595                 // assume we have a float
00596             {
00597                 float zefloat=tofloat(token);
00598                 program.append(opcodes["__insertconstant"]);
00599                 program.append(*(int*)&zefloat);
00600             }
00601             else if (token[0]=='%')
00602             {
00603                 vector<string> parts=split(token,":");
00604                 if (parts.size()==1) // expecting e.g. %10 for column 10
00605                 {
00606                     program.append(opcodes["__getfieldval"]);
00607                     int val=toint(token.substr(1));
00608                     program.append(val);
00609                 }
00610                 else if (parts.size()==2) // expecting e.g. %10-%20 for columns 10 to 20 inclusive
00611                 {
00612                     program.append(opcodes["__getfieldsrange"]);
00613                     int a=toint(parts[0].substr(1));
00614                     int b=toint(parts[1].substr(1));
00615                     program.append(a);
00616                     program.append(b);
00617                 }
00618             }
00619             else
00620             {
00621                 pos=opcodes.find(token);
00622                 if(pos!=opcodes.end())
00623                     program.append(pos->second);
00624                 else PLERROR("Undefined keyword : '%s'",token.c_str());
00625             }
00626         }
00627         car=peekAfterSkipBlanks(processed_sourcecode);
00628     }
00629 }
00630 
00631 void VMatLanguage::generateCode(const string& processed_sourcecode)
00632 {
00633     PStream in = openString(processed_sourcecode, PStream::raw_ascii);
00634     generateCode(in);
00635 }
00636 
00637 //void declareField(int fieldindex, const string& fieldname, VMField::FieldType fieldtype=VMField::UnknownType)
00638 
00639 void VMatLanguage::compileString(const string & code, vector<string>& fieldnames)
00640 {
00641     PStream in = openString(code, PStream::raw_ascii);
00642     compileStream(in,fieldnames);
00643 }
00644 
00645 void VMatLanguage::compileString(const string & code, TVec<string>& fieldnames)
00646 {
00647     vector<string> names;
00648     compileString(code, names);
00649     fieldnames.resize(int(names.size()));
00650     for(unsigned int i=0; i< names.size(); i++)
00651         fieldnames[i] = names[i];
00652 }
00653 
00654 void VMatLanguage::compileFile(const PPath& filename, vector<string>& fieldnames)
00655 {
00656     PStream in = openFile(filename, PStream::raw_ascii, "r");
00657     compileStream(in,fieldnames);
00658 }
00659 
00660 void VMatLanguage::compileStream(PStream & in, vector<string>& fieldnames)
00661 {
00662     map<string,string> defines;
00663     string processed_sourcecode;
00664 
00665     program.resize(0);
00666     mappings.resize(0);
00667 
00668     // first, warn user if a fieldname appears twice or more in the source matrix
00669     string fname;
00670     for(int i=0;i<srcfieldnames.length();i++)
00671     {
00672         fname = srcfieldnames[i];
00673         if (!fname.empty())
00674         {
00675             fname = string("@") + fname;
00676             if(defines.find(fname) != defines.end())
00677                 PLERROR("fieldname '%s' is duplicate in processed matrix", fname.c_str());
00678             defines[fname]=string("%")+tostring(i);
00679         }
00680     }
00681 
00682     // the filednames parameter is an output vector in which we put the fieldnames of the final VMat
00683     fieldnames.clear();
00684     preprocess(in, defines, processed_sourcecode, fieldnames);
00685     outputfieldnames.resize(int(fieldnames.size()));
00686     for(unsigned int k=0; k < fieldnames.size(); k++)
00687         outputfieldnames[k] = fieldnames[k];
00688 
00689     if(output_preproc)
00690     {
00691         perr<<"Preprocessed code:"<<endl<<processed_sourcecode<<endl;
00692         perr<<"FieldNames : "<<endl<<fieldnames<<endl;
00693     }
00694     generateCode(processed_sourcecode);
00695 }
00696 
00697 
00698 
00699 void VMatLanguage::getOutputFieldNamesFromString(const string & code, vector<string>& fieldnames)
00700 {
00701     PStream in = openString(code, PStream::raw_ascii);
00702     getOutputFieldNamesFromStream(in, fieldnames);
00703 }
00704 
00705 void VMatLanguage::getOutputFieldNamesFromString(const string & code, TVec<string>& fieldnames)
00706 {
00707     vector<string> names;
00708     getOutputFieldNamesFromString(code, names);
00709     fieldnames.resize(int(names.size()));
00710     for(unsigned int i=0; i< names.size(); i++)
00711         fieldnames[i] = names[i];
00712 }
00713 
00714 void VMatLanguage::getOutputFieldNamesFromStream(PStream &in, vector<string>& fieldnames)
00715 {
00716 
00717     map<string,string> defines;
00718     string processed_sourcecode;
00719     fieldnames.clear();
00720     staticPreprocess(in, defines, processed_sourcecode, fieldnames);
00721 }
00722 
00723 void VMatLanguage::staticPreprocess(PStream& in, map<string, string>& defines,
00724                                     string& processed_sourcecode, vector<string>& fieldnames)
00725 {
00726     string token;
00727     map<string,string>::iterator pos;
00728     while(in.good())
00729     {
00730         in >> token;
00731         pos=defines.find(token);
00732 
00733         // are we sitting on a mapping declaration?
00734         if(token[0]=='{')
00735         {
00736             //skip mapping to avoid brackets conflicts with fieldcopy macro syntax
00737             char car;
00738             processed_sourcecode+=token;
00739             // if the token is only a part of the mapping...
00740             if(token.find("}")==string::npos)
00741             {
00742                 // just eat till the end of the mapping
00743                 while((car=in.get())!='}' && !in.eof())
00744                     processed_sourcecode+=car;
00745                 processed_sourcecode+="}";
00746             }
00747         }
00748         // did we find a fieldName declaration?
00749         // format is either :myField or :myField:a:b
00750         else if(token[0]==':')
00751         {
00752             if(isBlank(token.substr(1)))
00753                 PLERROR("Found a ':' with no fieldname. Do not put a whitespace after the ':'");
00754             vector<string> parts=split(token,":");
00755             if(parts.size()==3)
00756             {
00757                 int a=toint(parts[1]);
00758                 int b=0;
00759                 // let the chance for the second interval boundary to be a "DEFINE"
00760                 // this is used with onehot and @myfield.ranges10.nbins
00761                 // ie: @myfield.onehot10 :myfieldonehot:0:@myfield.ranges10.nbins
00762                 if(pl_isnumber(parts[2]))
00763                     b=toint(parts[2]);
00764                 else
00765                 {
00766                     if(defines.find(parts[2])!=defines.end())
00767                         b=toint(defines[parts[2]]);
00768                     else
00769                         PLERROR("Found an undefined non-numeric boundary in multifield declaration : '%s'",parts[2].c_str());
00770                 }
00771 
00772                 for(int i=a;i<=b;i++)
00773                     fieldnames.push_back(parts[0]+tostring(i));
00774             }
00775             else if (parts.size()==1)
00776                 fieldnames.push_back(token.substr(1));
00777             else PLERROR("Strange fieldname format (multiple declaration format is :label:0:10");
00778         }
00779         // Did we find a fieldcopy macro?
00780         else if(token[0]=='[')
00781             PLERROR("fieldcopy macro not supported in VMatLanguage::staticPreprocess");
00782         // did we find a comment?
00783         else if(token[0]=='#')
00784             skipRestOfLine(in);
00785         // include declaration
00786         else if(token=="INCLUDE")
00787         {
00788             in >> token;
00789             // Try to be intelligent and find out if the file belongs directly to another .?mat (the case of a
00790             // stats file for example) and warn if the file is out of date
00791 
00792             // Mhhh.. is this still pertinent? This "stats" and "bins" thing is semi-standard I think
00793             size_t idx_meta  =  token.find(".metadata");
00794             size_t idx_stats =  token.find("stats.");
00795             size_t idx_bins  =  token.find("bins.");
00796             if(idx_meta!=string::npos && (idx_stats!=string::npos || idx_bins!=string::npos))
00797             {
00798                 string file=token.substr(0,idx_meta);
00799                 if(getDataSetDate(file) > mtime(token))
00800                     PLWARNING("File %s seems out of date with parent matrix %s",token.c_str(),file.c_str());
00801             }
00802 
00803             PStream incfile = openFile(token, PStream::raw_ascii, "r");
00804             // process recursively this included file
00805             // **POSSIBLE DRAWBACK : defines done in this file will be used in the next recursion level
00806             staticPreprocess(incfile,defines, processed_sourcecode,fieldnames);
00807         
00808         }
00809         // define declaration
00810         else if(token=="DEFINE")
00811         {
00812             in >> token;
00813             string str_buf;
00814             in.getline(str_buf);
00815             defines[token.c_str()] = str_buf;
00816         }
00817         else if(pos!=defines.end())
00818         {
00819             // the token is a macro (define) so we process it recursively until it's stable
00820             // (necessary since the define macro can use defines recursively)
00821             string oldstr=pos->second,newstr;
00822             bool unstable=true;
00823             while(unstable)
00824             {
00825                 PStream strm = openString(oldstr, PStream::raw_ascii);
00826                 newstr="";
00827                 staticPreprocess(strm,defines,newstr,fieldnames);
00828                 if(removeblanks(oldstr)==removeblanks(newstr))
00829                     unstable=false;
00830                 oldstr=newstr;
00831             }
00832             processed_sourcecode+=newstr + " ";
00833         }
00834         // did we find a reference to a string value of a VMatrix that has overloaded getStringVal(..) e.g.:StrTableVMatrix
00835         // In VPL, you can push on the stack the value of a string according to the string map of a particular column
00836         // e.g. : to push value of string "WBush" from field MostSuspectAmericanPresidents, write @MostSuspectsAmericanPresidents."WBush"
00837         else if ((token[0]=='@' || token[0]=='%') && token[token.length()-1]=='"' && token.find(".\"")!=string::npos)
00838             PLERROR("string values not supported in VMatLanguage::staticPreprocess");
00839         else processed_sourcecode+=token + " ";
00840     }
00841 }
00842 
00844 void VMatLanguage::build_opcodes_map()
00845 {
00846     if(opcodes.empty())
00847     {
00848         opcodes["__insertconstant"] = 0; // followed by a floating point number (4 bytes, just like int)
00849         opcodes["__getfieldval"] = 1; // followed by field#
00850         opcodes["__applymapping"] = 2; // followed by mapping#
00851         opcodes["pop"] = 3;
00852         opcodes["dup"] = 4;
00853         opcodes["exch"] = 5;
00854         opcodes["onehot"] = 6;
00855         opcodes["+"]  = 7;
00856         opcodes["-"] = 8;
00857         opcodes["*"] = 9;
00858         opcodes["/"] = 10;
00859         opcodes["=="] = 11;
00860         opcodes["!="] = 12;
00861         opcodes[">"] = 13;
00862         opcodes[">="] = 14;
00863         opcodes["<"] = 15;
00864         opcodes["<="] = 16;
00865         opcodes["and"] = 17;
00866         opcodes["or"] = 18;
00867         opcodes["not"] = 19;
00868         opcodes["ifelse"] = 20;
00869         opcodes["fabs"] = 21;
00870         opcodes["rint"] = 22;
00871         opcodes["floor"] = 23;
00872         opcodes["ceil"] = 24;
00873         opcodes["log"] = 25;
00874         opcodes["exp"] = 26;
00875         opcodes["rowindex"] = 27; // pushes the rownum on the stack
00876         opcodes["isnan"] = 28;
00877         opcodes["year"] = 29; // from format CYYMMDD -> YYYY
00878         opcodes["month"] = 30; // CYYMMDD -> MM
00879         opcodes["day"] = 31;  // CYYMMDD -> DD
00880         opcodes["daydiff"] = 32; // nb. days
00881         opcodes["monthdiff"] = 33; // continuous: nb. days / (365.25/12)
00882         opcodes["yeardiff"] = 34; // continuous: nb. days / 365.25
00883         opcodes["year_month_day"] = 35; // CYYMMDD -> YYYY MM DD
00884         opcodes["todate"] = 36; // YYYY MM DD -> CYYMMDD
00885         opcodes["dayofweek"] = 37; // from CYYMMDD -> [0..6] (0=monday; 6=sunday)
00886         opcodes["today"] = 38; // today's date CYYMMDD
00887         opcodes["date2julian"] = 39; // CYYMMDD -> nb. days
00888         opcodes["julian2date"] = 40; // nb. days -> CYYMMDD
00889         opcodes["min"] = 41; // a b -> (a<b? a : b)
00890         opcodes["max"] = 42; // a b -> (a<b? b : a)
00891         opcodes["sqrt"] = 43;
00892         opcodes["^"] = 44;
00893         opcodes["mod"] = 45;
00894         opcodes["vecscalmul"] = 46; // x1 ... xn alpha n --> (x1*alpha) ... (xn*alpha)
00895         opcodes["__getfieldsrange"] = 47; // %N:%M pushes field %N up to %M. M must be >= N.
00896         opcodes["select"] = 48; // v0 v1 v2 v3 ... vn-1 n i --> vi
00897         opcodes["length"] = 49; // the length of the currently processed column.
00898         opcodes["sign"]   = 50; // a --> sign(a)  (0 -1 or +1)
00899         opcodes["get"]    = 51; // pos --> value_of_stack_at_pos (if pos is negative then it's relative to stacke end ex: -1 get will get the previous element of stack)
00900         opcodes["memput"] = 52; // a mempos -->    ( a is saved in memory position mempos)
00901         opcodes["memget"] = 53; // mempos --> val  ( gets val from memory in position mempos)
00902         opcodes["neg"]    = 54; // a --> -a
00903         opcodes["missing"] = 55;  // a missing value
00904         opcodes["sumabs"] = 56;  // v0 v1 v2 ... vn --> sum_i |vi| (no pop, and starts from the beginning of the stack)
00905         opcodes["weeknumber"] = 57;  // CYYMMDD -> week number in the year between 0 and 52 incl. (ISO 8601 minus 1)
00906         opcodes["dayofyear"] = 58;  // CYYMMDD ->  number of days since january 1 of year CYY
00907         opcodes["nextincal"] = 59;  // cal# JDate -> next jdate on or after given jdate in global calendar cal#
00908         opcodes["previncal"] = 60;  // cal# JDate -> previous jdate on or before given jdate in global calendar cal#
00909         opcodes["gausshot"]  = 61;  // index nclasses sigma --> smooth one-hot
00910         opcodes["sigmoid"]  = 62;   // a -> sigmoid(a)
00911         opcodes["cos"]  = 63;   // a -> cos(a)
00912         opcodes["varproduct"] = 64; // a0 a1 ... an n+1 b0 b1 ... bm m+1 ... num_vars -> res0 ... (product of encoded variables)
00913         opcodes["thermometer"] = 65; // index nclasses -> thermometer encoding
00914         opcodes["erf"] = 66;
00915     }
00916 }
00917 
00918 void VMatLanguage::run(const Vec& srcvec, const Vec& result, int rowindex) const
00919 {
00920     if (program.length() == 0 && sourcecode != "")
00921     {
00922         TVec<string> outnames;
00923         const_cast<VMatLanguage*>(this)->compileString(sourcecode, outnames);
00924     }
00925     
00926     
00927     if (srcvec.length()!=srcfieldnames.length())
00928         PLERROR("In VMatLanguage::run, srcvec should have length %d, not %d.",srcfieldnames.length(),srcvec.length());
00929     
00930     pstack.resize(0);
00931     TVec<int>::iterator pptr = program.begin();
00932     const TVec<int>::iterator pptrend = program.end();
00933     real* pfieldvalues = srcvec.data();
00934     real a,b,c;    
00935 
00936     while (pptr != pptrend)
00937     {
00938         const int op = *pptr++;
00939         switch(op)
00940         {
00941         case 0: // insertconstant
00942             pstack.push(*((float*)pptr++));
00943             break;
00944         case 1: // getfieldval
00945             // XXX Question: why is the next PLERROR line commented? Is if made
00946             // obsolete by another bound check earlier in the code? Or is it
00947             // temporarily disabled? If the former, please *delete* the line,
00948             // together with this comment. If the latter, please reenable the
00949             // line or replace this comment by one explaining what needs to be
00950             // done before the line can be reenabled. (Bound checking in
00951             // general is a good idea...)
00952             //
00953             //if(*pptr > fieldvalues.width()) PLERROR("Tried to acces an out of bound field in VPL code");
00954             pstack.push(pfieldvalues[*pptr++]);
00955             break;
00956         case 2: // applymapping
00957             pstack.push(mappings[*pptr++].map(pstack.pop()));
00958             break;
00959         case 3: // pop
00960             pstack.pop();
00961             break;
00962         case 4: // dup
00963             pstack.push(pstack.top());
00964             break;
00965         case 5: // exch
00966             b = pstack.pop();
00967             a = pstack.pop();
00968             pstack.push(b);
00969             pstack.push(a);
00970             break;
00971         case 6: // onehot
00972         {
00973             const int nclasses = int(pstack.pop());
00974             const int index = int(pstack.pop());
00975             for (int i = 0; i < nclasses; i++)
00976                 pstack.push(i == index ? 1 : 0);
00977             break;
00978         }
00979         case 7: // +
00980             b = pstack.pop();
00981             a = pstack.pop();
00982             pstack.push(a+b);
00983             break;
00984         case 8: // -
00985             b = pstack.pop();
00986             a = pstack.pop();
00987             pstack.push(a-b);
00988             break;
00989         case 9: // *
00990             b = pstack.pop();
00991             a = pstack.pop();
00992             pstack.push(a*b);
00993             break;
00994         case 10: // /
00995             b = pstack.pop();
00996             a = pstack.pop();
00997             pstack.push(a/b);
00998             break;
00999         case 11: // ==
01000             b = pstack.pop();
01001             a = pstack.pop();
01002             pstack.push(fast_exact_is_equal((float)a, (float)b) ?1 :0);
01003             break;
01004         case 12: // !=
01005             b = pstack.pop();
01006             a = pstack.pop();
01007             pstack.push(!fast_exact_is_equal((float)a, (float)b) ?1 :0);
01008             break;
01009         case 13: // >
01010             b = pstack.pop();
01011             a = pstack.pop();
01012             pstack.push(((float)a>(float)b) ?1 :0);
01013             break;
01014         case 14: // >=
01015             b = pstack.pop();
01016             a = pstack.pop();
01017             pstack.push(((float)a>=(float)b) ?1 :0);
01018             break;
01019         case 15: // <
01020             b = pstack.pop();
01021             a = pstack.pop();
01022             pstack.push(((float)a<(float)b) ?1 :0);
01023             break;
01024         case 16: // <=
01025             b = pstack.pop();
01026             a = pstack.pop();
01027             pstack.push(((float)a<=(float)b) ?1 :0);
01028             break;
01029         case 17: // and
01030             b = pstack.pop();
01031             a = pstack.pop();
01032             pstack.push((!fast_exact_is_equal(a, 0) &&
01033                          !fast_exact_is_equal(b, 0)) ?1 :0);
01034             break;
01035         case 18: // or
01036             b = pstack.pop();
01037             a = pstack.pop();
01038             pstack.push((!fast_exact_is_equal(a, 0) ||
01039                          !fast_exact_is_equal(b, 0)) ?1 :0);
01040             break;
01041         case 19: // not
01042             pstack.push(fast_exact_is_equal(pstack.pop(), 0) ?1 :0);
01043             break;
01044         case 20: // ifelse
01045             c = pstack.pop();
01046             b = pstack.pop();
01047             a = pstack.pop();
01048             pstack.push(!fast_exact_is_equal(a, 0)?b:c);
01049             break;
01050         case 21: // fabs
01051             pstack.push(fabs(pstack.pop()));
01052             break;
01053         case 22: // rint
01054             pstack.push(rint(pstack.pop()));
01055             break;
01056         case 23: // floor
01057             pstack.push(floor(pstack.pop()));
01058             break;
01059         case 24: // ceil
01060             pstack.push(ceil(pstack.pop()));
01061             break;
01062         case 25: // log
01063             pstack.push(pl_log(pstack.pop()));
01064             break;
01065         case 26: // exp
01066             pstack.push(exp(pstack.pop()));
01067             break;
01068         case 27: // rowindex
01069             pstack.push(real(rowindex));
01070             break;
01071         case 28: // isnan
01072             pstack.push(isnan(pstack.pop())?1:0);
01073             break;
01074         case 29: //year
01075             pstack.push(float_to_date(pstack.pop()).year);
01076             break;
01077         case 30: //month
01078             pstack.push(float_to_date(pstack.pop()).month);
01079             break;
01080         case 31: //day
01081             pstack.push(float_to_date(pstack.pop()).day);
01082             break;
01083         case 32: //daydiff
01084             b= pstack.pop();
01085             a= pstack.pop();
01086             if (!isnan(a) && !isnan(b)) {
01087                 pstack.push(float_to_date(a)-float_to_date(b));
01088             }
01089             else {
01090                 pstack.push(MISSING_VALUE);
01091             }
01092             break;
01093         case 33: //monthdiff
01094             b= pstack.pop();
01095             a= pstack.pop();
01096             pstack.push((float_to_date(a)-float_to_date(b))*(12.0/365.25));
01097             break;
01098         case 34: //yeardiff
01099             b= pstack.pop();
01100             a= pstack.pop();
01101             if (is_missing(a) || is_missing(b))
01102                 pstack.push(MISSING_VALUE);
01103             else
01104                 pstack.push((float_to_date(a)-float_to_date(b))/365.25);
01105             break;
01106         case 35: //year_month_day
01107         {
01108             PDate d(float_to_date(pstack.pop()));
01109             pstack.push(d.year);
01110             pstack.push(d.month);
01111             pstack.push(d.day);
01112             break;
01113         }
01114         case 36: //todate
01115             c = pstack.pop();
01116             b = pstack.pop();
01117             a = pstack.pop();
01118             pstack.push(date_to_float(PDate((int)a, (int)b, (int)c)));
01119             break;
01120         case 37: //dayofweek
01121             pstack.push(float_to_date(pstack.pop()).dayOfWeek());
01122             break;
01123         case 38: //today
01124             pstack.push(date_to_float(PDate::today()));
01125             break;
01126         case 39: //date2julian
01127             pstack.push(float_to_date(pstack.pop()).toJulianDay());
01128             break;
01129         case 40: //julian2date
01130             pstack.push(date_to_float(PDate((int)pstack.pop())));
01131             break;
01132         case 41: //min
01133             a= pstack.pop();
01134             b= pstack.pop();
01135             pstack.push(a<b? a : b);
01136             break;
01137         case 42: //max
01138             a= pstack.pop();
01139             b= pstack.pop();
01140             pstack.push(a<b? b : a);
01141             break;
01142         case 43: // sqrt
01143             pstack.push(sqrt(pstack.pop()));
01144             break;
01145         case 44: // ^
01146             b= pstack.pop();
01147             a= pstack.pop();
01148             pstack.push(pow(a,b));
01149             break;
01150         case 45: // mod
01151             a= pstack.pop();
01152             b= pstack.pop();
01153             pstack.push((int)b % (int)a);
01154             break;
01155         case 46: // vecscalmul
01156         {
01157             a = pstack.pop(); // n
01158             b = pstack.pop(); // alpha
01159             int start = int(pstack.length()-a);
01160             for (int i=0;i<a;i++)
01161                 pstack[start+i] *= b;
01162             break;
01163         }
01164         case 47: // __getfieldsrange         %M:%N       pushes fields %N to %M inclusively on the stack
01165         {
01166             int M = *pptr++;
01167             int N = *pptr++;
01168             for (int i=M;i<=N;i++)
01169                 pstack.push(pfieldvalues[i]);
01170             break;
01171         }
01172         case 48: // select:    v0 v1 v2 v3 ... vn-1 n i --> vi
01173         {
01174             int i = (int)pstack.pop();
01175             int n = (int)pstack.pop();
01176             a = MISSING_VALUE;
01177             while(--n>=0)
01178             {
01179                 if(n==i)
01180                     a = pstack.pop();
01181                 else
01182                     pstack.pop();
01183             }
01184             pstack.push(a);
01185             break;
01186         }
01187         case 49: // length
01188         {
01189             pstack.push(srcvec.length());
01190             break;
01191         }
01192         case 50: // sign
01193         {
01194             a = pstack.pop();
01195             if(a>0)
01196                 pstack.push(1.);
01197             else if(a<0)
01198                 pstack.push(-1.);
01199             else
01200                 pstack.push(0.);
01201             break;
01202         }
01203         case 51: // get
01204         {
01205             const int i = int(pstack.pop());
01206             if (i >= 0)
01207                 pstack.push(pstack[i]);
01208             else
01209                 pstack.push(pstack.length() + i);
01210             break;
01211         }
01212         case 52: // memput
01213         {
01214             const int i = int(pstack.pop());
01215             a = pstack.pop();
01216             if (mem.size()<i+1)
01217                 mem.resize(i+1);
01218             mem[i] = a;
01219             break;
01220         }
01221         case 53: // memget
01222         {
01223             const int i = int(pstack.pop());
01224             pstack.push(mem[i]);
01225             break;
01226         }
01227         case 54: // neg
01228             pstack.push(-pstack.pop());
01229             break;
01230         case 55: // missing
01231             pstack.push(MISSING_VALUE);
01232             break;
01233         case 56: // sumabs
01234         {
01235             real sumabs = 0;
01236             for (int i = 0; i < pstack.length(); i++)
01237                 sumabs += fabs(pstack[i]);
01238             pstack.push(sumabs);
01239             break;
01240         }
01241         case 57: // weeknumber
01242             pstack.push(float_to_date(pstack.pop()).weekNumber());
01243             break;
01244         case 58: // dayofyear
01245             pstack.push(float_to_date(pstack.pop()).dayOfYear());
01246             break;
01247         case 59: // nextincal
01248         {
01249             const string cal_name = tostring(pstack.pop());
01250             PDate d = float_to_date(pstack.pop());
01251             JTime date = d.toJulianDay();
01252             const Calendar* cal = Calendar::getGlobalCalendar(cal_name);
01253             if (cal)
01254             {
01255                 JTime next = cal->calendarTimeOnOrAfter(date);
01256                 if(next<0)
01257                     PLERROR("VMatLanguage :: attempting 'nextincal' for date '%s' on "
01258                             "calendar '%s' but no next-date found",
01259                             d.info().c_str(), cal_name.c_str());
01260                 else
01261                     pstack.push(date_to_float(PDate((int)next)));
01262             }
01263             else
01264                 PLERROR("Global calendar '%s' does not exist", cal_name.c_str());
01265             break;
01266         }
01267         case 60: // previncal
01268         {
01269             const string cal_name = tostring(pstack.pop());
01270             PDate d = float_to_date(pstack.pop());
01271             JTime date = d.toJulianDay();
01272             const Calendar* cal = Calendar::getGlobalCalendar(cal_name);
01273             if (cal)
01274             {
01275                 JTime next = cal->calendarTimeOnOrBefore(date);
01276                 if(next<0)
01277                     PLERROR("VMatLanguage :: attempting 'previncal' for date '%s' on "
01278                             "calendar '%s' but no previous-date found",
01279                             d.info().c_str(), cal_name.c_str());
01280                 else
01281                     pstack.push(date_to_float(PDate((int)next)));
01282             }
01283             else
01284                 PLERROR("Global calendar '%s' does not exist", cal_name.c_str());
01285             break;
01286         }
01287         case 61: // gausshot
01288         {
01289             const real sigma = pstack.pop();
01290             const int nclasses = int(pstack.pop());
01291             const int index = int(pstack.pop());
01292             for (int i = 0; i < nclasses; i++) {
01293                 const real diff_index = i - index;
01294                 pstack.push(exp(- diff_index*diff_index / sigma));
01295             }
01296             break;
01297         }
01298         case 62: // sigmoid
01299         {
01300             pstack.push(sigmoid(pstack.pop()));
01301             break;
01302         }
01303         case 63: // cos
01304         {
01305             pstack.push(cos(pstack.pop()));
01306             break;
01307         }
01308         case 64: // varproduct
01309         {
01310             const real num_vars_real = pstack.pop();
01311             if (num_vars_real <= 0)
01312                 PLERROR("VMatLanguage: varproduct: num_vars must be a strictly positive number.");
01313             const int num_vars = (int)num_vars_real;
01314             if (num_vars != num_vars_real)
01315                 PLERROR("VMatLanguage: varproduct: num_vars must be an integer.");
01316             TVec<Vec> vars(num_vars);
01317             int result_size = 1;
01318 
01319             // Store all the encoded variables on the stack into "vars"
01320             for (int i = num_vars - 1; i >= 0; i--) {
01321                 real var_size_real = pstack.pop();
01322                 if (var_size_real <= 0)
01323                     PLERROR("VMatLanguage: varproduct: var_size must be a strictly positive number.");
01324                 int var_size = (int)var_size_real;
01325                 if (var_size != var_size_real)
01326                     PLERROR("VMatLanguage: varproduct: var_size must be an integer.");
01327 
01328                 result_size *= var_size;
01329                 vars[i].resize(var_size);
01330                 for (int j = var_size - 1; j >= 0 ; j--)
01331                     vars[i][j] = pstack.pop();
01332             }
01333 
01334             Vec res(result_size, 1);
01335             int length_of_run = 1;
01336             int gap_between_runs = 1;
01337 
01338             // Accumulate variable product into "res" variable.
01339             for (int var_index = num_vars - 1; var_index >= 0; var_index--) {
01340                 const int current_var_size = vars[var_index].size();
01341                 gap_between_runs *= current_var_size;
01342                 int start_dest_index = 0;
01343 
01344                 for (int var_data_index = 0; var_data_index < current_var_size; var_data_index++) {
01345                     int dest_index = start_dest_index;
01346                     while (dest_index < result_size) {
01347                         int start_of_run = dest_index;
01348                         while (dest_index - start_of_run < length_of_run) {
01349                             res[dest_index++] *= vars[var_index][var_data_index];
01350                         }
01351                         dest_index += gap_between_runs - 1;
01352                     }
01353                     start_dest_index += length_of_run;
01354                 }
01355                 length_of_run = gap_between_runs;
01356             }
01357 
01358             // Put the result onto the stack
01359             for (int i = 0; i < result_size; i++)
01360                 pstack.push(res[i]);
01361             break;
01362         }
01363         case 65: // thermometer
01364         {
01365             const int nclasses = int(pstack.pop());
01366             const int index = int(pstack.pop());
01367             for (int i = 0; i < nclasses; i++)
01368                 pstack.push(i > index ? 1 : 0);
01369             
01370             break;
01371         }        
01372         case 66: // erf
01373             pstack.push(pl_erf(pstack.pop()));
01374             break;
01375         default:
01376             PLASSERT_MSG(false, "BUG IN VMatLanguage::run while running program: unexpected opcode: " +
01377                          tostring(op));
01378         }
01379     }
01380 
01381     if (pstack.length() > result.length())
01382         PLERROR("Parsing VMatLanguage: left with %d too many items on the stack!",
01383                 pstack.length()-result.length());
01384     if (pstack.length() < result.length())
01385         PLERROR("Parsing VMatLanguage: left with %d missing items on the stack!",
01386                 result.length()-pstack.length());
01387 
01388     pstack >> result;
01389 }
01390 
01391 void VMatLanguage::run(int rowindex, const Vec& result) const
01392 {
01393     vmsource->getRow(rowindex,myvec);
01394     run(myvec, result, rowindex);
01395 }
01396 
01397 void VMatLanguage::setMemory(const Vec& new_mem) const
01398 {
01399     mem.resize(new_mem.size());
01400     mem << new_mem;
01401 }
01402 
01403 void VMatLanguage::makeDeepCopyFromShallowCopy(CopiesMap& copies)
01404 {
01405     inherited::makeDeepCopyFromShallowCopy(copies);
01406 
01407     deepCopyField(vmsource, copies);
01408     deepCopyField(srcfieldnames, copies);
01409     deepCopyField(outputfieldnames, copies);
01410     deepCopyField(program, copies);
01411     deepCopyField(mappings, copies);
01412     deepCopyField(pstack, copies);
01413     deepCopyField(myvec, copies);
01414     deepCopyField(mem, copies);
01415 }
01416 
01417 
01418 
01419 void  PreprocessingVMatrix::getNewRow(int i, const Vec& v) const
01420 {
01421     program->run(i,v);
01422 }
01423 
01424 PLEARN_IMPLEMENT_OBJECT(PreprocessingVMatrix, "DEPRECATED: use ProcessingVMatrix instead", "NO HELP");
01425 
01426 
01427 PreprocessingVMatrix::PreprocessingVMatrix(VMat the_source, const string& program_string)
01428     : source(the_source), program(new VMatLanguage(the_source))
01429 {
01430     program->compileString(program_string,fieldnames);
01431     build();
01432 }
01433 
01434 void
01435 PreprocessingVMatrix::build()
01436 {
01437     inherited::build();
01438     build_();
01439 }
01440 
01441 void
01442 PreprocessingVMatrix::build_()
01443 {
01444     if (source) {
01445         fieldinfos.resize((int)fieldnames.size());
01446         for(unsigned int j=0; j<fieldnames.size(); j++)
01447             fieldinfos[j] = VMField(fieldnames[j]);
01448 
01449         sourcevec.resize(source->width());
01450         width_ = (int)fieldnames.size();
01451         length_ = source.length();
01452     }
01453 }
01454 
01455 void
01456 PreprocessingVMatrix::declareOptions(OptionList &ol)
01457 {
01458     inherited::declareOptions(ol);
01459     declareOption(ol, "source", &PreprocessingVMatrix::source, OptionBase::buildoption, "");
01460     declareOption(ol, "program", &PreprocessingVMatrix::program, OptionBase::buildoption, "");
01461     declareOption(ol, "fieldnames", &PreprocessingVMatrix::fieldnames, OptionBase::buildoption, "");
01462 }
01463 
01464 
01465 } // end of namespace PLearn
01466 
01467 
01468 /*
01469   Local Variables:
01470   mode:c++
01471   c-basic-offset:4
01472   c-file-style:"stroustrup"
01473   c-file-offsets:((innamespace . 0)(inline-open . 0))
01474   indent-tabs-mode:nil
01475   fill-column:79
01476   End:
01477 */
01478 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines