PLearn 0.1
VVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // VVMatrix.cc
00004 // Copyright (C) 2002 Pascal Vincent and Julien Keable
00005 // Copyright (C) 2003 Olivier Delalleau
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: VVMatrix.cc 8613 2008-02-29 21:35:25Z nouiz $
00037  * This file is part of the PLearn library.
00038  ******************************************************* */
00039 
00040 #include <plearn/base/Object.h>
00041 #include <plearn/db/getDataSet.h>
00042 #include <plearn/io/IntVecFile.h>
00043 #include <plearn/math/random.h>
00044 #include "ConcatColumnsVMatrix.h"
00045 #include "ConcatRowsVMatrix.h"
00046 #include "DiskVMatrix.h"
00047 #include "FileVMatrix.h"
00048 #include "JoinVMatrix.h"
00049 #include "SelectRowsFileIndexVMatrix.h"
00050 #include "VMatLanguage.h"
00051 #include "VVMatrix.h"
00052 
00053 #ifdef WIN32
00054 #include <time.h>
00055 #else
00056 #include <unistd.h>
00057 #endif
00058 
00059 #define NEW_SYNTAX_CHAR '@'
00060 
00061 namespace PLearn {
00062 using namespace std;
00063 
00064 PLEARN_IMPLEMENT_OBJECT(VVMatrix,
00065                         "A VMat that reads a '.vmat' file.",
00066                         ""
00067     );
00068 
00070 // declareOptions //
00072 void VVMatrix::declareOptions(OptionList &ol)
00073 {
00074     declareOption(ol, "filename", &VVMatrix::the_filename, OptionBase::buildoption, "Path to the .vmat file");
00075     inherited::declareOptions(ol);
00076 }
00077 
00078 
00079 // vmat genfilterindex source.pmat toto.pvm toto.indexes
00080 
00081 void VVMatrix::generateFilterIndexFile(VMat source, const string & code, const string& ivfname)
00082 {
00083     rm(ivfname);
00084     IntVecFile ivf(ivfname,true);
00085     VMatLanguage filt(source);
00086     vector<string> fn;
00087     for(int i=0;i<source.width();i++)
00088         fn.push_back(source->fieldName(i));
00089     filt.compileString(code,fn);
00090     Vec bla(1);
00091     Vec src(source.width());
00092     ProgressBar pb(cerr,"Filtering",source.length());
00093     for(int i=0;i<source.length();i++)
00094     {
00095         filt.run(i,bla);
00096         if(!fast_exact_is_equal(bla[0], 0))
00097             ivf.append(i);
00098         pb(i);
00099     }
00100     ivf.close();
00101 }
00102 
00103 VMat VVMatrix::buildFilteredVMatFromVPL(VMat source, const string & code, const string& ivfname, time_t date_of_code)
00104 {
00105     if(getNonBlankLines(code).empty())
00106         return source;
00107     if(!isfile(ivfname) || mtime(ivfname) < date_of_code)
00108         generateFilterIndexFile(source, code, ivfname);
00109     IntVecFile ivf(ivfname,true);
00110     return new SelectRowsFileIndexVMatrix(source, ivfname);
00111 }
00112 
00113 
00114 vector<vector<string> > VVMatrix::extractSourceMatrix(const string & str,const string& filename)
00115 {
00116     vector<vector<string> > mstr;
00117     if (str[0] == NEW_SYNTAX_CHAR) {
00118         // We are using the new syntax : we don't care about this for now,
00119         // too bad for the getDate method !
00120         return mstr;
00121     }
00122     vector<string> lines=getNonBlankLines(str);
00123     for(unsigned int i=0;i<lines.size();i++)
00124         mstr.push_back(split(lines[i],"|"));
00125 
00126     for(unsigned int i=0;i<mstr.size();i++)
00127         for(unsigned int j=0;j<mstr[i].size();j++)
00128         {
00129             string srcstr = removeblanks(mstr[i][j]);
00130             if(srcstr[0]!=slash_char)
00131             {
00132                 string potential_dir = extract_directory(filename);
00133                 size_t p = srcstr.find(":");
00134                 string potential_path = potential_dir + srcstr.substr(0,p);
00135                 if(pathexists(potential_path))
00136                     srcstr=potential_dir+srcstr;
00137             }
00138             mstr[i][j] = srcstr;
00139         }
00140     return mstr;
00141 }
00142 
00143 time_t VVMatrix::getDateOfVMat(const string& filename)
00144 {
00145     time_t latest = 0;
00146     string in=readFileAndMacroProcess(filename, latest);
00147     size_t idx_source = in.find("<SOURCES>");
00148     size_t cidx_source;
00149     latest=max(getDateOfCode(filename),latest);
00150 
00151     if(idx_source!=string::npos)
00152     {
00153         idx_source += strlen("<SOURCES>");     // skip beyond
00154         cidx_source=in.find("</SOURCES>");
00155         if(cidx_source==string::npos)
00156             PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str());
00157         string sec=in.substr(idx_source,cidx_source-idx_source);
00158         vector<vector<string> > mstr = extractSourceMatrix(sec,filename);
00159         // TODO Make it work with the new syntax
00160         for(unsigned int i=0;i<mstr.size();i++)
00161             for(unsigned int j=0;j<mstr[i].size();j++)
00162             {
00163                 vector<string> vecstr;
00164                 vecstr=split(mstr[i][j],":");
00165                 if(vecstr.size()==2 || vecstr.size()==4)
00166                 {
00167                     // also check for date of possible IntVecFile
00168                     if(vecstr[1][0]!=slash_char)
00169                         vecstr[1]=extract_directory(filename)+vecstr[1];
00170                     latest=max(mtime(vecstr[1]),latest);
00171                 }
00172                 latest=max(getDataSetDate(vecstr[0]),latest);
00173             }
00174     }
00175     return latest;
00176 }
00177 
00178 // returns the result from the join operation
00179 void VVMatrix::processJoinSection(const vector<string> & code, VMat & tmpsource)
00180 {
00181     unsigned int i=0;
00182     while(i<code.size())
00183     {
00184         while(isBlank(code[i]))i++;
00185         VMat slave=getDataSet(code[i++]);
00186         vector<string> mess=split(code[i++],"[]");
00187         if(mess.size()!=3)
00188             PLERROR("In JOIN section, field correspondance syntax is : [master1,master2] -> [slave1,slave2]");
00189         vector<string> mfields=split(mess[0],",");
00190         vector<string> sfields=split(mess[2],",");
00191         TVec<int> midx(int(mfields.size())), sidx(int(sfields.size()));
00192         for(unsigned int j=0;j<mfields.size();j++)
00193             midx[j]=tmpsource->fieldIndex(mfields[j]);
00194         for(unsigned int j=0;j<sfields.size();j++)
00195             sidx[j]=slave->fieldIndex(sfields[j]);
00196 
00197         JoinVMatrix * jvm= new JoinVMatrix(tmpsource,slave,midx,sidx);
00198 
00199         // browse field declarations of the <JOIN> section to declare new fields in the resulting JoinVMatrix
00200 
00201         while(i<code.size() && code[i].find(":")!=string::npos)
00202         {
00203             vector<string> crunch=split(code[i],":");
00204             if(crunch.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname");
00205             vector<string> st=split(removeblanks(crunch[0]),"()");
00206             if(st.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname");
00207             jvm->addStatField(removeblanks(st[0]),removeblanks(st[1]),removeblanks(crunch[1]));
00208             i++;
00209         }
00210         while(i<code.size() && isBlank(code[i]))i++;
00211         tmpsource=jvm;
00212     }
00213 }
00214 
00215 // generate a file (ivfname) containing indexes of rows of 'source' that remain after filtering with
00216 // the *every* possible step that changes the index of rows (i.e : prefilter, shuffle.. postfiltering)
00217 // -- Not optimal, since it will first *precompute* if any postfilter is required
00218 void VVMatrix::generateVMatIndex(VMat source, const string& meta_data_dir,
00219                                  const string & filename, time_t date_of_code,const string & in,
00220                                  size_t idx_prefilter, size_t cidx_prefilter,
00221                                  size_t idx_postfilter, size_t cidx_postfilter,
00222                                  size_t idx_process, size_t cidx_process,
00223                                  size_t idx_shuffle, size_t cidx_shuffle,
00224                                  size_t idx_join, size_t cidx_join)
00225 {
00226     VMat tmpsource = source;
00227     rm(meta_data_dir+slash+"source.indexes");
00228 
00229     if(idx_prefilter!=string::npos)
00230     {
00231         idx_prefilter+=11;
00232         cidx_prefilter=in.find("</PREFILTER>");
00233         if(cidx_prefilter==string::npos)
00234             PLERROR("Cannot find closing tag </PREFILTER>. File is %s",filename.c_str());
00235         string code=in.substr(idx_prefilter,cidx_prefilter-idx_prefilter);
00236         cout<<"Prefiltering.. "<<endl;
00237         tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.prefilter.indexes",date_of_code);
00238     }
00239 
00240     if(idx_join!=string::npos)
00241     {
00242         cidx_join=in.find("</JOIN>");
00243         if(cidx_join==string::npos)
00244             PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str());
00245         vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n");
00246         processJoinSection(code,tmpsource);
00247     }
00248 
00249     if(idx_process!=string::npos)
00250     {
00251         cidx_process=in.find("</PROCESSING>");
00252         if(cidx_process==string::npos)
00253             PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str());
00254         string code=in.substr(idx_process+12,cidx_process-idx_process-12);
00255         tmpsource = new PreprocessingVMatrix(tmpsource,code);
00256     }
00257 
00258     if(idx_postfilter!=string::npos)
00259     {
00260         idx_postfilter+=12;
00261         cidx_postfilter=in.find("</POSTFILTER>");
00262         if(cidx_postfilter==string::npos)
00263             PLERROR("Cannot find closing tag </POSTFILTER>. File is %s",filename.c_str());
00264         string code=in.substr(idx_postfilter,cidx_postfilter-idx_postfilter);
00265         cout<<"Postfiltering.. "<<endl;
00266         tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.postfilter.indexes",date_of_code);
00267     }
00268 
00269     // combines pre and postfilter index file in a single one
00270     if(isfile(meta_data_dir+slash+"incomplete.source.prefilter.indexes"))
00271         if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes"))
00272         {
00273             IntVecFile ivf(meta_data_dir+slash+"source.indexes",true);
00274             IntVecFile preivf(meta_data_dir+slash+"incomplete.source.prefilter.indexes");
00275             IntVecFile postivf(meta_data_dir+slash+"incomplete.source.postfilter.indexes");
00276             for(int i=0;i<postivf.length();i++)
00277                 ivf.append(preivf[postivf[i]]);
00278         }
00279         else
00280             mv(meta_data_dir+slash+"incomplete.source.prefilter.indexes",
00281                meta_data_dir+slash+"source.indexes");
00282 
00283     else
00284         if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes"))
00285             mv(meta_data_dir+slash+"incomplete.source.postfilter.indexes",
00286                meta_data_dir+slash+"source.indexes");
00287 
00288     if(idx_shuffle!=string::npos)
00289     {
00290         idx_shuffle+=9;
00291         cidx_shuffle=in.find("</SHUFFLE>");
00292         if(cidx_shuffle==string::npos)
00293             PLERROR("Cannot find closing tag </SHUFFLE>. File is %s",filename.c_str());
00294         string seedstr=removeblanks(in.substr(idx_shuffle,cidx_shuffle-idx_shuffle));
00295         if(seedstr=="")
00296             manual_seed(clock());
00297         else
00298         {
00299             int seed = toint(seedstr);
00300             manual_seed(seed);
00301         }
00302         // if a source.indexes file exists, shuffle it, otherwise, create it
00303         TVec<int> idx;
00304         if(isfile(meta_data_dir+slash+"source.indexes"))
00305         {
00306             IntVecFile ivf(meta_data_dir+slash+"source.indexes");
00307             idx=ivf.getVec();
00308         }
00309         else idx=TVec<int>(0,tmpsource->length()-1,1);
00310         shuffleElements(idx);
00311         rm(meta_data_dir+slash+"source.indexes");
00312         IntVecFile newivf(meta_data_dir+slash+"source.indexes",true);
00313         newivf.append(idx);
00314     }
00315     // remove intermediate files
00316     rm(meta_data_dir+slash+"incomplete.source.prefilter.indexes");
00317     rm(meta_data_dir+slash+"incomplete.source.postfilter.indexes");
00318 }
00319 
00320 bool VVMatrix::isPrecomputedAndUpToDate()
00321 {
00322     string meta_data_dir=remove_trailing_slash(the_filename)+".metadata";
00323     if(isfile(meta_data_dir+slash+"precomputed.pmat") &&
00324        mtime(meta_data_dir+slash+"precomputed.pmat") > getMtime())
00325         return true;
00326     if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash) &&
00327        mtime(meta_data_dir+slash+"precomputed.dmat"+slash+"indexfile") > getMtime())
00328         return true;
00329     return false;
00330 }
00331 
00332 string VVMatrix::getPrecomputedDataName()
00333 {
00334     string meta_data_dir=remove_trailing_slash(the_filename)+".metadata";
00335     if(isfile(meta_data_dir+slash+"precomputed.pmat"))
00336         return meta_data_dir+slash+"precomputed.pmat";
00337     if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash))
00338         return meta_data_dir+slash+"precomputed.dmat";
00339     return "** no precomputed data found **";
00340 }
00341 
00342 
00343 VMat VVMatrix::createPreproVMat(const string & filename)
00344 {
00345     time_t date_of_code = 0;
00346     string in=readFileAndMacroProcess(filename,date_of_code);
00347     size_t idx_source     = in.find("<SOURCES>");
00348     size_t idx_prefilter  = in.find("<PREFILTER>");
00349     size_t idx_postfilter = in.find("<POSTFILTER>");
00350     size_t idx_process    = in.find("<PROCESSING>");
00351     size_t idx_shuffle    = in.find("<SHUFFLE>");
00352     size_t idx_join       = in.find("<JOIN>");
00353     size_t idx_precompute = in.find("<PRECOMPUTE>");
00354     size_t idx_sizes      = in.find("<SIZES>");
00355     size_t cidx_source    = 0, cidx_prefilter = 0,  cidx_postfilter = 0,
00356         cidx_process = 0, cidx_shuffle   = 0,  cidx_precompute = 0,
00357         cidx_join    = 0, cidx_sizes     = 0;
00358     string precomp;
00359     VMat source;
00360 
00361     bool olddebugval=VMatLanguage::output_preproc;
00362     VMatLanguage::output_preproc = in.find("<DEBUG>")!=string::npos;
00363 
00364     if( VMatLanguage::output_preproc )
00365         cerr<<"DEBUG is on (remove <DEBUG> in "+filename+" to turn it off)"<<endl;
00366 
00367     PPath meta_data_dir = remove_trailing_slash(filename)+".metadata";
00368     force_mkdir(meta_data_dir);
00369     date_of_code=max(date_of_code,getDateOfVMat(filename));
00370 
00371     // remove pollution : all stuff that has possibly been interrupted during computation
00372     rm ( meta_data_dir / "incomplete.*" );
00373 
00374     bool sizes_spec = false;
00375     int inputsize = -1;
00376     int targetsize = -1;
00377     int weightsize = -1;
00378     // Check if sizes are specified.
00379     if(idx_sizes != string::npos)
00380     {
00381         sizes_spec = true;
00382         idx_sizes += 7;
00383         cidx_sizes = in.find("</SIZES>");
00384         if(cidx_sizes == string::npos)
00385             PLERROR("Cannot find closing tag </SIZES>. File is %s",filename.c_str());
00386         vector<string> sizes = split(in.substr(idx_sizes, cidx_sizes - idx_sizes),"\n");
00387         if (sizes.size() != 3) {
00388             PLERROR("You should specify exactly 3 sizes between <SIZES> and </SIZES> (file is %s)", filename.c_str());
00389         }
00390         inputsize = toint(sizes[0]);
00391         targetsize = toint(sizes[1]);
00392         weightsize = toint(sizes[2]);
00393     }
00394 
00395     if(isfile(meta_data_dir / "precomputed.pmat"))
00396     {
00397         // Precomputed version exist in pmat format.
00398         // If it seems old, we display a warning (one may still want to use it
00399         // if he knows the changes made to the vmat code do not alter the data).
00400         if(mtime(meta_data_dir / "precomputed.pmat") < date_of_code) {
00401             PLWARNING("In VVMatrix::createPreproVMat - The precomputed data (in %s) is older than current code, you may want to recompute again", meta_data_dir.c_str());
00402         }
00403         source = new FileVMatrix(meta_data_dir / "precomputed.pmat");
00404         source->setMetaDataDir(meta_data_dir);
00405         source->updateMtime(date_of_code);
00406         source->defineSizes(inputsize, targetsize, weightsize);
00407         return source;
00408     }
00409 
00410     if( pathexists(meta_data_dir / "precomputed.dmat") )
00411     {
00412         // precomputed version exist in DiskVMatrix format,
00413         // so we use it *if it is up to date*
00414         if ( mtime(meta_data_dir/"precomputed.dmat/indexfile") > date_of_code )
00415         {
00416             source = new DiskVMatrix(meta_data_dir/"precomputed.dmat");
00417             source->setMetaDataDir(meta_data_dir);
00418             source->updateMtime(date_of_code);
00419             source->defineSizes(inputsize, targetsize, weightsize);
00420             return source;
00421         }
00422     }
00423 
00424     // if index_section is true, then this dataset needs a file containing the index of the rows to keep
00425     bool index_section = idx_prefilter!=string::npos || idx_postfilter!=string::npos || idx_shuffle!=string::npos ;
00426 
00427     // if true, index file lacks or is out of date
00428     bool must_regen_index = index_section &&
00429         (!isfile(meta_data_dir/"source.indexes") || date_of_code > mtime(meta_data_dir/"source.indexes"));
00430 
00431     // erase obsolete source.index if necessary
00432     if(isfile(meta_data_dir/"source.indexes") && !index_section)
00433         rm (meta_data_dir/"source.indexes");
00434 
00435     if(idx_source!=string::npos)
00436     {
00437         // go over tag
00438         idx_source+=9;
00439         cidx_source=in.find("</SOURCES>");
00440         if(cidx_source==string::npos)
00441             PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str());
00442         // 'sec' is the text content of the <SOURCES> section
00443         string sec=in.substr(idx_source,cidx_source-idx_source);
00444 
00445         if (sec[1] == NEW_SYNTAX_CHAR) {
00446             // We are using the new syntax
00447             sec[1] = ' ';  // remove the special character indicating the new syntax
00448             source = dynamic_cast<VMatrix*>(newObject(sec));
00449             if(source.isNull()) {
00450                 PLERROR("In VVMatrix::createPreproVMat '%s' is not a valid VMatrix subclass",sec.c_str());
00451             }
00452         } else {
00453 
00454             vector<vector<string> > mstr = extractSourceMatrix(sec,filename);
00455             Array<VMat> vmrows(int(mstr.size()));
00456             // we need to build a VMat that is the concatenation of the datasets contained in 'mstr'
00457 
00458             for(unsigned int i=0;i<mstr.size();i++)
00459             {
00460                 Array<VMat> ar( (int)mstr[i].size() );
00461                 for(unsigned int j=0;j<mstr[i].size();j++)
00462                 {
00463                     vector<string> vecstr;
00464                     vecstr=split(mstr[i][j],":");
00465                     ar[j]=getDataSet(vecstr[0]);
00466 
00467                     // handle different cases of dataset specification
00468                     // legal formats are:
00469                     // 1- simple dataset filename
00470                     // 2- intVecFile specification : filename:intVecFile_Filename
00471                     // 3- range specifiaction : filename:start:length
00472                     // 4- range specifiaction + intVecFile: filename:intVecFile_Filename:start:length
00473                     switch(vecstr.size())
00474                     {
00475                     case 1: // only dataset name so we do nothing
00476                         break;
00477                     case 2: // we have an intVecFile specification
00478                         // prefix with the path to the current vmat
00479                         if(vecstr[1][0]!=slash_char)
00480                             vecstr[1]=extract_directory(filename)+vecstr[1];
00481                         ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]);
00482                         break;
00483                     case 3: // submatRows range specification
00484                         ar[j]=ar[j].subMatRows(toint(vecstr[1]),toint(vecstr[2]));
00485                         break;
00486                     case 4: // intVecFile + submatRows
00487                         if(vecstr[1][0]!=slash_char)
00488                             vecstr[1]=extract_directory(filename)+vecstr[1];
00489                         ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]);
00490                         ar[j]=ar[j].subMatRows(toint(vecstr[2]),toint(vecstr[3]));
00491                         break;
00492                     default:
00493                         PLERROR("Strange number of semicolumns... Format of source must be something.vmat[:indexfile.index][:start_row:length]. File is %s",filename.c_str());
00494                         break;
00495                     }
00496                 }
00497                 // if we have more than one filename in this row, we use hconcat to consolidate 'ar'.
00498                 vmrows[i] = ar.size()==1?ar[0]:hconcat(ar);
00499                 if(vmrows[i].length()==-1)
00500                     PLERROR("Trying to hconcat matrix with different lengths! File is %s",filename.c_str());
00501             }
00502 
00503             source = vconcat(vmrows);
00504             if(mstr.size()==0)
00505                 PLERROR("No source matrix found in <SOURCES> section! File is %s",filename.c_str());
00506         }
00507     }
00508     else PLERROR("Need at least a <SOURCES> section ! File is %s",filename.c_str());
00509 
00510     if(must_regen_index)
00511         generateVMatIndex(source, meta_data_dir, filename, date_of_code, in, idx_prefilter, cidx_prefilter,
00512                           idx_postfilter, cidx_postfilter, idx_process, cidx_process,
00513                           idx_shuffle, cidx_shuffle, idx_join, cidx_join);
00514 
00515     if(idx_join!=string::npos)
00516     {
00517         cidx_join=in.find("</JOIN>");
00518         if(cidx_join==string::npos)
00519             PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str());
00520         vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n");
00521         processJoinSection(code,source);
00522     }
00523 
00524     if(idx_process!=string::npos)
00525     {
00526         cidx_process=in.find("</PROCESSING>");
00527         if(cidx_process==string::npos)
00528             PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str());
00529         string code=in.substr(idx_process+12,cidx_process-idx_process-12);
00530         source = new PreprocessingVMatrix(source,code);
00531     }
00532 
00533     // if source.index exists at this point, we need to apply it
00534     if(isfile(meta_data_dir/"source.indexes"))
00535         source = new SelectRowsFileIndexVMatrix(source,meta_data_dir/"source.indexes");
00536 
00537     // next lines handle precomputation of matrix
00538     if(idx_precompute!=string::npos)
00539     {
00540         idx_precompute+=12;
00541         cidx_precompute=in.find("</PRECOMPUTE>");
00542         if(cidx_precompute==string::npos)
00543             PLERROR("Cannot find closing tag </PRECOMPUTE>. File is %s",filename.c_str());
00544         precomp=removeblanks(in.substr(idx_precompute,cidx_precompute-idx_precompute));
00545         if(precomp ==  "dmat")
00546         {
00547             cout << "Rendering DMAT : " << meta_data_dir/"precomputed.dmat" << endl;
00548             source->saveDMAT(meta_data_dir/"incomplete.precomputed.dmat");
00549             int cnt=0;
00550             if (isdir(meta_data_dir/"precomputed.dmat")) {
00551                 while(cnt++ < 5 &&
00552                       !force_rmdir(meta_data_dir/"precomputed.dmat"))
00553                 {
00554                     cerr<<"Could not rm -rf '"+meta_data_dir+
00555                         slash+"precomputed.dmat/'. Maybe 'Stale NFS file handle' curse again. Retrying in 2 sec."
00556                         <<endl;
00557                     sleep(2);
00558                 }
00559             }
00560 
00561             mvforce(meta_data_dir/"incomplete.precomputed.dmat",
00562                     meta_data_dir/"precomputed.dmat");
00563             if(pathexists(meta_data_dir/"incomplete.precomputed.dmat.metadata"))
00564             {
00565                 rm(meta_data_dir/"precomputed.dmat.metadata");
00566                 mvforce(meta_data_dir/"incomplete.precomputed.dmat.metadata",
00567                         meta_data_dir/"precomputed.dmat.metadata");
00568             }
00569             source=new DiskVMatrix(meta_data_dir/"precomputed.dmat");
00570         }
00571         else if(precomp ==  "pmat")
00572         {
00573             cout<<"Rendering PMAT : "<<meta_data_dir+slash+"precomputed.pmat"<<endl;
00574             source->savePMAT(meta_data_dir+slash+"incomplete.precomputed.pmat");
00575             mvforce(meta_data_dir/"incomplete.precomputed.pmat",
00576                     meta_data_dir/"precomputed.pmat");
00577             if(pathexists(meta_data_dir/"incomplete.precomputed.pmat.metadata"))
00578                 mvforce(meta_data_dir/"incomplete.precomputed.pmat.metadata",
00579                         meta_data_dir/"precomputed.pmat.metadata");
00580 
00581             // Save the string mappings.
00582             source->setMetaDataDir(meta_data_dir);
00583             source->saveAllStringMappings();
00584             source = new FileVMatrix(meta_data_dir/"precomputed.pmat");
00585             source->setMetaDataDir(meta_data_dir);
00586             source->setMtime(date_of_code);
00587             source->defineSizes(inputsize, targetsize, weightsize);
00588             // TODO The 3 lines above are duplicated throughout the code, this is ugly.
00589         }
00590         else if(precomp!="")
00591             PLERROR("Unsupported precomputing format : %s.  File is %s", precomp.c_str(),filename.c_str());
00592     }
00593 
00594     VMatLanguage::output_preproc=olddebugval;
00595     source->setMetaDataDir(meta_data_dir);
00596     source->updateMtime(date_of_code);
00597     if (sizes_spec) {
00598         source->defineSizes(inputsize, targetsize, weightsize);
00599     }
00600     return source;
00601 }
00602 
00603 void VVMatrix::build()
00604 {
00605     inherited::build();
00606     build_();
00607 }
00608 
00609 void VVMatrix::build_()
00610 {
00611     if (the_filename != "") {
00612         setMetaDataDir( PPath(the_filename+".metadata").absolute() );
00613         force_mkdir( getMetaDataDir() );
00614         time_t latest=0;
00615         code = readFileAndMacroProcess(the_filename,latest);
00616         if(removeblanks(code)[0]=='<') // old xml-like format
00617             the_mat = createPreproVMat(the_filename);
00618         else  // New standard PLearn object description format
00619         {
00620             the_mat = dynamic_cast<VMatrix*>(newObject(code));
00621             if(the_mat.isNull())
00622                 PLERROR("Object described in %s is not a VMatrix subclass",the_filename.c_str());
00623             the_mat->setMetaDataDir(getMetaDataDir());
00624         }
00625 
00626         updateMtime(the_mat->getMtime());
00627         updateMtime(latest);
00628         length_ = the_mat.length();
00629         width_ = the_mat.width();
00630 
00631         // Copy the sizes.
00632         copySizesFrom(the_mat);
00633 
00634         // Copy the string mappings.
00635         copyStringMappingsFrom(the_mat);
00636 
00637         // Copy the parent field names
00638         fieldinfos.resize(width_);
00639         if (the_mat->getFieldInfos().size() > 0)
00640             for(int j=0; j<width_; j++)
00641                 fieldinfos[j] = the_mat->getFieldInfos()[j];
00642     }
00643 }
00644 
00645 // string maps are those loaded from the .vmat metadatadir, not those of the source vmatrix anymore
00646 // could be changed..
00647 
00648 // real VVMatrix::getStringVal(int col, const string & str) const
00649 // {
00650 //   return the_mat->getStringVal(col, str);
00651 // }
00652 // string VVMatrix::getValString(int col, real val) const
00653 // {
00654 //   return the_mat->getValString(col, val);
00655 // }
00656 // string VVMatrix::getString(int row,int col) const
00657 // {
00658 //   return the_mat->getString(row, col);
00659 // }
00660 
00661 // const map<string,real>& VVMatrix::getStringToRealMapping(int col) const
00662 // {
00663 //   return the_mat->getStringToRealMapping(col);
00664 // }
00665 
00666 
00668 // makeDeepCopyFromShallowCopy //
00670 void VVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) {
00671     inherited::makeDeepCopyFromShallowCopy(copies);
00672     deepCopyField(the_mat, copies);
00673 }
00674 
00675 } // end of namespace PLearn
00676 
00677 
00678 /*
00679   Local Variables:
00680   mode:c++
00681   c-basic-offset:4
00682   c-file-style:"stroustrup"
00683   c-file-offsets:((innamespace . 0)(inline-open . 0))
00684   indent-tabs-mode:nil
00685   fill-column:79
00686   End:
00687 */
00688 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines