PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // VVMatrix.cc 00004 // Copyright (C) 2002 Pascal Vincent and Julien Keable 00005 // Copyright (C) 2003 Olivier Delalleau 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: VVMatrix.cc 8613 2008-02-29 21:35:25Z nouiz $ 00037 * This file is part of the PLearn library. 00038 ******************************************************* */ 00039 00040 #include <plearn/base/Object.h> 00041 #include <plearn/db/getDataSet.h> 00042 #include <plearn/io/IntVecFile.h> 00043 #include <plearn/math/random.h> 00044 #include "ConcatColumnsVMatrix.h" 00045 #include "ConcatRowsVMatrix.h" 00046 #include "DiskVMatrix.h" 00047 #include "FileVMatrix.h" 00048 #include "JoinVMatrix.h" 00049 #include "SelectRowsFileIndexVMatrix.h" 00050 #include "VMatLanguage.h" 00051 #include "VVMatrix.h" 00052 00053 #ifdef WIN32 00054 #include <time.h> 00055 #else 00056 #include <unistd.h> 00057 #endif 00058 00059 #define NEW_SYNTAX_CHAR '@' 00060 00061 namespace PLearn { 00062 using namespace std; 00063 00064 PLEARN_IMPLEMENT_OBJECT(VVMatrix, 00065 "A VMat that reads a '.vmat' file.", 00066 "" 00067 ); 00068 00070 // declareOptions // 00072 void VVMatrix::declareOptions(OptionList &ol) 00073 { 00074 declareOption(ol, "filename", &VVMatrix::the_filename, OptionBase::buildoption, "Path to the .vmat file"); 00075 inherited::declareOptions(ol); 00076 } 00077 00078 00079 // vmat genfilterindex source.pmat toto.pvm toto.indexes 00080 00081 void VVMatrix::generateFilterIndexFile(VMat source, const string & code, const string& ivfname) 00082 { 00083 rm(ivfname); 00084 IntVecFile ivf(ivfname,true); 00085 VMatLanguage filt(source); 00086 vector<string> fn; 00087 for(int i=0;i<source.width();i++) 00088 fn.push_back(source->fieldName(i)); 00089 filt.compileString(code,fn); 00090 Vec bla(1); 00091 Vec src(source.width()); 00092 ProgressBar pb(cerr,"Filtering",source.length()); 00093 for(int i=0;i<source.length();i++) 00094 { 00095 filt.run(i,bla); 00096 if(!fast_exact_is_equal(bla[0], 0)) 00097 ivf.append(i); 00098 pb(i); 00099 } 00100 ivf.close(); 00101 } 00102 00103 VMat VVMatrix::buildFilteredVMatFromVPL(VMat source, const string & code, const string& ivfname, time_t date_of_code) 00104 { 00105 if(getNonBlankLines(code).empty()) 00106 return source; 00107 if(!isfile(ivfname) || mtime(ivfname) < date_of_code) 00108 generateFilterIndexFile(source, code, ivfname); 00109 IntVecFile ivf(ivfname,true); 00110 return new SelectRowsFileIndexVMatrix(source, ivfname); 00111 } 00112 00113 00114 vector<vector<string> > VVMatrix::extractSourceMatrix(const string & str,const string& filename) 00115 { 00116 vector<vector<string> > mstr; 00117 if (str[0] == NEW_SYNTAX_CHAR) { 00118 // We are using the new syntax : we don't care about this for now, 00119 // too bad for the getDate method ! 00120 return mstr; 00121 } 00122 vector<string> lines=getNonBlankLines(str); 00123 for(unsigned int i=0;i<lines.size();i++) 00124 mstr.push_back(split(lines[i],"|")); 00125 00126 for(unsigned int i=0;i<mstr.size();i++) 00127 for(unsigned int j=0;j<mstr[i].size();j++) 00128 { 00129 string srcstr = removeblanks(mstr[i][j]); 00130 if(srcstr[0]!=slash_char) 00131 { 00132 string potential_dir = extract_directory(filename); 00133 size_t p = srcstr.find(":"); 00134 string potential_path = potential_dir + srcstr.substr(0,p); 00135 if(pathexists(potential_path)) 00136 srcstr=potential_dir+srcstr; 00137 } 00138 mstr[i][j] = srcstr; 00139 } 00140 return mstr; 00141 } 00142 00143 time_t VVMatrix::getDateOfVMat(const string& filename) 00144 { 00145 time_t latest = 0; 00146 string in=readFileAndMacroProcess(filename, latest); 00147 size_t idx_source = in.find("<SOURCES>"); 00148 size_t cidx_source; 00149 latest=max(getDateOfCode(filename),latest); 00150 00151 if(idx_source!=string::npos) 00152 { 00153 idx_source += strlen("<SOURCES>"); // skip beyond 00154 cidx_source=in.find("</SOURCES>"); 00155 if(cidx_source==string::npos) 00156 PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str()); 00157 string sec=in.substr(idx_source,cidx_source-idx_source); 00158 vector<vector<string> > mstr = extractSourceMatrix(sec,filename); 00159 // TODO Make it work with the new syntax 00160 for(unsigned int i=0;i<mstr.size();i++) 00161 for(unsigned int j=0;j<mstr[i].size();j++) 00162 { 00163 vector<string> vecstr; 00164 vecstr=split(mstr[i][j],":"); 00165 if(vecstr.size()==2 || vecstr.size()==4) 00166 { 00167 // also check for date of possible IntVecFile 00168 if(vecstr[1][0]!=slash_char) 00169 vecstr[1]=extract_directory(filename)+vecstr[1]; 00170 latest=max(mtime(vecstr[1]),latest); 00171 } 00172 latest=max(getDataSetDate(vecstr[0]),latest); 00173 } 00174 } 00175 return latest; 00176 } 00177 00178 // returns the result from the join operation 00179 void VVMatrix::processJoinSection(const vector<string> & code, VMat & tmpsource) 00180 { 00181 unsigned int i=0; 00182 while(i<code.size()) 00183 { 00184 while(isBlank(code[i]))i++; 00185 VMat slave=getDataSet(code[i++]); 00186 vector<string> mess=split(code[i++],"[]"); 00187 if(mess.size()!=3) 00188 PLERROR("In JOIN section, field correspondance syntax is : [master1,master2] -> [slave1,slave2]"); 00189 vector<string> mfields=split(mess[0],","); 00190 vector<string> sfields=split(mess[2],","); 00191 TVec<int> midx(int(mfields.size())), sidx(int(sfields.size())); 00192 for(unsigned int j=0;j<mfields.size();j++) 00193 midx[j]=tmpsource->fieldIndex(mfields[j]); 00194 for(unsigned int j=0;j<sfields.size();j++) 00195 sidx[j]=slave->fieldIndex(sfields[j]); 00196 00197 JoinVMatrix * jvm= new JoinVMatrix(tmpsource,slave,midx,sidx); 00198 00199 // browse field declarations of the <JOIN> section to declare new fields in the resulting JoinVMatrix 00200 00201 while(i<code.size() && code[i].find(":")!=string::npos) 00202 { 00203 vector<string> crunch=split(code[i],":"); 00204 if(crunch.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname"); 00205 vector<string> st=split(removeblanks(crunch[0]),"()"); 00206 if(st.size()!=2)PLERROR("In JOIN section : field declaration section syntax incorrect. EG: MEAN(master1) :newfieldname"); 00207 jvm->addStatField(removeblanks(st[0]),removeblanks(st[1]),removeblanks(crunch[1])); 00208 i++; 00209 } 00210 while(i<code.size() && isBlank(code[i]))i++; 00211 tmpsource=jvm; 00212 } 00213 } 00214 00215 // generate a file (ivfname) containing indexes of rows of 'source' that remain after filtering with 00216 // the *every* possible step that changes the index of rows (i.e : prefilter, shuffle.. postfiltering) 00217 // -- Not optimal, since it will first *precompute* if any postfilter is required 00218 void VVMatrix::generateVMatIndex(VMat source, const string& meta_data_dir, 00219 const string & filename, time_t date_of_code,const string & in, 00220 size_t idx_prefilter, size_t cidx_prefilter, 00221 size_t idx_postfilter, size_t cidx_postfilter, 00222 size_t idx_process, size_t cidx_process, 00223 size_t idx_shuffle, size_t cidx_shuffle, 00224 size_t idx_join, size_t cidx_join) 00225 { 00226 VMat tmpsource = source; 00227 rm(meta_data_dir+slash+"source.indexes"); 00228 00229 if(idx_prefilter!=string::npos) 00230 { 00231 idx_prefilter+=11; 00232 cidx_prefilter=in.find("</PREFILTER>"); 00233 if(cidx_prefilter==string::npos) 00234 PLERROR("Cannot find closing tag </PREFILTER>. File is %s",filename.c_str()); 00235 string code=in.substr(idx_prefilter,cidx_prefilter-idx_prefilter); 00236 cout<<"Prefiltering.. "<<endl; 00237 tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.prefilter.indexes",date_of_code); 00238 } 00239 00240 if(idx_join!=string::npos) 00241 { 00242 cidx_join=in.find("</JOIN>"); 00243 if(cidx_join==string::npos) 00244 PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str()); 00245 vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n"); 00246 processJoinSection(code,tmpsource); 00247 } 00248 00249 if(idx_process!=string::npos) 00250 { 00251 cidx_process=in.find("</PROCESSING>"); 00252 if(cidx_process==string::npos) 00253 PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str()); 00254 string code=in.substr(idx_process+12,cidx_process-idx_process-12); 00255 tmpsource = new PreprocessingVMatrix(tmpsource,code); 00256 } 00257 00258 if(idx_postfilter!=string::npos) 00259 { 00260 idx_postfilter+=12; 00261 cidx_postfilter=in.find("</POSTFILTER>"); 00262 if(cidx_postfilter==string::npos) 00263 PLERROR("Cannot find closing tag </POSTFILTER>. File is %s",filename.c_str()); 00264 string code=in.substr(idx_postfilter,cidx_postfilter-idx_postfilter); 00265 cout<<"Postfiltering.. "<<endl; 00266 tmpsource = buildFilteredVMatFromVPL(tmpsource,code,meta_data_dir+slash+"incomplete.source.postfilter.indexes",date_of_code); 00267 } 00268 00269 // combines pre and postfilter index file in a single one 00270 if(isfile(meta_data_dir+slash+"incomplete.source.prefilter.indexes")) 00271 if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes")) 00272 { 00273 IntVecFile ivf(meta_data_dir+slash+"source.indexes",true); 00274 IntVecFile preivf(meta_data_dir+slash+"incomplete.source.prefilter.indexes"); 00275 IntVecFile postivf(meta_data_dir+slash+"incomplete.source.postfilter.indexes"); 00276 for(int i=0;i<postivf.length();i++) 00277 ivf.append(preivf[postivf[i]]); 00278 } 00279 else 00280 mv(meta_data_dir+slash+"incomplete.source.prefilter.indexes", 00281 meta_data_dir+slash+"source.indexes"); 00282 00283 else 00284 if(isfile(meta_data_dir+slash+"incomplete.source.postfilter.indexes")) 00285 mv(meta_data_dir+slash+"incomplete.source.postfilter.indexes", 00286 meta_data_dir+slash+"source.indexes"); 00287 00288 if(idx_shuffle!=string::npos) 00289 { 00290 idx_shuffle+=9; 00291 cidx_shuffle=in.find("</SHUFFLE>"); 00292 if(cidx_shuffle==string::npos) 00293 PLERROR("Cannot find closing tag </SHUFFLE>. File is %s",filename.c_str()); 00294 string seedstr=removeblanks(in.substr(idx_shuffle,cidx_shuffle-idx_shuffle)); 00295 if(seedstr=="") 00296 manual_seed(clock()); 00297 else 00298 { 00299 int seed = toint(seedstr); 00300 manual_seed(seed); 00301 } 00302 // if a source.indexes file exists, shuffle it, otherwise, create it 00303 TVec<int> idx; 00304 if(isfile(meta_data_dir+slash+"source.indexes")) 00305 { 00306 IntVecFile ivf(meta_data_dir+slash+"source.indexes"); 00307 idx=ivf.getVec(); 00308 } 00309 else idx=TVec<int>(0,tmpsource->length()-1,1); 00310 shuffleElements(idx); 00311 rm(meta_data_dir+slash+"source.indexes"); 00312 IntVecFile newivf(meta_data_dir+slash+"source.indexes",true); 00313 newivf.append(idx); 00314 } 00315 // remove intermediate files 00316 rm(meta_data_dir+slash+"incomplete.source.prefilter.indexes"); 00317 rm(meta_data_dir+slash+"incomplete.source.postfilter.indexes"); 00318 } 00319 00320 bool VVMatrix::isPrecomputedAndUpToDate() 00321 { 00322 string meta_data_dir=remove_trailing_slash(the_filename)+".metadata"; 00323 if(isfile(meta_data_dir+slash+"precomputed.pmat") && 00324 mtime(meta_data_dir+slash+"precomputed.pmat") > getMtime()) 00325 return true; 00326 if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash) && 00327 mtime(meta_data_dir+slash+"precomputed.dmat"+slash+"indexfile") > getMtime()) 00328 return true; 00329 return false; 00330 } 00331 00332 string VVMatrix::getPrecomputedDataName() 00333 { 00334 string meta_data_dir=remove_trailing_slash(the_filename)+".metadata"; 00335 if(isfile(meta_data_dir+slash+"precomputed.pmat")) 00336 return meta_data_dir+slash+"precomputed.pmat"; 00337 if(pathexists(meta_data_dir+slash+"precomputed.dmat"+slash)) 00338 return meta_data_dir+slash+"precomputed.dmat"; 00339 return "** no precomputed data found **"; 00340 } 00341 00342 00343 VMat VVMatrix::createPreproVMat(const string & filename) 00344 { 00345 time_t date_of_code = 0; 00346 string in=readFileAndMacroProcess(filename,date_of_code); 00347 size_t idx_source = in.find("<SOURCES>"); 00348 size_t idx_prefilter = in.find("<PREFILTER>"); 00349 size_t idx_postfilter = in.find("<POSTFILTER>"); 00350 size_t idx_process = in.find("<PROCESSING>"); 00351 size_t idx_shuffle = in.find("<SHUFFLE>"); 00352 size_t idx_join = in.find("<JOIN>"); 00353 size_t idx_precompute = in.find("<PRECOMPUTE>"); 00354 size_t idx_sizes = in.find("<SIZES>"); 00355 size_t cidx_source = 0, cidx_prefilter = 0, cidx_postfilter = 0, 00356 cidx_process = 0, cidx_shuffle = 0, cidx_precompute = 0, 00357 cidx_join = 0, cidx_sizes = 0; 00358 string precomp; 00359 VMat source; 00360 00361 bool olddebugval=VMatLanguage::output_preproc; 00362 VMatLanguage::output_preproc = in.find("<DEBUG>")!=string::npos; 00363 00364 if( VMatLanguage::output_preproc ) 00365 cerr<<"DEBUG is on (remove <DEBUG> in "+filename+" to turn it off)"<<endl; 00366 00367 PPath meta_data_dir = remove_trailing_slash(filename)+".metadata"; 00368 force_mkdir(meta_data_dir); 00369 date_of_code=max(date_of_code,getDateOfVMat(filename)); 00370 00371 // remove pollution : all stuff that has possibly been interrupted during computation 00372 rm ( meta_data_dir / "incomplete.*" ); 00373 00374 bool sizes_spec = false; 00375 int inputsize = -1; 00376 int targetsize = -1; 00377 int weightsize = -1; 00378 // Check if sizes are specified. 00379 if(idx_sizes != string::npos) 00380 { 00381 sizes_spec = true; 00382 idx_sizes += 7; 00383 cidx_sizes = in.find("</SIZES>"); 00384 if(cidx_sizes == string::npos) 00385 PLERROR("Cannot find closing tag </SIZES>. File is %s",filename.c_str()); 00386 vector<string> sizes = split(in.substr(idx_sizes, cidx_sizes - idx_sizes),"\n"); 00387 if (sizes.size() != 3) { 00388 PLERROR("You should specify exactly 3 sizes between <SIZES> and </SIZES> (file is %s)", filename.c_str()); 00389 } 00390 inputsize = toint(sizes[0]); 00391 targetsize = toint(sizes[1]); 00392 weightsize = toint(sizes[2]); 00393 } 00394 00395 if(isfile(meta_data_dir / "precomputed.pmat")) 00396 { 00397 // Precomputed version exist in pmat format. 00398 // If it seems old, we display a warning (one may still want to use it 00399 // if he knows the changes made to the vmat code do not alter the data). 00400 if(mtime(meta_data_dir / "precomputed.pmat") < date_of_code) { 00401 PLWARNING("In VVMatrix::createPreproVMat - The precomputed data (in %s) is older than current code, you may want to recompute again", meta_data_dir.c_str()); 00402 } 00403 source = new FileVMatrix(meta_data_dir / "precomputed.pmat"); 00404 source->setMetaDataDir(meta_data_dir); 00405 source->updateMtime(date_of_code); 00406 source->defineSizes(inputsize, targetsize, weightsize); 00407 return source; 00408 } 00409 00410 if( pathexists(meta_data_dir / "precomputed.dmat") ) 00411 { 00412 // precomputed version exist in DiskVMatrix format, 00413 // so we use it *if it is up to date* 00414 if ( mtime(meta_data_dir/"precomputed.dmat/indexfile") > date_of_code ) 00415 { 00416 source = new DiskVMatrix(meta_data_dir/"precomputed.dmat"); 00417 source->setMetaDataDir(meta_data_dir); 00418 source->updateMtime(date_of_code); 00419 source->defineSizes(inputsize, targetsize, weightsize); 00420 return source; 00421 } 00422 } 00423 00424 // if index_section is true, then this dataset needs a file containing the index of the rows to keep 00425 bool index_section = idx_prefilter!=string::npos || idx_postfilter!=string::npos || idx_shuffle!=string::npos ; 00426 00427 // if true, index file lacks or is out of date 00428 bool must_regen_index = index_section && 00429 (!isfile(meta_data_dir/"source.indexes") || date_of_code > mtime(meta_data_dir/"source.indexes")); 00430 00431 // erase obsolete source.index if necessary 00432 if(isfile(meta_data_dir/"source.indexes") && !index_section) 00433 rm (meta_data_dir/"source.indexes"); 00434 00435 if(idx_source!=string::npos) 00436 { 00437 // go over tag 00438 idx_source+=9; 00439 cidx_source=in.find("</SOURCES>"); 00440 if(cidx_source==string::npos) 00441 PLERROR("Cannot find closing tag </SOURCES>. File is %s",filename.c_str()); 00442 // 'sec' is the text content of the <SOURCES> section 00443 string sec=in.substr(idx_source,cidx_source-idx_source); 00444 00445 if (sec[1] == NEW_SYNTAX_CHAR) { 00446 // We are using the new syntax 00447 sec[1] = ' '; // remove the special character indicating the new syntax 00448 source = dynamic_cast<VMatrix*>(newObject(sec)); 00449 if(source.isNull()) { 00450 PLERROR("In VVMatrix::createPreproVMat '%s' is not a valid VMatrix subclass",sec.c_str()); 00451 } 00452 } else { 00453 00454 vector<vector<string> > mstr = extractSourceMatrix(sec,filename); 00455 Array<VMat> vmrows(int(mstr.size())); 00456 // we need to build a VMat that is the concatenation of the datasets contained in 'mstr' 00457 00458 for(unsigned int i=0;i<mstr.size();i++) 00459 { 00460 Array<VMat> ar( (int)mstr[i].size() ); 00461 for(unsigned int j=0;j<mstr[i].size();j++) 00462 { 00463 vector<string> vecstr; 00464 vecstr=split(mstr[i][j],":"); 00465 ar[j]=getDataSet(vecstr[0]); 00466 00467 // handle different cases of dataset specification 00468 // legal formats are: 00469 // 1- simple dataset filename 00470 // 2- intVecFile specification : filename:intVecFile_Filename 00471 // 3- range specifiaction : filename:start:length 00472 // 4- range specifiaction + intVecFile: filename:intVecFile_Filename:start:length 00473 switch(vecstr.size()) 00474 { 00475 case 1: // only dataset name so we do nothing 00476 break; 00477 case 2: // we have an intVecFile specification 00478 // prefix with the path to the current vmat 00479 if(vecstr[1][0]!=slash_char) 00480 vecstr[1]=extract_directory(filename)+vecstr[1]; 00481 ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]); 00482 break; 00483 case 3: // submatRows range specification 00484 ar[j]=ar[j].subMatRows(toint(vecstr[1]),toint(vecstr[2])); 00485 break; 00486 case 4: // intVecFile + submatRows 00487 if(vecstr[1][0]!=slash_char) 00488 vecstr[1]=extract_directory(filename)+vecstr[1]; 00489 ar[j]=new SelectRowsFileIndexVMatrix(ar[j],vecstr[1]); 00490 ar[j]=ar[j].subMatRows(toint(vecstr[2]),toint(vecstr[3])); 00491 break; 00492 default: 00493 PLERROR("Strange number of semicolumns... Format of source must be something.vmat[:indexfile.index][:start_row:length]. File is %s",filename.c_str()); 00494 break; 00495 } 00496 } 00497 // if we have more than one filename in this row, we use hconcat to consolidate 'ar'. 00498 vmrows[i] = ar.size()==1?ar[0]:hconcat(ar); 00499 if(vmrows[i].length()==-1) 00500 PLERROR("Trying to hconcat matrix with different lengths! File is %s",filename.c_str()); 00501 } 00502 00503 source = vconcat(vmrows); 00504 if(mstr.size()==0) 00505 PLERROR("No source matrix found in <SOURCES> section! File is %s",filename.c_str()); 00506 } 00507 } 00508 else PLERROR("Need at least a <SOURCES> section ! File is %s",filename.c_str()); 00509 00510 if(must_regen_index) 00511 generateVMatIndex(source, meta_data_dir, filename, date_of_code, in, idx_prefilter, cidx_prefilter, 00512 idx_postfilter, cidx_postfilter, idx_process, cidx_process, 00513 idx_shuffle, cidx_shuffle, idx_join, cidx_join); 00514 00515 if(idx_join!=string::npos) 00516 { 00517 cidx_join=in.find("</JOIN>"); 00518 if(cidx_join==string::npos) 00519 PLERROR("Cannot find closing tag </JOIN>. File is %s",filename.c_str()); 00520 vector<string> code=split(in.substr(idx_join+6,cidx_join-idx_join-6),"\n"); 00521 processJoinSection(code,source); 00522 } 00523 00524 if(idx_process!=string::npos) 00525 { 00526 cidx_process=in.find("</PROCESSING>"); 00527 if(cidx_process==string::npos) 00528 PLERROR("Cannot find closing tag </PROCESSING>. File is %s",filename.c_str()); 00529 string code=in.substr(idx_process+12,cidx_process-idx_process-12); 00530 source = new PreprocessingVMatrix(source,code); 00531 } 00532 00533 // if source.index exists at this point, we need to apply it 00534 if(isfile(meta_data_dir/"source.indexes")) 00535 source = new SelectRowsFileIndexVMatrix(source,meta_data_dir/"source.indexes"); 00536 00537 // next lines handle precomputation of matrix 00538 if(idx_precompute!=string::npos) 00539 { 00540 idx_precompute+=12; 00541 cidx_precompute=in.find("</PRECOMPUTE>"); 00542 if(cidx_precompute==string::npos) 00543 PLERROR("Cannot find closing tag </PRECOMPUTE>. File is %s",filename.c_str()); 00544 precomp=removeblanks(in.substr(idx_precompute,cidx_precompute-idx_precompute)); 00545 if(precomp == "dmat") 00546 { 00547 cout << "Rendering DMAT : " << meta_data_dir/"precomputed.dmat" << endl; 00548 source->saveDMAT(meta_data_dir/"incomplete.precomputed.dmat"); 00549 int cnt=0; 00550 if (isdir(meta_data_dir/"precomputed.dmat")) { 00551 while(cnt++ < 5 && 00552 !force_rmdir(meta_data_dir/"precomputed.dmat")) 00553 { 00554 cerr<<"Could not rm -rf '"+meta_data_dir+ 00555 slash+"precomputed.dmat/'. Maybe 'Stale NFS file handle' curse again. Retrying in 2 sec." 00556 <<endl; 00557 sleep(2); 00558 } 00559 } 00560 00561 mvforce(meta_data_dir/"incomplete.precomputed.dmat", 00562 meta_data_dir/"precomputed.dmat"); 00563 if(pathexists(meta_data_dir/"incomplete.precomputed.dmat.metadata")) 00564 { 00565 rm(meta_data_dir/"precomputed.dmat.metadata"); 00566 mvforce(meta_data_dir/"incomplete.precomputed.dmat.metadata", 00567 meta_data_dir/"precomputed.dmat.metadata"); 00568 } 00569 source=new DiskVMatrix(meta_data_dir/"precomputed.dmat"); 00570 } 00571 else if(precomp == "pmat") 00572 { 00573 cout<<"Rendering PMAT : "<<meta_data_dir+slash+"precomputed.pmat"<<endl; 00574 source->savePMAT(meta_data_dir+slash+"incomplete.precomputed.pmat"); 00575 mvforce(meta_data_dir/"incomplete.precomputed.pmat", 00576 meta_data_dir/"precomputed.pmat"); 00577 if(pathexists(meta_data_dir/"incomplete.precomputed.pmat.metadata")) 00578 mvforce(meta_data_dir/"incomplete.precomputed.pmat.metadata", 00579 meta_data_dir/"precomputed.pmat.metadata"); 00580 00581 // Save the string mappings. 00582 source->setMetaDataDir(meta_data_dir); 00583 source->saveAllStringMappings(); 00584 source = new FileVMatrix(meta_data_dir/"precomputed.pmat"); 00585 source->setMetaDataDir(meta_data_dir); 00586 source->setMtime(date_of_code); 00587 source->defineSizes(inputsize, targetsize, weightsize); 00588 // TODO The 3 lines above are duplicated throughout the code, this is ugly. 00589 } 00590 else if(precomp!="") 00591 PLERROR("Unsupported precomputing format : %s. File is %s", precomp.c_str(),filename.c_str()); 00592 } 00593 00594 VMatLanguage::output_preproc=olddebugval; 00595 source->setMetaDataDir(meta_data_dir); 00596 source->updateMtime(date_of_code); 00597 if (sizes_spec) { 00598 source->defineSizes(inputsize, targetsize, weightsize); 00599 } 00600 return source; 00601 } 00602 00603 void VVMatrix::build() 00604 { 00605 inherited::build(); 00606 build_(); 00607 } 00608 00609 void VVMatrix::build_() 00610 { 00611 if (the_filename != "") { 00612 setMetaDataDir( PPath(the_filename+".metadata").absolute() ); 00613 force_mkdir( getMetaDataDir() ); 00614 time_t latest=0; 00615 code = readFileAndMacroProcess(the_filename,latest); 00616 if(removeblanks(code)[0]=='<') // old xml-like format 00617 the_mat = createPreproVMat(the_filename); 00618 else // New standard PLearn object description format 00619 { 00620 the_mat = dynamic_cast<VMatrix*>(newObject(code)); 00621 if(the_mat.isNull()) 00622 PLERROR("Object described in %s is not a VMatrix subclass",the_filename.c_str()); 00623 the_mat->setMetaDataDir(getMetaDataDir()); 00624 } 00625 00626 updateMtime(the_mat->getMtime()); 00627 updateMtime(latest); 00628 length_ = the_mat.length(); 00629 width_ = the_mat.width(); 00630 00631 // Copy the sizes. 00632 copySizesFrom(the_mat); 00633 00634 // Copy the string mappings. 00635 copyStringMappingsFrom(the_mat); 00636 00637 // Copy the parent field names 00638 fieldinfos.resize(width_); 00639 if (the_mat->getFieldInfos().size() > 0) 00640 for(int j=0; j<width_; j++) 00641 fieldinfos[j] = the_mat->getFieldInfos()[j]; 00642 } 00643 } 00644 00645 // string maps are those loaded from the .vmat metadatadir, not those of the source vmatrix anymore 00646 // could be changed.. 00647 00648 // real VVMatrix::getStringVal(int col, const string & str) const 00649 // { 00650 // return the_mat->getStringVal(col, str); 00651 // } 00652 // string VVMatrix::getValString(int col, real val) const 00653 // { 00654 // return the_mat->getValString(col, val); 00655 // } 00656 // string VVMatrix::getString(int row,int col) const 00657 // { 00658 // return the_mat->getString(row, col); 00659 // } 00660 00661 // const map<string,real>& VVMatrix::getStringToRealMapping(int col) const 00662 // { 00663 // return the_mat->getStringToRealMapping(col); 00664 // } 00665 00666 00668 // makeDeepCopyFromShallowCopy // 00670 void VVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) { 00671 inherited::makeDeepCopyFromShallowCopy(copies); 00672 deepCopyField(the_mat, copies); 00673 } 00674 00675 } // end of namespace PLearn 00676 00677 00678 /* 00679 Local Variables: 00680 mode:c++ 00681 c-basic-offset:4 00682 c-file-style:"stroustrup" 00683 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00684 indent-tabs-mode:nil 00685 fill-column:79 00686 End: 00687 */ 00688 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :