PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // Copyright (C) 2004 Rejean Ducharme 00007 // 00008 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 /******************************************************** 00038 * $Id: VMatrix.cc 10321 2010-02-15 19:22:34Z ducharme $ 00039 ******************************************************* */ 00040 00041 #include "VMatrix.h" 00042 #include "CompactFileVMatrix.h" 00043 #include "DiskVMatrix.h" 00044 #include "FileVMatrix.h" 00045 #include "SubVMatrix.h" 00046 #include "VMat_computeStats.h" 00047 #include <plearn/base/tostring.h> 00048 #include <plearn/base/lexical_cast.h> 00049 #include <plearn/base/stringutils.h> 00050 #include <plearn/io/fileutils.h> 00051 #include <plearn/base/tostring.h> 00052 #include <plearn/io/load_and_save.h> 00053 #include <plearn/math/random.h> 00054 #include <plearn/base/RemoteDeclareMethod.h> 00055 #include <nspr/prenv.h> 00056 #include <plearn/math/TMat_maths.h> 00057 #include <plearn/sys/procinfo.h> 00058 #include <limits> 00059 00060 namespace PLearn { 00061 using namespace std; 00062 00063 // TODO-PPath : this class is now PPath compliant 00064 // TODO-PStream : this class is now PStream compliant 00065 00066 PLEARN_IMPLEMENT_ABSTRACT_OBJECT( 00067 VMatrix, 00068 "Base classes for virtual matrices", 00069 "VMatrix provides an abstraction for a virtual matrix, namely a matrix wherein\n" 00070 "all element access operations are virtual. This enables a wide variety of\n" 00071 "matrix-like objects to be implemented, from simple data containers (e.g.\n" 00072 "MemoryVMatrix), to large-scale matrices that don't fit in memory (e.g.\n" 00073 "FileVMatrix), to on-the-fly calculations that are implemented through \n" 00074 "various processing VMatrices.\n" 00075 "\n" 00076 "For implementers, a simple class to derive from is RowBufferedVMatrix, which\n" 00077 "implements most of the functionalities of the abstract VMatrix interface in terms\n" 00078 "of a few simple virtual functions to be overridden by the user."); 00079 00080 VMatrix::VMatrix(bool call_build_): 00081 inherited (call_build_), 00082 mtime_ (0), 00083 mtime_update(0), 00084 length_ (-1), 00085 width_ (-1), 00086 inputsize_ (-1), 00087 targetsize_ (-1), 00088 weightsize_ (-1), 00089 extrasize_ (0), 00090 writable (false) 00091 { 00092 lockf_ = PStream(); 00093 if (call_build_) 00094 build_(); 00095 } 00096 00097 VMatrix::VMatrix(int the_length, int the_width, bool call_build_): 00098 inherited (call_build_), 00099 mtime_ (0), 00100 mtime_update (0), 00101 length_ (the_length), 00102 width_ (the_width), 00103 inputsize_ (-1), 00104 targetsize_ (-1), 00105 weightsize_ (-1), 00106 extrasize_ (0), 00107 writable (false), 00108 map_sr(TVec<map<string,real> > (the_width)), 00109 map_rs(TVec<map<real,string> > (the_width)), 00110 fieldstats (0) 00111 { 00112 lockf_ = PStream(); 00113 if (call_build_) 00114 build_(); 00115 } 00116 00118 // declareOptions // 00120 void VMatrix::declareOptions(OptionList & ol) 00121 { 00122 declareOption( 00123 ol, "writable", &VMatrix::writable, OptionBase::buildoption, 00124 "Are write operations permitted?"); 00125 00126 declareOption( 00127 ol, "length", &VMatrix::length_, OptionBase::buildoption, 00128 "Length of the matrix (number of rows)"); 00129 00130 declareOption( 00131 ol, "width", &VMatrix::width_, OptionBase::buildoption, 00132 "Width of the matrix (number of columns; -1 indicates this varies\n" 00133 "from sample to sample...)"); 00134 00135 declareOption( 00136 ol, "inputsize", &VMatrix::inputsize_, OptionBase::buildoption, 00137 "Size of input part (-1 if variable or unspecified, 0 if no input)"); 00138 00139 declareOption( 00140 ol, "targetsize", &VMatrix::targetsize_, OptionBase::buildoption, 00141 "Size of target part (-1 if variable or unspecified, 0 if no target)"); 00142 00143 declareOption( 00144 ol, "weightsize", &VMatrix::weightsize_, OptionBase::buildoption, 00145 "Size of weights (-1 if unspecified, 0 if no weight, 1 for sample\n" 00146 "weight, >1 currently not supported)."); 00147 00148 declareOption( 00149 ol, "extrasize", &VMatrix::extrasize_, OptionBase::buildoption, 00150 "Size of extra fields (additional info). Defaults to 0"); 00151 00152 declareOption( 00153 ol, "metadatadir", &VMatrix::metadatadir, OptionBase::buildoption, 00154 "A directory in which to store meta-information for this matrix \n" 00155 "You don't always have to give this explicitly. For ex. if your \n" 00156 "VMat is the outer VMatrix in a .vmat file, the metadatadir will \n" 00157 "automatically be set to name_of_vmat_file.metadata/ \n" 00158 "And if it is the source inside another VMatrix that sets its \n" 00159 "metadatadir, it will often be set from that surrounding vmat's metadata.\n"); 00160 00161 declareOption( 00162 ol, "mtime", &VMatrix::mtime_update, 00163 OptionBase::buildoption|OptionBase::nosave, 00164 "DO NOT play with this if you don't know the implementation!\n" 00165 "This add a dependency mtime to the gived value.\n" 00166 "Use -1 to set permanently that we do not know the mtime."); 00167 00168 declareOption( 00169 ol, "fieldinfos", &VMatrix::fieldinfos, OptionBase::buildoption, 00170 "Field infos.\n"); 00171 00172 inherited::declareOptions(ol); 00173 } 00174 00175 void VMatrix::declareMethods(RemoteMethodMap& rmm) 00176 { 00177 // Insert a backpointer to remote methods; note that this 00178 // different than for declareOptions() 00179 rmm.inherited(inherited::_getRemoteMethodMap_()); 00180 00181 declareMethod( 00182 rmm, "getRow", &VMatrix::getRowVec, 00183 (BodyDoc("Returns a row of a matrix \n"), 00184 ArgDoc ("i", "Position of the row to get.\n"), 00185 RetDoc ("row i vector"))); 00186 00187 declareMethod( 00188 rmm, "getExample", &VMatrix::remote_getExample, 00189 (BodyDoc("Returns the input, target and weight parts of a row.\n"), 00190 ArgDoc ("i", "Position of the row to get.\n"), 00191 RetDoc ("An (input, target, weight) tuple."))); 00192 00193 declareMethod( 00194 rmm, "getExtra", &VMatrix::remote_getExtra, 00195 (BodyDoc("Returns the extra part of a row.\n"), 00196 ArgDoc ("i", "Position of the row to get.\n"), 00197 RetDoc ("Values for extrafields."))); 00198 00199 declareMethod( 00200 rmm, "getColumn", &VMatrix::remote_getColumn, 00201 (BodyDoc("Returns a row of a matrix \n"), 00202 ArgDoc ("i", "Position of the row to get.\n"), 00203 RetDoc ("row i vector"))); 00204 00205 00206 declareMethod( 00207 rmm, "getString", &VMatrix::getString, 00208 (BodyDoc("Returns an element of a matrix as a string\n"), 00209 ArgDoc ("i", "Position of the row to get.\n"), 00210 ArgDoc ("j", "Position of the column to get.\n"), 00211 RetDoc ("string value"))); 00212 00213 declareMethod( 00214 rmm, "getMat", &VMatrix::toMat, 00215 (BodyDoc("Returns the content of the vmat as a Mat\n"), 00216 RetDoc ("The content of this VMatrix as a Mat"))); 00217 00218 declareMethod( 00219 rmm, "getLength", &VMatrix::length, 00220 (BodyDoc("Return length of this VMatrix.\n"), 00221 RetDoc("The length of this VMatrix."))); 00222 00223 declareMethod( 00224 rmm, "declareField", &VMatrix::declareField, 00225 (BodyDoc("Declares the field infos for a given column (index).\n"), 00226 ArgDoc ("fieldindex", "The column index.\n"), 00227 ArgDoc ("fieldname", "The field name of this column.\n"), 00228 ArgDoc ("fieldtype", "The field type of this column.\n"))); 00229 00230 declareMethod( 00231 rmm, "declareFieldNames", &VMatrix::declareFieldNames, 00232 (BodyDoc("Declares the field names.\n"), 00233 ArgDoc ("fnames", "TVec of field names.\n"))); 00234 00235 declareMethod( 00236 rmm, "fieldNames", &VMatrix::fieldNames, 00237 (BodyDoc("Returns the field names.\n"), 00238 RetDoc ("TVec of field names.\n"))); 00239 00240 declareMethod( 00241 rmm, "fieldName", &VMatrix::fieldName, 00242 (BodyDoc("Returns the field name for a given column.\n"), 00243 ArgDoc ("col", "column index.\n"), 00244 RetDoc ("Field name.\n"))); 00245 00246 declareMethod( 00247 rmm, "findFieldIndex", &VMatrix::fieldIndex, 00248 (BodyDoc("Returns the index of a field, or -1 if the field does not " 00249 "exist.\n"), 00250 ArgDoc ("fname", 00251 "Field name of the field.\n"), 00252 RetDoc ("Index of the field (-1 if not found)\n"))); 00253 00254 declareMethod( 00255 rmm, "getFieldIndex", &VMatrix::remote_getFieldIndex, 00256 (BodyDoc("Returns the index of a field. " 00257 "Throws an error if the field is not found.\n"), 00258 ArgDoc ("fname_or_num", 00259 "Field name or index (as a string) of the field.\n"), 00260 RetDoc ("Index of the field.\n"))); 00261 00262 declareMethod( 00263 rmm, "appendRow", &VMatrix::appendRow, 00264 (BodyDoc("Appends a row to the VMatrix.\n"), 00265 ArgDoc ("v", "Vec with values (row) to append.\n"))); 00266 00267 declareMethod( 00268 rmm, "appendRows", &VMatrix::appendRows, 00269 (BodyDoc("Appends rows to the VMatrix.\n"), 00270 ArgDoc ("rows", "A matrix containing the rows to append.\n"))); 00271 00272 declareMethod( 00273 rmm, "putRow", &VMatrix::putRow, 00274 (BodyDoc("Store a row into the VMatrix.\n"), 00275 ArgDoc ("i", "Index of the row being modified.\n"), 00276 ArgDoc ("v", "Vec with values (row) to store.\n"))); 00277 00278 declareMethod( 00279 rmm, "saveFieldInfos", &VMatrix::saveFieldInfos, 00280 (BodyDoc("Saves field names, etc. in metadatadir.\n"))); 00281 00282 declareMethod( 00283 rmm, "flush", &VMatrix::flush, 00284 (BodyDoc("Flush mods. to disk.\n"))); 00285 00286 declareMethod( 00287 rmm, "getBoundingBox", &VMatrix::getBoundingBox, 00288 (BodyDoc("Returns the (possibly enlarged) bounding box of the data."), 00289 ArgDoc ("extra_percent", "if non 0, then the box is enlarged in both ends\n" 00290 "of every direction by that given percentage"), 00291 RetDoc ("bounding box as as a vector of (min,max) pairs"))); 00292 00293 declareMethod( 00294 rmm, "fill", &VMatrix::fill, 00295 (BodyDoc("Appends fills the VMatrix with a constant value.\n"), 00296 ArgDoc ("value", "The fill value.\n"))); 00297 00298 declareMethod( 00299 rmm, "dot", &VMatrix::dot, 00300 (BodyDoc("dot product between row i1 and row i2, w/ inputsize first elements."), 00301 ArgDoc ("i1", "First row to consider."), 00302 ArgDoc ("i2", "Second row to consider."), 00303 ArgDoc ("inputsize", "nb. elements to consider."), 00304 RetDoc ("dot product"))); 00305 00306 declareMethod( 00307 rmm, "saveAMAT", &VMatrix::saveAMAT, 00308 (BodyDoc("Saves this matrix as an .amat file."), 00309 ArgDoc ("amatfile", "Path of the file to create."), 00310 ArgDoc ("verbose", "output details?"), 00311 ArgDoc ("no_header", "save data only"), 00312 ArgDoc ("save_strings", "save string instead of real values"))); 00313 00314 declareMethod( 00315 rmm, "savePMAT", &VMatrix::remote_savePMAT, 00316 (BodyDoc("Saves this matrix as a .pmat file."), 00317 ArgDoc ("pmatfile", "Path of the file to create."))); 00318 00319 declareMethod( 00320 rmm, "savePMAT_float", &VMatrix::remote_savePMAT_float, 00321 (BodyDoc("Saves this matrix as a .pmat file in float format."), 00322 ArgDoc ("pmatfile", "Path of the file to create."))); 00323 00324 declareMethod( 00325 rmm, "saveDMAT", &VMatrix::saveDMAT, 00326 (BodyDoc("Saves this matrix as a .dmat directory."), 00327 ArgDoc ("dmatdir", "Path of the dir to create."))); 00328 00329 declareMethod( 00330 rmm, "subMat", &VMatrix::subMat, 00331 (BodyDoc("Return a sub-matrix from a VMatrix\n"), 00332 ArgDoc ("i", "start row"), 00333 ArgDoc ("j", "start col"), 00334 ArgDoc ("l", "length"), 00335 ArgDoc ("w", "width"), 00336 RetDoc ("The sub-VMatrix"))); 00337 00338 declareMethod( 00339 rmm, "get", &VMatrix::get, 00340 (BodyDoc("Returns the element at position (i,j)\n"), 00341 ArgDoc ("i", "row"), 00342 ArgDoc ("j", "col"), 00343 RetDoc ("Value at (i,j)"))); 00344 00345 00346 declareMethod( 00347 rmm, "getStats", &VMatrix::remote_getStats, 00348 (BodyDoc("Returns the unconditonal statistics for all fields\n"), 00349 RetDoc ("Stats vector"))); 00350 00351 declareMethod( 00352 rmm, "defineSizes", &VMatrix::defineSizes, 00353 (BodyDoc("Define this vmatrix's sizes\n"), 00354 ArgDoc ("inputsize", "inputsize"), 00355 ArgDoc ("targetsize", "targetsize"), 00356 ArgDoc ("weightsize", "weightsize"), 00357 ArgDoc ("extrasize", "extrasize"))); 00358 00359 declareMethod( 00360 rmm, "copySizesFrom", &VMatrix::copySizesFrom, 00361 (BodyDoc("Define this vmatrix's sizes from another vmatrix\n"), 00362 ArgDoc ("vm", "the other vmatrix"))); 00363 00364 declareMethod( 00365 rmm, "addStringMapping", static_cast<void (VMatrix::*)(int, string, real)>(&VMatrix::addStringMapping), 00366 (BodyDoc("Add or replace a string mapping for a column\n"), 00367 ArgDoc ("col", "column number"), 00368 ArgDoc ("str", "string value"), 00369 ArgDoc ("val", "numeric value"))); 00370 00371 declareMethod( 00372 rmm, "setStringMapping", &VMatrix::setStringMapping, 00373 (BodyDoc("Set the string->real mapping for a given column.\n"), 00374 ArgDoc ("col", "column number"), 00375 ArgDoc ("map", "map of string->real"))); 00376 00377 declareMethod( 00378 rmm, "getStringToRealMapping", &VMatrix::getStringToRealMapping, 00379 (BodyDoc("Get the string->real mapping for a given column.\n"), 00380 ArgDoc ("col", "column number"), 00381 RetDoc ("map of string->real"))); 00382 00383 declareMethod( 00384 rmm, "getRealToStringMapping", &VMatrix::getRealToStringMapping, 00385 (BodyDoc("Get the real->string mapping for a given column.\n"), 00386 ArgDoc ("col", "column number"), 00387 RetDoc ("map of real->string"))); 00388 00389 declareMethod( 00390 rmm, "setMetaInfoFrom", &VMatrix::setMetaInfoFrom, 00391 (BodyDoc("Set this vmatrix's meta-info from another vmatrix\n"), 00392 ArgDoc ("vm", "the other vmatrix"))); 00393 00394 declareMethod( 00395 rmm, "saveAllStringMappings", &VMatrix::saveAllStringMappings, 00396 (BodyDoc("Save this vmatrix's string mapping infos\n"))); 00397 00398 } 00399 00400 00402 // makeDeepCopyFromShallowCopy // 00404 void VMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00405 { 00406 inherited::makeDeepCopyFromShallowCopy(copies); 00407 deepCopyField(get_row, copies); 00408 deepCopyField(dotrow_1, copies); 00409 deepCopyField(dotrow_2, copies); 00410 deepCopyField(field_stats, copies); 00411 deepCopyField(map_sr, copies); 00412 deepCopyField(map_rs, copies); 00413 deepCopyField(fieldinfos, copies); 00414 deepCopyField(fieldstats, copies); 00415 00416 // TODO See if we can deep-copy a PStream (and what it means). 00417 } 00418 00420 // init_map_sr // 00422 void VMatrix::init_map_sr() const 00423 { 00424 if (map_sr.length()==0 || map_sr.length() != width()) { 00425 map_sr.resize(width()); 00426 map_rs.resize(width()); 00427 } 00428 } 00429 00430 00432 // getFieldInfos // 00434 Array<VMField>& VMatrix::getFieldInfos() const 00435 { 00436 if(fieldinfos.size()==0 && hasMetaDataDir()) 00437 { 00438 PPath fname = getMetaDataDir() / "fieldnames"; 00439 if(isfile(fname)) // file exists 00440 loadFieldInfos(); 00441 } 00442 00443 int ninfos = fieldinfos.size(); 00444 int w = width(); 00445 if(ninfos!=w && w > 0) 00446 { 00447 fieldinfos.resize(w); 00448 for(int j=ninfos; j<w; j++) 00449 fieldinfos[j] = VMField(tostring(j)); 00450 } 00451 00452 return fieldinfos; 00453 } 00454 00456 // setFieldInfos // 00458 void VMatrix::setFieldInfos(const Array<VMField>& finfo) const 00459 { 00460 fieldinfos=finfo; 00461 } 00462 00464 // hasFieldInfos // 00466 bool VMatrix::hasFieldInfos() const 00467 { 00468 if (fieldinfos.length() != width()) 00469 return false; 00470 // If there are some field infos, we check them to see whether they are 00471 // default ones (i.e. 0, 1, ..., width-1), in which case 'false' is 00472 // returned. 00473 double x; 00474 for (int i = 0; i < width(); i++) { 00475 string name = fieldName(i); 00476 if (!pl_isnumber(name, &x) || !is_equal(x, i)) 00477 return true; 00478 } 00479 return false; 00480 } 00481 00483 // unduplicateFieldNames // 00485 void VMatrix::unduplicateFieldNames() 00486 { 00487 map<string,vector<int> > mp; 00488 for(int i=0;i<width();i++) 00489 mp[getFieldInfos(i).name].push_back(i); 00490 map<string,vector<int> >::iterator it; 00491 for(it=mp.begin();it!=mp.end();++it) 00492 if(it->second.size()!=1) 00493 { 00494 vector<int> v=it->second; 00495 for(unsigned int j=0;j<v.size();j++) 00496 fieldinfos[v[j]].name+="."+tostring(j); 00497 } 00498 } 00499 00501 // fieldNames // 00503 TVec<string> VMatrix::fieldNames() const 00504 { 00505 int d = width(); 00506 if (d < 0) 00507 return TVec<string>(); 00508 TVec<string> names(d); 00509 for(int i=0; i<d; i++) 00510 names[i] = fieldName(i); 00511 return names; 00512 } 00513 00514 TVec<string> VMatrix::inputFieldNames() const 00515 { return fieldNames().subVec(0,inputsize_); } 00516 00517 TVec<string> VMatrix::targetFieldNames() const 00518 { return fieldNames().subVec(inputsize_, targetsize_); } 00519 00520 TVec<string> VMatrix::weightFieldNames() const 00521 { return fieldNames().subVec(inputsize_+targetsize_, weightsize_); } 00522 00523 TVec<string> VMatrix::extraFieldNames() const 00524 { return fieldNames().subVec(inputsize_+targetsize_+weightsize_,extrasize_); } 00525 00527 // fieldIndex // 00529 int VMatrix::fieldIndex(const string& fieldname) const 00530 { 00531 Array<VMField>& infos = getFieldInfos(); 00532 for(int i=0; i<width(); i++) 00533 if(infos[i].name==fieldname) 00534 return i; 00535 return -1; 00536 } 00537 00539 // getFieldIndex // 00541 int VMatrix::getFieldIndex(const string& fieldname_or_num, bool error) const 00542 { 00543 int i = fieldIndex(fieldname_or_num); 00544 if(i==-1 && pl_islong(fieldname_or_num)) { 00545 i = toint(fieldname_or_num); 00546 00547 // Now ensure that THE WHOLE FIELD has been converted, because we want 00548 // to ensure that stuff that starts with a number but contains other 00549 // things is not silently converted to the starting number 00550 if (tostring(i) != fieldname_or_num) 00551 i = -1; 00552 } 00553 if ((i < 0 || i >= width()) && error) 00554 PLERROR("In VMatrix::getFieldIndex - Asked for an invalid column number: '%s'", 00555 fieldname_or_num.c_str()); 00556 return i; 00557 } 00558 00560 // build_ // 00562 void VMatrix::build_() 00563 { 00564 if(!metadatadir.isEmpty()) 00565 setMetaDataDir(metadatadir); // make sure we perform all necessary operations 00566 if(mtime_update == time_t(-1)) 00567 updateMtime(0); 00568 else if(mtime_update!=0) 00569 updateMtime(mtime_update); 00570 } 00571 00573 // build // 00575 void VMatrix::build() 00576 { 00577 inherited::build(); 00578 build_(); 00579 } 00580 00582 // printFieldInfo // 00584 void VMatrix::printFieldInfo(PStream& out, int fieldnum, bool print_binning) const 00585 { 00586 VMField fi = getFieldInfos(fieldnum); 00587 StatsCollector& s = getStats(fieldnum); 00588 00589 out << "Field #" << fieldnum << ": "; 00590 out << fi.name << "\t type: "; 00591 switch(fi.fieldtype) 00592 { 00593 case VMField::UnknownType: 00594 out << "UnknownType\n"; 00595 break; 00596 case VMField::Continuous: 00597 out << "Continuous\n"; 00598 break; 00599 case VMField::DiscrGeneral: 00600 out << "DiscrGeneral\n"; 00601 break; 00602 case VMField::DiscrMonotonic: 00603 out << "DiscrMonotonic\n"; 00604 break; 00605 case VMField::DiscrFloat: 00606 out << "DiscrFloat\n"; 00607 break; 00608 case VMField::Date: 00609 out << "Date\n"; 00610 break; 00611 default: 00612 PLERROR("Can't write name of type"); 00613 } 00614 00615 map<real,StatsCollectorCounts>::const_iterator it = s.counts.begin(); 00616 map<real,StatsCollectorCounts>::const_iterator countsend = s.counts.end(); 00617 int n_values = 0; 00618 //some value(FLT_MAX, meaby others) are used for others purpose. 00619 //We must not cont then. 00620 while(it!=countsend) 00621 { 00622 real val = it->first; 00623 const StatsCollectorCounts& co = it->second; 00624 string str = getValString(fieldnum, val); 00625 if(co.n>0) 00626 n_values++; 00627 ++it; 00628 } 00629 char plus = ' '; 00630 if (n_values==s.maxnvalues) 00631 plus = '+'; 00632 00633 out << "nmissing: " << s.nmissing() << '\n'; 00634 out << "nnonmissing: " << s.nnonmissing() << '\n'; 00635 out << "sum: " << s.sum() << '\n'; 00636 out << "mean: " << s.mean() << '\n'; 00637 out << "stddev: " << s.stddev() << '\n'; 00638 out << "min: " << s.min() << '\n'; 00639 out << "max: " << s.max() << '\n'; 00640 out << "ndiffvalue: " << n_values << plus << '\n'; 00641 00642 if(!s.counts.empty() && print_binning) 00643 { 00644 out << "\nCOUNTS: \n"; 00645 map<real,StatsCollectorCounts>::const_iterator it = s.counts.begin(); 00646 map<real,StatsCollectorCounts>::const_iterator countsend = s.counts.end(); 00647 while(it!=countsend) 00648 { 00649 real val = it->first; 00650 const StatsCollectorCounts& co = it->second; 00651 string str = getValString(fieldnum, val); 00652 ostringstream os; 00653 os.setf(ios::left); 00654 os << " " << setw(12) << val 00655 << " " << setw(12) << str 00656 << " n=" << setw(10) << co.n 00657 << " nbelow=" << setw(10) << co.nbelow 00658 << " sumbelow=" << setw(10) << co.sum 00659 << endl; 00660 out << os.str(); 00661 ++it; 00662 } 00663 } 00664 out << endl << endl; 00665 } 00666 00668 // printFieldInfo // 00670 void VMatrix::printFieldInfo(PStream& out, const string& fieldname_or_num, 00671 bool print_binning) const 00672 { 00673 printFieldInfo(out, getFieldIndex(fieldname_or_num), print_binning); 00674 } 00675 00677 // printFields // 00679 void VMatrix::printFields(PStream& out) const 00680 { 00681 for(int j=0; j<width(); j++) 00682 { 00683 printFieldInfo(out,j); 00684 out << "-----------------------------------------------------" << endl; 00685 } 00686 } 00687 00689 // getExample // 00691 void VMatrix::getExample(int i, Vec& input, Vec& target, real& weight) 00692 { 00693 if(inputsize_<0) 00694 PLERROR("In VMatrix::getExample, inputsize_ not defined for this vmat"); 00695 input.resize(inputsize_); 00696 getSubRow(i,0,input); 00697 if(targetsize_<0) 00698 PLERROR("In VMatrix::getExample, targetsize_ not defined for this vmat"); 00699 target.resize(targetsize_); 00700 if (targetsize_ > 0) { 00701 getSubRow(i,inputsize_,target); 00702 } 00703 00704 if(weightsize_==0) 00705 weight = 1; 00706 else if(weightsize_<0) 00707 PLERROR("In VMatrix::getExample, weightsize_ not defined for this vmat"); 00708 else if(weightsize_>1) 00709 PLERROR("In VMatrix::getExample, weightsize_ >1 not supported by this call"); 00710 else 00711 weight = get(i,inputsize_+targetsize_); 00712 } 00713 00715 // remote_getExample // 00717 boost::tuple<Vec, Vec, real> VMatrix::remote_getExample(int i) 00718 { 00719 Vec input, target; 00720 real weight; 00721 getExample(i, input, target, weight); 00722 return boost::tuple<Vec, Vec, real>(input, target, weight); 00723 } 00724 00726 // getExamples // 00728 void VMatrix::getExamples(int i_start, int length, Mat& inputs, Mat& targets, 00729 Vec& weights, Mat* extras, bool allow_circular) 00730 { 00731 inputs.resize(length, inputsize()); 00732 targets.resize(length, targetsize()); 00733 weights.resize(length); 00734 if (extras) 00735 extras->resize(length, extrasize()); 00736 Vec input, target, extra; 00737 int total_length = this->length(); 00738 PLASSERT( i_start < total_length ); 00739 for (int k = 0; k < length; k++) { 00740 input = inputs(k); 00741 target = targets(k); 00742 int idx = i_start + k; 00743 if (allow_circular) 00744 idx %= total_length; 00745 PLASSERT( idx >= 0 && idx < total_length ); 00746 getExample(idx, input, target, weights[k]); 00747 if (extras) { 00748 extra = (*extras)(k); 00749 getExtra(idx, extra); 00750 } 00751 } 00752 } 00753 00755 // getExtra // 00757 void VMatrix::getExtra(int i, Vec& extra) 00758 { 00759 if(inputsize_<0 || targetsize_<0 || weightsize_<0 || extrasize_<0) 00760 PLERROR("In VMatrix::getExtra, sizes not properly defined for this vmat"); 00761 00762 extra.resize(extrasize_); 00763 if(extrasize_>0) 00764 getSubRow(i,inputsize_+targetsize_+weightsize_, extra); 00765 } 00766 00767 Vec VMatrix::remote_getExtra(int i) 00768 { 00769 Vec extra; 00770 getExtra(i, extra); 00771 return extra; 00772 } 00773 00774 00776 // computeStats // 00778 void VMatrix::computeStats() 00779 { 00780 fieldstats = Array<VMFieldStat>(width()); 00781 Vec row(width()); 00782 for(int i=0; i<length(); i++) 00783 { 00784 getRow(i,row); 00785 for(int j=0; j<width(); j++) 00786 fieldstats[j].update(row[j]); 00787 } 00788 } 00789 00791 // loadStats // 00793 void VMatrix::loadStats(const PPath& filename) 00794 { 00795 PStream in = openFile(filename, PStream::raw_ascii, "r"); 00796 int nfields; 00797 in >> nfields; 00798 if(nfields!=width()) 00799 PLWARNING("In VMatrix::loadStats - nfields differs from VMat width"); 00800 00801 fieldstats.resize(nfields); 00802 for(int j=0; j<fieldstats.size(); j++) 00803 fieldstats[j].read(in); 00804 } 00805 00807 // saveStats // 00809 void VMatrix::saveStats(const PPath& filename) const 00810 { 00811 PStream out = openFile(filename, PStream::raw_ascii, "w"); 00812 out << fieldstats.size() << endl; 00813 for(int j=0; j<fieldstats.size(); j++) 00814 { 00815 fieldstats[j].write(out); 00816 out << endl; 00817 } 00818 } 00819 00821 // declareField // 00823 void VMatrix::declareField(int fieldindex, const string& fieldname, VMField::FieldType fieldtype) 00824 { 00825 getFieldInfos(fieldindex) = VMField(fieldname,fieldtype); 00826 } 00827 00829 // declareFieldNames // 00831 void VMatrix::declareFieldNames(const TVec<string>& fnames) 00832 { 00833 if(fnames.length()!=width()) 00834 PLERROR("In VMatrix::declareFieldNames length of fnames differs from width() of VMatrix"); 00835 for(int i=0; i<fnames.length(); i++) 00836 declareField(i,fnames[i]); 00837 } 00838 00840 // saveFieldInfos // 00842 void VMatrix::saveFieldInfos() const 00843 { 00844 // check if we need to save the fieldinfos 00845 if(fieldinfos.size() > 0) { 00846 Array<VMField> current_fieldinfos; 00847 try{ 00848 current_fieldinfos = getSavedFieldInfos(); 00849 }catch(PLearnError){} 00850 if (current_fieldinfos != fieldinfos) { 00851 00852 // Ensure that the metadatadir exists 00853 if(!force_mkdir(getMetaDataDir())) 00854 PLERROR("In VMatrix::saveFieldInfos: could not create directory %s", 00855 getMetaDataDir().absolute().c_str()); 00856 00857 PPath filename = getMetaDataDir() / "fieldnames"; 00858 PStream out = openFile(filename, PStream::raw_ascii, "w"); 00859 for(int i= 0; i < fieldinfos.length(); ++i) 00860 out << fieldinfos[i].name << '\t' << fieldinfos[i].fieldtype << endl; 00861 } 00862 } 00863 00864 // check if we need to save the sizes 00865 int inp, targ, weight, extr; 00866 bool sizes_exist = getSavedSizes(inp, targ, weight, extr); 00867 if ((! sizes_exist && (inputsize_ != -1 || targetsize_ != -1 || weightsize_ != -1 || extrasize_ > 0)) || 00868 (sizes_exist && (inp != inputsize_ || targ != targetsize_ || weight != weightsize_ || extr!=extrasize_))) 00869 { 00870 // Slightly hackish phenomenon :: if the sizes file doesn't previously 00871 // exist and we cannot write them, THIS IS NOT AN ERROR. In this case, 00872 // just catch the error and continue 00873 try { 00874 // Ensure that the metadatadir exists 00875 if(!force_mkdir(getMetaDataDir())) 00876 PLERROR("In VMatrix::saveFieldInfos: could not create directory %s", 00877 getMetaDataDir().absolute().c_str()); 00878 00879 PPath filename = getMetaDataDir() / "sizes"; 00880 PStream out = openFile(filename, PStream::plearn_ascii, "w"); 00881 out << inputsize_ << targetsize_ << weightsize_ << extrasize_ << endl; 00882 } 00883 catch (const PLearnError&) { 00884 if (sizes_exist) 00885 throw; 00886 } 00887 } 00888 } 00889 00891 // loadFieldInfos // 00893 void VMatrix::loadFieldInfos() const 00894 { 00895 Array<VMField> current_fieldinfos = getSavedFieldInfos(); 00896 setFieldInfos(current_fieldinfos); 00897 00898 // Update only if they can successfully be read from the saved metadata 00899 // and they don't already exist in the VMatrix 00900 int inp, tar, weight, extr; 00901 if (inputsize_ == -1 && targetsize_ == -1 && weightsize_ == -1 00902 && getSavedSizes(inp,tar,weight,extr)) 00903 { 00904 inputsize_ = inp; 00905 targetsize_ = tar; 00906 weightsize_ = weight; 00907 extrasize_ = extr; 00908 } 00909 } 00910 00912 // getSavedFieldInfos // 00914 Array<VMField> VMatrix::getSavedFieldInfos() const 00915 { 00916 PPath filename = getMetaDataDir() / "fieldnames"; 00917 if (!isfile(filename)) // no current fieldinfos saved 00918 { 00919 Array<VMField> no_fieldinfos(0); 00920 return no_fieldinfos; 00921 } 00922 PStream in = openFile(filename, PStream::raw_ascii, "r"); 00923 int w = width(); 00924 Array<VMField> current_fieldinfos(w); 00925 for(int i=0; i<w; ++i) 00926 { 00927 string line = in.getline(); 00928 vector<string> v(split(line)); 00929 switch(v.size()) 00930 { 00931 case 1: current_fieldinfos[i] = VMField(v[0]); break; 00932 case 2: current_fieldinfos[i] = VMField(v[0], VMField::FieldType(toint(v[1]))); break; 00933 default: PLERROR("In VMatrix::getSavedFieldInfos Format not recognized in file %s.\n" 00934 "Each line should be '<name> {<type>}'.\n" 00935 "Got: '%s'. Check for a space in <name>", 00936 filename.absolute().c_str(),line.c_str()); 00937 } 00938 } 00939 return current_fieldinfos; 00940 } 00941 00943 // getSavedSizes // 00945 bool VMatrix::getSavedSizes(int& inputsize, int& targetsize, int& weightsize, int& extrasize) const 00946 { 00947 PPath filename = getMetaDataDir() / "sizes"; 00948 inputsize = targetsize = weightsize = extrasize = -1; 00949 if (isfile(filename)) 00950 { 00951 PStream in = openFile(filename, PStream::plearn_ascii, "r"); 00952 // perr << "In loadFieldInfos() loading sizes from " << filename << endl; 00953 in >> inputsize >> targetsize >> weightsize; 00954 in.skipBlanks(); 00955 extrasize = 0; 00956 if(in.peek()!=EOF) 00957 in >> extrasize; 00958 return true; // Successfully loaded 00959 } 00960 return false; 00961 } 00962 00963 00965 // resolveFieldInfoLink // 00967 string VMatrix::resolveFieldInfoLink(const PPath& target, const PPath& source) 00968 { 00969 PPath contents = removeblanks( loadFileAsString(source) ); 00970 if ( contents == source ) 00971 return "ERROR"; 00972 00973 if( isdir(contents) ) 00974 { 00975 if ( isfile(contents/target+".lnk") ) 00976 return resolveFieldInfoLink(target,contents/target+".lnk"); 00977 00978 else if ( isfile(contents/target) ) 00979 return contents/target; 00980 00981 else if( isfile(contents/"__default.lnk") ) 00982 return resolveFieldInfoLink(target, contents/"__default.lnk"); 00983 00984 // assume target is there, but file is empty thus inexistant 00985 else return contents/target; 00986 } 00987 00988 else if( contents.extension() == "lnk" ) 00989 return resolveFieldInfoLink(target,contents); 00990 00991 else return contents; 00992 } 00993 00995 // getSFIFDirectory // 00997 PPath VMatrix::getSFIFDirectory() const 00998 { 00999 PPath meta = getMetaDataDir(); 01000 if (meta.empty()) 01001 PLERROR("%s: cannot have a SFIFDirectory if there is no metadatadir", 01002 __FUNCTION__); 01003 return meta / "FieldInfo"; 01004 } 01005 01007 // setSFIFFilename // 01009 void VMatrix::setSFIFFilename(int col, string ext, const PPath& filepath) 01010 { 01011 setSFIFFilename(fieldName(col),ext,filepath); 01012 } 01013 01014 void VMatrix::setSFIFFilename(string fieldname, string ext, const PPath& filepath) 01015 { 01016 PPath target = makeFileNameValid(fieldname+ext); 01017 PPath normalfname = getSFIFDirectory() / target; 01018 PPath normalfname_lnk = normalfname + ".lnk"; 01019 01020 rm(normalfname_lnk); 01021 if(filepath==normalfname || filepath=="") 01022 { 01023 rm(normalfname_lnk); 01024 return; 01025 } 01026 01027 force_mkdir_for_file(normalfname); 01028 PStream o = openFile(normalfname_lnk, PStream::raw_ascii, "w"); 01029 o<<filepath<<endl; 01030 } 01031 01033 // getSFIFFilename // 01035 PPath VMatrix::getSFIFFilename(int col, string ext) 01036 { 01037 return getSFIFFilename(fieldName(col),ext); 01038 } 01039 01040 PPath VMatrix::getSFIFFilename(string fieldname, string ext) 01041 { 01042 PPath target = makeFileNameValid(fieldname+ext); 01043 PPath normalfname = getSFIFDirectory() / target; 01044 string defaultlinkfname = getSFIFDirectory() / "__default.lnk"; 01045 01046 if(isfile(normalfname)) 01047 return normalfname; 01048 else if(isfile(normalfname+".lnk")) 01049 return resolveFieldInfoLink(target, normalfname+".lnk"); 01050 else if(isfile(defaultlinkfname)) 01051 return resolveFieldInfoLink(target, defaultlinkfname); 01052 // assume target is here, but file is empty thus inexistant 01053 else return normalfname; 01054 } 01055 01057 // isSFIFDirect // 01059 bool VMatrix::isSFIFDirect(int col, string ext) 01060 { 01061 return isSFIFDirect(fieldName(col), ext); 01062 } 01063 01064 bool VMatrix::isSFIFDirect(string fieldname, string ext) 01065 { 01066 PPath target = makeFileNameValid(fieldname+ext); 01067 PPath normalfname = getSFIFDirectory() / target; 01068 return getSFIFFilename(fieldname,ext) == normalfname; 01069 } 01070 01072 // addStringMapping // 01074 void VMatrix::addStringMapping(int col, string str, real val) 01075 { 01076 init_map_sr(); 01077 map_sr[col][str]=val; 01078 map_rs[col][val]=str; 01079 } 01080 01082 // addStringMapping // 01084 real VMatrix::addStringMapping(int col, string str) 01085 { 01086 init_map_sr(); 01087 map<string,real>& m = map_sr[col]; 01088 map<string,real>::iterator it = m.find(str); 01089 01090 real val = 0; 01091 if(it != m.end()) // str was found in map 01092 val = it->second; 01093 else // str not found in map: add a new mapping 01094 { 01095 val = - real(m.size()) - 100; 01096 addStringMapping(col, str, val); 01097 } 01098 return val; 01099 } 01100 01102 // removeAllStringMappings // 01104 void VMatrix::removeAllStringMappings() 01105 { 01106 init_map_sr(); 01107 for(int i=0;i<width();i++) 01108 { 01109 map_sr[i].clear(); 01110 map_rs[i].clear(); 01111 } 01112 } 01113 01115 // removeColumnStringMappings // 01117 void VMatrix::removeColumnStringMappings(int c) 01118 { 01119 init_map_sr(); 01120 map_sr[c].clear(); 01121 map_rs[c].clear(); 01122 } 01123 01125 // saveAllStringMappings // 01127 void VMatrix::saveAllStringMappings() 01128 { 01129 PPath fname; 01130 map<string, real> the_map; 01131 for(int i=0;i<width();i++) 01132 { 01133 the_map = getStringToRealMapping(i); 01134 if (!the_map.empty()) { 01135 fname = getSFIFFilename(i,".smap"); 01136 saveStringMappings(i, fname, &the_map); 01137 } 01138 } 01139 } 01140 01142 // saveStringMappings // 01144 void VMatrix::saveStringMappings(int col, const PPath& fname, map<string, real>* str_to_real) 01145 { 01146 map<string, real> the_map; 01147 if (!str_to_real) { 01148 the_map = getStringToRealMapping(col); 01149 str_to_real = &the_map; 01150 } 01151 if(str_to_real->empty()) 01152 { 01153 rm(fname); 01154 return; 01155 } 01156 force_mkdir_for_file(fname); 01157 PStream o = openFile(fname, PStream::plearn_ascii, "w"); 01158 for(map<string,real>::iterator it = str_to_real->begin(); 01159 it != str_to_real->end(); ++it) 01160 o << it->first << it->second << endl; 01161 } 01162 01164 // removeStringMapping // 01166 void VMatrix::removeStringMapping(int col, string str) 01167 { 01168 init_map_sr(); 01169 map<string,real>::iterator sriterator; 01170 // Check if the mapping actually exists. 01171 if((sriterator = map_sr[col].find(str)) == map_sr[col].end()) 01172 return; 01173 real val = map_sr[col][str]; 01174 map_sr[col].erase(sriterator); 01175 map_rs[col].erase(map_rs[col].find(val)); 01176 } 01177 01179 // setStringMapping // 01181 void VMatrix::setStringMapping(int col, const map<string,real> & zemap) 01182 { 01183 init_map_sr(); 01184 map_sr[col]=zemap; 01185 map_rs[col].clear(); 01186 for(map<string,real>::iterator it = map_sr[col].begin();it!=map_sr[col].end();++it) 01187 map_rs[col][it->second]=it->first; 01188 } 01189 01191 // deleteStringMapping // 01193 void VMatrix::deleteStringMapping(int col) 01194 { 01195 init_map_sr(); 01196 if(col>=map_sr.size() || 01197 col>=map_rs.size()) 01198 PLERROR("deleteStringMapping : out of bounds for col=%i in string mapping array (size=%i).\n Current VMatrix\nclass"\ 01199 "is '%s' (or maybe derivated class?). be sure to set\n map_sr(rs) to appropriate sizes as soon as you know the width of the matrix\n"\ 01200 "(in constructor or elsewhere)",col,map_sr.size(),classname().c_str()); 01201 map_sr[col].clear(); 01202 map_rs[col].clear(); 01203 } 01204 01206 // getValString // 01208 string VMatrix::getValString(int col, real val) const 01209 { 01210 if(is_missing(val)) 01211 return ""; 01212 init_map_sr(); 01213 if(map_rs[col].find(val)==map_rs[col].end()) 01214 return ""; 01215 else return map_rs[col][val]; 01216 } 01217 01219 // getStringVal // 01221 real VMatrix::getStringVal(int col,const string & str) const 01222 { 01223 if(map_sr.length()==0 || map_sr[col].find(str)==map_sr[col].end()) 01224 return MISSING_VALUE; 01225 else return map_sr[col][str]; 01226 } 01227 01229 // getString // 01231 string VMatrix::getString(int row,int col) const 01232 { 01233 real val = get(row,col); 01234 string str = getValString(col, val); 01235 if (str == "") 01236 // There is no string mapping associated to this value. 01237 return tostring(val); 01238 else 01239 return str; 01240 } 01241 01243 // getRowAsStrings // 01245 void VMatrix::getRowAsStrings(int i, TVec<string>& v_str) const { 01246 v_str.resize(width()); 01247 for (int j = 0; j < width(); j++) 01248 v_str[j] = getString(i, j); 01249 } 01250 01251 01253 // getStringToRealMapping // 01255 const map<string,real>& VMatrix::getStringToRealMapping(int col) const { 01256 init_map_sr(); 01257 return map_sr[col]; 01258 } 01259 01261 // getRealToStringMapping // 01263 const map<real,string>& VMatrix::getRealToStringMapping(int col) const { 01264 init_map_sr(); 01265 return map_rs[col]; 01266 } 01267 01269 // setMetaDataDir // 01271 void VMatrix::setMetaDataDir(const PPath& the_metadatadir) 01272 { 01273 if (the_metadatadir.isEmpty()) 01274 PLERROR("In VMatrix::setMetaDataDir - Called setMetaDataDir with an empty PPath"); 01275 metadatadir = the_metadatadir.absolute() / ""; 01276 // We do not create the metadata directory here anymore. 01277 // This is to prevent the proliferation of useless directories. 01278 // A VMatrix's subclass should now create the metadatadir itself if it needs it. 01279 01280 // Load string mappings from the metadatadir. 01281 loadAllStringMappings(); 01282 } 01283 01285 // getDictionary // 01287 PP<Dictionary> VMatrix::getDictionary(int col) const 01288 { 01289 return 0; 01290 } 01291 01293 // getValues // 01295 void VMatrix::getValues(int row, int col, Vec& values) const 01296 { 01297 values.resize(0); 01298 } 01299 01300 void VMatrix::getValues(const Vec& input, int col, Vec& values) const 01301 { 01302 values.resize(0); 01303 } 01304 01305 01307 // copySizesFrom // 01309 void VMatrix::copySizesFrom(const VMat& m) { 01310 defineSizes(m->inputsize(), m->targetsize(), m->weightsize(), m->extrasize()); 01311 } 01312 01314 // setMetaInfoFrom // 01316 void VMatrix::setMetaInfoFrom(const VMatrix* vm) 01317 { 01318 updateMtime(vm->getMtime()); 01319 01320 // copy length and width from vm if not set 01321 if(length_<0) 01322 length_ = vm->length(); 01323 if(width_<0) 01324 width_ = vm->width(); 01325 01326 // Copy sizes from vm if not set and they do not conflict with the width. 01327 int current_w = max(0, inputsize_) + max(0, targetsize_) + 01328 max(0, weightsize_) + max(0, extrasize_); 01329 int is = vm->inputsize(); 01330 if(inputsize_<0 && is>=0) { 01331 if (is + current_w <= width_) { 01332 inputsize_ = is; 01333 current_w += is; 01334 } 01335 } 01336 int ts = vm->targetsize(); 01337 if(targetsize_<0 && ts>=0) { 01338 if (ts + current_w <= width_) { 01339 targetsize_ = ts; 01340 current_w += ts; 01341 } 01342 } 01343 int ws = vm->weightsize(); 01344 if(weightsize_<0 && ws>=0) { 01345 if (ws + current_w <= width_) { 01346 // We must also ensure the total sum of sizes (if available) 01347 // will match the width. Otherwise we may end up with sizes 01348 // conflicting with the width. 01349 if (inputsize_ < 0 || targetsize_ < 0 || extrasize_ < 0 || 01350 inputsize_ + targetsize_ + extrasize_ + ws == width_) 01351 { 01352 weightsize_ = ws; 01353 current_w += ws; 01354 } 01355 } 01356 } 01357 int es = vm->extrasize(); 01358 if(extrasize_<=0 && es>=0) { 01359 if (es + current_w <= width_) { 01360 // Same as above. 01361 if (inputsize_ < 0 || targetsize_ < 0 || weightsize_ < 0 || 01362 inputsize_ + targetsize_ + weightsize_ + es == width_) 01363 { 01364 extrasize_ = es; 01365 current_w += es; 01366 } 01367 } 01368 } 01369 01370 // Fill missing size if possible, also display warning when sizes are not 01371 // compatible with the width. 01372 computeMissingSizeValue(false); 01373 01374 // Copy fieldnames from vm if not set and they look good. 01375 bool same_fields_as_source = 01376 (!hasFieldInfos() && (width() == vm->width()) && vm->hasFieldInfos()); 01377 if(same_fields_as_source) 01378 setFieldInfos(vm->getFieldInfos()); 01379 01380 // Copy string <-> real mappings for fields which have the same name (or for 01381 // all fields if it looks like the fields are the same as the source). 01382 TVec<string> fnames = fieldNames(); 01383 for (int i = 0; i < width_; i++) { 01384 int vm_index = -1; 01385 if (same_fields_as_source) 01386 vm_index = i; 01387 else if (!pl_isnumber(fnames[i])) 01388 vm_index = vm->fieldIndex(fnames[i]); 01389 if (vm_index >= 0) 01390 // The source VMatrix has a field with the same name (which is not a 01391 // number): we can get its string mapping. 01392 setStringMapping(i, vm->getStringToRealMapping(vm_index)); 01393 } 01394 01395 //we save it now in case the program crash 01396 if(hasMetaDataDir()) 01397 saveFieldInfos(); 01398 } 01399 01401 // looksTheSameAs // 01403 bool VMatrix::looksTheSameAs(const VMat& m) { 01404 return !( 01405 this->width() != m->width() 01406 || this->length() != m->length() 01407 || this->inputsize() != m->inputsize() 01408 || this->weightsize() != m->weightsize() 01409 || this->targetsize() != m->targetsize() 01410 || this->extrasize() != m->extrasize() ); 01411 } 01412 01414 // compatibleSizeError // 01416 void VMatrix::compatibleSizeError(const VMat& m, const string& extra_msg) { 01417 #define MY_PRINT_ERROR_MST(NAME) PLERROR("In VMatrix::compatibleSizeError " \ 01418 " - in class %s - The matrices are not compatible!\n" \ 01419 "m1."#NAME"=%d and m2."#NAME"=%d. \n%s", \ 01420 classname().c_str(), this->NAME(), m->NAME(), extra_msg.c_str()); 01421 01422 if(this->width() != m->width()) 01423 MY_PRINT_ERROR_MST(width) 01424 else if(this->inputsize() != m->inputsize()) 01425 MY_PRINT_ERROR_MST(inputsize) 01426 else if(this->weightsize() != m->weightsize()) 01427 MY_PRINT_ERROR_MST(weightsize) 01428 else if(this->targetsize() != m->targetsize()) 01429 MY_PRINT_ERROR_MST(targetsize) 01430 else if(this->extrasize() != m->extrasize() ) 01431 MY_PRINT_ERROR_MST(extrasize) 01432 #undef MY_PRINT_ERROR_MST 01433 } 01434 01436 // lockMetaDataDir // 01438 void VMatrix::lockMetaDataDir(time_t max_lock_age, bool verbose) const 01439 { 01440 #ifndef DISABLE_VMATRIX_LOCK 01441 if(!hasMetaDataDir()) 01442 PLERROR("In VMatrix::lockMetaDataDir() subclass %s -" 01443 " metadatadir was not set", classname().c_str()); 01444 if(lockf_.good()) // Already locked by this object! 01445 PLERROR("VMatrix::lockMetaDataDir() subclass %s -" 01446 " called while already locked by this object.", 01447 classname().c_str()); 01448 if(!pathexists(metadatadir)) 01449 force_mkdir(metadatadir); 01450 01451 PPath lockfile = metadatadir / ".lock"; 01452 while (isfile(lockfile) && (max_lock_age == 0 || mtime(lockfile) + max_lock_age > time(0))) { 01453 // There is a lock file, and it is not older than 'max_lock_age'. 01454 string bywho; 01455 try{ 01456 PStream st = openFile(lockfile, PStream::raw_ascii, "r", false); 01457 if(st.good()) 01458 st.read(bywho, streamsize(filesize(lockfile))); 01459 } 01460 catch(const PLearnError& e) { 01461 PLERROR("In VMatrix::lockMetaDataDir - Catching exceptions is" 01462 " dangerous in PLearn (memory" 01463 " leaks may occur), thus I prefer to stop here. " 01464 " Comment this line if you don't care." 01465 " The error message is: %s",e.message().c_str()); 01466 bywho = "UNKNOWN (could not read .lock file)" ; 01467 } catch(...) { 01468 PLERROR("In VMatrix::lockMetaDataDir - Catching exceptions is dangerous in PLearn (memory " 01469 "leaks may occur), thus I prefer to stop here. Comment this line if you don't care."); 01470 bywho = "UNKNOWN (could not read .lock file)"; 01471 } 01472 01473 if (verbose) 01474 perr << "Waiting for .lock in directory " << metadatadir 01475 << " created by " << bywho << endl; 01476 sleep(uniform_multinomial_sample(10) + 1); // Random wait for more safety. 01477 } 01478 lockf_ = openFile(lockfile, PStream::raw_ascii, "w"); 01479 string lock_content = "host " + hostname() + ", pid " + tostring(getPid()) + ", user " + getUser(); 01480 lockf_ << lock_content; 01481 lockf_.flush(); 01482 #endif 01483 } 01484 01486 // unlockMetaDataDir // 01488 void VMatrix::unlockMetaDataDir() const 01489 { 01490 #ifndef DISABLE_VMATRIX_LOCK 01491 if(!lockf_) 01492 PLERROR("In VMatrix::unlockMetaDataDir() was called while no lock is held by this object"); 01493 lockf_ = PStream(); // Release the lock. 01494 PPath lockfile = metadatadir / ".lock"; 01495 rm(lockfile); // Remove the file. 01496 #endif 01497 } 01498 01500 // getMetaDataDir // 01502 PPath VMatrix::getMetaDataDir() const 01503 { 01504 // TODO Remove ? 01505 // if(!hasMetaDataDir()) 01506 // PLERROR("In VMatrix::getMetaDataDir(): metadatadir was not set"); 01507 return metadatadir; 01508 } 01509 01511 // loadAllStringMappings // 01513 void VMatrix::loadAllStringMappings() 01514 { 01515 if (! hasMetaDataDir() || ! isdir(getSFIFDirectory())) 01516 return; 01517 01518 for(int i=0;i<width();i++) 01519 loadStringMapping(i); 01520 } 01521 01523 // loadStringMapping // 01525 void VMatrix::loadStringMapping(int col) 01526 { 01527 if(!hasMetaDataDir()) 01528 return; 01529 PPath fname = getSFIFFilename(col,".smap"); 01530 init_map_sr(); 01531 string SFIFdir= getSFIFDirectory(); 01532 if(!pathexists(SFIFdir)) 01533 force_mkdir(SFIFdir); 01534 if(!isfile(fname)) 01535 return; 01536 01537 deleteStringMapping(col); 01538 01539 // smap file exists, open it 01540 PStream f = openFile(fname, PStream::plearn_ascii); 01541 01542 // TODO Remove ? 01543 #if 0 01544 string pref; 01545 f>>pref; 01546 if(string(pref)!="#SMAP") 01547 PLERROR( string("File "+fname+" is not a valid String mapping file.\nShould start with #SMAP on first line (this is to prevent inopportunely overwritting another type of file)").c_str()); 01548 #endif 01549 01550 while(f.good()) 01551 { 01552 string s; 01553 real val; 01554 f >> s >> val; 01555 if(f.good()) 01556 { 01557 map_sr[col][s] = val; 01558 map_rs[col][val] = s; 01559 f.skipBlanks(); 01560 } 01561 } 01562 } 01563 01565 // copyStringMappingsFrom // 01567 void VMatrix::copyStringMappingsFrom(const VMat& source) { 01568 if (width_ != source->width()) { 01569 PLERROR("In VMatrix::copyStringMappingsFrom - The source VMatrix doesn't have the same width"); 01570 } 01571 map_rs.resize(width_); 01572 map_sr.resize(width_); 01573 for (int i = 0; i < width_; i++) { 01574 setStringMapping(i, source->getStringToRealMapping(i)); 01575 } 01576 } 01577 01579 // getStats // 01581 TVec<StatsCollector> VMatrix::getStats(bool progress_bar) const 01582 { 01583 if(!field_stats) 01584 field_stats = getPrecomputedStatsFromFile("stats.psave", 2000, 01585 progress_bar); 01586 return field_stats; 01587 } 01588 01590 // updateMtime // 01592 void VMatrix::updateMtime(time_t t) 01593 { 01594 if(t>mtime_ && mtime_!=numeric_limits<time_t>::max()) 01595 mtime_=t; 01596 else if(t==0) 01597 mtime_=numeric_limits<time_t>::max(); 01598 } 01599 void VMatrix::updateMtime(const PPath& p){if(!p.isEmpty())updateMtime(mtime(p));} 01600 01601 void VMatrix::updateMtime(VMat v){if(v)updateMtime(v->getMtime());} 01602 01604 // isUpToDate // 01606 bool VMatrix::isUpToDate(const PPath& path, bool warning_mtime0, 01607 bool warning_older) const 01608 { 01609 bool exist = isfile(path); 01610 bool uptodate = false; 01611 if(exist) 01612 uptodate = getMtime() < mtime(path); 01613 if (warning_mtime0 && exist && uptodate && getMtime()==0) 01614 PLWARNING("In VMatrix::isUpToDate - for class '%s'" 01615 " File '%s' will be used, but " 01616 "this VMat's last modification time is undefined: we cannot " 01617 "be sure the file is up-to-date.", 01618 classname().c_str(), path.absolute().c_str()); 01619 if(warning_older && exist && !uptodate) 01620 PLWARNING("In VMatrix::isUpToDate - for class '%s'" 01621 " File '%s' is older than this " 01622 "VMat's mtime of %ld, and should not be re-used.", 01623 classname().c_str(), path.absolute().c_str(), long(getMtime())); 01624 01625 return exist && uptodate; 01626 } 01627 01629 // isUpToDate // 01631 bool VMatrix::isUpToDate(VMat vm, bool warning_mtime0, 01632 bool warning_older) const 01633 { 01634 time_t my_time = getMtime(); 01635 time_t vm_time = vm->getMtime(); 01636 bool uptodate = my_time < vm_time || 01637 my_time == 0 || 01638 vm_time == 0; 01639 if (warning_mtime0 && uptodate && (my_time == 0 || vm_time == 0)) 01640 PLWARNING("In VMatrix::isUpToDate - for class '%s'" 01641 " When comparing the VMats' last modification times, at " 01642 "least one was found to be undefined: we cannot be sure " 01643 "the VMat is up-to-date.", 01644 classname().c_str()); 01645 if(warning_older && !uptodate) 01646 PLWARNING("In VMatrix::isUpToDate - for class '%s'" 01647 " The VMat with mtime of %ld is older than this " 01648 "VMat's with mtime of %ld, and should not be re-used.", 01649 classname().c_str(), long(vm->getMtime()), long(getMtime())); 01650 01651 return uptodate; 01652 } 01653 01655 // getPrecomputedStatsFromFile // 01657 TVec<StatsCollector> VMatrix::getPrecomputedStatsFromFile( 01658 const string& filename, int maxnvalues, bool progress_bar) const 01659 { 01660 TVec<StatsCollector> stats; 01661 PPath metadatadir = getMetaDataDir(); 01662 PPath statsfile; 01663 bool uptodate = false; 01664 if (hasMetaDataDir()) { 01665 lockMetaDataDir(); 01666 statsfile = metadatadir / filename; 01667 uptodate = isUpToDate(statsfile); 01668 } 01669 try{ 01670 if (uptodate){ 01671 PLearn::load(statsfile, stats); 01672 if(stats.length()!=width()){ 01673 uptodate=false; 01674 PLWARNING("In VMatrix::getPrecomputedStatsFromFile() for class" 01675 " %s - The file %s don't have the good number of" 01676 " stats. We regenerate it.", 01677 classname().c_str(), statsfile.c_str()); 01678 } 01679 } 01680 if(!uptodate){ 01681 VMat vm = const_cast<VMatrix*>(this); 01682 stats = PLearn::computeStats(vm, maxnvalues, progress_bar); 01683 if(hasMetaDataDir()) 01684 PLearn::save(statsfile, stats); 01685 } 01686 }catch(const PLearnError& e){ 01687 if(!metadatadir.isEmpty()) 01688 unlockMetaDataDir(); 01689 //we erase the file if we are creating it 01690 // as it can be partilly saved. 01691 if(!uptodate && isfile(statsfile)) 01692 rm(statsfile); 01693 throw e; 01694 } 01695 if (!metadatadir.isEmpty()) 01696 unlockMetaDataDir(); 01697 return stats; 01698 } 01699 01701 // remote_getStats // 01703 TVec<PP<StatsCollector> > VMatrix::remote_getStats() const 01704 { 01705 if(field_p_stats.isEmpty()) 01706 { 01707 TVec<StatsCollector> st= getStats(); 01708 field_p_stats.resize(st.length()); 01709 CopiesMap cm; 01710 for(int i= 0; i < st.length(); ++i) 01711 field_p_stats[i]= st[i].deepCopy(cm); 01712 } 01713 return field_p_stats; 01714 } 01715 01717 // getBoundingBox // 01719 TVec< pair<real,real> > VMatrix::getBoundingBox(real extra_percent) const 01720 { 01721 TVec<StatsCollector> stats = getStats(); 01722 int n = stats.length(); 01723 TVec< pair<real,real> > bbox(n); 01724 for(int k=0; k<n; k++) 01725 { 01726 StatsCollector& st = stats[k]; 01727 bbox[k] = pair<real,real>(st.min()-extra_percent*st.range(), st.max()+extra_percent*st.range()); 01728 } 01729 return bbox; 01730 } 01731 01733 // getRanges // 01735 TVec<RealMapping> VMatrix::getRanges() 01736 { 01737 TVec<RealMapping> ranges; 01738 PPath rangefile = getMetaDataDir() / "ranges.psave"; 01739 if(isfile(rangefile)) 01740 PLearn::load(rangefile, ranges); 01741 else 01742 { 01743 ranges = computeRanges(getStats(),std::max(10,length()/200),std::max(10,length()/100) ); 01744 PLearn::save(rangefile, ranges); 01745 } 01746 return ranges; 01747 } 01748 01750 // put // 01752 void VMatrix::put(int i, int j, real value) 01753 { 01754 PLERROR("In VMatrix::put - Method not implemented for this VMat(%s), please implement.", 01755 classname().c_str()); 01756 } 01757 01759 // getColumn // 01761 void VMatrix::getColumn(int j, Vec v) const 01762 { 01763 #ifdef BOUNDCHECK 01764 if(v.length() != length()) 01765 PLERROR("In VMatrix::getColumn - v must have the same length as the VMatrix"); 01766 #endif 01767 for(int i=0; i<v.length(); i++) 01768 v[i] = get(i,j); 01769 } 01770 01771 Vec VMatrix::remote_getColumn(int i) const 01772 { 01773 Vec v(length()); 01774 getColumn(i,v); 01775 return v; 01776 } 01777 01778 01780 // getSubRow // 01782 void VMatrix::getSubRow(int i, int j, Vec v) const 01783 { 01784 for(int k=0; k<v.length(); k++) 01785 v[k] = get(i,j+k); 01786 } 01787 01789 // putSubRow // 01791 void VMatrix::putSubRow(int i, int j, Vec v) 01792 { 01793 for(int k=0; k<v.length(); k++) 01794 put(i, j+k, v[k]); 01795 } 01796 01798 // getRow // 01800 void VMatrix::getRow(int i, Vec v) const 01801 { 01802 #ifdef BOUNDCHECK 01803 if(v.length() != width()) 01804 PLERROR("In VMatrix::getRow(i,v) length of v and width of VMatrix differ"); 01805 #endif 01806 getSubRow(i,0,v); 01807 } 01808 01810 // putRow // 01812 void VMatrix::putRow(int i, Vec v) 01813 { 01814 #ifdef BOUNDCHECK 01815 if(v.length() != width()) 01816 PLERROR("In VMatrix::putRow(i,v) length of v and width of VMatrix differ"); 01817 #endif 01818 putSubRow(i,0,v); 01819 } 01820 01822 // fill // 01824 void VMatrix::fill(real value) 01825 { 01826 Vec v(width(), value); 01827 for (int i=0; i<length(); i++) putRow(i,v); 01828 } 01829 01831 // appendRow // 01833 void VMatrix::appendRow(Vec v) 01834 { 01835 PLERROR("In VMatrix::appendRow - Not implemented by VMatrix subclass '%s'", 01836 classname().c_str()); 01837 } 01838 01840 // insertRow // 01842 void VMatrix::insertRow(int i, Vec v) 01843 { 01844 if (i<0 || i>length_) 01845 PLERROR("In VMatrix::insertRow: row index (%d) outside valid range [%d,%d]", i, 0, length_); 01846 else if (i == length_) 01847 appendRow(v); 01848 else 01849 { 01850 appendRow(v); // dummy operation to increase VMat length 01851 Vec row(width_); 01852 for (int j=length_-1; j>i; --j) 01853 { 01854 getRow(j-1, row); 01855 putRow(j, row); 01856 } 01857 putRow(i, v); 01858 } 01859 } 01860 01862 // flush // 01864 void VMatrix::flush() 01865 {} 01866 01868 // putOrAppendRow // 01870 void VMatrix::putOrAppendRow(int i, Vec v) 01871 { 01872 if (v.length() != width()) 01873 PLERROR("In putOrAppendRow, Vec to append must have same length (%d) as VMatrix width (%d)", v.length(), width()); 01874 01875 if(i==length()) 01876 appendRow(v); 01877 else if(i<length()) 01878 putRow(i,v); 01879 else 01880 PLERROR("In putOrAppendRow, index %d out of range",i); 01881 } 01882 01884 // forcePutRow // 01886 void VMatrix::forcePutRow(int i, Vec v) 01887 { 01888 if (v.length() != width()) 01889 PLERROR("In forcePutRow, Vec to append must have same length (%d) as VMatrix width (%d)", v.length(), width()); 01890 01891 if(i<length()) 01892 putRow(i,v); 01893 else 01894 { 01895 Vec emptyrow(width()); 01896 emptyrow.clear(); 01897 while(length()<i) 01898 appendRow(emptyrow); 01899 appendRow(v); 01900 } 01901 } 01902 01904 // getMat // 01906 void VMatrix::getMat(int i, int j, Mat m) const 01907 { 01908 #ifdef BOUNDCHECK 01909 if(i<0 || j<0 || i+m.length()>length() || j+m.width()>width()) 01910 PLERROR("In VMatrix::getMat(i,j,m) OUT OF BOUNDS"); 01911 #endif 01912 for(int ii=0; ii<m.length(); ii++) 01913 { 01914 getSubRow(i+ii, j, m(ii)); 01915 } 01916 } 01917 01919 // putMat // 01921 void VMatrix::putMat(int i, int j, Mat m) 01922 { 01923 #ifdef BOUNDCHECK 01924 if(i<0 || j<0 || i+m.length()>length() || j+m.width()>width()) 01925 PLERROR("In VMatrix::putMat(i,j,m) OUT OF BOUNDS"); 01926 #endif 01927 for(int ii=0; ii<m.length(); ii++) 01928 { 01929 putSubRow(i+ii, j, m(ii)); 01930 } 01931 } 01932 01934 // compacify // 01936 void VMatrix::compacify() {} 01937 01939 // toMat // 01941 Mat VMatrix::toMat() const 01942 { 01943 return toMatCopy(); 01944 } 01945 01946 Mat VMatrix::toMatCopy() const 01947 { 01948 Mat m(length(),width()); 01949 getMat(0,0,m); 01950 return m; 01951 } 01952 01954 // subMat // 01956 VMat VMatrix::subMat(int i, int j, int l, int w) 01957 { return new SubVMatrix(this,i,j,l,w); } 01958 01960 // dot // 01962 real VMatrix::dot(int i1, int i2, int inputsize) const 01963 { 01964 dotrow_1.resize(inputsize); 01965 dotrow_2.resize(inputsize); 01966 getSubRow(i1, 0, dotrow_1); 01967 getSubRow(i2, 0, dotrow_2); 01968 return PLearn::dot(dotrow_1, dotrow_2); 01969 } 01970 01971 real VMatrix::dot(int i, const Vec& v) const 01972 { 01973 dotrow_1.resize(v.length()); 01974 getSubRow(i, 0, dotrow_1); 01975 return PLearn::dot(dotrow_1, v); 01976 } 01977 01978 01980 // find // 01982 bool VMatrix::find(const Vec& input, real tolerance, int* i, int i_start) const 01983 { 01984 get_row.resize(inputsize()); 01985 #ifdef BOUNDCHECK 01986 if (input.length() != inputsize()) 01987 PLERROR("In VMatrix::find - The given vector must be the same size as " 01988 "inputsize"); 01989 #endif 01990 int n = length(); 01991 for (int j = 0; j < n; j++) { 01992 int row = (j + i_start) % n; 01993 getSubRow(row, 0, get_row); 01994 if (powdistance(input, get_row, 2.0) < tolerance) { 01995 if (i) 01996 *i = row; 01997 return true; 01998 } 01999 } 02000 if (i) 02001 *i = -1; 02002 return false; 02003 } 02004 02006 // newwrite // 02008 void VMatrix::newwrite(PStream& out) const 02009 { 02010 /* 02011 switch(out.outmode) 02012 { 02013 case PStream::raw_ascii: 02014 case PStream::pretty_ascii: 02015 { 02016 Vec v(width()); 02017 for(int i=0; i<length(); i++) { 02018 getRow(i,v); 02019 out << v << endl; 02020 } 02021 break; 02022 } 02023 default: 02024 inherited::newwrite(out); 02025 } 02026 */ 02027 inherited::newwrite(out); 02028 } 02029 02031 // ~ // 02033 VMatrix::~VMatrix() 02034 {} 02035 02037 // save // 02039 void VMatrix::save(const PPath& filename) const 02040 { 02041 PLDEPRECATED( "This method overloads the Object::save method which is " 02042 "deprecated. This method is therefore deprecated and you should call " 02043 "directly the savePMAT() method." ); 02044 02045 savePMAT(filename); 02046 } 02047 02049 // savePMAT // 02051 void VMatrix::savePMAT(const PPath& pmatfile, bool force_float, 02052 bool auto_float) const 02053 { 02054 if (width() == -1) 02055 PLERROR("In VMat::save - Saving in a pmat file is only possible for constant width VMats (where width()!=-1)"); 02056 02057 if(force_float && auto_float) 02058 PLERROR("VMatrix::savePMAT() - force_float an auto_float are incompatible option"); 02059 02060 int nsamples = length(); 02061 PPath pmatfiletmp=pmatfile+".tmp"; 02062 if(auto_float){ 02063 #ifdef USEFLOAT 02064 PLERROR("VMatrix::savePMAT() - auto_float can't reliably select float or double when compiled in float. Compile it in double."); 02065 #endif 02066 Vec v(width()); 02067 bool found_not_equal=false; 02068 for(int i=0;i<length();i++){ 02069 getRow(i,v); 02070 for(int j=0;j<width();j++){ 02071 if( ((double)((float)(v[j])))!=v[j] ){ 02072 found_not_equal=true;break; 02073 } 02074 } 02075 } 02076 if(!found_not_equal){ 02077 force_float=true; 02078 pout<<"We will store the result matrix in FLOAT format."<<endl; 02079 } 02080 else 02081 pout<<"We will store the result matrix in DOUBLE format."<<endl; 02082 } 02083 { 02084 FileVMatrix m(pmatfiletmp,nsamples,width(),force_float); 02085 m.setMetaInfoFrom(this); 02086 // m.setFieldInfos(getFieldInfos()); 02087 // m.copySizesFrom(this); 02088 Vec tmpvec(width()); 02089 02090 ProgressBar pb(cout, "Saving to pmat", nsamples); 02091 02092 for(int i=0; i<nsamples; i++) 02093 { 02094 getRow(i,tmpvec); 02095 m.putRow(i,tmpvec); 02096 pb(i); 02097 } 02098 m.saveFieldInfos(); 02099 m.saveAllStringMappings(); 02100 }// to ensure that m is deleted? 02101 02102 rm(pmatfile); 02103 force_rmdir(pmatfile+".metadata"); 02104 mv(pmatfiletmp,pmatfile); 02105 mv(pmatfiletmp+".metadata",pmatfile+".metadata"); 02106 } 02107 02109 // saveDMAT // 02111 void VMatrix::saveDMAT(const PPath& dmatdir) const 02112 { 02113 force_rmdir(dmatdir); 02114 DiskVMatrix vm(dmatdir,width()); 02115 vm.setMetaInfoFrom(this); 02116 // vm.setFieldInfos(getFieldInfos()); 02117 // vm.copySizesFrom(this); 02118 Vec v(width()); 02119 02120 ProgressBar pb(cout, "Saving to dmat", length()); 02121 02122 for(int i=0;i<length();i++) 02123 { 02124 getRow(i,v); 02125 vm.appendRow(v); 02126 pb(i); 02127 } 02128 vm.saveFieldInfos(); 02129 vm.saveAllStringMappings(); 02130 } 02131 02133 // saveAMAT // 02135 void VMatrix::saveAMAT(const PPath& amatfile, bool verbose, bool no_header, bool save_strings) const 02136 { 02137 int l = length(); 02138 int w = width(); 02139 PStream out = openFile(amatfile, PStream::raw_ascii, "w"); 02140 if (!no_header) { 02141 out << "#size: "<< l << ' ' << w << endl; 02142 } 02143 if(w>0 && !no_header) 02144 { 02145 out << "#: "; 02146 for(int k=0; k<w; k++) 02147 //there must not be any space in a field name... 02148 out << space_to_underscore(fieldName(k)) << ' '; 02149 out << "\n"; 02150 } 02151 if(!no_header) 02152 out << "#sizes: " << inputsize() << ' ' << targetsize() << ' ' << weightsize() << ' ' << extrasize() << endl; 02153 02154 PP<ProgressBar> pb; 02155 if (verbose) 02156 pb = new ProgressBar(cout, "Saving to amat", length()); 02157 02158 if (save_strings) { 02159 TVec<string> v(w); 02160 for (int i = 0; i < l; i++) { 02161 getRowAsStrings(i, v); 02162 out << v << endl; 02163 if (verbose) 02164 pb->update(i+1); 02165 } 02166 02167 } else { 02168 Vec v(w); 02169 for(int i=0;i<l;i++) 02170 { 02171 getRow(i,v); 02172 for(int j=0; j<w; j++) 02173 out << v[j] << ' '; 02174 out << "\n"; 02175 if (verbose) 02176 pb->update(i + 1); 02177 } 02178 } 02179 } 02180 02181 void VMatrix::saveCMAT(const PPath& filename) const 02182 { 02183 PLWARNING("VMatrix::saveCMAT() - NOT FULLY IMPLEMENTED"); 02184 02185 //calculate the datatype needed 02186 TVec<StatsCollector> stats = getStats(true); 02187 int max_bits=0; 02188 for(int i=0;i<stats.size();i++){ 02189 StatsCollector stat = stats[i]; 02190 if(! stat.isinteger()) 02191 PLERROR("VMatrix::saveCMAT() currently the source need to contain only integer."); 02192 if(stat.min()>=0){ 02193 int bits=(int)ceil(sqrt(stat.max())); 02194 if(max_bits<bits)max_bits=bits; 02195 }else{ 02196 PLERROR("not implemented to store negatif number."); 02197 } 02198 02199 } 02200 //example 12000000 u:784:1:8 u:1:1:8 02201 //write the header 02202 if(max_bits>8) PLERROR("VMatrix::saveCMAT() currently we convert to cmat with a maximum of 8 bits by fields!"); 02203 if(max_bits > 1 && max_bits<8){ 02204 max_bits=8; 02205 PLWARNING("VMatrix::saveCMAT() currently when we need less then 8 bits(except for 1), we upgrade to 8 bits."); 02206 } 02207 if(max_bits==0){ 02208 PLERROR("VMatrix::saveCMAT() - their was only 0 in the matrix! This is not supported as we don't think this can happen in real case!"); 02209 } 02210 //write the data 02211 if(max_bits==8){ 02212 PStream out = openFile(filename, PStream::raw_ascii, "w"); 02213 out<<length()<<" u:"<<width()<<":1:"<<max_bits<<endl; 02214 Vec v(width()); 02215 for(int i=0;i<length();i++){ 02216 getRow(i,v); 02217 for(int j=0;j<width();j++){ 02218 out.put((char)v[j]); 02219 } 02220 } 02221 }else if(max_bits==1){ 02222 PStream out = openFile(filename, PStream::raw_ascii, "w"); 02223 int w2=width()%8; 02224 int w1=width()-w2; 02225 PLCHECK(w2+w1==width()); 02226 PLCHECK(w1%8==0); 02227 PLCHECK(w1>0 && w2>=0); 02228 out<<length()<<" u:"<<w1<<":1:"<<max_bits; 02229 if(w2!=0) 02230 out<<" u:"<<w2<<":1:8"; 02231 out<<endl; 02232 Vec v(width()); 02233 02234 for(int i=0;i<length();i++){ 02235 getRow(i,v); 02236 int j; 02237 for(j=0;j<w1;){ 02238 char c=0; 02239 for(int k=0;k<8;j++,k++){ 02240 c=c<<1; 02241 c|=((bool)v[j]); 02242 } 02243 //revert the bits 02244 char value=c; 02245 value = (value & 0x0f) << 4 | (value & 0xf0) >> 4; 02246 value = (value & 0x33) << 2 | (value & 0xcc) >> 2; 02247 value = (value & 0x55) << 1 | (value & 0xaa) >> 1; 02248 out.put(value); 02249 } 02250 PLCHECK(width()-j==w2); 02251 for(;j<width();j++){ 02252 out.put((char)v[j]); 02253 } 02254 } 02255 } 02256 else 02257 PLERROR("VMatrix::saveCMAT() - %d bits are not supported!",max_bits); 02258 02259 CompactFileVMatrix m = CompactFileVMatrix(filename); 02260 m.setMetaDataDir(filename + ".metadata"); 02261 m.setMetaInfoFrom(this); 02262 m.saveFieldInfos(); 02263 m.saveAllStringMappings(); 02264 02265 pout<<"generated the file " <<filename <<endl; 02266 } 02268 // accumulateXtY // 02270 void VMatrix::accumulateXtY(int X_startcol, int X_ncols, int Y_startcol, int Y_ncols, 02271 Mat& result, int startrow, int nrows, int ignore_this_row) const 02272 { 02273 int endrow = (nrows>0) ?startrow+nrows :length_; 02274 Vec x(X_ncols); 02275 Vec y(Y_ncols); 02276 for(int i=startrow; i<endrow; i++) 02277 if(i!=ignore_this_row) 02278 { 02279 getSubRow(i,X_startcol,x); 02280 getSubRow(i,Y_startcol,y); 02281 externalProductAcc(result, x,y); 02282 } 02283 } 02284 02286 // accumulateXtX // 02288 void VMatrix::accumulateXtX(int X_startcol, int X_ncols, 02289 Mat& result, int startrow, int nrows, int ignore_this_row) const 02290 { 02291 Vec x(X_ncols); 02292 int endrow = (nrows>0) ?startrow+nrows :length_; 02293 for(int i=startrow; i<endrow; i++) 02294 if(i!=ignore_this_row) 02295 { 02296 getSubRow(i,X_startcol,x); 02297 externalProductAcc(result, x,x); 02298 } 02299 } 02300 02302 // getRowVec // 02304 Vec VMatrix::getRowVec(int i) const 02305 { 02306 Vec v(width()); 02307 getRow(i,v); 02308 return v; 02309 } 02310 02312 // appendRows // 02314 void VMatrix::appendRows(Mat rows) 02315 { 02316 for(int i=0; i<rows.length(); i++) 02317 appendRow(rows(i)); 02318 } 02319 02320 02322 // compareStats // 02324 void VMatrix::compareStats(VMat target, 02325 real stderror_threshold, 02326 real missing_threshold, 02327 Vec stderror, 02328 Vec missing) 02329 { 02330 if(target->width()!=width()) 02331 PLERROR("In VecStatsCollector:: compareStats() - This VMatrix has " 02332 "width %d which differs from the target width of %d", 02333 width(), target->width()); 02334 02335 for(int i=0;i<width();i++) 02336 { 02337 const StatsCollector tstats = target->getStats(i); 02338 const StatsCollector lstats = getStats(i); 02339 02340 real tmissing = tstats.nmissing()/tstats.n(); 02341 real lmissing = lstats.nmissing()/lstats.n(); 02342 real terr = sqrt(tmissing*(1-tmissing)+lmissing*(1-lmissing)); 02343 real th_missing = fabs(tmissing-lmissing)/terr; 02344 if(fast_is_equal(terr,0)) 02345 { 02346 if(!fast_is_equal(tmissing,0)||!fast_is_equal(lmissing,0)) 02347 PLWARNING("In VMatrix::compareStats - field %d(%s)terr=%f," 02348 " tmissing=%f, lmissing=%f!",i, fieldName(i).c_str(), 02349 terr, tmissing, lmissing); 02350 PLCHECK((fast_is_equal(tmissing,0)||fast_is_equal(tmissing,1)) 02351 && (fast_is_equal(lmissing,0)||fast_is_equal(lmissing,1))); 02352 } 02353 else if(isnan(th_missing)) 02354 PLWARNING("In VMatrix::compareStats - should not happen!"); 02355 02356 real tmean = tstats.mean(); 02357 real lmean = lstats.mean(); 02358 real tstderror = sqrt(pow(tstats.stderror(), 2) + 02359 pow(lstats.stderror(), 2)); 02360 real th_stderror = fabs(lmean-tmean)/tstderror; 02361 if(tstderror==0) 02362 PLWARNING("In VMatrix::compareStats - field %d(%s) have a" 02363 " stderror of 0 for both matrice.", 02364 i, fieldName(i).c_str()); 02365 stderror[i]=th_stderror; 02366 missing[i]=th_missing; 02367 } 02368 return; 02369 } 02370 02372 // maxFieldNamesSize // 02374 int VMatrix::maxFieldNamesSize() const 02375 { 02376 uint size_fieldnames=0; 02377 for(int i=0;i<width();i++) 02378 if(fieldName(i).size()>size_fieldnames) 02379 size_fieldnames=fieldName(i).size(); 02380 return size_fieldnames; 02381 } 02382 02384 // computeMissingSizeValue // 02386 void VMatrix::computeMissingSizeValue(bool warn_if_cannot_compute, 02387 bool warn_if_size_mismatch) 02388 { 02389 int v=min(inputsize_,0) + min(targetsize_,0) 02390 + min(weightsize_,0) + min(extrasize_,0); 02391 02392 if (width_ < 0 && v <= -1) { 02393 if (warn_if_cannot_compute) 02394 PLWARNING("In VMatrix::computeMissingSizeValue for %s - Cannot " 02395 "compute the missing size value when the width is undefined", 02396 classname().c_str()); 02397 return; 02398 } 02399 02400 if(v < -1){ 02401 if(warn_if_cannot_compute) 02402 PLWARNING("In VMatrix::computeMissingSizeValue() - in class %s" 02403 " more then one of" 02404 " inputsize(%d), targetsize(%d), weightsize(%d) and" 02405 " extrasize(%d) is unknow so we cannot compute them with" 02406 " the width(%d)", 02407 classname().c_str(), inputsize_, targetsize_, weightsize_, 02408 extrasize_, width_); 02409 return; 02410 }else if(v==0 && warn_if_size_mismatch && width_ >= 0 && 02411 width_ != inputsize_ + targetsize_ + weightsize_ + extrasize_) 02412 PLWARNING("In VMatrix::computeMissingSizeValue() for class %s - " 02413 "inputsize_(%d) + targetsize_(%d) + weightsize_(%d) + " 02414 "extrasize_(%d) != width_(%d) !", 02415 classname().c_str(), inputsize_, targetsize_, weightsize_, 02416 extrasize_, width_); 02417 02418 else if(inputsize_<0) 02419 inputsize_ = width_- targetsize_ - weightsize_ - extrasize_; 02420 else if(targetsize_ < 0) 02421 targetsize_ = width_- inputsize_ - weightsize_ - extrasize_; 02422 else if(weightsize_ < 0) 02423 weightsize_ = width_- inputsize_ - targetsize_ - extrasize_; 02424 else if(extrasize_ < 0) 02425 extrasize_ = width_- inputsize_ - targetsize_ - weightsize_; 02426 } 02427 02428 } // end of namespace PLearn 02429 02430 02431 /* 02432 Local Variables: 02433 mode:c++ 02434 c-basic-offset:4 02435 c-file-style:"stroustrup" 02436 c-file-offsets:((innamespace . 0)(inline-open . 0)) 02437 indent-tabs-mode:nil 02438 fill-column:79 02439 End: 02440 */ 02441 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :