PLearn 0.1
VMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 // Copyright (C) 2004 Rejean Ducharme
00007 //
00008 
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 //
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 //
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 //
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 //
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 //
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 /********************************************************
00038  * $Id: VMatrix.cc 10321 2010-02-15 19:22:34Z ducharme $
00039  ******************************************************* */
00040 
00041 #include "VMatrix.h"
00042 #include "CompactFileVMatrix.h"
00043 #include "DiskVMatrix.h"
00044 #include "FileVMatrix.h"
00045 #include "SubVMatrix.h"
00046 #include "VMat_computeStats.h"
00047 #include <plearn/base/tostring.h>
00048 #include <plearn/base/lexical_cast.h>
00049 #include <plearn/base/stringutils.h> 
00050 #include <plearn/io/fileutils.h>     
00051 #include <plearn/base/tostring.h>     
00052 #include <plearn/io/load_and_save.h>
00053 #include <plearn/math/random.h>      
00054 #include <plearn/base/RemoteDeclareMethod.h>
00055 #include <nspr/prenv.h>
00056 #include <plearn/math/TMat_maths.h> 
00057 #include <plearn/sys/procinfo.h> 
00058 #include <limits>
00059 
00060 namespace PLearn {
00061 using namespace std;
00062 
00063 // TODO-PPath   : this class is now PPath compliant
00064 // TODO-PStream : this class is now PStream compliant
00065 
00066 PLEARN_IMPLEMENT_ABSTRACT_OBJECT(
00067     VMatrix,
00068     "Base classes for virtual matrices",
00069     "VMatrix provides an abstraction for a virtual matrix, namely a matrix wherein\n"
00070     "all element access operations are virtual. This enables a wide variety of\n"
00071     "matrix-like objects to be implemented, from simple data containers (e.g.\n"
00072     "MemoryVMatrix), to large-scale matrices that don't fit in memory (e.g.\n"
00073     "FileVMatrix), to on-the-fly calculations that are implemented through \n"
00074     "various processing VMatrices.\n"
00075     "\n"
00076     "For implementers, a simple class to derive from is RowBufferedVMatrix, which\n"
00077     "implements most of the functionalities of the abstract VMatrix interface in terms\n"
00078     "of a few simple virtual functions to be overridden by the user.");
00079 
00080 VMatrix::VMatrix(bool call_build_):
00081     inherited   (call_build_),
00082     mtime_      (0),
00083     mtime_update(0),
00084     length_     (-1),
00085     width_      (-1),
00086     inputsize_  (-1),
00087     targetsize_ (-1),
00088     weightsize_ (-1),
00089     extrasize_  (0),
00090     writable    (false)
00091 {
00092     lockf_ = PStream();
00093     if (call_build_)
00094         build_();
00095 }
00096 
00097 VMatrix::VMatrix(int the_length, int the_width, bool call_build_):
00098     inherited                       (call_build_),
00099     mtime_                          (0),
00100     mtime_update                    (0),
00101     length_                         (the_length),
00102     width_                          (the_width),
00103     inputsize_                      (-1),
00104     targetsize_                     (-1),
00105     weightsize_                     (-1),
00106     extrasize_                      (0),
00107     writable                        (false),
00108     map_sr(TVec<map<string,real> >  (the_width)),
00109     map_rs(TVec<map<real,string> >  (the_width)),
00110     fieldstats                      (0)
00111 {
00112     lockf_ = PStream();
00113     if (call_build_)
00114         build_();
00115 }
00116 
00118 // declareOptions //
00120 void VMatrix::declareOptions(OptionList & ol)
00121 {
00122     declareOption(
00123         ol, "writable", &VMatrix::writable, OptionBase::buildoption,
00124         "Are write operations permitted?");
00125 
00126     declareOption(
00127         ol, "length", &VMatrix::length_, OptionBase::buildoption,
00128         "Length of the matrix (number of rows)");
00129 
00130     declareOption(
00131         ol, "width", &VMatrix::width_, OptionBase::buildoption,
00132         "Width of the matrix (number of columns; -1 indicates this varies\n"
00133         "from sample to sample...)");
00134 
00135     declareOption(
00136         ol, "inputsize", &VMatrix::inputsize_, OptionBase::buildoption,
00137         "Size of input part (-1 if variable or unspecified, 0 if no input)");
00138 
00139     declareOption(
00140         ol, "targetsize", &VMatrix::targetsize_, OptionBase::buildoption,
00141         "Size of target part (-1 if variable or unspecified, 0 if no target)");
00142 
00143     declareOption(
00144         ol, "weightsize", &VMatrix::weightsize_, OptionBase::buildoption,
00145         "Size of weights (-1 if unspecified, 0 if no weight, 1 for sample\n"
00146         "weight, >1 currently not supported).");
00147 
00148     declareOption(
00149         ol, "extrasize", &VMatrix::extrasize_, OptionBase::buildoption,
00150         "Size of extra fields (additional info). Defaults to 0");
00151 
00152     declareOption(
00153         ol, "metadatadir", &VMatrix::metadatadir, OptionBase::buildoption,
00154         "A directory in which to store meta-information for this matrix \n"
00155         "You don't always have to give this explicitly. For ex. if your \n"
00156         "VMat is the outer VMatrix in a .vmat file, the metadatadir will \n"
00157         "automatically be set to name_of_vmat_file.metadata/ \n"
00158         "And if it is the source inside another VMatrix that sets its \n"
00159         "metadatadir, it will often be set from that surrounding vmat's metadata.\n");
00160 
00161     declareOption(
00162         ol, "mtime", &VMatrix::mtime_update, 
00163         OptionBase::buildoption|OptionBase::nosave,
00164         "DO NOT play with this if you don't know the implementation!\n"
00165         "This add a dependency mtime to the gived value.\n"
00166         "Use -1 to set permanently that we do not know the mtime.");
00167 
00168     declareOption(
00169         ol, "fieldinfos", &VMatrix::fieldinfos, OptionBase::buildoption,
00170         "Field infos.\n");
00171 
00172     inherited::declareOptions(ol);
00173 }
00174 
00175 void VMatrix::declareMethods(RemoteMethodMap& rmm)
00176 {
00177     // Insert a backpointer to remote methods; note that this
00178     // different than for declareOptions()
00179     rmm.inherited(inherited::_getRemoteMethodMap_());
00180 
00181     declareMethod(
00182         rmm, "getRow", &VMatrix::getRowVec,
00183         (BodyDoc("Returns a row of a matrix \n"),
00184          ArgDoc ("i", "Position of the row to get.\n"),
00185          RetDoc ("row i vector")));
00186 
00187     declareMethod(
00188         rmm, "getExample", &VMatrix::remote_getExample,
00189         (BodyDoc("Returns the input, target and weight parts of a row.\n"),
00190          ArgDoc ("i", "Position of the row to get.\n"),
00191          RetDoc ("An (input, target, weight) tuple.")));
00192 
00193     declareMethod(
00194         rmm, "getExtra", &VMatrix::remote_getExtra,
00195         (BodyDoc("Returns the extra part of a row.\n"),
00196          ArgDoc ("i", "Position of the row to get.\n"),
00197          RetDoc ("Values for extrafields.")));
00198 
00199     declareMethod(
00200         rmm, "getColumn", &VMatrix::remote_getColumn,
00201         (BodyDoc("Returns a row of a matrix \n"),
00202          ArgDoc ("i", "Position of the row to get.\n"),
00203          RetDoc ("row i vector")));
00204 
00205 
00206     declareMethod(
00207         rmm, "getString", &VMatrix::getString,
00208         (BodyDoc("Returns an element of a matrix as a string\n"),
00209          ArgDoc ("i", "Position of the row to get.\n"),
00210          ArgDoc ("j", "Position of the column to get.\n"),
00211          RetDoc ("string value")));
00212 
00213     declareMethod(
00214         rmm, "getMat", &VMatrix::toMat,
00215         (BodyDoc("Returns the content of the vmat as a Mat\n"),
00216          RetDoc ("The content of this VMatrix as a Mat")));
00217 
00218     declareMethod(
00219             rmm, "getLength", &VMatrix::length,
00220             (BodyDoc("Return length of this VMatrix.\n"),
00221              RetDoc("The length of this VMatrix.")));
00222 
00223     declareMethod(
00224         rmm, "declareField", &VMatrix::declareField,
00225         (BodyDoc("Declares the field infos for a given column (index).\n"),
00226          ArgDoc ("fieldindex", "The column index.\n"),
00227          ArgDoc ("fieldname", "The field name of this column.\n"),
00228          ArgDoc ("fieldtype", "The field type of this column.\n")));
00229 
00230     declareMethod(
00231         rmm, "declareFieldNames", &VMatrix::declareFieldNames,
00232         (BodyDoc("Declares the field names.\n"),
00233          ArgDoc ("fnames", "TVec of field names.\n")));
00234 
00235     declareMethod(
00236         rmm, "fieldNames", &VMatrix::fieldNames,
00237         (BodyDoc("Returns the field names.\n"),
00238          RetDoc ("TVec of field names.\n")));
00239 
00240     declareMethod(
00241         rmm, "fieldName", &VMatrix::fieldName,
00242         (BodyDoc("Returns the field name for a given column.\n"),
00243          ArgDoc ("col", "column index.\n"),
00244          RetDoc ("Field name.\n")));
00245 
00246      declareMethod(
00247         rmm, "findFieldIndex", &VMatrix::fieldIndex,
00248         (BodyDoc("Returns the index of a field, or -1 if the field does not "
00249                  "exist.\n"),
00250          ArgDoc ("fname",
00251              "Field name of the field.\n"),
00252          RetDoc ("Index of the field (-1 if not found)\n")));
00253 
00254       declareMethod(
00255         rmm, "getFieldIndex", &VMatrix::remote_getFieldIndex,
00256         (BodyDoc("Returns the index of a field. "
00257                  "Throws an error if the field is not found.\n"),
00258          ArgDoc ("fname_or_num",
00259              "Field name or index (as a string) of the field.\n"),
00260          RetDoc ("Index of the field.\n")));
00261     
00262     declareMethod(
00263         rmm, "appendRow", &VMatrix::appendRow,
00264         (BodyDoc("Appends a row to the VMatrix.\n"),
00265          ArgDoc ("v", "Vec with values (row) to append.\n")));
00266 
00267     declareMethod(
00268         rmm, "appendRows", &VMatrix::appendRows,
00269         (BodyDoc("Appends rows to the VMatrix.\n"),
00270          ArgDoc ("rows", "A matrix containing the rows to append.\n")));
00271 
00272     declareMethod(
00273         rmm, "putRow", &VMatrix::putRow,
00274         (BodyDoc("Store a row into the VMatrix.\n"),
00275          ArgDoc ("i", "Index of the row being modified.\n"),
00276          ArgDoc ("v", "Vec with values (row) to store.\n")));
00277 
00278     declareMethod(
00279         rmm, "saveFieldInfos", &VMatrix::saveFieldInfos,
00280         (BodyDoc("Saves field names, etc. in metadatadir.\n")));
00281 
00282     declareMethod(
00283         rmm, "flush", &VMatrix::flush,
00284         (BodyDoc("Flush mods. to disk.\n")));
00285 
00286     declareMethod(
00287         rmm, "getBoundingBox", &VMatrix::getBoundingBox,
00288         (BodyDoc("Returns the (possibly enlarged) bounding box of the data."),
00289          ArgDoc ("extra_percent", "if non 0, then the box is enlarged in both ends\n"
00290                  "of every direction by that given percentage"),
00291          RetDoc ("bounding box as as a vector of (min,max) pairs")));         
00292 
00293     declareMethod(
00294         rmm, "fill", &VMatrix::fill,
00295         (BodyDoc("Appends fills the VMatrix with a constant value.\n"),
00296          ArgDoc ("value", "The fill value.\n")));
00297 
00298     declareMethod(
00299         rmm, "dot", &VMatrix::dot,
00300         (BodyDoc("dot product between row i1 and row i2, w/ inputsize first elements."),
00301          ArgDoc ("i1", "First row to consider."),
00302          ArgDoc ("i2", "Second row to consider."),
00303          ArgDoc ("inputsize", "nb. elements to consider."),
00304          RetDoc ("dot product")));         
00305 
00306     declareMethod(
00307         rmm, "saveAMAT", &VMatrix::saveAMAT,
00308         (BodyDoc("Saves this matrix as an .amat file."),
00309          ArgDoc ("amatfile", "Path of the file to create."),
00310          ArgDoc ("verbose", "output details?"),
00311          ArgDoc ("no_header", "save data only"),
00312          ArgDoc ("save_strings", "save string instead of real values")));
00313 
00314     declareMethod(
00315         rmm, "savePMAT", &VMatrix::remote_savePMAT,
00316         (BodyDoc("Saves this matrix as a .pmat file."),
00317          ArgDoc ("pmatfile", "Path of the file to create.")));
00318 
00319     declareMethod(
00320         rmm, "savePMAT_float", &VMatrix::remote_savePMAT_float,
00321         (BodyDoc("Saves this matrix as a .pmat file in float format."),
00322          ArgDoc ("pmatfile", "Path of the file to create.")));
00323 
00324     declareMethod(
00325         rmm, "saveDMAT", &VMatrix::saveDMAT,
00326         (BodyDoc("Saves this matrix as a .dmat directory."),
00327          ArgDoc ("dmatdir", "Path of the dir to create.")));
00328 
00329     declareMethod(
00330         rmm, "subMat", &VMatrix::subMat,
00331         (BodyDoc("Return a sub-matrix from a VMatrix\n"),
00332          ArgDoc ("i", "start row"),
00333          ArgDoc ("j", "start col"),
00334          ArgDoc ("l", "length"),
00335          ArgDoc ("w", "width"),
00336          RetDoc ("The sub-VMatrix")));
00337 
00338     declareMethod(
00339         rmm, "get", &VMatrix::get,
00340         (BodyDoc("Returns the element at position (i,j)\n"),
00341          ArgDoc ("i", "row"),
00342          ArgDoc ("j", "col"),
00343          RetDoc ("Value at (i,j)")));
00344 
00345 
00346     declareMethod(
00347         rmm, "getStats", &VMatrix::remote_getStats,
00348         (BodyDoc("Returns the unconditonal statistics for all fields\n"),
00349          RetDoc ("Stats vector")));
00350 
00351     declareMethod(
00352         rmm, "defineSizes", &VMatrix::defineSizes,
00353         (BodyDoc("Define this vmatrix's sizes\n"),
00354          ArgDoc ("inputsize", "inputsize"),
00355          ArgDoc ("targetsize", "targetsize"),
00356          ArgDoc ("weightsize", "weightsize"),
00357          ArgDoc ("extrasize", "extrasize")));
00358 
00359     declareMethod(
00360         rmm, "copySizesFrom", &VMatrix::copySizesFrom,
00361         (BodyDoc("Define this vmatrix's sizes from another vmatrix\n"),
00362          ArgDoc ("vm", "the other vmatrix")));
00363 
00364     declareMethod(
00365         rmm, "addStringMapping", static_cast<void (VMatrix::*)(int, string, real)>(&VMatrix::addStringMapping),
00366         (BodyDoc("Add or replace a string mapping for a column\n"),
00367          ArgDoc ("col", "column number"),
00368          ArgDoc ("str", "string value"),
00369          ArgDoc ("val", "numeric value")));
00370 
00371     declareMethod(
00372         rmm, "setStringMapping", &VMatrix::setStringMapping,
00373         (BodyDoc("Set the string->real mapping for a given column.\n"),
00374          ArgDoc ("col", "column number"),
00375          ArgDoc ("map", "map of string->real")));
00376 
00377     declareMethod(
00378         rmm, "getStringToRealMapping", &VMatrix::getStringToRealMapping,
00379         (BodyDoc("Get the string->real mapping for a given column.\n"),
00380          ArgDoc ("col", "column number"),
00381          RetDoc ("map of string->real")));
00382 
00383     declareMethod(
00384         rmm, "getRealToStringMapping", &VMatrix::getRealToStringMapping,
00385         (BodyDoc("Get the real->string mapping for a given column.\n"),
00386          ArgDoc ("col", "column number"),
00387          RetDoc ("map of real->string")));
00388 
00389     declareMethod(
00390         rmm, "setMetaInfoFrom", &VMatrix::setMetaInfoFrom,
00391         (BodyDoc("Set this vmatrix's meta-info from another vmatrix\n"),
00392          ArgDoc ("vm", "the other vmatrix")));
00393 
00394     declareMethod(
00395         rmm, "saveAllStringMappings", &VMatrix::saveAllStringMappings,
00396         (BodyDoc("Save this vmatrix's string mapping infos\n")));
00397 
00398 }
00399 
00400 
00402 // makeDeepCopyFromShallowCopy //
00404 void VMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00405 {
00406     inherited::makeDeepCopyFromShallowCopy(copies);
00407     deepCopyField(get_row,     copies);
00408     deepCopyField(dotrow_1,    copies);
00409     deepCopyField(dotrow_2,    copies);
00410     deepCopyField(field_stats, copies);
00411     deepCopyField(map_sr,      copies);
00412     deepCopyField(map_rs,      copies);
00413     deepCopyField(fieldinfos,  copies);
00414     deepCopyField(fieldstats,  copies);
00415 
00416     // TODO See if we can deep-copy a PStream (and what it means).
00417 }
00418 
00420 // init_map_sr //
00422 void VMatrix::init_map_sr() const
00423 {
00424     if (map_sr.length()==0 || map_sr.length() != width()) {
00425         map_sr.resize(width());
00426         map_rs.resize(width());
00427     }
00428 }
00429 
00430 
00432 // getFieldInfos //
00434 Array<VMField>& VMatrix::getFieldInfos() const
00435 {
00436     if(fieldinfos.size()==0 && hasMetaDataDir())
00437     {
00438         PPath fname =  getMetaDataDir() / "fieldnames";
00439         if(isfile(fname)) // file exists
00440             loadFieldInfos();
00441     }
00442 
00443     int ninfos = fieldinfos.size();
00444     int w = width();
00445     if(ninfos!=w && w > 0)
00446     {
00447         fieldinfos.resize(w);
00448         for(int j=ninfos; j<w; j++)
00449             fieldinfos[j] = VMField(tostring(j));
00450     }
00451 
00452     return fieldinfos;
00453 }
00454 
00456 // setFieldInfos //
00458 void VMatrix::setFieldInfos(const Array<VMField>& finfo) const
00459 {
00460     fieldinfos=finfo;
00461 }
00462 
00464 // hasFieldInfos //
00466 bool VMatrix::hasFieldInfos() const
00467 {
00468     if (fieldinfos.length() != width())
00469         return false;
00470     // If there are some field infos, we check them to see whether they are
00471     // default ones (i.e. 0, 1, ..., width-1), in which case 'false' is
00472     // returned.
00473     double x;
00474     for (int i = 0; i < width(); i++) {
00475         string name = fieldName(i);
00476         if (!pl_isnumber(name, &x)  || !is_equal(x, i))
00477             return true;
00478     }
00479     return false;
00480 }
00481 
00483 // unduplicateFieldNames //
00485 void VMatrix::unduplicateFieldNames()
00486 {
00487     map<string,vector<int> > mp;
00488     for(int i=0;i<width();i++)
00489         mp[getFieldInfos(i).name].push_back(i);
00490     map<string,vector<int> >::iterator it;
00491     for(it=mp.begin();it!=mp.end();++it)
00492         if(it->second.size()!=1)
00493         {
00494             vector<int> v=it->second;
00495             for(unsigned int j=0;j<v.size();j++)
00496                 fieldinfos[v[j]].name+="."+tostring(j);
00497         }
00498 }
00499 
00501 // fieldNames //
00503 TVec<string> VMatrix::fieldNames() const
00504 {
00505     int d = width();
00506     if (d < 0)
00507         return TVec<string>();
00508     TVec<string> names(d);
00509     for(int i=0; i<d; i++)
00510         names[i] = fieldName(i);
00511     return names;
00512 }
00513 
00514 TVec<string> VMatrix::inputFieldNames() const
00515 { return fieldNames().subVec(0,inputsize_); }
00516 
00517 TVec<string> VMatrix::targetFieldNames() const
00518 { return fieldNames().subVec(inputsize_, targetsize_); }
00519 
00520 TVec<string> VMatrix::weightFieldNames() const
00521 { return fieldNames().subVec(inputsize_+targetsize_, weightsize_); }
00522 
00523 TVec<string> VMatrix::extraFieldNames() const
00524 { return fieldNames().subVec(inputsize_+targetsize_+weightsize_,extrasize_); }
00525 
00527 // fieldIndex //
00529 int VMatrix::fieldIndex(const string& fieldname) const
00530 {
00531     Array<VMField>& infos = getFieldInfos();
00532     for(int i=0; i<width(); i++)
00533         if(infos[i].name==fieldname)
00534             return i;
00535     return -1;
00536 }
00537 
00539 // getFieldIndex //
00541 int VMatrix::getFieldIndex(const string& fieldname_or_num, bool error) const
00542 {
00543     int i = fieldIndex(fieldname_or_num);
00544     if(i==-1 && pl_islong(fieldname_or_num)) {
00545         i = toint(fieldname_or_num);
00546 
00547         // Now ensure that THE WHOLE FIELD has been converted, because we want
00548         // to ensure that stuff that starts with a number but contains other
00549         // things is not silently converted to the starting number
00550         if (tostring(i) != fieldname_or_num)
00551             i = -1;
00552     }
00553     if ((i < 0 || i >= width()) && error)
00554         PLERROR("In VMatrix::getFieldIndex - Asked for an invalid column number: '%s'",
00555                 fieldname_or_num.c_str());
00556     return i;
00557 }
00558 
00560 // build_ //
00562 void VMatrix::build_()
00563 {
00564     if(!metadatadir.isEmpty())
00565         setMetaDataDir(metadatadir); // make sure we perform all necessary operations
00566     if(mtime_update == time_t(-1))
00567         updateMtime(0);
00568     else if(mtime_update!=0)
00569         updateMtime(mtime_update);
00570 }
00571 
00573 // build //
00575 void VMatrix::build()
00576 {
00577     inherited::build();
00578     build_();
00579 }
00580 
00582 // printFieldInfo //
00584 void VMatrix::printFieldInfo(PStream& out, int fieldnum, bool print_binning) const
00585 {
00586     VMField fi = getFieldInfos(fieldnum);
00587     StatsCollector& s = getStats(fieldnum);
00588 
00589     out << "Field #" << fieldnum << ":  ";
00590     out << fi.name << "\t type: ";
00591     switch(fi.fieldtype)
00592     {
00593     case VMField::UnknownType:
00594         out << "UnknownType\n";
00595         break;
00596     case VMField::Continuous:
00597         out << "Continuous\n";
00598         break;
00599     case VMField::DiscrGeneral:
00600         out << "DiscrGeneral\n";
00601         break;
00602     case VMField::DiscrMonotonic:
00603         out << "DiscrMonotonic\n";
00604         break;
00605     case VMField::DiscrFloat:
00606         out << "DiscrFloat\n";
00607         break;
00608     case VMField::Date:
00609         out << "Date\n";
00610         break;
00611     default:
00612         PLERROR("Can't write name of type");
00613     }
00614 
00615     map<real,StatsCollectorCounts>::const_iterator it = s.counts.begin();
00616     map<real,StatsCollectorCounts>::const_iterator countsend = s.counts.end();
00617     int n_values = 0;
00618     //some value(FLT_MAX, meaby others) are used for others purpose.
00619     //We must not cont then.
00620     while(it!=countsend)
00621     {
00622         real val = it->first;
00623         const StatsCollectorCounts& co = it->second;
00624         string str = getValString(fieldnum, val);
00625         if(co.n>0)
00626             n_values++;
00627         ++it;
00628         }
00629     char plus = ' ';
00630     if (n_values==s.maxnvalues)
00631         plus = '+';
00632 
00633     out << "nmissing: " << s.nmissing() << '\n';
00634     out << "nnonmissing: " << s.nnonmissing() << '\n';
00635     out << "sum: " << s.sum() << '\n';
00636     out << "mean: " << s.mean() << '\n';
00637     out << "stddev: " << s.stddev() << '\n';
00638     out << "min: " << s.min() << '\n';
00639     out << "max: " << s.max() << '\n';
00640     out << "ndiffvalue: " << n_values << plus << '\n';
00641 
00642     if(!s.counts.empty() && print_binning)
00643     {
00644         out << "\nCOUNTS: \n";
00645         map<real,StatsCollectorCounts>::const_iterator it = s.counts.begin();
00646         map<real,StatsCollectorCounts>::const_iterator countsend = s.counts.end();
00647         while(it!=countsend)
00648         {
00649             real val = it->first;
00650             const StatsCollectorCounts& co = it->second;
00651             string str = getValString(fieldnum, val);
00652             ostringstream os;
00653             os.setf(ios::left);
00654             os << "  "          << setw(12) << val
00655                << "  "          << setw(12) << str
00656                << "  n="        << setw(10) << co.n
00657                << "  nbelow="   << setw(10) << co.nbelow
00658                << "  sumbelow=" << setw(10) << co.sum
00659                << endl;
00660             out << os.str();
00661             ++it;
00662         }
00663     }
00664     out << endl << endl;
00665 }
00666 
00668 // printFieldInfo //
00670 void VMatrix::printFieldInfo(PStream& out, const string& fieldname_or_num,
00671                              bool print_binning) const
00672 {
00673     printFieldInfo(out, getFieldIndex(fieldname_or_num), print_binning);
00674 }
00675 
00677 // printFields //
00679 void VMatrix::printFields(PStream& out) const
00680 {
00681     for(int j=0; j<width(); j++)
00682     {
00683         printFieldInfo(out,j);
00684         out << "-----------------------------------------------------" << endl;
00685     }
00686 }
00687 
00689 // getExample //
00691 void VMatrix::getExample(int i, Vec& input, Vec& target, real& weight)
00692 {
00693     if(inputsize_<0)
00694         PLERROR("In VMatrix::getExample, inputsize_ not defined for this vmat");
00695     input.resize(inputsize_);
00696     getSubRow(i,0,input);
00697     if(targetsize_<0)
00698         PLERROR("In VMatrix::getExample, targetsize_ not defined for this vmat");
00699     target.resize(targetsize_);
00700     if (targetsize_ > 0) {
00701         getSubRow(i,inputsize_,target);
00702     }
00703 
00704     if(weightsize_==0)
00705         weight = 1;
00706     else if(weightsize_<0)
00707         PLERROR("In VMatrix::getExample, weightsize_ not defined for this vmat");
00708     else if(weightsize_>1)
00709         PLERROR("In VMatrix::getExample, weightsize_ >1 not supported by this call");
00710     else
00711         weight = get(i,inputsize_+targetsize_);
00712 }
00713 
00715 // remote_getExample //
00717 boost::tuple<Vec, Vec, real> VMatrix::remote_getExample(int i)
00718 {
00719     Vec input, target;
00720     real weight;
00721     getExample(i, input, target, weight);
00722     return boost::tuple<Vec, Vec, real>(input, target, weight);
00723 }
00724 
00726 // getExamples //
00728 void VMatrix::getExamples(int i_start, int length, Mat& inputs, Mat& targets,
00729                           Vec& weights, Mat* extras, bool allow_circular)
00730 {
00731     inputs.resize(length, inputsize());
00732     targets.resize(length, targetsize());
00733     weights.resize(length);
00734     if (extras)
00735         extras->resize(length, extrasize());
00736     Vec input, target, extra;
00737     int total_length = this->length();
00738     PLASSERT( i_start < total_length );
00739     for (int k = 0; k < length; k++) {
00740         input = inputs(k);
00741         target = targets(k);
00742         int idx = i_start + k;
00743         if (allow_circular)
00744             idx %= total_length;
00745         PLASSERT( idx >= 0 && idx < total_length );
00746         getExample(idx, input, target, weights[k]);
00747         if (extras) {
00748             extra = (*extras)(k);
00749             getExtra(idx, extra);
00750         }
00751     }
00752 }
00753 
00755 // getExtra //
00757 void VMatrix::getExtra(int i, Vec& extra)
00758 {
00759     if(inputsize_<0 || targetsize_<0 || weightsize_<0 || extrasize_<0)
00760         PLERROR("In VMatrix::getExtra, sizes not properly defined for this vmat");
00761 
00762     extra.resize(extrasize_);
00763     if(extrasize_>0)
00764         getSubRow(i,inputsize_+targetsize_+weightsize_, extra);
00765 }
00766 
00767 Vec VMatrix::remote_getExtra(int i)
00768 {
00769     Vec extra;
00770     getExtra(i, extra);
00771     return extra;
00772 }
00773 
00774 
00776 // computeStats //
00778 void VMatrix::computeStats()
00779 {
00780     fieldstats = Array<VMFieldStat>(width());
00781     Vec row(width());
00782     for(int i=0; i<length(); i++)
00783     {
00784         getRow(i,row);
00785         for(int j=0; j<width(); j++)
00786             fieldstats[j].update(row[j]);
00787     }
00788 }
00789 
00791 // loadStats //
00793 void VMatrix::loadStats(const PPath& filename)
00794 {
00795     PStream in = openFile(filename, PStream::raw_ascii, "r");
00796     int nfields;
00797     in >> nfields;
00798     if(nfields!=width())
00799         PLWARNING("In VMatrix::loadStats - nfields differs from VMat width");
00800 
00801     fieldstats.resize(nfields);
00802     for(int j=0; j<fieldstats.size(); j++)
00803         fieldstats[j].read(in);
00804 }
00805 
00807 // saveStats //
00809 void VMatrix::saveStats(const PPath& filename) const
00810 {
00811     PStream out = openFile(filename, PStream::raw_ascii, "w");
00812     out << fieldstats.size() << endl;
00813     for(int j=0; j<fieldstats.size(); j++)
00814     {
00815         fieldstats[j].write(out);
00816         out << endl;
00817     }
00818 }
00819 
00821 // declareField //
00823 void VMatrix::declareField(int fieldindex, const string& fieldname, VMField::FieldType fieldtype)
00824 {
00825     getFieldInfos(fieldindex) = VMField(fieldname,fieldtype);
00826 }
00827 
00829 // declareFieldNames //
00831 void VMatrix::declareFieldNames(const TVec<string>& fnames)
00832 {
00833     if(fnames.length()!=width())
00834         PLERROR("In VMatrix::declareFieldNames length of fnames differs from width() of VMatrix");
00835     for(int i=0; i<fnames.length(); i++)
00836         declareField(i,fnames[i]);
00837 }
00838 
00840 // saveFieldInfos //
00842 void VMatrix::saveFieldInfos() const
00843 {
00844     // check if we need to save the fieldinfos
00845     if(fieldinfos.size() > 0) {
00846         Array<VMField> current_fieldinfos;
00847         try{
00848             current_fieldinfos = getSavedFieldInfos();
00849         }catch(PLearnError){}
00850         if (current_fieldinfos != fieldinfos) {
00851 
00852             // Ensure that the metadatadir exists
00853             if(!force_mkdir(getMetaDataDir()))
00854                 PLERROR("In VMatrix::saveFieldInfos: could not create directory %s",
00855                         getMetaDataDir().absolute().c_str());
00856 
00857             PPath filename = getMetaDataDir() / "fieldnames";
00858             PStream out = openFile(filename, PStream::raw_ascii, "w");
00859             for(int i= 0; i < fieldinfos.length(); ++i)
00860                 out << fieldinfos[i].name << '\t' << fieldinfos[i].fieldtype << endl;
00861         }
00862     }
00863 
00864     // check if we need to save the sizes
00865     int inp, targ, weight, extr;
00866     bool sizes_exist = getSavedSizes(inp, targ, weight, extr);
00867     if ((! sizes_exist && (inputsize_ != -1 || targetsize_ != -1 || weightsize_ != -1 || extrasize_ > 0)) ||
00868         (sizes_exist && (inp != inputsize_ || targ != targetsize_ || weight != weightsize_ || extr!=extrasize_)))
00869     {
00870         // Slightly hackish phenomenon :: if the sizes file doesn't previously
00871         // exist and we cannot write them, THIS IS NOT AN ERROR.  In this case,
00872         // just catch the error and continue
00873         try {
00874             // Ensure that the metadatadir exists
00875             if(!force_mkdir(getMetaDataDir()))
00876                 PLERROR("In VMatrix::saveFieldInfos: could not create directory %s",
00877                         getMetaDataDir().absolute().c_str());
00878 
00879             PPath filename = getMetaDataDir() / "sizes";
00880             PStream out = openFile(filename, PStream::plearn_ascii, "w");
00881             out << inputsize_ << targetsize_ << weightsize_ << extrasize_ << endl;
00882         }
00883         catch (const PLearnError&) {
00884             if (sizes_exist)
00885                 throw;
00886         }
00887     }
00888 }
00889 
00891 // loadFieldInfos //
00893 void VMatrix::loadFieldInfos() const
00894 {
00895     Array<VMField> current_fieldinfos = getSavedFieldInfos();
00896     setFieldInfos(current_fieldinfos);
00897 
00898     // Update only if they can successfully be read from the saved metadata
00899     // and they don't already exist in the VMatrix
00900     int inp, tar, weight, extr;
00901     if (inputsize_ == -1 && targetsize_ == -1 && weightsize_ == -1 
00902         && getSavedSizes(inp,tar,weight,extr))
00903     {
00904         inputsize_  = inp;
00905         targetsize_ = tar;
00906         weightsize_ = weight;
00907         extrasize_  = extr;
00908     }
00909 }
00910 
00912 // getSavedFieldInfos //
00914 Array<VMField> VMatrix::getSavedFieldInfos() const
00915 {
00916     PPath filename = getMetaDataDir() / "fieldnames";
00917     if (!isfile(filename)) // no current fieldinfos saved
00918     {
00919         Array<VMField> no_fieldinfos(0);
00920         return no_fieldinfos;
00921     }
00922     PStream in = openFile(filename, PStream::raw_ascii, "r");
00923     int w = width();
00924     Array<VMField> current_fieldinfos(w);
00925     for(int i=0; i<w; ++i)
00926     {
00927         string line = in.getline();
00928         vector<string> v(split(line));
00929         switch(v.size())
00930         {
00931         case 1: current_fieldinfos[i] = VMField(v[0]); break;
00932         case 2: current_fieldinfos[i] = VMField(v[0], VMField::FieldType(toint(v[1]))); break;
00933         default: PLERROR("In VMatrix::getSavedFieldInfos Format not recognized in file %s.\n"
00934                          "Each line should be '<name> {<type>}'.\n"
00935                          "Got: '%s'. Check for a space in <name>",
00936                          filename.absolute().c_str(),line.c_str());
00937         }
00938     }
00939     return current_fieldinfos;
00940 }
00941 
00943 // getSavedSizes //
00945 bool VMatrix::getSavedSizes(int& inputsize, int& targetsize, int& weightsize, int& extrasize) const
00946 {
00947     PPath filename = getMetaDataDir() / "sizes";
00948     inputsize = targetsize = weightsize = extrasize = -1;
00949     if (isfile(filename))
00950     {
00951         PStream in = openFile(filename, PStream::plearn_ascii, "r");
00952         // perr << "In loadFieldInfos() loading sizes from " << filename << endl;
00953         in >> inputsize >> targetsize >> weightsize;
00954         in.skipBlanks();
00955         extrasize = 0;
00956         if(in.peek()!=EOF)
00957             in >> extrasize;
00958         return true;                         // Successfully loaded
00959     }
00960     return false;
00961 }
00962 
00963 
00965 // resolveFieldInfoLink //
00967 string VMatrix::resolveFieldInfoLink(const PPath& target, const PPath& source)
00968 {
00969     PPath contents = removeblanks( loadFileAsString(source) );
00970     if ( contents == source )
00971         return "ERROR";
00972 
00973     if( isdir(contents) )
00974     {
00975         if ( isfile(contents/target+".lnk") )
00976             return resolveFieldInfoLink(target,contents/target+".lnk");
00977 
00978         else if ( isfile(contents/target) )
00979             return contents/target;
00980 
00981         else if( isfile(contents/"__default.lnk") )
00982             return resolveFieldInfoLink(target, contents/"__default.lnk");
00983 
00984         // assume target is there, but file is empty thus inexistant
00985         else return contents/target;
00986     }
00987 
00988     else if( contents.extension() == "lnk" )
00989         return resolveFieldInfoLink(target,contents);
00990 
00991     else return contents;
00992 }
00993 
00995 // getSFIFDirectory //
00997 PPath VMatrix::getSFIFDirectory() const
00998 {
00999     PPath meta = getMetaDataDir();
01000     if (meta.empty())
01001         PLERROR("%s: cannot have a SFIFDirectory if there is no metadatadir",
01002                 __FUNCTION__);
01003     return meta / "FieldInfo";
01004 }
01005 
01007 // setSFIFFilename //
01009 void VMatrix::setSFIFFilename(int col, string ext, const PPath& filepath)
01010 {
01011     setSFIFFilename(fieldName(col),ext,filepath);
01012 }
01013 
01014 void VMatrix::setSFIFFilename(string fieldname, string ext, const PPath& filepath)
01015 {
01016     PPath target      = makeFileNameValid(fieldname+ext);
01017     PPath normalfname = getSFIFDirectory() / target;
01018     PPath normalfname_lnk = normalfname + ".lnk";
01019 
01020     rm(normalfname_lnk);
01021     if(filepath==normalfname || filepath=="")
01022     {
01023         rm(normalfname_lnk);
01024         return;
01025     }
01026 
01027     force_mkdir_for_file(normalfname);
01028     PStream o = openFile(normalfname_lnk, PStream::raw_ascii, "w");
01029     o<<filepath<<endl;
01030 }
01031 
01033 // getSFIFFilename //
01035 PPath VMatrix::getSFIFFilename(int col, string ext)
01036 {
01037     return getSFIFFilename(fieldName(col),ext);
01038 }
01039 
01040 PPath VMatrix::getSFIFFilename(string fieldname, string ext)
01041 {
01042     PPath  target           = makeFileNameValid(fieldname+ext);
01043     PPath  normalfname      = getSFIFDirectory() / target;
01044     string defaultlinkfname = getSFIFDirectory() / "__default.lnk";
01045 
01046     if(isfile(normalfname))
01047         return normalfname;
01048     else if(isfile(normalfname+".lnk"))
01049         return resolveFieldInfoLink(target, normalfname+".lnk");
01050     else if(isfile(defaultlinkfname))
01051         return resolveFieldInfoLink(target, defaultlinkfname);
01052     // assume target is here, but file is empty thus inexistant
01053     else return normalfname;
01054 }
01055 
01057 // isSFIFDirect //
01059 bool VMatrix::isSFIFDirect(int col, string ext)
01060 {
01061     return isSFIFDirect(fieldName(col), ext);
01062 }
01063 
01064 bool VMatrix::isSFIFDirect(string fieldname, string ext)
01065 {
01066     PPath target      = makeFileNameValid(fieldname+ext);
01067     PPath normalfname = getSFIFDirectory() / target;
01068     return getSFIFFilename(fieldname,ext) == normalfname;
01069 }
01070 
01072 // addStringMapping //
01074 void VMatrix::addStringMapping(int col, string str, real val)
01075 {
01076     init_map_sr();
01077     map_sr[col][str]=val;
01078     map_rs[col][val]=str;
01079 }
01080 
01082 // addStringMapping //
01084 real VMatrix::addStringMapping(int col, string str)
01085 {
01086     init_map_sr();
01087     map<string,real>& m = map_sr[col];
01088     map<string,real>::iterator it = m.find(str);
01089 
01090     real val = 0;
01091     if(it != m.end()) // str was found in map
01092         val = it->second;
01093     else // str not found in map: add a new mapping
01094     {
01095         val = - real(m.size()) - 100;
01096         addStringMapping(col, str, val);
01097     }
01098     return val;
01099 }
01100 
01102 // removeAllStringMappings //
01104 void VMatrix::removeAllStringMappings()
01105 {
01106     init_map_sr();
01107     for(int i=0;i<width();i++)
01108     {
01109         map_sr[i].clear();
01110         map_rs[i].clear();
01111     }
01112 }
01113 
01115 // removeColumnStringMappings //
01117 void VMatrix::removeColumnStringMappings(int c)
01118 {
01119     init_map_sr();
01120     map_sr[c].clear();
01121     map_rs[c].clear();
01122 }
01123 
01125 // saveAllStringMappings //
01127 void VMatrix::saveAllStringMappings()
01128 {
01129     PPath fname;
01130     map<string, real> the_map;
01131     for(int i=0;i<width();i++)
01132     {
01133         the_map = getStringToRealMapping(i);
01134         if (!the_map.empty()) {
01135             fname = getSFIFFilename(i,".smap");
01136             saveStringMappings(i, fname, &the_map);
01137         }
01138     }
01139 }
01140 
01142 // saveStringMappings //
01144 void VMatrix::saveStringMappings(int col, const PPath& fname, map<string, real>* str_to_real)
01145 {
01146     map<string, real> the_map;
01147     if (!str_to_real) {
01148         the_map = getStringToRealMapping(col);
01149         str_to_real = &the_map;
01150     }
01151     if(str_to_real->empty())
01152     {
01153         rm(fname);
01154         return;
01155     }
01156     force_mkdir_for_file(fname);
01157     PStream o = openFile(fname, PStream::plearn_ascii, "w");
01158     for(map<string,real>::iterator it  = str_to_real->begin();
01159         it != str_to_real->end();   ++it)
01160         o << it->first << it->second << endl;
01161 }
01162 
01164 // removeStringMapping //
01166 void VMatrix::removeStringMapping(int col, string str)
01167 {
01168     init_map_sr();
01169     map<string,real>::iterator sriterator;
01170     // Check if the mapping actually exists.
01171     if((sriterator = map_sr[col].find(str)) == map_sr[col].end())
01172         return;
01173     real val = map_sr[col][str];
01174     map_sr[col].erase(sriterator);
01175     map_rs[col].erase(map_rs[col].find(val));
01176 }
01177 
01179 // setStringMapping //
01181 void VMatrix::setStringMapping(int col, const map<string,real> & zemap)
01182 {
01183     init_map_sr();
01184     map_sr[col]=zemap;
01185     map_rs[col].clear();
01186     for(map<string,real>::iterator it = map_sr[col].begin();it!=map_sr[col].end();++it)
01187         map_rs[col][it->second]=it->first;
01188 }
01189 
01191 // deleteStringMapping //
01193 void VMatrix::deleteStringMapping(int col)
01194 {
01195     init_map_sr();
01196     if(col>=map_sr.size() ||
01197        col>=map_rs.size())
01198         PLERROR("deleteStringMapping : out of bounds for col=%i in string mapping array (size=%i).\n Current VMatrix\nclass"\
01199                 "is '%s' (or maybe derivated class?). be sure to set\n map_sr(rs) to appropriate sizes as soon as you know the width of the matrix\n"\
01200                 "(in constructor or elsewhere)",col,map_sr.size(),classname().c_str());
01201     map_sr[col].clear();
01202     map_rs[col].clear();
01203 }
01204 
01206 // getValString //
01208 string VMatrix::getValString(int col, real val) const
01209 {
01210     if(is_missing(val))
01211         return "";
01212     init_map_sr();
01213     if(map_rs[col].find(val)==map_rs[col].end())
01214         return "";
01215     else return map_rs[col][val];
01216 }
01217 
01219 // getStringVal //
01221 real VMatrix::getStringVal(int col,const string & str) const
01222 {
01223     if(map_sr.length()==0 || map_sr[col].find(str)==map_sr[col].end())
01224         return MISSING_VALUE;
01225     else return map_sr[col][str];
01226 }
01227 
01229 // getString //
01231 string VMatrix::getString(int row,int col) const
01232 {
01233     real val = get(row,col);
01234     string str = getValString(col, val);
01235     if (str == "")
01236         // There is no string mapping associated to this value.
01237         return tostring(val);
01238     else
01239         return str;
01240 }
01241 
01243 // getRowAsStrings //
01245 void VMatrix::getRowAsStrings(int i, TVec<string>& v_str) const {
01246     v_str.resize(width());
01247     for (int j = 0; j < width(); j++)
01248         v_str[j] = getString(i, j);
01249 }
01250 
01251 
01253 // getStringToRealMapping //
01255 const map<string,real>& VMatrix::getStringToRealMapping(int col) const {
01256     init_map_sr();
01257     return map_sr[col];
01258 }
01259 
01261 // getRealToStringMapping //
01263 const map<real,string>& VMatrix::getRealToStringMapping(int col) const {
01264     init_map_sr();
01265     return map_rs[col];
01266 }
01267 
01269 // setMetaDataDir //
01271 void VMatrix::setMetaDataDir(const PPath& the_metadatadir)
01272 {
01273     if (the_metadatadir.isEmpty())
01274         PLERROR("In VMatrix::setMetaDataDir - Called setMetaDataDir with an empty PPath");
01275     metadatadir = the_metadatadir.absolute() / "";
01276     // We do not create the metadata directory here anymore.
01277     // This is to prevent the proliferation of useless directories.
01278     // A VMatrix's subclass should now create the metadatadir itself if it needs it.
01279 
01280     // Load string mappings from the metadatadir.
01281     loadAllStringMappings();
01282 }
01283 
01285 // getDictionary //
01287 PP<Dictionary> VMatrix::getDictionary(int col) const
01288 {
01289     return 0;
01290 }
01291 
01293 // getValues //
01295 void VMatrix::getValues(int row, int col, Vec& values) const 
01296 { 
01297     values.resize(0);
01298 }
01299 
01300 void VMatrix::getValues(const Vec& input, int col, Vec& values) const 
01301 { 
01302     values.resize(0);
01303 }
01304 
01305 
01307 // copySizesFrom //
01309 void VMatrix::copySizesFrom(const VMat& m) {
01310     defineSizes(m->inputsize(), m->targetsize(), m->weightsize(), m->extrasize());
01311 }
01312 
01314 // setMetaInfoFrom //
01316 void VMatrix::setMetaInfoFrom(const VMatrix* vm)
01317 {
01318     updateMtime(vm->getMtime());
01319 
01320     // copy length and width from vm if not set
01321     if(length_<0)
01322         length_ = vm->length();
01323     if(width_<0)
01324         width_ = vm->width();
01325 
01326     // Copy sizes from vm if not set and they do not conflict with the width.
01327     int current_w = max(0, inputsize_) + max(0, targetsize_) +
01328                     max(0, weightsize_) + max(0, extrasize_);
01329     int is = vm->inputsize();
01330     if(inputsize_<0 && is>=0) {
01331         if (is + current_w <= width_) {
01332             inputsize_ = is;
01333             current_w += is;
01334         }
01335     }
01336     int ts = vm->targetsize();
01337     if(targetsize_<0 && ts>=0) {
01338         if (ts + current_w <= width_) {
01339             targetsize_ = ts;
01340             current_w += ts;
01341         }
01342     }
01343     int ws = vm->weightsize();
01344     if(weightsize_<0 && ws>=0) {
01345         if (ws + current_w <= width_) {
01346             // We must also ensure the total sum of sizes (if available)
01347             // will match the width. Otherwise we may end up with sizes
01348             // conflicting with the width.
01349             if (inputsize_ < 0 || targetsize_ < 0 || extrasize_ < 0 ||
01350                 inputsize_ + targetsize_ + extrasize_ + ws == width_)
01351             {
01352                 weightsize_ = ws;
01353                 current_w += ws;
01354             }
01355         }
01356     }
01357     int es = vm->extrasize();
01358     if(extrasize_<=0 && es>=0) {
01359         if (es + current_w <= width_) {
01360             // Same as above.
01361             if (inputsize_ < 0 || targetsize_ < 0 || weightsize_ < 0 ||
01362                 inputsize_ + targetsize_ + weightsize_ + es == width_)
01363             {
01364                 extrasize_ = es;
01365                 current_w += es;
01366             }
01367         }
01368     }
01369 
01370     // Fill missing size if possible, also display warning when sizes are not
01371     // compatible with the width.
01372     computeMissingSizeValue(false);
01373 
01374     // Copy fieldnames from vm if not set and they look good.
01375     bool same_fields_as_source =
01376         (!hasFieldInfos() && (width() == vm->width()) && vm->hasFieldInfos());
01377     if(same_fields_as_source)
01378         setFieldInfos(vm->getFieldInfos());
01379 
01380     // Copy string <-> real mappings for fields which have the same name (or for
01381     // all fields if it looks like the fields are the same as the source).
01382     TVec<string> fnames = fieldNames();
01383     for (int i = 0; i < width_; i++) {
01384         int vm_index = -1;
01385         if (same_fields_as_source)
01386             vm_index = i;
01387         else if (!pl_isnumber(fnames[i]))
01388             vm_index = vm->fieldIndex(fnames[i]);
01389         if (vm_index >= 0)
01390             // The source VMatrix has a field with the same name (which is not a
01391             // number): we can get its string mapping.
01392             setStringMapping(i, vm->getStringToRealMapping(vm_index));
01393     }
01394 
01395     //we save it now in case the program crash
01396     if(hasMetaDataDir())
01397         saveFieldInfos();
01398 }
01399 
01401 // looksTheSameAs //
01403 bool VMatrix::looksTheSameAs(const VMat& m) {
01404     return !(
01405         this->width()      != m->width()
01406         || this->length()     != m->length()
01407         || this->inputsize()  != m->inputsize()
01408         || this->weightsize() != m->weightsize()
01409         || this->targetsize() != m->targetsize()
01410         || this->extrasize()  != m->extrasize() );
01411 }
01412 
01414 // compatibleSizeError //
01416 void VMatrix::compatibleSizeError(const VMat& m, const string& extra_msg) {
01417 #define MY_PRINT_ERROR_MST(NAME) PLERROR("In VMatrix::compatibleSizeError " \
01418         " - in class %s - The matrices are not compatible!\n"               \
01419         "m1."#NAME"=%d and m2."#NAME"=%d. \n%s",                            \
01420         classname().c_str(), this->NAME(), m->NAME(), extra_msg.c_str());
01421 
01422     if(this->width()      != m->width())
01423         MY_PRINT_ERROR_MST(width)
01424     else if(this->inputsize()  != m->inputsize())
01425         MY_PRINT_ERROR_MST(inputsize)
01426     else if(this->weightsize() != m->weightsize())
01427         MY_PRINT_ERROR_MST(weightsize)
01428     else if(this->targetsize() != m->targetsize())
01429         MY_PRINT_ERROR_MST(targetsize)
01430     else if(this->extrasize()  != m->extrasize() )
01431         MY_PRINT_ERROR_MST(extrasize)
01432 #undef MY_PRINT_ERROR_MST
01433 }
01434 
01436 // lockMetaDataDir //
01438 void VMatrix::lockMetaDataDir(time_t max_lock_age, bool verbose) const
01439 {
01440 #ifndef DISABLE_VMATRIX_LOCK
01441     if(!hasMetaDataDir())
01442         PLERROR("In VMatrix::lockMetaDataDir() subclass %s -"
01443                 " metadatadir was not set", classname().c_str());
01444     if(lockf_.good()) // Already locked by this object!
01445         PLERROR("VMatrix::lockMetaDataDir() subclass %s -"
01446                 " called while already locked by this object.",
01447                 classname().c_str());
01448     if(!pathexists(metadatadir))
01449         force_mkdir(metadatadir);
01450 
01451     PPath lockfile = metadatadir / ".lock";
01452     while (isfile(lockfile) && (max_lock_age == 0 || mtime(lockfile) + max_lock_age > time(0))) {
01453         // There is a lock file, and it is not older than 'max_lock_age'.
01454         string bywho;
01455         try{ 
01456             PStream st = openFile(lockfile, PStream::raw_ascii, "r", false);
01457             if(st.good())
01458                 st.read(bywho, streamsize(filesize(lockfile)));
01459         }
01460         catch(const PLearnError& e) {
01461             PLERROR("In VMatrix::lockMetaDataDir - Catching exceptions is"
01462                     " dangerous in PLearn (memory"
01463                     " leaks may occur), thus I prefer to stop here. "
01464                     " Comment this line if you don't care."
01465                     " The error message is: %s",e.message().c_str());
01466             bywho = "UNKNOWN (could not read .lock file)" ;
01467         } catch(...) {
01468             PLERROR("In VMatrix::lockMetaDataDir - Catching exceptions is dangerous in PLearn (memory "
01469                     "leaks may occur), thus I prefer to stop here. Comment this line if you don't care.");
01470             bywho = "UNKNOWN (could not read .lock file)";
01471         }
01472 
01473         if (verbose)
01474             perr << "Waiting for .lock in directory " << metadatadir
01475                  << " created by " << bywho << endl;
01476         sleep(uniform_multinomial_sample(10) + 1); // Random wait for more safety.
01477     }
01478     lockf_ = openFile(lockfile, PStream::raw_ascii, "w");
01479     string lock_content = "host " + hostname() + ", pid " + tostring(getPid()) + ", user " + getUser();
01480     lockf_ << lock_content;
01481     lockf_.flush();
01482 #endif
01483 }
01484 
01486 // unlockMetaDataDir //
01488 void VMatrix::unlockMetaDataDir() const
01489 {
01490 #ifndef DISABLE_VMATRIX_LOCK
01491     if(!lockf_)
01492         PLERROR("In VMatrix::unlockMetaDataDir() was called while no lock is held by this object");
01493     lockf_ = PStream();   // Release the lock.
01494     PPath lockfile = metadatadir / ".lock";
01495     rm(lockfile); // Remove the file.
01496 #endif
01497 }
01498 
01500 // getMetaDataDir //
01502 PPath VMatrix::getMetaDataDir() const
01503 {
01504     // TODO Remove ?
01505     //  if(!hasMetaDataDir())
01506     //  PLERROR("In VMatrix::getMetaDataDir(): metadatadir was not set");
01507     return metadatadir;
01508 }
01509 
01511 // loadAllStringMappings //
01513 void VMatrix::loadAllStringMappings()
01514 {
01515     if (! hasMetaDataDir() || ! isdir(getSFIFDirectory()))
01516         return;
01517     
01518     for(int i=0;i<width();i++)
01519         loadStringMapping(i);
01520 }
01521 
01523 // loadStringMapping //
01525 void VMatrix::loadStringMapping(int col)
01526 {
01527     if(!hasMetaDataDir())
01528         return;
01529     PPath fname = getSFIFFilename(col,".smap");
01530     init_map_sr();
01531     string SFIFdir= getSFIFDirectory();
01532     if(!pathexists(SFIFdir))
01533         force_mkdir(SFIFdir);
01534     if(!isfile(fname))
01535         return;
01536 
01537     deleteStringMapping(col);
01538 
01539     // smap file exists, open it
01540     PStream f = openFile(fname, PStream::plearn_ascii);
01541 
01542     // TODO Remove ?
01543 #if 0
01544     string pref;
01545     f>>pref;
01546     if(string(pref)!="#SMAP")
01547         PLERROR( string("File "+fname+" is not a valid String mapping file.\nShould start with #SMAP on first line (this is to prevent inopportunely overwritting another type of file)").c_str());
01548 #endif
01549 
01550     while(f.good())
01551     {
01552         string s;
01553         real val;
01554         f >> s >> val;
01555         if(f.good())
01556         {
01557             map_sr[col][s]   = val;
01558             map_rs[col][val] = s;
01559             f.skipBlanks();
01560         }
01561     }
01562 }
01563 
01565 // copyStringMappingsFrom //
01567 void VMatrix::copyStringMappingsFrom(const VMat& source) {
01568     if (width_ != source->width()) {
01569         PLERROR("In VMatrix::copyStringMappingsFrom - The source VMatrix doesn't have the same width");
01570     }
01571     map_rs.resize(width_);
01572     map_sr.resize(width_);
01573     for (int i = 0; i < width_; i++) {
01574         setStringMapping(i, source->getStringToRealMapping(i));
01575     }
01576 }
01577 
01579 // getStats //
01581 TVec<StatsCollector> VMatrix::getStats(bool progress_bar) const
01582 {
01583     if(!field_stats)
01584         field_stats = getPrecomputedStatsFromFile("stats.psave", 2000, 
01585                                                   progress_bar);
01586     return field_stats;
01587 }
01588 
01590 // updateMtime //
01592 void VMatrix::updateMtime(time_t t)
01593 {
01594     if(t>mtime_ && mtime_!=numeric_limits<time_t>::max())
01595         mtime_=t;
01596     else if(t==0)
01597         mtime_=numeric_limits<time_t>::max();
01598 }
01599 void VMatrix::updateMtime(const PPath& p){if(!p.isEmpty())updateMtime(mtime(p));}
01600 
01601 void VMatrix::updateMtime(VMat v){if(v)updateMtime(v->getMtime());}
01602 
01604 // isUpToDate //
01606 bool VMatrix::isUpToDate(const PPath& path, bool warning_mtime0,
01607                          bool warning_older) const
01608 {
01609     bool exist = isfile(path);
01610     bool uptodate = false;
01611     if(exist)
01612         uptodate = getMtime() < mtime(path);
01613     if (warning_mtime0 && exist && uptodate && getMtime()==0)
01614         PLWARNING("In VMatrix::isUpToDate - for class '%s'"
01615                   " File '%s' will be used, but "
01616                   "this VMat's last modification time is undefined: we cannot "
01617                   "be sure the file is up-to-date.",
01618                   classname().c_str(), path.absolute().c_str());
01619     if(warning_older && exist && !uptodate)
01620         PLWARNING("In VMatrix::isUpToDate - for class '%s'"
01621                   " File '%s' is older than this "
01622                   "VMat's mtime of %ld, and should not be re-used.",
01623                   classname().c_str(), path.absolute().c_str(), long(getMtime()));
01624 
01625     return exist && uptodate;
01626 }
01627 
01629 // isUpToDate //
01631 bool VMatrix::isUpToDate(VMat vm, bool warning_mtime0,
01632                          bool warning_older) const
01633 {
01634     time_t my_time = getMtime();
01635     time_t vm_time = vm->getMtime();
01636     bool uptodate = my_time < vm_time ||
01637                     my_time == 0      ||
01638                     vm_time == 0;
01639     if (warning_mtime0 && uptodate && (my_time == 0 || vm_time == 0))
01640         PLWARNING("In VMatrix::isUpToDate - for class '%s'"
01641                   " When comparing the VMats' last modification times, at "
01642                   "least one was found to be undefined: we cannot be sure "
01643                   "the VMat is up-to-date.",
01644                   classname().c_str());
01645     if(warning_older && !uptodate)
01646         PLWARNING("In VMatrix::isUpToDate - for class '%s'"
01647                   " The VMat with mtime of %ld is older than this "
01648                   "VMat's with mtime of %ld, and should not be re-used.",
01649                   classname().c_str(), long(vm->getMtime()), long(getMtime()));
01650 
01651     return uptodate;
01652 }
01653 
01655 // getPrecomputedStatsFromFile //
01657 TVec<StatsCollector> VMatrix::getPrecomputedStatsFromFile(
01658         const string& filename, int maxnvalues, bool progress_bar) const
01659 {
01660     TVec<StatsCollector> stats;
01661     PPath metadatadir = getMetaDataDir();
01662     PPath statsfile;
01663     bool uptodate = false;
01664     if (hasMetaDataDir()) {
01665         lockMetaDataDir();
01666         statsfile =  metadatadir / filename;
01667         uptodate = isUpToDate(statsfile);
01668     }
01669     try{
01670         if (uptodate){
01671             PLearn::load(statsfile, stats);
01672             if(stats.length()!=width()){
01673                 uptodate=false;
01674                 PLWARNING("In VMatrix::getPrecomputedStatsFromFile() for class"
01675                           " %s - The file %s don't have the good number of"
01676                           " stats. We regenerate it.",
01677                         classname().c_str(), statsfile.c_str());
01678             }
01679         }
01680         if(!uptodate){
01681             VMat vm = const_cast<VMatrix*>(this);
01682             stats = PLearn::computeStats(vm, maxnvalues, progress_bar);
01683             if(hasMetaDataDir())
01684                 PLearn::save(statsfile, stats);
01685         }
01686     }catch(const PLearnError& e){
01687         if(!metadatadir.isEmpty())
01688             unlockMetaDataDir();
01689         //we erase the file if we are creating it
01690         // as it can be partilly saved.
01691         if(!uptodate && isfile(statsfile))
01692             rm(statsfile);
01693         throw e;
01694     }
01695     if (!metadatadir.isEmpty())
01696         unlockMetaDataDir();
01697     return stats;
01698 }
01699 
01701 // remote_getStats //
01703 TVec<PP<StatsCollector> > VMatrix::remote_getStats() const
01704 {
01705     if(field_p_stats.isEmpty())
01706     {
01707         TVec<StatsCollector> st= getStats();
01708         field_p_stats.resize(st.length());
01709         CopiesMap cm;
01710         for(int i= 0; i < st.length(); ++i)
01711             field_p_stats[i]= st[i].deepCopy(cm);
01712     }
01713     return field_p_stats;
01714 }
01715 
01717 // getBoundingBox //
01719 TVec< pair<real,real> > VMatrix::getBoundingBox(real extra_percent) const
01720 {
01721     TVec<StatsCollector> stats = getStats();
01722     int n = stats.length();
01723     TVec< pair<real,real> > bbox(n);
01724     for(int k=0; k<n; k++)
01725     {
01726         StatsCollector& st = stats[k];
01727         bbox[k] = pair<real,real>(st.min()-extra_percent*st.range(), st.max()+extra_percent*st.range());
01728     }
01729     return bbox;
01730 }
01731 
01733 // getRanges //
01735 TVec<RealMapping> VMatrix::getRanges()
01736 {
01737     TVec<RealMapping> ranges;
01738     PPath rangefile = getMetaDataDir() / "ranges.psave";
01739     if(isfile(rangefile))
01740         PLearn::load(rangefile, ranges);
01741     else
01742     {
01743         ranges = computeRanges(getStats(),std::max(10,length()/200),std::max(10,length()/100) );
01744         PLearn::save(rangefile, ranges);
01745     }
01746     return ranges;
01747 }
01748 
01750 // put //
01752 void VMatrix::put(int i, int j, real value)
01753 {
01754     PLERROR("In VMatrix::put - Method not implemented for this VMat(%s), please implement.",
01755             classname().c_str());
01756 }
01757 
01759 // getColumn //
01761 void VMatrix::getColumn(int j, Vec v) const
01762 {
01763 #ifdef BOUNDCHECK
01764     if(v.length() != length())
01765         PLERROR("In VMatrix::getColumn - v must have the same length as the VMatrix");
01766 #endif
01767     for(int i=0; i<v.length(); i++)
01768         v[i] = get(i,j);
01769 }
01770 
01771 Vec VMatrix::remote_getColumn(int i) const
01772 {
01773     Vec v(length());
01774     getColumn(i,v);
01775     return v;
01776 }
01777 
01778 
01780 // getSubRow //
01782 void VMatrix::getSubRow(int i, int j, Vec v) const
01783 {
01784     for(int k=0; k<v.length(); k++)
01785         v[k] = get(i,j+k);
01786 }
01787 
01789 // putSubRow //
01791 void VMatrix::putSubRow(int i, int j, Vec v)
01792 {
01793     for(int k=0; k<v.length(); k++)
01794         put(i, j+k, v[k]);
01795 }
01796 
01798 // getRow //
01800 void VMatrix::getRow(int i, Vec v) const
01801 {
01802 #ifdef BOUNDCHECK
01803     if(v.length() != width())
01804         PLERROR("In VMatrix::getRow(i,v) length of v and width of VMatrix differ");
01805 #endif
01806     getSubRow(i,0,v);
01807 }
01808 
01810 // putRow //
01812 void VMatrix::putRow(int i, Vec v)
01813 {
01814 #ifdef BOUNDCHECK
01815     if(v.length() != width())
01816         PLERROR("In VMatrix::putRow(i,v) length of v and width of VMatrix differ");
01817 #endif
01818     putSubRow(i,0,v);
01819 }
01820 
01822 // fill //
01824 void VMatrix::fill(real value)
01825 {
01826     Vec v(width(), value);
01827     for (int i=0; i<length(); i++) putRow(i,v);
01828 }
01829 
01831 // appendRow //
01833 void VMatrix::appendRow(Vec v)
01834 {
01835     PLERROR("In VMatrix::appendRow - Not implemented by VMatrix subclass '%s'",
01836             classname().c_str());
01837 }
01838 
01840 // insertRow //
01842 void VMatrix::insertRow(int i, Vec v)
01843 {
01844     if (i<0 || i>length_)
01845         PLERROR("In VMatrix::insertRow: row index (%d) outside valid range [%d,%d]", i, 0, length_);
01846     else if (i == length_)
01847         appendRow(v);
01848     else
01849     {
01850         appendRow(v); // dummy operation to increase VMat length
01851         Vec row(width_);
01852         for (int j=length_-1; j>i; --j)
01853         {
01854             getRow(j-1, row);
01855             putRow(j, row);
01856         }
01857         putRow(i, v);
01858     }
01859 }
01860 
01862 // flush //
01864 void VMatrix::flush()
01865 {}
01866 
01868 // putOrAppendRow //
01870 void VMatrix::putOrAppendRow(int i, Vec v)
01871 {
01872     if (v.length() != width())
01873         PLERROR("In putOrAppendRow, Vec to append must have same length (%d) as VMatrix width (%d)", v.length(), width());
01874 
01875     if(i==length())
01876         appendRow(v);
01877     else if(i<length())
01878         putRow(i,v);
01879     else
01880         PLERROR("In putOrAppendRow, index %d out of range",i);
01881 }
01882 
01884 // forcePutRow //
01886 void VMatrix::forcePutRow(int i, Vec v)
01887 {
01888     if (v.length() != width())
01889         PLERROR("In forcePutRow, Vec to append must have same length (%d) as VMatrix width (%d)", v.length(), width());
01890 
01891     if(i<length())
01892         putRow(i,v);
01893     else
01894     {
01895         Vec emptyrow(width());
01896         emptyrow.clear();
01897         while(length()<i)
01898             appendRow(emptyrow);
01899         appendRow(v);
01900     }
01901 }
01902 
01904 // getMat //
01906 void VMatrix::getMat(int i, int j, Mat m) const
01907 {
01908 #ifdef BOUNDCHECK
01909     if(i<0 || j<0 || i+m.length()>length() || j+m.width()>width())
01910         PLERROR("In VMatrix::getMat(i,j,m) OUT OF BOUNDS");
01911 #endif
01912     for(int ii=0; ii<m.length(); ii++)
01913     {
01914         getSubRow(i+ii, j, m(ii));
01915     }
01916 }
01917 
01919 // putMat //
01921 void VMatrix::putMat(int i, int j, Mat m)
01922 {
01923 #ifdef BOUNDCHECK
01924     if(i<0 || j<0 || i+m.length()>length() || j+m.width()>width())
01925         PLERROR("In VMatrix::putMat(i,j,m) OUT OF BOUNDS");
01926 #endif
01927     for(int ii=0; ii<m.length(); ii++)
01928     {
01929         putSubRow(i+ii, j, m(ii));
01930     }
01931 }
01932 
01934 // compacify //
01936 void VMatrix::compacify() {}
01937 
01939 // toMat //
01941 Mat VMatrix::toMat() const
01942 {
01943     return toMatCopy();
01944 }
01945 
01946 Mat VMatrix::toMatCopy() const
01947 {
01948     Mat m(length(),width());
01949     getMat(0,0,m);
01950     return m;
01951 }
01952 
01954 // subMat //
01956 VMat VMatrix::subMat(int i, int j, int l, int w)
01957 { return new SubVMatrix(this,i,j,l,w); }
01958 
01960 // dot //
01962 real VMatrix::dot(int i1, int i2, int inputsize) const
01963 {
01964     dotrow_1.resize(inputsize);
01965     dotrow_2.resize(inputsize);
01966     getSubRow(i1, 0, dotrow_1);
01967     getSubRow(i2, 0, dotrow_2);
01968     return PLearn::dot(dotrow_1, dotrow_2);
01969 }
01970 
01971 real VMatrix::dot(int i, const Vec& v) const
01972 {
01973     dotrow_1.resize(v.length());
01974     getSubRow(i, 0, dotrow_1);
01975     return PLearn::dot(dotrow_1, v);
01976 }
01977 
01978 
01980 // find //
01982 bool VMatrix::find(const Vec& input, real tolerance, int* i, int i_start) const
01983 {
01984     get_row.resize(inputsize());
01985 #ifdef BOUNDCHECK
01986     if (input.length() != inputsize())
01987         PLERROR("In VMatrix::find - The given vector must be the same size as "
01988                 "inputsize");
01989 #endif
01990     int n = length();
01991     for (int j = 0; j < n; j++) {
01992         int row = (j + i_start) % n;
01993         getSubRow(row, 0, get_row);
01994         if (powdistance(input, get_row, 2.0) < tolerance) {
01995             if (i)
01996                 *i = row;
01997             return true;
01998         }
01999     }
02000     if (i)
02001         *i = -1;
02002     return false;
02003 }
02004 
02006 // newwrite //
02008 void VMatrix::newwrite(PStream& out) const
02009 {
02010     /*
02011       switch(out.outmode)
02012       {
02013       case PStream::raw_ascii:
02014       case PStream::pretty_ascii:
02015       {
02016       Vec v(width());
02017       for(int i=0; i<length(); i++) {
02018       getRow(i,v);
02019       out << v << endl;
02020       }
02021       break;
02022       }
02023       default:
02024       inherited::newwrite(out);
02025       }
02026     */
02027     inherited::newwrite(out);
02028 }
02029 
02031 // ~ //
02033 VMatrix::~VMatrix()
02034 {}
02035 
02037 // save //
02039 void VMatrix::save(const PPath& filename) const
02040 {
02041     PLDEPRECATED( "This method overloads the Object::save method which is "
02042                   "deprecated. This method is therefore deprecated and you should call "
02043                   "directly the savePMAT() method." );
02044 
02045     savePMAT(filename);
02046 }
02047 
02049 // savePMAT //
02051 void VMatrix::savePMAT(const PPath& pmatfile, bool force_float, 
02052                        bool auto_float) const
02053 {
02054     if (width() == -1)
02055         PLERROR("In VMat::save - Saving in a pmat file is only possible for constant width VMats (where width()!=-1)");
02056 
02057     if(force_float && auto_float)
02058         PLERROR("VMatrix::savePMAT() - force_float an auto_float are incompatible option");
02059 
02060     int nsamples = length();
02061     PPath pmatfiletmp=pmatfile+".tmp";
02062     if(auto_float){
02063 #ifdef USEFLOAT
02064         PLERROR("VMatrix::savePMAT() - auto_float can't reliably select  float or double when compiled in float. Compile it in double.");
02065 #endif
02066         Vec v(width());
02067         bool found_not_equal=false;
02068         for(int i=0;i<length();i++){
02069             getRow(i,v);
02070             for(int j=0;j<width();j++){
02071                 if( ((double)((float)(v[j])))!=v[j] ){
02072                     found_not_equal=true;break;
02073                 }
02074             }
02075         }
02076         if(!found_not_equal){
02077             force_float=true;
02078             pout<<"We will store the result matrix in FLOAT format."<<endl;
02079         }
02080         else
02081             pout<<"We will store the result matrix in DOUBLE format."<<endl;
02082     }
02083     {
02084     FileVMatrix m(pmatfiletmp,nsamples,width(),force_float);
02085     m.setMetaInfoFrom(this);
02086     // m.setFieldInfos(getFieldInfos());
02087     // m.copySizesFrom(this);
02088     Vec tmpvec(width());
02089 
02090     ProgressBar pb(cout, "Saving to pmat", nsamples);
02091 
02092     for(int i=0; i<nsamples; i++)
02093     { 
02094         getRow(i,tmpvec);
02095         m.putRow(i,tmpvec);
02096         pb(i);
02097     }
02098     m.saveFieldInfos();
02099     m.saveAllStringMappings();
02100     }// to ensure that m is deleted?
02101     
02102     rm(pmatfile);
02103     force_rmdir(pmatfile+".metadata");
02104     mv(pmatfiletmp,pmatfile);
02105     mv(pmatfiletmp+".metadata",pmatfile+".metadata");
02106 }
02107 
02109 // saveDMAT //
02111 void VMatrix::saveDMAT(const PPath& dmatdir) const
02112 {
02113     force_rmdir(dmatdir);
02114     DiskVMatrix vm(dmatdir,width());
02115     vm.setMetaInfoFrom(this);
02116     // vm.setFieldInfos(getFieldInfos());
02117     // vm.copySizesFrom(this);
02118     Vec v(width());
02119 
02120     ProgressBar pb(cout, "Saving to dmat", length());
02121 
02122     for(int i=0;i<length();i++)
02123     {
02124         getRow(i,v);
02125         vm.appendRow(v);
02126         pb(i);
02127     }
02128     vm.saveFieldInfos();
02129     vm.saveAllStringMappings();
02130 }
02131 
02133 // saveAMAT //
02135 void VMatrix::saveAMAT(const PPath& amatfile, bool verbose, bool no_header, bool save_strings) const
02136 {
02137     int l = length();
02138     int w = width();
02139     PStream out = openFile(amatfile, PStream::raw_ascii, "w");
02140     if (!no_header) {
02141         out << "#size: "<< l << ' ' << w << endl;
02142     }
02143     if(w>0 && !no_header)
02144     {
02145         out << "#: ";
02146         for(int k=0; k<w; k++)
02147             //there must not be any space in a field name...
02148             out << space_to_underscore(fieldName(k)) << ' ';
02149         out << "\n";
02150     }
02151     if(!no_header)
02152         out << "#sizes: " << inputsize() << ' ' << targetsize() << ' ' << weightsize() << ' ' << extrasize() << endl;
02153 
02154     PP<ProgressBar> pb;
02155     if (verbose)
02156         pb = new ProgressBar(cout, "Saving to amat", length());
02157 
02158     if (save_strings) {
02159         TVec<string> v(w);
02160         for (int i = 0; i < l; i++) {
02161             getRowAsStrings(i, v);
02162             out << v << endl;
02163             if (verbose)
02164                 pb->update(i+1);
02165         }
02166 
02167     } else {
02168         Vec v(w);
02169         for(int i=0;i<l;i++)
02170         {
02171             getRow(i,v);
02172             for(int j=0; j<w; j++)
02173                 out << v[j] << ' ';
02174             out << "\n";
02175             if (verbose)
02176                 pb->update(i + 1);
02177         }
02178     }
02179 }
02180 
02181 void VMatrix::saveCMAT(const PPath& filename) const
02182 {
02183     PLWARNING("VMatrix::saveCMAT() - NOT FULLY IMPLEMENTED");
02184 
02185     //calculate the datatype needed
02186     TVec<StatsCollector> stats = getStats(true);
02187     int max_bits=0;
02188     for(int i=0;i<stats.size();i++){
02189         StatsCollector stat = stats[i];
02190         if(! stat.isinteger())
02191             PLERROR("VMatrix::saveCMAT() currently the source need to contain only integer.");
02192         if(stat.min()>=0){
02193             int bits=(int)ceil(sqrt(stat.max()));
02194             if(max_bits<bits)max_bits=bits;
02195         }else{
02196             PLERROR("not implemented to store negatif number.");
02197         }
02198         
02199     }
02200     //example 12000000 u:784:1:8 u:1:1:8
02201     //write the header
02202      if(max_bits>8) PLERROR("VMatrix::saveCMAT() currently we convert to cmat with a maximum of 8 bits by fields!");
02203     if(max_bits > 1 && max_bits<8){
02204         max_bits=8;
02205         PLWARNING("VMatrix::saveCMAT() currently when we need less then 8 bits(except for 1), we upgrade to 8 bits.");
02206     }
02207     if(max_bits==0){
02208         PLERROR("VMatrix::saveCMAT() - their was only 0 in the matrix! This is not supported as we don't think this can happen in real case!");
02209     }
02210     //write the data
02211     if(max_bits==8){
02212         PStream out = openFile(filename, PStream::raw_ascii, "w");
02213         out<<length()<<" u:"<<width()<<":1:"<<max_bits<<endl;
02214         Vec v(width());
02215         for(int i=0;i<length();i++){
02216             getRow(i,v);
02217             for(int j=0;j<width();j++){
02218                 out.put((char)v[j]);
02219             }
02220         }
02221     }else if(max_bits==1){
02222         PStream out = openFile(filename, PStream::raw_ascii, "w");
02223         int w2=width()%8;
02224         int w1=width()-w2;
02225         PLCHECK(w2+w1==width());
02226         PLCHECK(w1%8==0);
02227         PLCHECK(w1>0 && w2>=0);
02228         out<<length()<<" u:"<<w1<<":1:"<<max_bits;
02229         if(w2!=0)
02230             out<<" u:"<<w2<<":1:8";
02231         out<<endl;
02232         Vec v(width());
02233 
02234         for(int i=0;i<length();i++){
02235             getRow(i,v);
02236             int j;
02237             for(j=0;j<w1;){
02238                 char c=0;
02239                 for(int k=0;k<8;j++,k++){
02240                     c=c<<1;
02241                     c|=((bool)v[j]);
02242                 }
02243                 //revert the bits
02244                 char value=c;
02245                 value = (value & 0x0f) << 4 | (value & 0xf0) >> 4;
02246                 value = (value & 0x33) << 2 | (value & 0xcc) >> 2;
02247                 value = (value & 0x55) << 1 | (value & 0xaa) >> 1;
02248                 out.put(value);
02249             }
02250             PLCHECK(width()-j==w2);
02251             for(;j<width();j++){
02252                 out.put((char)v[j]);
02253             }
02254         }
02255     }
02256     else
02257         PLERROR("VMatrix::saveCMAT() - %d bits are not supported!",max_bits);
02258 
02259     CompactFileVMatrix m = CompactFileVMatrix(filename);
02260     m.setMetaDataDir(filename + ".metadata");
02261     m.setMetaInfoFrom(this);
02262     m.saveFieldInfos();
02263     m.saveAllStringMappings();
02264 
02265     pout<<"generated the file " <<filename <<endl;
02266 }
02268 // accumulateXtY //
02270 void VMatrix::accumulateXtY(int X_startcol, int X_ncols, int Y_startcol, int Y_ncols,
02271                             Mat& result, int startrow, int nrows, int ignore_this_row) const
02272 {
02273     int endrow = (nrows>0) ?startrow+nrows :length_;
02274     Vec x(X_ncols);
02275     Vec y(Y_ncols);
02276     for(int i=startrow; i<endrow; i++)
02277         if(i!=ignore_this_row)
02278         {
02279             getSubRow(i,X_startcol,x);
02280             getSubRow(i,Y_startcol,y);
02281             externalProductAcc(result, x,y);
02282         }
02283 }
02284 
02286 // accumulateXtX //
02288 void VMatrix::accumulateXtX(int X_startcol, int X_ncols,
02289                             Mat& result, int startrow, int nrows, int ignore_this_row) const
02290 {
02291     Vec x(X_ncols);
02292     int endrow = (nrows>0) ?startrow+nrows :length_;
02293     for(int i=startrow; i<endrow; i++)
02294         if(i!=ignore_this_row)
02295         {
02296             getSubRow(i,X_startcol,x);
02297             externalProductAcc(result, x,x);
02298         }
02299 }
02300 
02302 // getRowVec //
02304 Vec VMatrix::getRowVec(int i) const
02305 {
02306     Vec v(width());
02307     getRow(i,v);
02308     return v;
02309 }
02310 
02312 // appendRows //
02314 void VMatrix::appendRows(Mat rows)
02315 {
02316     for(int i=0; i<rows.length(); i++)
02317         appendRow(rows(i));
02318 }
02319 
02320 
02322 // compareStats //
02324 void VMatrix::compareStats(VMat target,
02325                            real stderror_threshold,
02326                            real missing_threshold,
02327                            Vec stderror,
02328                            Vec missing)
02329 {
02330     if(target->width()!=width())
02331         PLERROR("In VecStatsCollector:: compareStats() - This VMatrix has "
02332                 "width %d which differs from the target width of %d",
02333                 width(), target->width());
02334 
02335     for(int i=0;i<width();i++)
02336     {
02337         const StatsCollector tstats = target->getStats(i);
02338         const StatsCollector lstats = getStats(i);
02339 
02340         real tmissing = tstats.nmissing()/tstats.n();
02341         real lmissing = lstats.nmissing()/lstats.n();
02342         real terr = sqrt(tmissing*(1-tmissing)+lmissing*(1-lmissing));
02343         real th_missing = fabs(tmissing-lmissing)/terr;
02344         if(fast_is_equal(terr,0))
02345         {
02346             if(!fast_is_equal(tmissing,0)||!fast_is_equal(lmissing,0))
02347                 PLWARNING("In VMatrix::compareStats - field %d(%s)terr=%f,"
02348                           " tmissing=%f, lmissing=%f!",i, fieldName(i).c_str(),
02349                           terr, tmissing, lmissing);
02350             PLCHECK((fast_is_equal(tmissing,0)||fast_is_equal(tmissing,1))
02351                     && (fast_is_equal(lmissing,0)||fast_is_equal(lmissing,1)));
02352         }
02353         else if(isnan(th_missing))
02354             PLWARNING("In VMatrix::compareStats - should not happen!");
02355         
02356         real tmean = tstats.mean();
02357         real lmean = lstats.mean();
02358         real tstderror = sqrt(pow(tstats.stderror(), 2) + 
02359                               pow(lstats.stderror(), 2));
02360         real th_stderror = fabs(lmean-tmean)/tstderror;
02361         if(tstderror==0)
02362             PLWARNING("In VMatrix::compareStats - field %d(%s) have a"
02363                       " stderror of 0 for both matrice.",
02364                       i, fieldName(i).c_str());
02365         stderror[i]=th_stderror;
02366         missing[i]=th_missing;
02367     }
02368     return;
02369 }
02370 
02372 // maxFieldNamesSize //
02374 int VMatrix::maxFieldNamesSize() const
02375 {
02376     uint size_fieldnames=0;
02377     for(int i=0;i<width();i++)
02378         if(fieldName(i).size()>size_fieldnames)
02379             size_fieldnames=fieldName(i).size();
02380     return size_fieldnames;
02381 }
02382 
02384 // computeMissingSizeValue //
02386 void VMatrix::computeMissingSizeValue(bool warn_if_cannot_compute,
02387                                       bool warn_if_size_mismatch)
02388 {
02389     int v=min(inputsize_,0) + min(targetsize_,0)
02390         + min(weightsize_,0) + min(extrasize_,0);
02391 
02392     if (width_ < 0 && v <= -1) {
02393         if (warn_if_cannot_compute)
02394             PLWARNING("In VMatrix::computeMissingSizeValue for %s - Cannot "
02395                       "compute the missing size value when the width is undefined",
02396                       classname().c_str());
02397         return;
02398     }
02399 
02400     if(v < -1){
02401         if(warn_if_cannot_compute)
02402             PLWARNING("In VMatrix::computeMissingSizeValue() - in class %s"
02403                       " more then one of"
02404                       " inputsize(%d), targetsize(%d), weightsize(%d) and"
02405                       " extrasize(%d) is unknow so we cannot compute them with"
02406                       " the width(%d)",
02407                       classname().c_str(), inputsize_, targetsize_, weightsize_,
02408                       extrasize_, width_);
02409         return;
02410     }else if(v==0 && warn_if_size_mismatch && width_ >= 0 &&
02411             width_ != inputsize_ + targetsize_ + weightsize_ + extrasize_)
02412         PLWARNING("In VMatrix::computeMissingSizeValue() for class %s - "
02413                   "inputsize_(%d) + targetsize_(%d) + weightsize_(%d) + "
02414                   "extrasize_(%d) != width_(%d) !",
02415                   classname().c_str(), inputsize_, targetsize_, weightsize_,
02416                   extrasize_, width_);
02417 
02418     else if(inputsize_<0)
02419         inputsize_ = width_- targetsize_ - weightsize_ - extrasize_;
02420     else if(targetsize_ < 0)
02421         targetsize_ = width_- inputsize_ - weightsize_ - extrasize_;
02422     else if(weightsize_ < 0)
02423         weightsize_ = width_- inputsize_ - targetsize_ - extrasize_;
02424     else if(extrasize_ < 0)
02425         extrasize_  = width_- inputsize_ - targetsize_ - weightsize_;
02426 }
02427 
02428 } // end of namespace PLearn
02429 
02430 
02431 /*
02432   Local Variables:
02433   mode:c++
02434   c-basic-offset:4
02435   c-file-style:"stroustrup"
02436   c-file-offsets:((innamespace . 0)(inline-open . 0))
02437   indent-tabs-mode:nil
02438   fill-column:79
02439   End:
02440 */
02441 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines