PLearn 0.1
VMatrix.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 // PLearn (A C++ Machine Learning Library)
00003 // Copyright (C) 1998 Pascal Vincent
00004 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00005 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 
00036 /* *******************************************************
00037  * $Id: VMatrix.h 10280 2009-07-20 15:01:53Z nouiz $
00038  ******************************************************* */
00039 
00040 
00043 #ifndef VMatrix_INC
00044 #define VMatrix_INC
00045 
00046 //#include <cstdlib>
00047 #include <plearn/base/Object.h>
00048 #include <plearn/base/PP.h>
00049 #include <plearn/math/TMat.h>
00050 #include <plearn/math/StatsCollector.h>
00051 #include "VMField.h"
00052 #include <plearn/dict/Dictionary.h>
00053 #include <plearn/io/PPath.h>
00054 
00055 #include <map>
00056 
00057 namespace PLearn {
00058 using namespace std;
00059 
00060 class VMat;
00061 
00077 class VMatrix: public Object
00078 {
00079     typedef Object inherited;
00080     friend class VMat;
00081 
00083     mutable PStream lockf_; 
00084 
00086     mutable Vec get_row;
00087 
00089     mutable Vec dotrow_1;
00090     mutable Vec dotrow_2;
00091     time_t mtime_;          
00092     time_t mtime_update;    
00093 
00094 protected:
00095 
00096     mutable int length_;    
00097     int width_;             
00098 
00102     mutable int inputsize_;
00103     mutable int targetsize_;
00104     mutable int weightsize_;
00105     mutable int extrasize_;
00106 
00108     bool writable;
00109 
00112     PPath metadatadir;
00113 
00115     mutable TVec<StatsCollector> field_stats;  
00116     mutable TVec<PP<StatsCollector> > field_p_stats; //same, for remote calls
00117 
00119     mutable TVec<map<string,real> > map_sr;
00120     mutable TVec<map<real,string> > map_rs;
00121 
00122 private:
00124     void build_();
00125 
00126 protected:
00128     static void declareOptions(OptionList & ol);
00129 
00131     static void declareMethods(RemoteMethodMap& rmm);
00132 
00133 public:
00134     // @TODO Move to protected / private if we don't want to use it directly
00135     mutable Array<VMField> fieldinfos;
00136     Array<VMFieldStat> fieldstats;
00137 
00138 public:
00139     //#####  PLearn Object Protocol  ##########################################
00140 
00142     VMatrix(bool call_build_ = false);
00143 
00144     VMatrix(int the_length, int the_width, bool call_build_ = false);
00145 
00146     virtual ~VMatrix();
00147 
00149     virtual void build();
00150 
00151     PLEARN_DECLARE_ABSTRACT_OBJECT(VMatrix);
00152     void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00153 
00154 
00155     //#####  Field Types and Field Names  #####################################
00156     
00159     void setFieldInfos(const Array<VMField>& finfo) const;
00160     
00162     bool hasFieldInfos() const;
00163 
00166     Array<VMField>& getFieldInfos() const;
00167 
00170     Array<VMField> getSavedFieldInfos() const;
00171 
00173     VMField& getFieldInfos(int fieldindex) const
00174     {
00175         return getFieldInfos()[fieldindex];
00176     }
00177 
00180     void declareField(int fieldindex, const string& fieldname,
00181                       VMField::FieldType fieldtype=VMField::UnknownType);
00182 
00184     void declareFieldNames(const TVec<string>& fnames);
00185 
00188     int fieldIndex(const string& fieldname) const;
00189 
00197     int getFieldIndex(const string& fieldname_or_num,
00198                       bool throw_error = true) const;
00199 
00201     int remote_getFieldIndex(const string& fieldname_or_num) const
00202     {
00203         return getFieldIndex(fieldname_or_num);
00204     }
00205 
00207     string fieldName(int fieldindex) const
00208     {
00209         return getFieldInfos(fieldindex).name;
00210     }
00211 
00213     TVec<string> fieldNames() const;
00214 
00216     void unduplicateFieldNames(); 
00217 
00219     virtual TVec<string> inputFieldNames() const;
00220 
00222     virtual TVec<string> targetFieldNames() const;
00223 
00225     virtual TVec<string> weightFieldNames() const;
00226 
00228     virtual TVec<string> extraFieldNames() const;
00229 
00230     VMField::FieldType fieldType(int fieldindex) const
00231     {
00232         return getFieldInfos(fieldindex).fieldtype;
00233     } 
00234 
00235     VMField::FieldType fieldType(const string& fieldname) const
00236     {
00237         return fieldType(fieldIndex(fieldname));
00238     } 
00239 
00240     const VMFieldStat& fieldStat(int j) const
00241     {
00242         return fieldstats[j];
00243     } 
00244 
00245     const VMFieldStat& fieldStat(const string& fieldname) const
00246     {
00247         return fieldStat(fieldIndex(fieldname));
00248     }
00249 
00250     void printFields(PStream& out) const;
00251     void printFieldInfo(PStream& out, int fieldnum, bool print_binning = false) const;
00252     void printFieldInfo(PStream& out, const string& fieldname_or_num,
00253                         bool print_binning = false) const;
00254 
00256     void saveFieldInfos() const;
00257     void loadFieldInfos() const;
00258 
00259     
00260     //#####  Meta Data  #######################################################
00261 
00270     virtual void setMetaInfoFrom(const VMatrix* vm);
00271 
00274     bool looksTheSameAs(const VMat& m);
00275 
00280     void compatibleSizeError(const VMat& m, const string& extra_msg = "");
00281 
00287     virtual void setMetaDataDir(const PPath& the_metadatadir);
00288 
00290     bool hasMetaDataDir() const { return !metadatadir.isEmpty(); }
00291 
00293     PPath getMetaDataDir() const;
00294 
00310     void lockMetaDataDir(time_t max_lock_age = 0, bool verbose = true) const;
00311 
00314     void unlockMetaDataDir() const;
00315 
00323     virtual void save(const PPath& filename) const;
00324 
00326     virtual void savePMAT(const PPath& pmatfile,
00327                           bool force_float=false,
00328                           bool auto_float=false) const;
00329     virtual void remote_savePMAT(const PPath& pmatfile) const{savePMAT(pmatfile,false);}
00330     virtual void remote_savePMAT_float(const PPath& pmatfile) const{savePMAT(pmatfile,true);}
00331 
00333     virtual void saveDMAT(const PPath& dmatdir) const;
00334 
00343     virtual void saveAMAT(const PPath& amatfile, bool verbose = true,
00344                           bool no_header = false, bool save_strings = false) const;
00345 
00346     virtual void saveCMAT(const PPath& filename) const;
00347 
00350     inline bool isWritable() const { return writable; }
00351 
00373     string resolveFieldInfoLink(const PPath& target, const PPath& source);
00374 
00382     inline time_t getMtime() const { return mtime_==numeric_limits<time_t>::max()? 0: mtime_; }
00383 
00392     void updateMtime(time_t t);
00393 
00394     void updateMtime(const PPath& p);
00395 
00396     void updateMtime(VMat v);
00397 
00405     inline void setMtime(time_t t) { mtime_ = t; }
00406 
00413     bool isUpToDate(const PPath& file, bool warning_mtime0 = true,
00414                     bool warning_older = false) const;
00415 
00423     bool isUpToDate(VMat vm, bool warning_mtime0 = true,
00424                     bool warning_older = false) const;
00425 
00426     //#####  Matrix Sizes  ####################################################
00427 
00429     inline int width() const
00430     {
00431 #ifdef BOUNDCHECK
00432         if (!this)
00433             PLERROR("VMatrix::width() This object has pointer this=NULL");
00434 #endif
00435         return width_;
00436     }
00437 
00439     inline int length() const
00440     {
00441 #ifdef BOUNDCHECK
00442         if (!this)
00443             PLERROR("VMatrix::length() This object has pointer this=NULL");
00444 #endif
00445         return length_;
00446     }
00447 
00449     inline void defineSizes(int inputsize, int targetsize, int weightsize=0, int extrasize=0)
00450     {
00451         inputsize_  = inputsize;
00452         targetsize_ = targetsize;
00453         weightsize_ = weightsize;
00454         extrasize_ = extrasize;
00455     }
00456 
00459     void copySizesFrom(const VMat& m);
00460 
00467     bool getSavedSizes(int& inputsize, int& targetsize, int& weightsize, int& extrasize) const;
00468 
00470     inline int inputsize() const { return inputsize_; }
00471 
00473     inline int targetsize() const { return targetsize_; }
00474 
00476     inline int weightsize() const { return weightsize_; }
00477 
00479     inline int extrasize() const { return extrasize_; }
00480 
00482     inline bool hasWeights() const { return weightsize_>0; }
00483 
00484     
00485     //#####  Numerical Data Access  ###########################################
00486 
00493     virtual void getExample(int i, Vec& input, Vec& target, real& weight);
00494 
00496     boost::tuple<Vec, Vec, real> remote_getExample(int i);
00497 
00505     void getExamples(int i_start, int length, Mat& inputs, Mat& targets,
00506                      Vec& weights, Mat* extra = NULL,
00507                      bool allow_circular = false);
00508 
00515     virtual void getExtra(int i, Vec& extra);
00516     Vec remote_getExtra(int i);
00517 
00519     virtual real get(int i, int j) const = 0; 
00520 
00522     virtual void put(int i, int j, real value); 
00523 
00530     virtual void getSubRow(int i, int j, Vec v) const;
00531 
00537     virtual void putSubRow(int i, int j, Vec v);
00538 
00540     virtual void appendRow(Vec v);
00541 
00543     virtual void insertRow(int i, Vec v);
00544 
00546     virtual void flush();
00547 
00549     void putOrAppendRow(int i, Vec v);
00550 
00554     void forcePutRow(int i, Vec v);
00555 
00558     virtual void getRow(int i, Vec v) const; 
00559 
00560     virtual void putRow(int i, Vec v);
00561     virtual void fill(real value);
00562 
00565     virtual void getMat(int i, int j, Mat m) const;
00566 
00568     virtual void putMat(int i, int j, Mat m);
00569 
00572     virtual void getColumn(int i, Vec v) const;
00573 
00575     Vec remote_getColumn(int i) const;
00576 
00585     bool find(const Vec& input, real tolerance, int* i = 0, int i_start = 0) const;
00586 
00592     virtual VMat subMat(int i, int j, int l, int w);
00593 
00594 
00595     //#####  Conversions  #####################################################
00596 
00609     virtual Mat toMat() const;
00610 
00613     Mat toMatCopy() const;
00614 
00618     virtual void compacify();
00619 
00621     virtual void reset_dimensions() { }
00622 
00629     operator Mat() const { return toMat(); }
00630 
00637     virtual void newwrite(PStream& out) const;
00638 
00639 
00640     //#####  Statistics  ######################################################
00641 
00642     virtual void computeStats();
00643     bool hasStats() const { return fieldstats.size()>0; }
00644     void saveStats(const PPath& filename) const;
00645     void loadStats(const PPath& filename);
00646 
00652     TVec<StatsCollector> getStats(bool progress_bar=false) const;
00653 
00657     TVec<StatsCollector> getPrecomputedStatsFromFile(const string& filename,
00658                                                      int maxnvalues,
00659                                                      bool progress_bar) const;
00660 
00661     TVec<PP<StatsCollector> > remote_getStats() const;
00662 
00663     StatsCollector& getStats(int fieldnum) const
00664     { return getStats()[fieldnum]; }
00665 
00666     
00675     void compareStats(VMat target,
00676                       real stderror_threshold ,
00677                       real missing_threshold,
00678                       Vec stderror,
00679                       Vec missing
00680                       );
00681                       
00685     int maxFieldNamesSize() const;
00686 
00696     void computeMissingSizeValue(bool warn_if_cannot_compute = true,
00697                                  bool warn_if_size_mismatch = true);
00698 
00706     TVec< pair<real,real> > getBoundingBox(real extra_percent=0.00) const;
00707 
00713     TVec<RealMapping> getRanges();
00714 
00715 
00716     //#####  Special Mathematical Operations  #################################
00717 
00728     virtual real dot(int i1, int i2, int inputsize) const;
00729 
00730     inline real dot(int i1, int i2) const { return dot(i1,i2,width()); }
00731 
00734     virtual real dot(int i, const Vec& v) const;
00735 
00741     virtual void accumulateXtY(
00742         int X_startcol, int X_ncols, int Y_startcol, int Y_ncols,
00743         Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;
00744 
00750     virtual void accumulateXtX(
00751         int X_startcol, int X_ncols,
00752         Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const;
00753 
00754     //#####  String Mappings  #################################################
00755 
00757     void init_map_sr() const ;
00758 
00760     void saveAllStringMappings();
00761 
00768     void saveStringMappings(int col, const PPath& fname,
00769                             map<string, real>* str_to_real = 0);
00771     void addStringMapping(int col, string str, real val);
00772 
00779     real addStringMapping(int col, string str);
00780 
00782     void removeAllStringMappings();
00783 
00785     void removeColumnStringMappings(int c);
00786 
00788     void removeStringMapping(int col, string str);
00789 
00792     void setStringMapping(int col, const map<string,real>& zemap);
00793 
00795     void deleteStringMapping(int col);
00796 
00798     void loadStringMapping(int col);
00799 
00802     virtual void loadAllStringMappings();
00803 
00805     void copyStringMappingsFrom(const VMat& source);
00806 
00809     virtual string getValString(int col, real val) const;
00810 
00812     virtual const map<string,real>& getStringToRealMapping(int col) const;
00813 
00815     virtual const map<real,string>& getRealToStringMapping(int col) const;
00816 
00819     virtual real getStringVal(int col, const string & str) const;
00820 
00823     virtual string getString(int row, int col) const;
00824 
00827     virtual void getRowAsStrings(int i, TVec<string>& v_str) const;
00828 
00831     virtual PP<Dictionary> getDictionary(int col) const;
00832 
00838     virtual void getValues(int row, int col, Vec& values) const;
00839 
00841     virtual void getValues(const Vec& input, int col, Vec& values) const;
00842 
00844     Vec getRowVec(int i) const; 
00845 
00847     void appendRows(Mat rows);
00848 
00849     //#####  SFIF Files  ######################################################
00850     
00871     void setSFIFFilename(int col, string ext, const PPath& filepath="");
00872     void setSFIFFilename(string fieldname, string ext, const PPath& filepath="");
00873 
00885     PPath getSFIFFilename(int col, string ext);
00886     PPath getSFIFFilename(string fieldname, string ext);
00887 
00889     PPath getSFIFDirectory() const;
00890     
00894     bool isSFIFDirect(int col, string ext);
00895     bool isSFIFDirect(string fieldname, string ext);
00896 
00897     // return the size on the disk. If the sub class don't implement it return -1. 
00898     virtual int64_t getSizeOnDisk(){return -1;}
00899 };
00900 
00901 DECLARE_OBJECT_PTR(VMatrix);
00902 
00917 #define SPECIAL_FORMAT ((real)3.1e36)
00918 
00919 
00920 } // end of namespace PLearn
00921 
00922 #endif
00923 
00924 
00925 /*
00926   Local Variables:
00927   mode:c++
00928   c-basic-offset:4
00929   c-file-style:"stroustrup"
00930   c-file-offsets:((innamespace . 0)(inline-open . 0))
00931   indent-tabs-mode:nil
00932   fill-column:79
00933   End:
00934 */
00935 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines