PLearn 0.1
|
00001 // -*- C++ -*- 00002 // PLearn (A C++ Machine Learning Library) 00003 // Copyright (C) 1998 Pascal Vincent 00004 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00005 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 00036 /* ******************************************************* 00037 * $Id: VMatrix.h 10280 2009-07-20 15:01:53Z nouiz $ 00038 ******************************************************* */ 00039 00040 00043 #ifndef VMatrix_INC 00044 #define VMatrix_INC 00045 00046 //#include <cstdlib> 00047 #include <plearn/base/Object.h> 00048 #include <plearn/base/PP.h> 00049 #include <plearn/math/TMat.h> 00050 #include <plearn/math/StatsCollector.h> 00051 #include "VMField.h" 00052 #include <plearn/dict/Dictionary.h> 00053 #include <plearn/io/PPath.h> 00054 00055 #include <map> 00056 00057 namespace PLearn { 00058 using namespace std; 00059 00060 class VMat; 00061 00077 class VMatrix: public Object 00078 { 00079 typedef Object inherited; 00080 friend class VMat; 00081 00083 mutable PStream lockf_; 00084 00086 mutable Vec get_row; 00087 00089 mutable Vec dotrow_1; 00090 mutable Vec dotrow_2; 00091 time_t mtime_; 00092 time_t mtime_update; 00093 00094 protected: 00095 00096 mutable int length_; 00097 int width_; 00098 00102 mutable int inputsize_; 00103 mutable int targetsize_; 00104 mutable int weightsize_; 00105 mutable int extrasize_; 00106 00108 bool writable; 00109 00112 PPath metadatadir; 00113 00115 mutable TVec<StatsCollector> field_stats; 00116 mutable TVec<PP<StatsCollector> > field_p_stats; //same, for remote calls 00117 00119 mutable TVec<map<string,real> > map_sr; 00120 mutable TVec<map<real,string> > map_rs; 00121 00122 private: 00124 void build_(); 00125 00126 protected: 00128 static void declareOptions(OptionList & ol); 00129 00131 static void declareMethods(RemoteMethodMap& rmm); 00132 00133 public: 00134 // @TODO Move to protected / private if we don't want to use it directly 00135 mutable Array<VMField> fieldinfos; 00136 Array<VMFieldStat> fieldstats; 00137 00138 public: 00139 //##### PLearn Object Protocol ########################################## 00140 00142 VMatrix(bool call_build_ = false); 00143 00144 VMatrix(int the_length, int the_width, bool call_build_ = false); 00145 00146 virtual ~VMatrix(); 00147 00149 virtual void build(); 00150 00151 PLEARN_DECLARE_ABSTRACT_OBJECT(VMatrix); 00152 void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00153 00154 00155 //##### Field Types and Field Names ##################################### 00156 00159 void setFieldInfos(const Array<VMField>& finfo) const; 00160 00162 bool hasFieldInfos() const; 00163 00166 Array<VMField>& getFieldInfos() const; 00167 00170 Array<VMField> getSavedFieldInfos() const; 00171 00173 VMField& getFieldInfos(int fieldindex) const 00174 { 00175 return getFieldInfos()[fieldindex]; 00176 } 00177 00180 void declareField(int fieldindex, const string& fieldname, 00181 VMField::FieldType fieldtype=VMField::UnknownType); 00182 00184 void declareFieldNames(const TVec<string>& fnames); 00185 00188 int fieldIndex(const string& fieldname) const; 00189 00197 int getFieldIndex(const string& fieldname_or_num, 00198 bool throw_error = true) const; 00199 00201 int remote_getFieldIndex(const string& fieldname_or_num) const 00202 { 00203 return getFieldIndex(fieldname_or_num); 00204 } 00205 00207 string fieldName(int fieldindex) const 00208 { 00209 return getFieldInfos(fieldindex).name; 00210 } 00211 00213 TVec<string> fieldNames() const; 00214 00216 void unduplicateFieldNames(); 00217 00219 virtual TVec<string> inputFieldNames() const; 00220 00222 virtual TVec<string> targetFieldNames() const; 00223 00225 virtual TVec<string> weightFieldNames() const; 00226 00228 virtual TVec<string> extraFieldNames() const; 00229 00230 VMField::FieldType fieldType(int fieldindex) const 00231 { 00232 return getFieldInfos(fieldindex).fieldtype; 00233 } 00234 00235 VMField::FieldType fieldType(const string& fieldname) const 00236 { 00237 return fieldType(fieldIndex(fieldname)); 00238 } 00239 00240 const VMFieldStat& fieldStat(int j) const 00241 { 00242 return fieldstats[j]; 00243 } 00244 00245 const VMFieldStat& fieldStat(const string& fieldname) const 00246 { 00247 return fieldStat(fieldIndex(fieldname)); 00248 } 00249 00250 void printFields(PStream& out) const; 00251 void printFieldInfo(PStream& out, int fieldnum, bool print_binning = false) const; 00252 void printFieldInfo(PStream& out, const string& fieldname_or_num, 00253 bool print_binning = false) const; 00254 00256 void saveFieldInfos() const; 00257 void loadFieldInfos() const; 00258 00259 00260 //##### Meta Data ####################################################### 00261 00270 virtual void setMetaInfoFrom(const VMatrix* vm); 00271 00274 bool looksTheSameAs(const VMat& m); 00275 00280 void compatibleSizeError(const VMat& m, const string& extra_msg = ""); 00281 00287 virtual void setMetaDataDir(const PPath& the_metadatadir); 00288 00290 bool hasMetaDataDir() const { return !metadatadir.isEmpty(); } 00291 00293 PPath getMetaDataDir() const; 00294 00310 void lockMetaDataDir(time_t max_lock_age = 0, bool verbose = true) const; 00311 00314 void unlockMetaDataDir() const; 00315 00323 virtual void save(const PPath& filename) const; 00324 00326 virtual void savePMAT(const PPath& pmatfile, 00327 bool force_float=false, 00328 bool auto_float=false) const; 00329 virtual void remote_savePMAT(const PPath& pmatfile) const{savePMAT(pmatfile,false);} 00330 virtual void remote_savePMAT_float(const PPath& pmatfile) const{savePMAT(pmatfile,true);} 00331 00333 virtual void saveDMAT(const PPath& dmatdir) const; 00334 00343 virtual void saveAMAT(const PPath& amatfile, bool verbose = true, 00344 bool no_header = false, bool save_strings = false) const; 00345 00346 virtual void saveCMAT(const PPath& filename) const; 00347 00350 inline bool isWritable() const { return writable; } 00351 00373 string resolveFieldInfoLink(const PPath& target, const PPath& source); 00374 00382 inline time_t getMtime() const { return mtime_==numeric_limits<time_t>::max()? 0: mtime_; } 00383 00392 void updateMtime(time_t t); 00393 00394 void updateMtime(const PPath& p); 00395 00396 void updateMtime(VMat v); 00397 00405 inline void setMtime(time_t t) { mtime_ = t; } 00406 00413 bool isUpToDate(const PPath& file, bool warning_mtime0 = true, 00414 bool warning_older = false) const; 00415 00423 bool isUpToDate(VMat vm, bool warning_mtime0 = true, 00424 bool warning_older = false) const; 00425 00426 //##### Matrix Sizes #################################################### 00427 00429 inline int width() const 00430 { 00431 #ifdef BOUNDCHECK 00432 if (!this) 00433 PLERROR("VMatrix::width() This object has pointer this=NULL"); 00434 #endif 00435 return width_; 00436 } 00437 00439 inline int length() const 00440 { 00441 #ifdef BOUNDCHECK 00442 if (!this) 00443 PLERROR("VMatrix::length() This object has pointer this=NULL"); 00444 #endif 00445 return length_; 00446 } 00447 00449 inline void defineSizes(int inputsize, int targetsize, int weightsize=0, int extrasize=0) 00450 { 00451 inputsize_ = inputsize; 00452 targetsize_ = targetsize; 00453 weightsize_ = weightsize; 00454 extrasize_ = extrasize; 00455 } 00456 00459 void copySizesFrom(const VMat& m); 00460 00467 bool getSavedSizes(int& inputsize, int& targetsize, int& weightsize, int& extrasize) const; 00468 00470 inline int inputsize() const { return inputsize_; } 00471 00473 inline int targetsize() const { return targetsize_; } 00474 00476 inline int weightsize() const { return weightsize_; } 00477 00479 inline int extrasize() const { return extrasize_; } 00480 00482 inline bool hasWeights() const { return weightsize_>0; } 00483 00484 00485 //##### Numerical Data Access ########################################### 00486 00493 virtual void getExample(int i, Vec& input, Vec& target, real& weight); 00494 00496 boost::tuple<Vec, Vec, real> remote_getExample(int i); 00497 00505 void getExamples(int i_start, int length, Mat& inputs, Mat& targets, 00506 Vec& weights, Mat* extra = NULL, 00507 bool allow_circular = false); 00508 00515 virtual void getExtra(int i, Vec& extra); 00516 Vec remote_getExtra(int i); 00517 00519 virtual real get(int i, int j) const = 0; 00520 00522 virtual void put(int i, int j, real value); 00523 00530 virtual void getSubRow(int i, int j, Vec v) const; 00531 00537 virtual void putSubRow(int i, int j, Vec v); 00538 00540 virtual void appendRow(Vec v); 00541 00543 virtual void insertRow(int i, Vec v); 00544 00546 virtual void flush(); 00547 00549 void putOrAppendRow(int i, Vec v); 00550 00554 void forcePutRow(int i, Vec v); 00555 00558 virtual void getRow(int i, Vec v) const; 00559 00560 virtual void putRow(int i, Vec v); 00561 virtual void fill(real value); 00562 00565 virtual void getMat(int i, int j, Mat m) const; 00566 00568 virtual void putMat(int i, int j, Mat m); 00569 00572 virtual void getColumn(int i, Vec v) const; 00573 00575 Vec remote_getColumn(int i) const; 00576 00585 bool find(const Vec& input, real tolerance, int* i = 0, int i_start = 0) const; 00586 00592 virtual VMat subMat(int i, int j, int l, int w); 00593 00594 00595 //##### Conversions ##################################################### 00596 00609 virtual Mat toMat() const; 00610 00613 Mat toMatCopy() const; 00614 00618 virtual void compacify(); 00619 00621 virtual void reset_dimensions() { } 00622 00629 operator Mat() const { return toMat(); } 00630 00637 virtual void newwrite(PStream& out) const; 00638 00639 00640 //##### Statistics ###################################################### 00641 00642 virtual void computeStats(); 00643 bool hasStats() const { return fieldstats.size()>0; } 00644 void saveStats(const PPath& filename) const; 00645 void loadStats(const PPath& filename); 00646 00652 TVec<StatsCollector> getStats(bool progress_bar=false) const; 00653 00657 TVec<StatsCollector> getPrecomputedStatsFromFile(const string& filename, 00658 int maxnvalues, 00659 bool progress_bar) const; 00660 00661 TVec<PP<StatsCollector> > remote_getStats() const; 00662 00663 StatsCollector& getStats(int fieldnum) const 00664 { return getStats()[fieldnum]; } 00665 00666 00675 void compareStats(VMat target, 00676 real stderror_threshold , 00677 real missing_threshold, 00678 Vec stderror, 00679 Vec missing 00680 ); 00681 00685 int maxFieldNamesSize() const; 00686 00696 void computeMissingSizeValue(bool warn_if_cannot_compute = true, 00697 bool warn_if_size_mismatch = true); 00698 00706 TVec< pair<real,real> > getBoundingBox(real extra_percent=0.00) const; 00707 00713 TVec<RealMapping> getRanges(); 00714 00715 00716 //##### Special Mathematical Operations ################################# 00717 00728 virtual real dot(int i1, int i2, int inputsize) const; 00729 00730 inline real dot(int i1, int i2) const { return dot(i1,i2,width()); } 00731 00734 virtual real dot(int i, const Vec& v) const; 00735 00741 virtual void accumulateXtY( 00742 int X_startcol, int X_ncols, int Y_startcol, int Y_ncols, 00743 Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const; 00744 00750 virtual void accumulateXtX( 00751 int X_startcol, int X_ncols, 00752 Mat& result, int startrow=0, int nrows=-1, int ignore_this_row=-1) const; 00753 00754 //##### String Mappings ################################################# 00755 00757 void init_map_sr() const ; 00758 00760 void saveAllStringMappings(); 00761 00768 void saveStringMappings(int col, const PPath& fname, 00769 map<string, real>* str_to_real = 0); 00771 void addStringMapping(int col, string str, real val); 00772 00779 real addStringMapping(int col, string str); 00780 00782 void removeAllStringMappings(); 00783 00785 void removeColumnStringMappings(int c); 00786 00788 void removeStringMapping(int col, string str); 00789 00792 void setStringMapping(int col, const map<string,real>& zemap); 00793 00795 void deleteStringMapping(int col); 00796 00798 void loadStringMapping(int col); 00799 00802 virtual void loadAllStringMappings(); 00803 00805 void copyStringMappingsFrom(const VMat& source); 00806 00809 virtual string getValString(int col, real val) const; 00810 00812 virtual const map<string,real>& getStringToRealMapping(int col) const; 00813 00815 virtual const map<real,string>& getRealToStringMapping(int col) const; 00816 00819 virtual real getStringVal(int col, const string & str) const; 00820 00823 virtual string getString(int row, int col) const; 00824 00827 virtual void getRowAsStrings(int i, TVec<string>& v_str) const; 00828 00831 virtual PP<Dictionary> getDictionary(int col) const; 00832 00838 virtual void getValues(int row, int col, Vec& values) const; 00839 00841 virtual void getValues(const Vec& input, int col, Vec& values) const; 00842 00844 Vec getRowVec(int i) const; 00845 00847 void appendRows(Mat rows); 00848 00849 //##### SFIF Files ###################################################### 00850 00871 void setSFIFFilename(int col, string ext, const PPath& filepath=""); 00872 void setSFIFFilename(string fieldname, string ext, const PPath& filepath=""); 00873 00885 PPath getSFIFFilename(int col, string ext); 00886 PPath getSFIFFilename(string fieldname, string ext); 00887 00889 PPath getSFIFDirectory() const; 00890 00894 bool isSFIFDirect(int col, string ext); 00895 bool isSFIFDirect(string fieldname, string ext); 00896 00897 // return the size on the disk. If the sub class don't implement it return -1. 00898 virtual int64_t getSizeOnDisk(){return -1;} 00899 }; 00900 00901 DECLARE_OBJECT_PTR(VMatrix); 00902 00917 #define SPECIAL_FORMAT ((real)3.1e36) 00918 00919 00920 } // end of namespace PLearn 00921 00922 #endif 00923 00924 00925 /* 00926 Local Variables: 00927 mode:c++ 00928 c-basic-offset:4 00929 c-file-style:"stroustrup" 00930 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00931 indent-tabs-mode:nil 00932 fill-column:79 00933 End: 00934 */ 00935 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :