PLearn 0.1
|
00001 00002 // -*- C++ -*- 00003 00004 // TextFilesVMatrix.h 00005 // 00006 // Copyright (C) 2003-2004 ApSTAT Technologies Inc. 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 /* ******************************************************* 00037 * $Id: TextFilesVMatrix.h 10009 2009-03-11 15:28:58Z nouiz $ 00038 ******************************************************* */ 00039 00040 // Author: Pascal Vincent 00041 00043 #ifndef TextFilesVMatrix_INC 00044 #define TextFilesVMatrix_INC 00045 00046 #include <plearn/vmat/RowBufferedVMatrix.h> 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00051 00068 class TextFilesVMatrix: public RowBufferedVMatrix 00069 { 00070 protected: 00071 // ********************* 00072 // * protected options * 00073 // ********************* 00074 00075 FILE* idxfile; 00076 TVec<FILE*> txtfiles; 00077 00080 00081 public: 00082 00085 TVec< pair<int,int> > colrange; 00086 00087 // ************************ 00088 // * public build options * 00089 // ************************ 00090 00093 PPath metadatapath; 00094 00096 TVec<PPath> txtfilenames; 00097 00102 string delimiter; 00103 00106 string quote_delimiter; 00107 00110 TVec<int> skipheader; 00111 00113 TVec< pair<string, string> > fieldspec; 00114 00117 bool auto_build_map; 00118 00121 bool auto_extend_map; 00122 00125 bool build_vmatrix_stringmap; 00126 00127 bool reorder_fieldspec_from_headers; 00128 bool partial_match; 00129 00132 string default_spec; 00133 00134 // **************** 00135 // * Constructors * 00136 // **************** 00137 00138 // Default constructor, make sure the implementation in the .cc 00139 // initializes all fields to reasonable default values. 00140 TextFilesVMatrix(); 00141 virtual ~TextFilesVMatrix(); 00142 00143 // ****************** 00144 // * Object methods * 00145 // ****************** 00146 00147 private: 00149 // (Please implement in .cc) 00150 void build_(); 00151 void setColumnNamesAndWidth(); 00152 void getFileAndPos(int i, unsigned char& fileno, int& pos) const; 00153 void buildIdx(); 00154 static void readAndCheckOptionName(PStream& in, const string& optionname, char buf[]); 00155 void closeCurrentFile(); 00156 00157 protected: 00159 // (Please implement in .cc) 00160 static void declareOptions(OptionList& ol); 00161 00167 mutable TVec< hash_map<string, real> > mapping; 00168 00169 mutable TVec<FILE*> mapfiles; 00170 00172 void autoBuildMappings(); 00173 00174 void loadMappings(); 00175 00176 virtual real getPostalEncoding(const string& strval, bool display_warning = true) const; 00177 00179 virtual bool isValidNonSkipFieldType(const string& ftype) const; 00180 00181 public: 00182 00183 typedef RowBufferedVMatrix inherited; 00184 PLEARN_DECLARE_OBJECT(TextFilesVMatrix); 00185 00190 real getMapping(int fieldnum, const string& strval) const; 00191 00192 string getMapFilePath(int fieldnum) const 00193 { return getMetaDataDir()/"mappings"/fieldspec[fieldnum].first+".map"; } 00194 00196 string getTextRow(int i) const; 00197 00202 int nTextFields() const 00203 { return fieldspec.length(); } 00204 00206 TVec<string> splitIntoFields(const string& raw_row) const; 00207 00209 TVec<string> getTextFields(int i) const; 00210 00213 int getIndexOfTextField(const string& fieldname) const; 00214 virtual void setMetaDataDir(const PPath& the_metadatadir); 00215 00218 virtual void transformStringToValue(int k, string strval, Vec dest) const; 00219 00220 virtual void getNewRow(int i, const Vec& v) const; 00221 00222 // simply calls inherited::build() then build_() 00223 virtual void build(); 00224 00228 void buildVMatrixStringMapping(); 00229 00232 void generateMapCounts(); 00233 00235 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00236 00237 }; 00238 00239 } // end of namespace PLearn 00240 #endif 00241 00242 00243 /* 00244 Local Variables: 00245 mode:c++ 00246 c-basic-offset:4 00247 c-file-style:"stroustrup" 00248 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00249 indent-tabs-mode:nil 00250 fill-column:79 00251 End: 00252 */ 00253 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :