PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2008 Xavier Saint-Mleux, ApSTAT Technologies inc. 00008 // 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ******************************************************* 00039 * $Id: DiskVMatrix.cc 10003 2009-03-10 17:06:50Z tihocan $ 00040 ******************************************************* */ 00041 00042 #include "DiskVMatrix.h" 00043 #include <errno.h> 00044 #include <errno.h> 00045 #include <plearn/base/stringutils.h> 00046 #include <plearn/io/pl_io.h> 00047 #include <plearn/io/fileutils.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 #if defined(WIN32) && !defined(__CYGWIN__) 00053 #include <io.h> 00054 #define unlink _unlink 00055 #endif 00056 00058 /* Format description 00059 00060 The directory in variable dirname, which should end in .dmat, 00061 contains a file named 'indexfile' and data files of up to roughly 00062 500Meg each named 0.data, 1.data, ... 00063 00064 The indexfile has a 4 byte header, of which currently only the first byte is checked. 00065 'L' means new little-endian format 00066 'B' means new big-endian format 00067 An ascii code of 16 means old little-endian format 00068 00069 In the new formats, the last 3 bytes of the 4 byte header are ' ' 00070 00071 All other raw-binary data (whether in the indexfile or the data files) 00072 is written in the specified endianness (and swapping will be performed if 00073 reading it on a machine with a different endianness). 00074 00075 Following the 4-byte header are two 4-byte ints: length and width of the matrix. 00076 00077 N.B. As of October 2008, the length is not always stored in the indexfile's 00078 header; instead, while the matrix is opened for writing, 00079 DiskVMatrix::NO_LENGTH_SENTINEL (=-42) is used to indicate that the actual 00080 matrix length should be calculated some other way. The sentinel is replaced by 00081 the actual length when the VMat is closed. It is an error to open a 00082 DiskVMatrix for writing while its header indicates a length of 00083 DiskVMatrix::NO_LENGTH_SENTINEL *but this is not a safe locking mechanism, so 00084 you still have to be careful!* 00085 00086 Following are 5 bytes for each row of the matrix. 00087 The first of those 5 bytes is an unsigned byte indicating the number (0-255) of 00088 the data file in which to find the row. 00089 The remaining 4 bytes are an unsigned int indicating the start position of the row in 00090 that data file (as passed to fseek). 00091 00092 The row is encoded in the thus indicated data file at the indicated position, 00093 using the new vector compression format (see new_read_compressed) 00094 [ the old compression format pf binread_compressed is supported 00095 for backward compatibility only. ] 00096 00097 */ 00098 00099 PLEARN_IMPLEMENT_OBJECT( 00100 DiskVMatrix, 00101 "VMatrix whose data are precomputed on disk (in a .dmat directory).", 00102 "" 00103 ); 00104 00105 // DiskVMatrix::NO_LENGTH_SENTINEL= -42; written in header 00106 // instead of actual length while the file is opened for writing 00107 const int DiskVMatrix::NO_LENGTH_SENTINEL; 00108 const int DiskVMatrix::HEADER_OFFSET_LENGTH; 00109 00110 DiskVMatrix::DiskVMatrix(): 00111 indexf (0), 00112 freshnewfile(false), 00113 old_format (false), 00114 swap_endians(false), 00115 last_op_was_append(false), 00116 tolerance (1e-6) 00117 { 00118 writable = false; 00119 } 00120 00121 DiskVMatrix::DiskVMatrix(const PPath& the_dirname, bool readwrite, 00122 bool call_build_): 00123 inherited (call_build_), 00124 indexf (0), 00125 freshnewfile(false), 00126 old_format (false), 00127 swap_endians(false), 00128 last_op_was_append(false), 00129 dirname (the_dirname), 00130 tolerance (1e-6) 00131 { 00132 writable = readwrite; 00133 dirname.removeTrailingSlash(); // For safety. 00134 if (call_build_) 00135 build_(); 00136 } 00137 00138 DiskVMatrix::DiskVMatrix(const PPath& the_dirname, int the_width, 00139 bool write_double_as_float): 00140 inherited (0, the_width, true), 00141 indexf (0), 00142 freshnewfile(true), 00143 old_format (false), 00144 swap_endians(false), 00145 last_op_was_append(false), 00146 dirname (the_dirname), 00147 tolerance (1e-6) 00148 { 00149 writable = true; 00150 dirname.removeTrailingSlash(); // For safety. 00151 build_(); 00152 } 00153 00154 void DiskVMatrix::build() 00155 { 00156 inherited::build(); 00157 build_(); 00158 } 00159 00160 void DiskVMatrix::build_() 00161 { 00162 closeCurrentFiles(); // All file pointers will be re-created. 00163 if(writable && width_ > 0 && !isdir(dirname)) 00164 freshnewfile= true; 00165 if(!freshnewfile) 00166 { 00167 if(!isdir(dirname)) 00168 PLERROR("In DiskVMatrix constructor, directory '%s' could not be found",dirname.c_str()); 00169 setMetaDataDir(dirname + ".metadata"); 00170 updateMtime( dirname/"indexfile" ); 00171 string omode; 00172 if(writable) 00173 omode = "r+b"; 00174 else // read-only 00175 omode = "rb"; 00176 00177 string indexfname = dirname/"indexfile"; 00178 indexf = fopen(indexfname.c_str(), omode.c_str()); 00179 if(!indexf) 00180 PLERROR("In DiskVMatrix constructor, could not open file %s in specified mode", indexfname.c_str()); 00181 00182 unsigned char header[4]; 00183 fread(header,1,4,indexf); 00184 if(header[0]=='L' || header[0]=='B') 00185 { // New format 00186 old_format = false; 00187 swap_endians = (header[0]!=byte_order()); 00188 } 00189 else if(header[0]==16) 00190 { // Old format 00191 old_format = true; 00192 if(byte_order()!='L') 00193 PLERROR("Old format DiskVMatrix can only be read from a little-endian machine.\n" 00194 "Convert it to a new format on a little-endian machine prior to attempt\n" 00195 "using it from a big endian machine.\n"); 00196 swap_endians = false; 00197 } 00198 else 00199 { 00200 PLERROR("Wrong header byte in index file %s: ascii code %d\n" 00201 "(should be 'L' or 'B' or '...')\n", indexfname.c_str(), header[0]); 00202 } 00203 00204 fread(&length_,sizeof(int),1,indexf); 00205 fread(&width_,sizeof(int),1,indexf); 00206 if(swap_endians) 00207 { 00208 endianswap(&length_); 00209 endianswap(&width_); 00210 } 00211 00212 // Calc. length of VMat from indexfile length 00213 fseek(indexf, 0, SEEK_END); 00214 int pos= ftell(indexf); 00215 // pos less 12 header bytes s/b a multiple of 5 since each index 00216 // entry is 5-byte long i.e. pos%5==2 00217 PLASSERT_MSG(pos%5 == 2, 00218 "Length of DiskVMatrix is absent from header and length" 00219 " of index file is " + tostring(pos) + ", not \\eqv 2 (mod 5)" ); 00220 int calculated_length= (pos-12) / 5; //nb. lines in matrix 00221 00222 if(length_ == NO_LENGTH_SENTINEL) // check after endianswap 00223 { 00224 PLASSERT_MSG(!writable, // already open for writing? not good... 00225 "Trying to open a DiskVMatrix for writing but length in header is " 00226 "NO_LENGTH_SENTINEL (="+tostring(NO_LENGTH_SENTINEL)+"); " 00227 "Is it already being written to by another process? " 00228 "dirname='" + dirname + "'."); 00229 length_= calculated_length; 00230 } 00231 else 00232 { 00233 PLASSERT_MSG(length_ == calculated_length, //length mismatch? not good... 00234 "Length in DiskVMatrix's header does not match the indexfile's length " 00235 "and is not NO_LENGTH_SENTINEL (="+tostring(NO_LENGTH_SENTINEL)+"); " 00236 "indexfile='" + indexfname + "', " 00237 "header len=" + tostring(length_) + ", " 00238 "calc. len=" + tostring(calculated_length) + "."); 00239 if(writable) 00240 {// put NO_LENGTH_SENTINEL in header to indicate that we are appending to this file. 00241 fseek(indexf, HEADER_OFFSET_LENGTH, SEEK_SET); 00242 fwrite(&NO_LENGTH_SENTINEL, 4, 1, indexf); 00243 // will seek to end before end of build_ 00244 } 00245 } 00246 int k=0; 00247 string fname = dirname/tostring(k)+".data"; 00248 while(isfile(fname)) 00249 { 00250 FILE* f = fopen(fname.c_str(), omode.c_str()); 00251 if(!f) 00252 PLERROR("In DiskVMatrix constructor, could not open file %s in specified mode", fname.c_str()); 00253 dataf.append(f); 00254 fname = dirname/(tostring(++k)+".data"); 00255 } 00256 // Stuff related to RowBufferedVMatrix, for consistency 00257 current_row_index = -1; 00258 current_row.resize(width_); 00259 other_row_index = -1; 00260 other_row.resize(width_); 00261 00262 //resize the string mappings 00263 map_sr = TVec<map<string,real> >(width_); 00264 map_rs = TVec<map<real,string> >(width_); 00265 00266 getFieldInfos(); 00267 if (writable) 00268 fseek(indexf, 0, SEEK_END); 00269 } 00270 else 00271 { 00272 if(isdir(dirname)) 00273 PLERROR("In DiskVMatrix constructor (with specified width), directory %s already exists",dirname.c_str()); 00274 setMetaDataDir(dirname + ".metadata"); 00275 updateMtime( dirname/"indexfile" ); 00276 00277 if(isfile(dirname)) // patch for running mkstemp (TmpFilenames) 00278 unlink(dirname.c_str()); 00279 if(!force_mkdir(dirname)) // force directory creation 00280 PLERROR("In DiskVMatrix constructor (with specified width), could not create directory %s Error was: %s",dirname.c_str(), strerror(errno)); 00281 00282 string indexfname = dirname/"indexfile"; 00283 indexf = fopen(indexfname.c_str(),"w+b"); 00284 00285 char header[4]; 00286 header[0] = byte_order(); 00287 header[1] = ' '; 00288 header[2] = ' '; 00289 header[3] = ' '; 00290 fwrite(header,1,4,indexf); 00291 //no more length in header while opened [was fwrite((char*)&length_,sizeof(int),1,indexf);] 00292 fwrite(&NO_LENGTH_SENTINEL, sizeof(int), 1, indexf); 00293 fwrite((char*)&width_,sizeof(int),1,indexf); 00294 length_= 0; 00295 00296 string fname = dirname/"0.data"; 00297 FILE* f = fopen(fname.c_str(), "w+b"); 00298 dataf.append(f); 00299 } 00300 freshnewfile=false; 00301 loadAllStringMappings();//make sure string mappings are loaded after width is set 00302 last_op_was_append= false; 00303 } 00304 00305 void DiskVMatrix::declareOptions(OptionList &ol) 00306 { 00307 declareOption(ol, "dirname", &DiskVMatrix::dirname, OptionBase::buildoption, "Directory name of the.dmat"); 00308 declareOption(ol, "tolerance", &DiskVMatrix::tolerance, OptionBase::buildoption, "The absolute error tolerance for storing doubles as floats"); 00309 inherited::declareOptions(ol); 00310 } 00311 00313 // closeCurrentFiles // 00315 void DiskVMatrix::closeCurrentFiles() 00316 { 00317 for (int i = 0; i < dataf.length(); i++) 00318 if(dataf[i]) { 00319 fclose(dataf[i]); 00320 dataf[i] = 0; 00321 } 00322 00323 dataf.resize(0); 00324 00325 if (indexf) { 00326 // write final length to indexfile before closing 00327 fseek(indexf, HEADER_OFFSET_LENGTH, SEEK_SET); // it is and will always be 4 bytes 00328 int le= length_; 00329 if(swap_endians) 00330 endianswap(&le); 00331 fwrite(&le,sizeof(int),1,indexf); 00332 00333 fclose(indexf); 00334 indexf = 0; 00335 } 00336 last_op_was_append= false; 00337 } 00338 00340 // getNewRow // 00342 void DiskVMatrix::getNewRow(int i, const Vec& v) const 00343 { 00344 #ifdef BOUNDCHECK 00345 if(i<0 || i>length()) 00346 PLERROR("In DiskVMatrix::getNewRow, bad row number %d",i); 00347 if(v.length() != width()) 00348 PLERROR("In DiskVMatrix::getNewRow, length of v (%d) does not match matrix width (%d)",v.length(),width()); 00349 #endif 00350 00351 unsigned char filenum; 00352 unsigned int position; 00353 fseek(indexf,3*sizeof(int) + i*(sizeof(unsigned char)+sizeof(unsigned int)), SEEK_SET); 00354 fread(&filenum,sizeof(unsigned char),1,indexf); 00355 fread(&position,sizeof(unsigned int),1,indexf); 00356 if(swap_endians) 00357 endianswap(&position); 00358 FILE* f = dataf[int(filenum)]; 00359 PLASSERT( f ); 00360 fseek(f,position,SEEK_SET); 00361 if(old_format) 00362 binread_compressed(f,v.data(),v.length()); 00363 else 00364 new_read_compressed(f, v.data(), v.length(), swap_endians); 00365 last_op_was_append= false; 00366 } 00367 00368 void DiskVMatrix::putRow(int i, Vec v) 00369 { 00370 PLERROR("putRow cannot in general be correctly and efficiently implemented for a DiskVMatrix.\n" 00371 "Use appendRow if you wish to write more rows."); 00372 } 00373 00374 void DiskVMatrix::appendRow(Vec v) 00375 { 00376 if(!writable) 00377 PLERROR("In DiskVMatrix::appendRow cannot append row in read only mode, set readwrite parameter to true when calling the constructor"); 00378 if(v.length() != width()) 00379 PLERROR("In DiskVMatrix::appendRow, length of v (%d) does not match matrix width (%d)",v.length(),width()); 00380 00381 int filenum = dataf.size()-1; 00382 FILE* f = dataf[filenum]; 00383 if(!last_op_was_append) //otherwise, we are already at end 00384 fseek(f,0,SEEK_END); 00385 unsigned int position = (unsigned int)ftell(f); 00386 if(position>500000000L) 00387 { 00388 fflush(f); 00389 filenum++; 00390 string filename = dirname / (tostring(filenum) + ".data"); 00391 f = fopen(filename.c_str(), "w+b"); 00392 dataf.append(f); 00393 position = 0; 00394 } 00395 if(old_format) 00396 binwrite_compressed(f,v.data(),v.length()); 00397 else 00398 new_write_compressed(f, v.data(),v.length(), tolerance, swap_endians); 00399 00400 if(!last_op_was_append) //otherwise, we are already at end 00401 fseek(indexf,0,SEEK_END); 00402 fputc(filenum,indexf); 00403 fwrite((char*)&position,sizeof(unsigned int),1,indexf); 00404 length_++; 00405 /* DO NOT write length in header; wait 'till file is closed 00406 fseek(indexf,sizeof(int),SEEK_SET); 00407 int le = length_; 00408 if(swap_endians) 00409 endianswap(&le); 00410 fwrite(&le,sizeof(int),1,indexf); 00411 */ 00412 last_op_was_append= true; 00413 } 00414 00416 // flush // 00418 void DiskVMatrix::flush() 00419 { 00420 int filenum = dataf.size()-1; 00421 FILE* f = dataf[filenum]; 00422 fflush(f); 00423 fflush(indexf); 00424 last_op_was_append= false; 00425 } 00426 00428 // makeDeepCopyFromShallowCopy // 00430 void DiskVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00431 { 00432 inherited::makeDeepCopyFromShallowCopy(copies); 00433 deepCopyField(dataf, copies); 00434 // Set pointers to zero, because we will open again the file (file pointers 00435 // should not be shared between copied objects). 00436 indexf = 0; 00437 dataf.fill(0); 00438 dataf.resize(0); 00439 build(); // Open again the dataset. 00440 } 00441 00442 00444 // ~DiskVMatrix // 00446 DiskVMatrix::~DiskVMatrix() 00447 { 00448 if (hasMetaDataDir()) 00449 saveFieldInfos(); 00450 DiskVMatrix::closeCurrentFiles(); 00451 } 00452 00453 00454 #ifdef WIN32 00455 #undef unlink 00456 #endif 00457 00458 } // end of namespace PLearn 00459 00460 00461 /* 00462 Local Variables: 00463 mode:c++ 00464 c-basic-offset:4 00465 c-file-style:"stroustrup" 00466 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00467 indent-tabs-mode:nil 00468 fill-column:79 00469 End: 00470 */ 00471 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :