PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: FileVMatrix.cc 10263 2009-07-09 14:51:31Z nouiz $ 00039 ******************************************************* */ 00040 00041 #include "FileVMatrix.h" 00042 #include <plearn/io/fileutils.h> 00043 #include <plearn/io/pl_NSPR_io.h> 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00048 00051 PLEARN_IMPLEMENT_OBJECT( 00052 FileVMatrix, 00053 "VMatrix whose data is stored on disk in an uncompressed '.pmat' file.", 00054 "" 00055 ); 00056 00058 // FileVMatrix // 00060 FileVMatrix::FileVMatrix(): 00061 filename_ (""), 00062 f (0), 00063 build_new_file (false) 00064 { 00065 writable=true; 00066 remove_when_done = track_ref = -1; 00067 } 00068 00069 FileVMatrix::FileVMatrix(const PPath& filename, bool writable_): 00070 inherited (true), 00071 filename_ (filename.absolute()), 00072 f (0), 00073 build_new_file (!isfile(filename)) 00074 { 00075 remove_when_done = track_ref = -1; 00076 writable = writable_; 00077 build_(); 00078 } 00079 00080 FileVMatrix::FileVMatrix(const PPath& filename, int the_length, int the_width, 00081 bool force_float, bool call_build_): 00082 inherited (the_length, the_width, true), 00083 filename_ (filename.absolute()), 00084 f (0), 00085 file_is_float (force_float), 00086 force_float (force_float), 00087 build_new_file (true) 00088 { 00089 remove_when_done = track_ref = -1; 00090 writable = true; 00091 if(call_build_) 00092 build_(); 00093 } 00094 00095 FileVMatrix::FileVMatrix(const PPath& filename, int the_length, 00096 const TVec<string>& fieldnames): 00097 inherited (the_length, fieldnames.length(), true), 00098 filename_ (filename.absolute()), 00099 f (0), 00100 build_new_file (true) 00101 { 00102 remove_when_done = track_ref = -1; 00103 writable = true; 00104 build_(); 00105 declareFieldNames(fieldnames); 00106 saveFieldInfos(); 00107 } 00108 00109 static int strlen(char* s) { 00110 int n=0; 00111 while (s[n]!=0) 00112 n++; 00113 return n; 00114 } 00115 00117 // build // 00119 void FileVMatrix::build() 00120 { 00121 inherited::build(); 00122 build_(); 00123 } 00124 00126 // build_ // 00128 void FileVMatrix::build_() 00129 { 00130 // Check for deprecated options. 00131 if (remove_when_done != -1) { 00132 PLDEPRECATED("In FileVMatrix::build_ - The option 'remove_when_done' " 00133 "is now deprecated, please use the " 00134 "TemporaryFileVMatrix class instead: this data file " 00135 "may thus not be properly deleted"); 00136 PLASSERT( remove_when_done == 0 || remove_when_done == 1 ); 00137 } 00138 if (track_ref != -1) { 00139 PLDEPRECATED("In FileVMatrix::build_ - The option 'track_ref' " 00140 "is now deprecated, please use the " 00141 "TemporaryFileVMatrix class instead: this data file " 00142 "may thus not be properly deleted"); 00143 PLASSERT( track_ref == 0 || track_ref == 1 ); 00144 } 00145 // Since we are going to re-create it, we can close the current f. 00146 closeCurrentFile(); 00147 00148 // If no file is given, we can stop here. 00149 if (filename_.isEmpty()) 00150 return; 00151 00152 char header[DATAFILE_HEADERLENGTH]; 00153 char matorvec[20]; 00154 char datatype[20]; 00155 char endiantype[20]; 00156 00157 if (build_new_file || !isfile(filename_)) 00158 force_mkdir_for_file(filename_); 00159 00160 if (build_new_file || !isfile(filename_)) 00161 { 00162 if (!writable) 00163 PLERROR("In FileVMatrix::build_ - You asked to create a new file (%s), but 'writable' is set to 0 !", filename_.c_str()); 00164 00165 openfile(filename_,"w+b", 00166 PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0666); 00167 if (!f) 00168 PLERROR("In FileVMatrix constructor, could not open file %s",filename_.c_str()); 00169 00170 #ifdef USEFLOAT 00171 file_is_float = true; 00172 #endif 00173 #ifdef USEDOUBLE 00174 file_is_float = false; 00175 #endif 00176 #ifdef LITTLEENDIAN 00177 file_is_bigendian = false; 00178 #endif 00179 #ifdef BIGENDIAN 00180 file_is_bigendian = true; 00181 #endif 00182 if(force_float) 00183 file_is_float = true; 00184 00185 updateHeader(); 00186 00187 if(length_ > 0 && width_ > 0) 00188 { 00189 // Ensure we can allocate enough space... if len>0 00190 #ifdef USE_NSPR_FILE 00191 moveto(length_-1,width_-1); 00192 char c = '\0'; 00193 PR_Write(f, &c, 1); 00194 PR_Sync(f); 00195 #else 00196 if( fseek(f, DATAFILE_HEADERLENGTH+length_*width_*sizeof(real)-1, SEEK_SET) <0 ) 00197 { 00198 perror(""); 00199 PLERROR("In FileVMatrix::build_ - Could not fseek to last byte"); 00200 } 00201 fputc('\0',f); 00202 #endif 00203 } 00204 00205 // Get rid of any pre-existing .metadata :: this ensures that old 00206 // fieldsizes do not pollute a newly-created FileVMatrix 00207 force_rmdir(filename_ + ".metadata"); 00208 } 00209 else 00210 { 00211 if (writable) 00212 openfile(filename_, "r+b", PR_RDWR | PR_CREATE_FILE, 0666); 00213 else 00214 openfile(filename_, "rb", PR_RDONLY, 0666); 00215 00216 if (! f) 00217 PLERROR("FileVMatrix::build: could not open file %s", filename_.c_str()); 00218 00219 #ifdef USE_NSPR_FILE 00220 PR_Read(f,header,DATAFILE_HEADERLENGTH); 00221 #else 00222 fread(header,DATAFILE_HEADERLENGTH,1,f); 00223 #endif 00224 if(header[DATAFILE_HEADERLENGTH-1]!='\n') 00225 PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(0)"); 00226 int file_length, file_width; 00227 bool need_update_header = false; 00228 sscanf(header, "%s%d%d%s%s", matorvec, &file_length, &file_width, datatype, endiantype); 00229 if (file_length == -1 && this->length_ >= 0 && writable) { 00230 // The length set in the file is not valid, but we have specified a length. 00231 // This can happen if build() has been called once before the sizes have 00232 // been specified. In this case we must modify the file's length. 00233 need_update_header = true; 00234 } else if (file_length >= 0 && this->length_ >= 0 && file_length != this->length_) { 00235 PLWARNING("In FileVMatrix::build_ - The length option of the VMatrix and the " 00236 "loaded file differ; using the loaded file length (%d instead of %d)", 00237 file_length, this->length_); 00238 this->length_ = file_length; 00239 } else { 00240 this->length_ = file_length; 00241 } 00242 00243 if (file_width == -1 && this->width_ >= 0 && writable) { 00244 // Same as above, but for the width. 00245 need_update_header = true; 00246 } else if (file_width >= 0 && this->width_ >= 0 && file_width != this->width_) { 00247 PLWARNING("In FileVMatrix::build_ - The width option of the VMatrix and the " 00248 "loaded file differ; using the loaded file width (%d instead of %d)", 00249 file_width, this->width_); 00250 this->width_ = file_width; 00251 } else { 00252 this->width_ = file_width; 00253 } 00254 00255 if (need_update_header) { 00256 updateHeader(); 00257 } 00258 00259 if (strcmp(matorvec,"MATRIX")!=0) 00260 PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(1)"); 00261 00262 if (strcmp(endiantype,"LITTLE_ENDIAN")==0) 00263 file_is_bigendian = false; 00264 else if (strcmp(endiantype,"BIG_ENDIAN")==0) 00265 file_is_bigendian = true; 00266 else 00267 PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(2)"); 00268 00269 if (strcmp(datatype,"FLOAT")==0) 00270 file_is_float = true; 00271 else if (strcmp(datatype,"DOUBLE")==0) 00272 file_is_float = false; 00273 else 00274 PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(3)"); 00275 00276 //resize the string mappings 00277 if (width_ >= 0) { 00278 map_sr = TVec<map<string,real> >(width_); 00279 map_rs = TVec<map<real,string> >(width_); 00280 } 00281 #ifdef USE_NSPR_FILE 00282 //check if the file have all data 00283 PRFileInfo64 info; 00284 if(PR_FAILURE==PR_GetFileInfo64(filename_.absolute().c_str(), &info)) 00285 PLERROR("In FileVMatrix::build_() - can't get file info for file '%s'", 00286 filename_.c_str()); 00287 PRInt64 elemsize = file_is_float ? sizeof(float) : sizeof(double); 00288 PRInt64 expectedsize=DATAFILE_HEADERLENGTH+length_*width_*elemsize; 00289 if(info.size!=expectedsize) 00290 PLWARNING("In FileVMatrix::build_() - The file '%s' have a size" 00291 " of %ld, expected %ld", 00292 filename_.c_str(), (long int)info.size, (long int)expectedsize); 00293 #endif 00294 00295 } 00296 00297 setMetaDataDir(filename_ + ".metadata"); 00298 setMtime(mtime(filename_)); 00299 00300 if (width_ >= 0) 00301 getFieldInfos(); 00302 00303 loadFieldInfos(); 00304 } 00305 00306 00308 // closeCurrentFile // 00310 void FileVMatrix::closeCurrentFile() 00311 { 00312 if (f) 00313 { 00314 #ifdef USE_NSPR_FILE 00315 PR_Close(f); 00316 #else 00317 fclose(f); 00318 #endif 00319 f = 0; 00320 } 00321 } 00322 00324 // declareOptions // 00326 void FileVMatrix::declareOptions(OptionList & ol) 00327 { 00328 declareOption(ol, "filename", &FileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix"); 00329 00330 declareOption(ol, "remove_when_done", &FileVMatrix::remove_when_done, 00331 OptionBase::learntoption, 00332 "Deprecated option! (use TemporaryFileVMatrix instead)."); 00333 00334 declareOption(ol, "track_ref", &FileVMatrix::track_ref, 00335 OptionBase::learntoption, 00336 "Deprecated option! (use TemporaryFileVMatrix instead)."); 00337 00338 inherited::declareOptions(ol); 00339 } 00340 00342 // makeDeepCopyFromShallowCopy // 00344 void FileVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00345 { 00346 inherited::makeDeepCopyFromShallowCopy(copies); 00347 00348 // it is unclear whether f should be shared or not... 00349 // if we allow multi-threading, we should probably not share it 00350 // so for now we will not share it. But a cleaner behavoiur would probably be 00351 // to share it in multiple threads but make sure that getRow, putRow, etc... operations 00352 // are atomic (no context switch to another thread). 00353 00354 f = 0; // Because we will open again the file (f should not be shared). 00355 // however reopening the file twice in write mode is certainly a VERY bad idea. 00356 // thus we switch to read-mode 00357 build_new_file = false; 00358 writable = false; 00359 00360 build(); // To open the file. 00361 } 00362 00364 // ~FileVMatrix // 00366 FileVMatrix::~FileVMatrix() 00367 { 00368 if (hasMetaDataDir() && isWritable()) 00369 saveFieldInfos(); 00370 FileVMatrix::closeCurrentFile(); 00371 } 00372 00373 00375 // getNewRow // 00377 void FileVMatrix::getNewRow(int i, const Vec& v) const 00378 { 00379 #ifdef BOUNDCHECK 00380 if(v.length()!=width_) 00381 PLERROR("In FileVMatrix::getNewRow length of v (%d) differs from matrix width (%d)",v.length(),width_); 00382 #endif 00383 00384 moveto(i); 00385 #ifdef USE_NSPR_FILE 00386 if(file_is_float) 00387 PR_Read_float(f, v.data(), v.length(), file_is_bigendian); 00388 else 00389 PR_Read_double(f, v.data(), v.length(), file_is_bigendian); 00390 #else 00391 if(file_is_float) 00392 fread_float(f, v.data(), v.length(), file_is_bigendian); 00393 else 00394 fread_double(f, v.data(), v.length(), file_is_bigendian); 00395 #endif 00396 } 00397 00399 // putSubRow // 00401 void FileVMatrix::putSubRow(int i, int j, Vec v) 00402 { 00403 moveto(i,j); 00404 #ifdef USE_NSPR_FILE 00405 if(file_is_float) 00406 PR_Write_float(f, v.data(), v.length(), file_is_bigendian); 00407 else 00408 PR_Write_double(f, v.data(), v.length(), file_is_bigendian); 00409 #else 00410 if(file_is_float) 00411 fwrite_float(f, v.data(), v.length(), file_is_bigendian); 00412 else 00413 fwrite_double(f, v.data(), v.length(), file_is_bigendian); 00414 #endif 00415 00416 invalidateBuffer(); 00417 } 00418 00419 void FileVMatrix::moveto(int i, int j) const 00420 { 00421 #ifdef USE_NSPR_FILE 00422 PRInt64 offset = i; 00423 int elemsize = file_is_float ? sizeof(float) : sizeof(double); 00424 offset *= width_; 00425 offset += j; 00426 offset *= elemsize; 00427 offset += DATAFILE_HEADERLENGTH; 00428 PR_Seek64(f, offset, PR_SEEK_SET); 00429 #else 00430 if(file_is_float) 00431 fseek(f, DATAFILE_HEADERLENGTH+(i*width_+j)*sizeof(float), SEEK_SET); 00432 else 00433 fseek(f, DATAFILE_HEADERLENGTH+(i*width_+j)*sizeof(double), SEEK_SET); 00434 #endif 00435 } 00436 00438 // put // 00440 void FileVMatrix::put(int i, int j, real value) 00441 { 00442 moveto(i,j); 00443 #ifdef USE_NSPR_FILE 00444 if(file_is_float) 00445 PR_Write_float(f,float(value),file_is_bigendian); 00446 else 00447 PR_Write_double(f,double(value),file_is_bigendian); 00448 #else 00449 if(file_is_float) 00450 fwrite_float(f,float(value),file_is_bigendian); 00451 else 00452 fwrite_double(f,double(value),file_is_bigendian); 00453 #endif 00454 invalidateBuffer(); 00455 } 00456 00458 // appendRow // 00460 void FileVMatrix::appendRow(Vec v) 00461 { 00462 moveto(length_,0); 00463 #ifdef USE_NSPR_FILE 00464 if(file_is_float) 00465 PR_Write_float(f, v.data(), v.length(), file_is_bigendian); 00466 else 00467 PR_Write_double(f, v.data(), v.length(), file_is_bigendian); 00468 #else 00469 if(file_is_float) 00470 fwrite_float(f, v.data(), v.length(), file_is_bigendian); 00471 else 00472 fwrite_double(f, v.data(), v.length(), file_is_bigendian); 00473 #endif 00474 00475 length_++; 00476 updateHeader(); 00477 } 00478 00480 // flush // 00482 void FileVMatrix::flush() 00483 { 00484 if(f) 00485 { 00486 #ifdef USE_NSPR_FILE 00487 PR_Sync(f); 00488 #else 00489 fflush(f); 00490 #endif 00491 } 00492 } 00493 00495 // updateHeader // 00497 void FileVMatrix::updateHeader() { 00498 char header[DATAFILE_HEADERLENGTH]; 00499 string real = "DOUBLE"; 00500 if(file_is_float) 00501 real = "FLOAT"; 00502 00503 #ifdef LITTLEENDIAN 00504 sprintf(header,"MATRIX %d %d %s LITTLE_ENDIAN", length_, width_, real.c_str()); 00505 #endif 00506 #ifdef BIGENDIAN 00507 sprintf(header,"MATRIX %d %d %s BIG_ENDIAN", length_, width_, real.c_str()); 00508 #endif 00509 00510 00511 int pos = strlen(header); 00512 for(; pos<DATAFILE_HEADERLENGTH; pos++) 00513 { 00514 header[pos] = ' '; 00515 } 00516 header[DATAFILE_HEADERLENGTH-1] = '\n'; 00517 00518 #ifdef USE_NSPR_FILE 00519 PR_Seek64(f, 0, PR_SEEK_SET); 00520 PR_Write(f, header, DATAFILE_HEADERLENGTH); 00521 #else 00522 fseek(f,0,SEEK_SET); 00523 fwrite(header,1,DATAFILE_HEADERLENGTH,f); 00524 #endif 00525 } 00526 00528 // updateHeader // 00530 void FileVMatrix::openfile(const PPath& path, const char *mode, 00531 PRIntn flags, PRIntn mode2) { 00532 #ifdef USE_NSPR_FILE 00533 f = PR_Open(path.absolute().c_str(), flags, mode2); 00534 #else 00535 f = fopen(path.absolute().c_str(), mode); 00536 #endif 00537 00538 } 00539 00540 00541 int64_t FileVMatrix::getSizeOnDisk(){ 00542 return DATAFILE_HEADERLENGTH + width_*length_*(file_is_float ? 4 : 8); 00543 } 00544 } // end of namespace PLearn 00545 00546 00547 /* 00548 Local Variables: 00549 mode:c++ 00550 c-basic-offset:4 00551 c-file-style:"stroustrup" 00552 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00553 indent-tabs-mode:nil 00554 fill-column:79 00555 End: 00556 */ 00557 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :