PLearn 0.1
FileVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 //
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 //
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 //
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 //
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 //
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 /* *******************************************************
00038  * $Id: FileVMatrix.cc 10263 2009-07-09 14:51:31Z nouiz $
00039  ******************************************************* */
00040 
00041 #include "FileVMatrix.h"
00042 #include <plearn/io/fileutils.h>
00043 #include <plearn/io/pl_NSPR_io.h>
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00048 
00051 PLEARN_IMPLEMENT_OBJECT(
00052     FileVMatrix,
00053     "VMatrix whose data is stored on disk in an uncompressed '.pmat' file.",
00054     ""
00055 );
00056 
00058 // FileVMatrix //
00060 FileVMatrix::FileVMatrix():
00061     filename_       (""),
00062     f               (0),
00063     build_new_file  (false)
00064 {
00065     writable=true;
00066     remove_when_done = track_ref = -1;
00067 }
00068 
00069 FileVMatrix::FileVMatrix(const PPath& filename, bool writable_):
00070     inherited       (true),
00071     filename_       (filename.absolute()),
00072     f               (0),
00073     build_new_file  (!isfile(filename))
00074 {
00075     remove_when_done = track_ref = -1;
00076     writable = writable_;
00077     build_();
00078 }
00079 
00080 FileVMatrix::FileVMatrix(const PPath& filename, int the_length, int the_width,
00081                          bool force_float, bool call_build_):
00082     inherited       (the_length, the_width, true),
00083     filename_       (filename.absolute()),
00084     f               (0),
00085     file_is_float   (force_float),
00086     force_float     (force_float),
00087     build_new_file  (true)
00088 {
00089     remove_when_done = track_ref = -1;
00090     writable = true;
00091     if(call_build_)
00092         build_();
00093 }
00094 
00095 FileVMatrix::FileVMatrix(const PPath& filename, int the_length,
00096                          const TVec<string>& fieldnames):
00097     inherited       (the_length, fieldnames.length(), true),
00098     filename_       (filename.absolute()),
00099     f               (0),
00100     build_new_file  (true)
00101 {
00102     remove_when_done = track_ref = -1;
00103     writable = true;
00104     build_();
00105     declareFieldNames(fieldnames);
00106     saveFieldInfos();
00107 }
00108 
00109 static int strlen(char* s) {
00110     int n=0;
00111     while (s[n]!=0)
00112         n++;
00113     return n;
00114 }
00115 
00117 // build //
00119 void FileVMatrix::build()
00120 {
00121     inherited::build();
00122     build_();
00123 }
00124 
00126 // build_ //
00128 void FileVMatrix::build_()
00129 {
00130     // Check for deprecated options.
00131     if (remove_when_done != -1) {
00132         PLDEPRECATED("In FileVMatrix::build_ - The option 'remove_when_done' "
00133                      "is now deprecated, please use the "
00134                      "TemporaryFileVMatrix class instead: this data file "
00135                      "may thus not be properly deleted");
00136         PLASSERT( remove_when_done == 0 || remove_when_done == 1 );
00137     }
00138     if (track_ref != -1) {
00139         PLDEPRECATED("In FileVMatrix::build_ - The option 'track_ref' "
00140                      "is now deprecated, please use the "
00141                      "TemporaryFileVMatrix class instead: this data file "
00142                      "may thus not be properly deleted");
00143         PLASSERT( track_ref == 0 || track_ref == 1 );
00144     }
00145     // Since we are going to re-create it, we can close the current f.
00146     closeCurrentFile();
00147 
00148     // If no file is given, we can stop here.
00149     if (filename_.isEmpty())
00150         return;
00151 
00152     char header[DATAFILE_HEADERLENGTH];
00153     char matorvec[20];
00154     char datatype[20];
00155     char endiantype[20];
00156 
00157     if (build_new_file || !isfile(filename_))
00158         force_mkdir_for_file(filename_);
00159 
00160     if (build_new_file || !isfile(filename_))
00161     {
00162         if (!writable)
00163             PLERROR("In FileVMatrix::build_ - You asked to create a new file (%s), but 'writable' is set to 0 !", filename_.c_str());
00164 
00165         openfile(filename_,"w+b", 
00166                  PR_RDWR | PR_CREATE_FILE | PR_TRUNCATE, 0666);
00167         if (!f)
00168             PLERROR("In FileVMatrix constructor, could not open file %s",filename_.c_str());
00169 
00170 #ifdef USEFLOAT
00171         file_is_float = true;
00172 #endif
00173 #ifdef USEDOUBLE
00174         file_is_float = false;
00175 #endif
00176 #ifdef LITTLEENDIAN
00177         file_is_bigendian = false;
00178 #endif
00179 #ifdef BIGENDIAN
00180         file_is_bigendian = true;
00181 #endif
00182         if(force_float)
00183             file_is_float = true;
00184 
00185         updateHeader();
00186 
00187         if(length_ > 0 && width_ > 0)
00188         {
00189             // Ensure we can allocate enough space... if len>0
00190 #ifdef USE_NSPR_FILE
00191             moveto(length_-1,width_-1);
00192             char c = '\0';
00193             PR_Write(f, &c, 1);
00194             PR_Sync(f);
00195 #else
00196             if( fseek(f, DATAFILE_HEADERLENGTH+length_*width_*sizeof(real)-1, SEEK_SET) <0 )
00197             {
00198                 perror("");
00199                 PLERROR("In FileVMatrix::build_ - Could not fseek to last byte");
00200             }
00201             fputc('\0',f);
00202 #endif
00203         }
00204 
00205         // Get rid of any pre-existing .metadata :: this ensures that old
00206         // fieldsizes do not pollute a newly-created FileVMatrix
00207         force_rmdir(filename_ + ".metadata");
00208     }
00209     else
00210     {
00211         if (writable)
00212             openfile(filename_, "r+b", PR_RDWR | PR_CREATE_FILE, 0666);
00213         else
00214             openfile(filename_, "rb", PR_RDONLY, 0666);
00215 
00216         if (! f)
00217             PLERROR("FileVMatrix::build: could not open file %s", filename_.c_str());
00218 
00219 #ifdef USE_NSPR_FILE
00220         PR_Read(f,header,DATAFILE_HEADERLENGTH);
00221 #else
00222         fread(header,DATAFILE_HEADERLENGTH,1,f);
00223 #endif
00224         if(header[DATAFILE_HEADERLENGTH-1]!='\n')
00225             PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(0)");
00226         int file_length, file_width;
00227         bool need_update_header = false;
00228         sscanf(header, "%s%d%d%s%s", matorvec, &file_length, &file_width, datatype, endiantype);
00229         if (file_length == -1 && this->length_ >= 0 && writable) {
00230             // The length set in the file is not valid, but we have specified a length.
00231             // This can happen if build() has been called once before the sizes have
00232             // been specified. In this case we must modify the file's length.
00233             need_update_header = true;
00234         } else if (file_length >= 0 && this->length_ >= 0 && file_length != this->length_) {
00235             PLWARNING("In FileVMatrix::build_ - The length option of the VMatrix and the "
00236                       "loaded file differ; using the loaded file length (%d instead of %d)",
00237                       file_length, this->length_);
00238             this->length_ = file_length;
00239         } else {
00240             this->length_ = file_length;
00241         }
00242 
00243         if (file_width == -1 && this->width_ >= 0 && writable) {
00244             // Same as above, but for the width.
00245             need_update_header = true;
00246         } else if (file_width >= 0 && this->width_ >= 0 && file_width != this->width_) {
00247             PLWARNING("In FileVMatrix::build_ - The width option of the VMatrix and the "
00248                       "loaded file differ; using the loaded file width (%d instead of %d)",
00249                       file_width, this->width_);
00250             this->width_ = file_width;
00251         } else {
00252             this->width_ = file_width;
00253         }
00254 
00255         if (need_update_header) {
00256             updateHeader();
00257         }
00258 
00259         if (strcmp(matorvec,"MATRIX")!=0)
00260             PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(1)");
00261 
00262         if (strcmp(endiantype,"LITTLE_ENDIAN")==0)
00263             file_is_bigendian = false;
00264         else if (strcmp(endiantype,"BIG_ENDIAN")==0)
00265             file_is_bigendian = true;
00266         else
00267             PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(2)");
00268 
00269         if (strcmp(datatype,"FLOAT")==0)
00270             file_is_float = true;
00271         else if (strcmp(datatype,"DOUBLE")==0)
00272             file_is_float = false;
00273         else
00274             PLERROR("In FileVMatrix constructor, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.(3)");
00275 
00276         //resize the string mappings
00277         if (width_ >= 0) {
00278             map_sr = TVec<map<string,real> >(width_);
00279             map_rs = TVec<map<real,string> >(width_);
00280         }
00281 #ifdef USE_NSPR_FILE
00282         //check if the file have all data
00283         PRFileInfo64 info;
00284         if(PR_FAILURE==PR_GetFileInfo64(filename_.absolute().c_str(), &info))
00285             PLERROR("In FileVMatrix::build_() - can't get file info for file '%s'",
00286                     filename_.c_str());
00287         PRInt64 elemsize = file_is_float ? sizeof(float) : sizeof(double);
00288         PRInt64 expectedsize=DATAFILE_HEADERLENGTH+length_*width_*elemsize;
00289         if(info.size!=expectedsize)
00290             PLWARNING("In FileVMatrix::build_() - The file '%s' have a size"
00291                       " of %ld, expected %ld",
00292                       filename_.c_str(), (long int)info.size, (long int)expectedsize);
00293 #endif
00294         
00295     }
00296 
00297     setMetaDataDir(filename_ + ".metadata");
00298     setMtime(mtime(filename_));
00299 
00300     if (width_ >= 0)
00301         getFieldInfos();
00302 
00303     loadFieldInfos();
00304 }
00305 
00306 
00308 // closeCurrentFile //
00310 void FileVMatrix::closeCurrentFile()
00311 {
00312     if (f)
00313     {
00314 #ifdef USE_NSPR_FILE
00315         PR_Close(f);
00316 #else
00317         fclose(f);
00318 #endif
00319         f = 0;
00320     }
00321 }
00322 
00324 // declareOptions //
00326 void FileVMatrix::declareOptions(OptionList & ol)
00327 {
00328     declareOption(ol, "filename", &FileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix");
00329 
00330     declareOption(ol, "remove_when_done", &FileVMatrix::remove_when_done,
00331             OptionBase::learntoption,
00332             "Deprecated option! (use TemporaryFileVMatrix instead).");
00333 
00334     declareOption(ol, "track_ref", &FileVMatrix::track_ref,
00335             OptionBase::learntoption,
00336             "Deprecated option! (use TemporaryFileVMatrix instead).");
00337 
00338     inherited::declareOptions(ol);
00339 }
00340 
00342 // makeDeepCopyFromShallowCopy //
00344 void FileVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00345 {
00346     inherited::makeDeepCopyFromShallowCopy(copies);
00347 
00348     // it is unclear whether f should be shared or not...
00349     // if we allow multi-threading, we should probably not share it
00350     // so for now we will not share it. But a cleaner behavoiur would probably be
00351     // to share it in multiple threads but make sure that getRow, putRow, etc... operations
00352     // are atomic (no context switch to another thread).
00353 
00354     f = 0;   // Because we will open again the file (f should not be shared).
00355     // however reopening the file twice in write mode is certainly a VERY bad idea.
00356     // thus we switch to read-mode 
00357     build_new_file = false;
00358     writable = false;
00359 
00360     build(); // To open the file.
00361 }
00362 
00364 // ~FileVMatrix //
00366 FileVMatrix::~FileVMatrix()
00367 {
00368     if (hasMetaDataDir() && isWritable())
00369         saveFieldInfos();
00370     FileVMatrix::closeCurrentFile();
00371 }
00372 
00373 
00375 // getNewRow //
00377 void FileVMatrix::getNewRow(int i, const Vec& v) const
00378 {
00379 #ifdef BOUNDCHECK
00380     if(v.length()!=width_)
00381         PLERROR("In FileVMatrix::getNewRow length of v (%d) differs from matrix width (%d)",v.length(),width_);
00382 #endif
00383 
00384     moveto(i);
00385 #ifdef USE_NSPR_FILE
00386     if(file_is_float)
00387         PR_Read_float(f, v.data(), v.length(), file_is_bigendian);
00388     else
00389         PR_Read_double(f, v.data(), v.length(), file_is_bigendian);
00390 #else
00391     if(file_is_float)
00392         fread_float(f, v.data(), v.length(), file_is_bigendian);
00393     else
00394         fread_double(f, v.data(), v.length(), file_is_bigendian);
00395 #endif
00396 }
00397 
00399 // putSubRow //
00401 void FileVMatrix::putSubRow(int i, int j, Vec v)
00402 {
00403     moveto(i,j);
00404 #ifdef USE_NSPR_FILE
00405     if(file_is_float)
00406         PR_Write_float(f, v.data(), v.length(), file_is_bigendian);
00407     else
00408         PR_Write_double(f, v.data(), v.length(), file_is_bigendian);
00409 #else
00410     if(file_is_float)
00411         fwrite_float(f, v.data(), v.length(), file_is_bigendian);
00412     else
00413         fwrite_double(f, v.data(), v.length(), file_is_bigendian);
00414 #endif
00415 
00416     invalidateBuffer();
00417 }
00418 
00419 void FileVMatrix::moveto(int i, int j) const
00420 {
00421 #ifdef USE_NSPR_FILE
00422     PRInt64 offset = i;
00423     int elemsize = file_is_float ? sizeof(float) : sizeof(double);
00424     offset *= width_;
00425     offset += j;
00426     offset *= elemsize;
00427     offset += DATAFILE_HEADERLENGTH;
00428     PR_Seek64(f, offset, PR_SEEK_SET);
00429 #else
00430     if(file_is_float)
00431         fseek(f, DATAFILE_HEADERLENGTH+(i*width_+j)*sizeof(float), SEEK_SET);
00432     else
00433         fseek(f, DATAFILE_HEADERLENGTH+(i*width_+j)*sizeof(double), SEEK_SET);
00434 #endif
00435 }
00436 
00438 // put //
00440 void FileVMatrix::put(int i, int j, real value)
00441 {
00442     moveto(i,j);
00443 #ifdef USE_NSPR_FILE
00444     if(file_is_float)
00445         PR_Write_float(f,float(value),file_is_bigendian);
00446     else
00447         PR_Write_double(f,double(value),file_is_bigendian);
00448 #else
00449     if(file_is_float)
00450         fwrite_float(f,float(value),file_is_bigendian);
00451     else
00452         fwrite_double(f,double(value),file_is_bigendian);
00453 #endif
00454     invalidateBuffer();
00455 }
00456 
00458 // appendRow //
00460 void FileVMatrix::appendRow(Vec v)
00461 {
00462     moveto(length_,0);
00463 #ifdef USE_NSPR_FILE
00464     if(file_is_float)
00465         PR_Write_float(f, v.data(), v.length(), file_is_bigendian);
00466     else
00467         PR_Write_double(f, v.data(), v.length(), file_is_bigendian);
00468 #else
00469     if(file_is_float)
00470         fwrite_float(f, v.data(), v.length(), file_is_bigendian);
00471     else
00472         fwrite_double(f, v.data(), v.length(), file_is_bigendian);
00473 #endif
00474 
00475     length_++;
00476     updateHeader();
00477 }
00478 
00480 // flush //
00482 void FileVMatrix::flush()
00483 {
00484     if(f)
00485     {
00486 #ifdef USE_NSPR_FILE
00487     PR_Sync(f);
00488 #else
00489     fflush(f);
00490 #endif
00491     }
00492 }
00493 
00495 // updateHeader //
00497 void FileVMatrix::updateHeader() {
00498     char header[DATAFILE_HEADERLENGTH];
00499     string real = "DOUBLE";
00500     if(file_is_float)
00501         real = "FLOAT";
00502 
00503 #ifdef LITTLEENDIAN
00504     sprintf(header,"MATRIX %d %d %s LITTLE_ENDIAN", length_, width_, real.c_str());
00505 #endif
00506 #ifdef BIGENDIAN
00507     sprintf(header,"MATRIX %d %d %s BIG_ENDIAN", length_, width_, real.c_str());
00508 #endif
00509 
00510 
00511     int pos = strlen(header);
00512     for(; pos<DATAFILE_HEADERLENGTH; pos++)
00513     {
00514         header[pos] = ' ';
00515     }
00516     header[DATAFILE_HEADERLENGTH-1] = '\n';
00517 
00518 #ifdef USE_NSPR_FILE
00519     PR_Seek64(f, 0, PR_SEEK_SET);
00520     PR_Write(f, header, DATAFILE_HEADERLENGTH);
00521 #else
00522     fseek(f,0,SEEK_SET);
00523     fwrite(header,1,DATAFILE_HEADERLENGTH,f);
00524 #endif
00525 }
00526 
00528 // updateHeader //
00530 void FileVMatrix::openfile(const PPath& path, const char *mode,
00531                            PRIntn flags, PRIntn mode2) {
00532 #ifdef USE_NSPR_FILE
00533         f = PR_Open(path.absolute().c_str(), flags, mode2);
00534 #else
00535         f = fopen(path.absolute().c_str(), mode);
00536 #endif
00537 
00538 }
00539 
00540 
00541 int64_t FileVMatrix::getSizeOnDisk(){
00542     return DATAFILE_HEADERLENGTH + width_*length_*(file_is_float ? 4 : 8);
00543 }
00544 } // end of namespace PLearn
00545 
00546 
00547 /*
00548   Local Variables:
00549   mode:c++
00550   c-basic-offset:4
00551   c-file-style:"stroustrup"
00552   c-file-offsets:((innamespace . 0)(inline-open . 0))
00553   indent-tabs-mode:nil
00554   fill-column:79
00555   End:
00556 */
00557 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines