PLearn 0.1
DiskVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2008 Xavier Saint-Mleux, ApSTAT Technologies inc.
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 //
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 //
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 //
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 //
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 //
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************
00039  * $Id: DiskVMatrix.cc 10003 2009-03-10 17:06:50Z tihocan $
00040  ******************************************************* */
00041 
00042 #include "DiskVMatrix.h"
00043 #include <errno.h>
00044 #include <errno.h>
00045 #include <plearn/base/stringutils.h>
00046 #include <plearn/io/pl_io.h>
00047 #include <plearn/io/fileutils.h>
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 #if defined(WIN32) && !defined(__CYGWIN__)
00053 #include <io.h>
00054 #define unlink _unlink
00055 #endif
00056 
00058 /* Format description
00059 
00060 The directory in variable dirname, which should end in .dmat,
00061 contains a file named 'indexfile' and data files of up to roughly
00062 500Meg each named 0.data, 1.data, ...
00063 
00064 The indexfile has a 4 byte header, of which currently only the first byte is checked.
00065 'L' means new little-endian format
00066 'B' means new big-endian format
00067 An ascii code of 16 means old little-endian format
00068 
00069 In the new formats, the last 3 bytes of the 4 byte header are ' '
00070 
00071 All other raw-binary data (whether in the indexfile or the data files)
00072 is written in the specified endianness (and swapping will be performed if
00073 reading it on a machine with a different endianness).
00074 
00075 Following the 4-byte header are two 4-byte ints: length and width of the matrix.  
00076 
00077 N.B. As of October 2008, the length is not always stored in the indexfile's
00078 header; instead, while the matrix is opened for writing,
00079 DiskVMatrix::NO_LENGTH_SENTINEL (=-42) is used to indicate that the actual
00080 matrix length should be calculated some other way.  The sentinel is replaced by
00081 the actual length when the VMat is closed.  It is an error to open a
00082 DiskVMatrix for writing while its header indicates a length of
00083 DiskVMatrix::NO_LENGTH_SENTINEL *but this is not a safe locking mechanism, so
00084 you still have to be careful!*
00085 
00086 Following are 5 bytes for each row of the matrix.
00087 The first of those 5 bytes is an unsigned byte indicating the number (0-255) of
00088 the data file in which to find the row.
00089 The remaining 4 bytes are an unsigned int indicating the start position of the row in
00090 that data file (as passed to fseek).
00091 
00092 The row is encoded in the thus indicated data file at the indicated position,
00093 using the new vector compression format (see new_read_compressed)
00094 [ the old compression format pf binread_compressed is supported
00095 for backward compatibility only. ]
00096 
00097 */
00098 
00099 PLEARN_IMPLEMENT_OBJECT(
00100     DiskVMatrix,
00101     "VMatrix whose data are precomputed on disk (in a .dmat directory).",
00102     ""
00103 );
00104 
00105 // DiskVMatrix::NO_LENGTH_SENTINEL= -42; written in header 
00106 // instead of actual length while the file is opened for writing
00107 const int DiskVMatrix::NO_LENGTH_SENTINEL;
00108 const int DiskVMatrix::HEADER_OFFSET_LENGTH;
00109 
00110 DiskVMatrix::DiskVMatrix():
00111     indexf      (0),
00112     freshnewfile(false),
00113     old_format  (false),
00114     swap_endians(false),
00115     last_op_was_append(false),
00116     tolerance   (1e-6)
00117 {
00118     writable = false;
00119 }
00120 
00121 DiskVMatrix::DiskVMatrix(const PPath& the_dirname, bool readwrite,
00122                          bool call_build_):
00123     inherited   (call_build_),
00124     indexf      (0),
00125     freshnewfile(false),
00126     old_format  (false),
00127     swap_endians(false),
00128     last_op_was_append(false),
00129     dirname     (the_dirname),
00130     tolerance   (1e-6)
00131 {
00132     writable = readwrite;
00133     dirname.removeTrailingSlash(); // For safety.
00134     if (call_build_)
00135         build_();
00136 }
00137 
00138 DiskVMatrix::DiskVMatrix(const PPath& the_dirname, int the_width,
00139                          bool write_double_as_float):
00140     inherited   (0, the_width, true),
00141     indexf      (0),
00142     freshnewfile(true),
00143     old_format  (false),
00144     swap_endians(false),
00145     last_op_was_append(false),
00146     dirname     (the_dirname),
00147     tolerance   (1e-6)
00148 {
00149     writable = true;
00150     dirname.removeTrailingSlash(); // For safety.
00151     build_();
00152 }
00153 
00154 void DiskVMatrix::build()
00155 {
00156     inherited::build();
00157     build_();
00158 }
00159 
00160 void DiskVMatrix::build_()
00161 {
00162     closeCurrentFiles(); // All file pointers will be re-created.
00163     if(writable && width_ > 0 && !isdir(dirname))
00164         freshnewfile= true;
00165     if(!freshnewfile)
00166     {
00167         if(!isdir(dirname))
00168             PLERROR("In DiskVMatrix constructor, directory '%s' could not be found",dirname.c_str());
00169         setMetaDataDir(dirname + ".metadata");
00170         updateMtime( dirname/"indexfile" );
00171         string omode;
00172         if(writable)
00173             omode = "r+b";
00174         else // read-only
00175             omode = "rb";
00176 
00177         string indexfname = dirname/"indexfile";
00178         indexf = fopen(indexfname.c_str(), omode.c_str());
00179         if(!indexf)
00180             PLERROR("In DiskVMatrix constructor, could not open file %s in specified mode", indexfname.c_str());
00181 
00182         unsigned char header[4];
00183         fread(header,1,4,indexf);
00184         if(header[0]=='L' || header[0]=='B')
00185         { // New format
00186             old_format = false;
00187             swap_endians = (header[0]!=byte_order());
00188         }
00189         else if(header[0]==16)
00190         { // Old format
00191             old_format = true;
00192             if(byte_order()!='L')
00193                 PLERROR("Old format DiskVMatrix can only be read from a little-endian machine.\n"
00194                         "Convert it to a new format on a little-endian machine prior to attempt\n"
00195                         "using it from a big endian machine.\n");
00196             swap_endians = false;
00197         }
00198         else
00199         {
00200             PLERROR("Wrong header byte in index file %s: ascii code %d\n"
00201                     "(should be 'L' or 'B' or '...')\n", indexfname.c_str(), header[0]);
00202         }
00203 
00204         fread(&length_,sizeof(int),1,indexf);
00205         fread(&width_,sizeof(int),1,indexf);
00206         if(swap_endians)
00207         {
00208             endianswap(&length_);
00209             endianswap(&width_);
00210         }
00211 
00212         // Calc. length of VMat from indexfile length
00213         fseek(indexf, 0, SEEK_END);
00214         int pos= ftell(indexf);
00215         // pos less 12 header bytes s/b a multiple of 5 since each index 
00216         // entry is 5-byte long i.e. pos%5==2
00217         PLASSERT_MSG(pos%5 == 2,
00218                      "Length of DiskVMatrix is absent from header and length"
00219                      " of index file is " + tostring(pos) + ", not \\eqv 2 (mod 5)" );
00220         int calculated_length= (pos-12) / 5; //nb. lines in matrix
00221 
00222         if(length_ == NO_LENGTH_SENTINEL) // check after endianswap
00223         {
00224             PLASSERT_MSG(!writable, // already open for writing? not good...
00225                          "Trying to open a DiskVMatrix for writing but length in header is "
00226                          "NO_LENGTH_SENTINEL (="+tostring(NO_LENGTH_SENTINEL)+"); "
00227                          "Is it already being written to by another process? "
00228                          "dirname='" + dirname + "'.");
00229             length_= calculated_length;
00230         }
00231         else
00232         {
00233             PLASSERT_MSG(length_ == calculated_length, //length mismatch? not good...
00234                          "Length in DiskVMatrix's header does not match the indexfile's length "
00235                          "and is not NO_LENGTH_SENTINEL (="+tostring(NO_LENGTH_SENTINEL)+"); "
00236                          "indexfile='" + indexfname + "', "
00237                          "header len=" + tostring(length_) + ", "
00238                          "calc. len=" + tostring(calculated_length) + ".");
00239             if(writable)
00240             {// put NO_LENGTH_SENTINEL in header to indicate that we are appending to this file.
00241                 fseek(indexf, HEADER_OFFSET_LENGTH, SEEK_SET);
00242                 fwrite(&NO_LENGTH_SENTINEL, 4, 1, indexf);
00243                 // will seek to end before end of build_
00244             }
00245         }
00246         int k=0;
00247         string fname = dirname/tostring(k)+".data";
00248         while(isfile(fname))
00249         {
00250             FILE* f = fopen(fname.c_str(), omode.c_str());
00251             if(!f)
00252                 PLERROR("In DiskVMatrix constructor, could not open file %s in specified mode", fname.c_str());
00253             dataf.append(f);
00254             fname = dirname/(tostring(++k)+".data");
00255         }
00256         // Stuff related to RowBufferedVMatrix, for consistency
00257         current_row_index = -1;
00258         current_row.resize(width_);
00259         other_row_index = -1;
00260         other_row.resize(width_);
00261 
00262         //resize the string mappings
00263         map_sr = TVec<map<string,real> >(width_);
00264         map_rs = TVec<map<real,string> >(width_);
00265 
00266         getFieldInfos();
00267         if (writable)
00268             fseek(indexf, 0, SEEK_END);
00269     }
00270     else
00271     {
00272         if(isdir(dirname))
00273             PLERROR("In DiskVMatrix constructor (with specified width), directory %s already exists",dirname.c_str());
00274         setMetaDataDir(dirname + ".metadata");
00275         updateMtime( dirname/"indexfile" );
00276 
00277         if(isfile(dirname)) // patch for running mkstemp (TmpFilenames)
00278             unlink(dirname.c_str());
00279         if(!force_mkdir(dirname)) // force directory creation
00280             PLERROR("In DiskVMatrix constructor (with specified width), could not create directory %s  Error was: %s",dirname.c_str(), strerror(errno));
00281 
00282         string indexfname = dirname/"indexfile";
00283         indexf = fopen(indexfname.c_str(),"w+b");
00284 
00285         char header[4];
00286         header[0] = byte_order();
00287         header[1] = ' ';
00288         header[2] = ' ';
00289         header[3] = ' ';
00290         fwrite(header,1,4,indexf);
00291         //no more length in header while opened [was fwrite((char*)&length_,sizeof(int),1,indexf);]
00292         fwrite(&NO_LENGTH_SENTINEL, sizeof(int), 1, indexf);
00293         fwrite((char*)&width_,sizeof(int),1,indexf);
00294         length_= 0;
00295 
00296         string fname = dirname/"0.data";
00297         FILE* f = fopen(fname.c_str(), "w+b");
00298         dataf.append(f);
00299     }
00300     freshnewfile=false;
00301     loadAllStringMappings();//make sure string mappings are loaded after width is set
00302     last_op_was_append= false;
00303 }
00304 
00305 void DiskVMatrix::declareOptions(OptionList &ol)
00306 {
00307     declareOption(ol, "dirname", &DiskVMatrix::dirname, OptionBase::buildoption, "Directory name of the.dmat");
00308     declareOption(ol, "tolerance", &DiskVMatrix::tolerance, OptionBase::buildoption, "The absolute error tolerance for storing doubles as floats");
00309     inherited::declareOptions(ol);
00310 }
00311 
00313 // closeCurrentFiles //
00315 void DiskVMatrix::closeCurrentFiles()
00316 {
00317     for (int i = 0; i < dataf.length(); i++)
00318         if(dataf[i]) {
00319             fclose(dataf[i]);
00320             dataf[i] = 0;
00321         }
00322 
00323     dataf.resize(0);
00324     
00325     if (indexf) {
00326         // write final length to indexfile before closing
00327         fseek(indexf, HEADER_OFFSET_LENGTH, SEEK_SET); // it is and will always be 4 bytes
00328         int le= length_;
00329         if(swap_endians)
00330             endianswap(&le);
00331         fwrite(&le,sizeof(int),1,indexf);
00332 
00333         fclose(indexf);
00334         indexf = 0;
00335     }
00336     last_op_was_append= false;
00337 }
00338 
00340 // getNewRow //
00342 void DiskVMatrix::getNewRow(int i, const Vec& v) const
00343 {
00344 #ifdef BOUNDCHECK
00345     if(i<0 || i>length())
00346         PLERROR("In DiskVMatrix::getNewRow, bad row number %d",i);
00347     if(v.length() != width())
00348         PLERROR("In DiskVMatrix::getNewRow, length of v (%d) does not match matrix width (%d)",v.length(),width());
00349 #endif
00350 
00351     unsigned char filenum;
00352     unsigned int position;
00353     fseek(indexf,3*sizeof(int) + i*(sizeof(unsigned char)+sizeof(unsigned int)), SEEK_SET);
00354     fread(&filenum,sizeof(unsigned char),1,indexf);
00355     fread(&position,sizeof(unsigned int),1,indexf);
00356     if(swap_endians)
00357         endianswap(&position);
00358     FILE* f = dataf[int(filenum)];
00359     PLASSERT( f );
00360     fseek(f,position,SEEK_SET);
00361     if(old_format)
00362         binread_compressed(f,v.data(),v.length());
00363     else
00364         new_read_compressed(f, v.data(), v.length(), swap_endians);
00365     last_op_was_append= false;
00366 }
00367 
00368 void DiskVMatrix::putRow(int i, Vec v)
00369 {
00370     PLERROR("putRow cannot in general be correctly and efficiently implemented for a DiskVMatrix.\n"
00371             "Use appendRow if you wish to write more rows.");
00372 }
00373 
00374 void DiskVMatrix::appendRow(Vec v)
00375 {
00376     if(!writable)
00377         PLERROR("In DiskVMatrix::appendRow cannot append row in read only mode, set readwrite parameter to true when calling the constructor");
00378     if(v.length() != width())
00379         PLERROR("In DiskVMatrix::appendRow, length of v (%d) does not match matrix width (%d)",v.length(),width());
00380 
00381     int filenum = dataf.size()-1;
00382     FILE* f = dataf[filenum];
00383     if(!last_op_was_append) //otherwise, we are already at end
00384         fseek(f,0,SEEK_END);
00385     unsigned int position = (unsigned int)ftell(f);
00386     if(position>500000000L)
00387     {
00388         fflush(f);
00389         filenum++;
00390         string filename = dirname / (tostring(filenum) + ".data");
00391         f = fopen(filename.c_str(), "w+b");
00392         dataf.append(f);
00393         position = 0;
00394     }
00395     if(old_format)
00396         binwrite_compressed(f,v.data(),v.length());
00397     else
00398         new_write_compressed(f, v.data(),v.length(), tolerance, swap_endians);
00399 
00400     if(!last_op_was_append) //otherwise, we are already at end
00401         fseek(indexf,0,SEEK_END);
00402     fputc(filenum,indexf);
00403     fwrite((char*)&position,sizeof(unsigned int),1,indexf);
00404     length_++;
00405     /* DO NOT write length in header; wait 'till file is closed
00406     fseek(indexf,sizeof(int),SEEK_SET);
00407     int le = length_;
00408     if(swap_endians)
00409         endianswap(&le);
00410     fwrite(&le,sizeof(int),1,indexf);
00411     */
00412     last_op_was_append= true;
00413 }
00414 
00416 // flush //
00418 void DiskVMatrix::flush()
00419 {
00420     int filenum = dataf.size()-1;
00421     FILE* f = dataf[filenum];
00422     fflush(f);
00423     fflush(indexf);
00424     last_op_was_append= false;
00425 }
00426 
00428 // makeDeepCopyFromShallowCopy //
00430 void DiskVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00431 {
00432     inherited::makeDeepCopyFromShallowCopy(copies);
00433     deepCopyField(dataf, copies);
00434     // Set pointers to zero, because we will open again the file (file pointers
00435     // should not be shared between copied objects).
00436     indexf = 0;
00437     dataf.fill(0);
00438     dataf.resize(0);
00439     build(); // Open again the dataset.
00440 }
00441 
00442 
00444 // ~DiskVMatrix //
00446 DiskVMatrix::~DiskVMatrix()
00447 {
00448     if (hasMetaDataDir())
00449         saveFieldInfos();
00450     DiskVMatrix::closeCurrentFiles();
00451 }
00452 
00453 
00454 #ifdef WIN32
00455 #undef unlink
00456 #endif
00457 
00458 } // end of namespace PLearn
00459 
00460 
00461 /*
00462   Local Variables:
00463   mode:c++
00464   c-basic-offset:4
00465   c-file-style:"stroustrup"
00466   c-file-offsets:((innamespace . 0)(inline-open . 0))
00467   indent-tabs-mode:nil
00468   fill-column:79
00469   End:
00470 */
00471 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines