PLearn 0.1
CompactFileVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // CompactFileVMatrix.h
00004 //
00005 // Copyright (C) 2007 Olivier Breuleux
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 
00036 #include "CompactFileVMatrix.h"
00037 #include <plearn/io/fileutils.h>
00038 #include <plearn/io/pl_NSPR_io.h>
00039 
00040 #include <iostream>
00041 #include <errno.h>
00042 
00043 namespace PLearn {
00044 using namespace std;
00045 
00046 
00049 PLEARN_IMPLEMENT_OBJECT(
00050     CompactFileVMatrix,
00051     "Compact VMatrix where fields are bitfields.",
00052     "VMatrix whose data is stored on disk in a compacted '.cmat' file. \n"
00053     "The file contains a certain amount of field groups (with supposed \n"
00054     "distinct semantics). Each field group has an encoding type, \n"
00055     "a length and a maximal value (the values range from 0 to that value). \n"
00056     "Each field in a field group is typically encoded on the minimal amount \n"
00057     "of bits required to hold it (it is possible to specify more). Values \n"
00058     "are converted to floats and normalized by max, unless they are \n"
00059     "onehot-encoded. \n \n"
00060     "Encoding types: 'i' = float between 0 and 1, 'o' = onehot, 'u' = unsigned integer"
00061 );
00062 
00064 // CompactFileVMatrix //
00066 CompactFileVMatrix::CompactFileVMatrix():
00067     inherited       (true),
00068     in_ram_         (false),
00069     f               (0)
00070 {
00071     writable=false;
00072 }
00073 
00074 CompactFileVMatrix::CompactFileVMatrix(const PPath& filename):
00075     inherited       (true),
00076     filename_       (filename.absolute()),
00077     in_ram_         (false),
00078     f               (0)
00079 {
00080     build_();
00081 }
00082 
00083 
00085 // build //
00087 void CompactFileVMatrix::build()
00088 {
00089     inherited::build();
00090     build_();
00091 }
00092 
00093 
00094 
00096 // build_ //
00098 void CompactFileVMatrix::build_()
00099 {
00100     // cleanup (close file, free structures, etc.)
00101     cleanup();
00102 
00103     // If no file is given, we can stop here.
00104     if (filename_.isEmpty())
00105         return;
00106 
00107     char header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX + 1];
00108     header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX] = '\n';
00109 
00110     openCurrentFile();
00111 
00112 
00113 #ifdef USE_NSPR_FILE
00114     PR_Read(f,header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX);
00115 #else
00116     fread(header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX,1,f);
00117 #endif
00118 
00119     int file_length;
00120 
00121     file_length = atoi(header);
00122 
00123     if (!file_length) {
00124         PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (0)");
00125     }
00126     this->length_ = file_length;
00127 
00128     int i = 0;
00129     while (header[i] >= '0' && header[i] <= '9')
00130         i++;
00131     i++;
00132 
00133     info = TVec<GroupInfo>();
00134 
00135     // Here we painfully read the header.
00136     while (1) {
00137         char type = header[i];
00138         if (type == '\n' || type == ' ')
00139             break;
00140         if (!type || !(type == 'o' || type == 'i' || type == 'u'))
00141             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (bad type: %c)", type);
00142         i++;
00143 
00144         if (header[i] != ':')
00145             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (2)");
00146         i++;
00147 
00148         int length = atoi(&(header[i]));
00149         if (!length)
00150             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (3)");
00151         while (header[i] >= '0' && header[i] <= '9')
00152             i++;
00153 
00154         if (header[i] != ':')
00155             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (4)");
00156         i++;
00157 
00158         int max = atoi(&(header[i]));
00159         if (!max)
00160             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (5)");
00161         while (header[i] >= '0' && header[i] <= '9')
00162             i++;
00163 
00164         if (header[i] != ':')
00165             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (6)");
00166         i++;
00167 
00168         int bits_per_value = atoi(&(header[i]));
00169         if (!bits_per_value)
00170             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (7)");
00171         while (header[i] >= '0' && header[i] <= '9')
00172             i++;
00173 
00174         if (bits_per_value > 8)
00175             PLERROR("In CompactFileVMatrix::build_ - each field must span at most 8 bits");
00176 
00177         if (header[i] != ' ' && header[i] != '\n')
00178             PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file");
00179 
00180         info.append(GroupInfo(type, length, max, bits_per_value));
00181 
00182         if (header[i] == '\n') {
00183             i++;
00184             break;
00185         }
00186         i++;
00187     }
00188 
00189     header_length = i;
00190 
00191 
00192     // If there is a non-empty list of active groups, activate only those.
00193     if (active_list_.length()) {
00194         for (int i=0; i<active_list_.length(); i++) {
00195             int next_active = (int)(active_list_[i]);
00196             if (i && active_list_[i-1] >= next_active)
00197                 PLERROR("In CompactFileVMatrix - Please give the active fields list in ascending order.");
00198             if (i < 0 || i >= info.length())
00199                 PLERROR("In CompactFileVMatrix - Bad list of active groups: group index out of bounds.");
00200             info[next_active].active = true;
00201         }
00202     }
00203     // else, activate all of them
00204     else {
00205         for(int i=0; i<info.length(); i++)
00206             info[i].active = true;
00207     }
00208 
00209     int width = 0;
00210     int file_width = 0; // real width of an entry in the file
00211     for (int i=0; i<info.length(); i++) {
00212         info[i].compact_length = (info[i].length * info[i].bits_per_value + 7) / 8;
00213         if (info[i].active) {
00214             if (info[i].type == 'o') { // one-hot encoding
00215                 width += info[i].length * (info[i].max + 1);
00216             }
00217             else if (info[i].type == 'i' || info[i].type == 'u') { // int (if max=1) or float encoding
00218                 width += info[i].length;
00219             }
00220         }
00221         file_width += info[i].compact_length;
00222     }
00223 
00224     this->width_ = width; // + 1;
00225     this->compact_width_ = file_width; // + 1;
00226 
00227     // If the option in_ram is specified, we allocate a buffer
00228     if (in_ram_) {
00229         int sz = length_ * compact_width_;
00230         cache = string(sz, '\0');
00231         cache_index = TVec<unsigned char>((length_ + 7) / 8);
00232     }
00233 
00234     updateMtime(filename_);
00235     loadFieldInfos();
00236 }
00237 
00238 
00239 
00241 // openCurrentFile //
00243 void CompactFileVMatrix::openCurrentFile()
00244 {
00245     if (f) {
00246         PLERROR("In CompactFileVMatrix::openCurrentFile - File already open (%s)", filename_.c_str());
00247     }
00248     else if (!isfile(filename_)) {
00249         PLERROR("In CompactFileVMatrix::openCurrentFile - File does not exist (%s)", filename_.c_str());
00250     }
00251     else {
00252 #ifdef USE_NSPR_FILE
00253         f = PR_Open(filename_.c_str(), PR_RDONLY, 0666);
00254 #else
00255         f = fopen(filename_.c_str(), "rb");
00256 #endif
00257         if (! f)
00258             PLERROR("CompactFileVMatrix::openCurrentFile: could not open file %s", filename_.c_str());
00259     }
00260 }
00261 
00262 
00263 
00265 // closeCurrentFile //
00267 void CompactFileVMatrix::closeCurrentFile()
00268 {
00269     if (f)
00270     {
00271 #ifdef USE_NSPR_FILE
00272         PR_Close(f);
00273 #else
00274         fclose(f);
00275 #endif
00276         f = 0;
00277     }
00278 }
00279 
00281 // declareOptions //
00283 void CompactFileVMatrix::declareOptions(OptionList & ol)
00284 {
00285     declareOption(ol, "filename", &CompactFileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix");
00286     declareOption(ol, "active_list", &CompactFileVMatrix::active_list_, OptionBase::buildoption, "comma-separated list of active field groups");
00287     declareOption(ol, "cache", &CompactFileVMatrix::in_ram_, OptionBase::buildoption, "if set to true, all rows read from the file will be cached");
00288 
00289     inherited::declareOptions(ol);
00290 }
00291 
00293 // makeDeepCopyFromShallowCopy //
00295 void CompactFileVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00296 {
00297     inherited::makeDeepCopyFromShallowCopy(copies);
00298 
00299     TVec<GroupInfo> temp = info;
00300     info = TVec<GroupInfo>();
00301     for (int i=0; i<temp.length(); i++) {
00302         info.append(temp[i]);
00303     }
00304 
00305     deepCopyField(cache_index, copies);
00306 
00307     // We don't want to share the file descriptor
00308     f = 0;
00309     openCurrentFile();
00310 }
00311 
00313 // ~CompactFileVMatrix //
00315 CompactFileVMatrix::~CompactFileVMatrix()
00316 {
00317     CompactFileVMatrix::cleanup();
00318 }
00319 
00321 // nFieldGroups //
00323 
00324 void CompactFileVMatrix::cleanup() {
00325     CompactFileVMatrix::closeCurrentFile();
00326     cache = string();
00327     cache_index = TVec<unsigned char>();
00328 }
00329 
00330 
00332 // nFieldGroups //
00334 
00335 int CompactFileVMatrix::nGroups() const
00336 {
00337     return info.length();
00338 }
00339 
00340 
00342 // getGroup //
00344 
00345 GroupInfo& CompactFileVMatrix::getGroup(int group) const
00346 {
00347     if (group < 0 || group >= info.length()) {
00348         PLERROR("In CompactFileVMatrix::getGroup - group index out of bounds.");
00349     }
00350     return info[group];
00351 }
00352 
00353 
00355 // groupEncoding //
00357 
00358 char CompactFileVMatrix::groupEncoding(int group) const
00359 {
00360     return getGroup(group).type;
00361 }
00362 
00363 
00364 
00366 // groupNFields //
00368 
00369 int CompactFileVMatrix::groupNFields(int group) const
00370 {
00371     return getGroup(group).length;
00372 }
00373 
00374 
00375 
00377 // groupNValues //
00379 
00380 int CompactFileVMatrix::groupNValues(int group) const
00381 {
00382     return getGroup(group).max;
00383 }
00384 
00385 
00386 
00388 // groupLength //
00390 
00391 int CompactFileVMatrix::groupLength(int group) const
00392 {
00393     GroupInfo& info = getGroup(group);
00394     if (info.type == 'i')
00395         return info.length;
00396     else
00397         return info.length * info.max;
00398 }
00399 
00400 
00401 
00403 // getNewRow //
00405 void CompactFileVMatrix::getNewRow(int i, const Vec& v) const
00406 {
00407 #ifdef BOUNDCHECK
00408     if(v.length()!=width_)
00409         PLERROR("In CompactFileVMatrix::getNewRow - length of v (%d) differs from matrix width (%d)",v.length(),width_);
00410 #endif
00411 
00412     unsigned char* buffer;
00413     bool m_flag=false; // to check if we allocated the buffer using malloc
00414 
00415     if (in_ram_ && (cache_index[i/8] & (1 << (i%8)))) {
00416         buffer = (unsigned char*)(cache.data() + (i*compact_width_));
00417     }
00418     else {
00419         buffer = (unsigned char*)malloc(compact_width_);
00420         m_flag = true;
00421 
00422 #ifdef USE_NSPR_FILE
00423         moveto(i);
00424         PR_Read(f, buffer, compact_width_);
00425 #else
00426         fseek(f, COMPACTFILEVMATRIX_HEADERLENGTH_MAX+(i*compact_width_), SEEK_SET);
00427         fread(buffer, compact_width_, 1, f);
00428 #endif
00429         if (in_ram_) {
00430             cache_index[i/8] |= 1 << (i%8);
00431             for (int j=0; j<compact_width_; j++)
00432                 cache[(i*compact_width_) + j] = buffer[j];
00433         }
00434     }
00435 
00436     int current_b = -1;
00437     int current_v = 0;
00438     int value_b = 0;
00439 
00440     for (int g=0; g<info.length(); g++) {
00441         if (!info[g].active) {
00442             current_b += info[g].compact_length;
00443             continue;
00444         }
00445         int nbits = info[g].bits_per_value;
00446         int max = info[g].max;
00447         int length = info[g].length;
00448         char type = info[g].type;
00449 
00450         if (type == 'o') {
00451             v.subVec(current_v, length * (max + 1)).clear();
00452         }
00453 
00454         int remainder = 0;
00455         for (int i = 0; i < length; i++) {
00456             if (!remainder) { // we're done with the current byte (this block is entered in the first iteration)
00457                 remainder = 8;
00458                 current_b++;
00459                 PLASSERT(current_b < compact_width_); // just in case
00460                 value_b = buffer[current_b];
00461             }
00462             if (remainder < nbits) { // the value overlaps two bytes
00463                 int val = value_b;
00464                 int overflow = nbits - remainder;
00465                 value_b = buffer[++current_b];
00466                 val |= (value_b & (1<<overflow)-1) << remainder;
00467                 value_b >>= overflow;
00468                 remainder = 8 - nbits + remainder;
00469                 if (type == 'i')
00470                     v[current_v] = (real)val / max;
00471                 else if (type == 'u')
00472                     v[current_v] = val;
00473                 else if (type == 'o' && val <= max)
00474                     v[current_v + val] = 1;
00475             }
00476             else { // the value is whole in the current byte
00477                 int val = (value_b & ((1<<nbits) - 1));
00478                 if (type == 'i')
00479                     v[current_v] = (real)val / max;
00480                 else if (type == 'u')
00481                     v[current_v] = val;
00482                 else if (type == 'o' && val <= max) {
00483                     v[current_v + val] = 1;
00484                 }
00485                 value_b >>= nbits;
00486                 remainder -= nbits;
00487             }
00488             if (type == 'i' || type == 'u')
00489                 current_v++;
00490             else if (type == 'o')
00491                 current_v += max + 1;
00492         }
00493     }
00494 
00495    if (m_flag)
00496         free(buffer);
00497 }
00498 
00500 // putSubRow //
00502 void CompactFileVMatrix::putSubRow(int i, int j, Vec v)
00503 {
00504     PLERROR("In CompactFileVMatrix::putSubRow - not implemented.");
00505 }
00506 
00507 void CompactFileVMatrix::moveto(int i, int j) const
00508 {
00509 #ifdef USE_NSPR_FILE
00510     PRInt64 offset = i;
00511     offset *= compact_width_;
00512     offset += j;
00513     offset += header_length;
00514     PR_Seek64(f, offset, PR_SEEK_SET);
00515 #else
00516     fseek(f, header_length+(i*compact_width_+j), SEEK_SET);
00517 #endif
00518 }
00519 
00521 // put //
00523 void CompactFileVMatrix::put(int i, int j, real value)
00524 {
00525     PLERROR("In CompactFileVMatrix::put - not implemented.");
00526 }
00527 
00529 // appendRow //
00531 void CompactFileVMatrix::appendRow(Vec v)
00532 {
00533     PLERROR("In CompactFileVMatrix::put - not implemented.");
00534 }
00535 
00537 // flush //
00539 void CompactFileVMatrix::flush()
00540 {
00541     if(f)
00542     {
00543 #ifdef USE_NSPR_FILE
00544     PR_Sync(f);
00545 #else
00546     fflush(f);
00547 #endif
00548     }
00549 }
00550 
00552 // updateHeader //
00554 void CompactFileVMatrix::updateHeader() {
00555     PLERROR("In CompactFileVMatrix::put - not implemented.");
00556 }
00557 
00558 VMatrixExtensionRegistrar* CompactFileVMatrix::extension_registrar =
00559     new VMatrixExtensionRegistrar(
00560         "cmat",
00561         &CompactFileVMatrix::instantiateFromPPath,
00562         "A compact file matrix");
00563 
00564 } // end of namespace PLearn
00565 
00566 
00567 /*
00568   Local Variables:
00569   mode:c++
00570   c-basic-offset:4
00571   c-file-style:"stroustrup"
00572   c-file-offsets:((innamespace . 0)(inline-open . 0))
00573   indent-tabs-mode:nil
00574   fill-column:79
00575   End:
00576 */
00577 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines