PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // CompactFileVMatrix.h 00004 // 00005 // Copyright (C) 2007 Olivier Breuleux 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 00036 #include "CompactFileVMatrix.h" 00037 #include <plearn/io/fileutils.h> 00038 #include <plearn/io/pl_NSPR_io.h> 00039 00040 #include <iostream> 00041 #include <errno.h> 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 00049 PLEARN_IMPLEMENT_OBJECT( 00050 CompactFileVMatrix, 00051 "Compact VMatrix where fields are bitfields.", 00052 "VMatrix whose data is stored on disk in a compacted '.cmat' file. \n" 00053 "The file contains a certain amount of field groups (with supposed \n" 00054 "distinct semantics). Each field group has an encoding type, \n" 00055 "a length and a maximal value (the values range from 0 to that value). \n" 00056 "Each field in a field group is typically encoded on the minimal amount \n" 00057 "of bits required to hold it (it is possible to specify more). Values \n" 00058 "are converted to floats and normalized by max, unless they are \n" 00059 "onehot-encoded. \n \n" 00060 "Encoding types: 'i' = float between 0 and 1, 'o' = onehot, 'u' = unsigned integer" 00061 ); 00062 00064 // CompactFileVMatrix // 00066 CompactFileVMatrix::CompactFileVMatrix(): 00067 inherited (true), 00068 in_ram_ (false), 00069 f (0) 00070 { 00071 writable=false; 00072 } 00073 00074 CompactFileVMatrix::CompactFileVMatrix(const PPath& filename): 00075 inherited (true), 00076 filename_ (filename.absolute()), 00077 in_ram_ (false), 00078 f (0) 00079 { 00080 build_(); 00081 } 00082 00083 00085 // build // 00087 void CompactFileVMatrix::build() 00088 { 00089 inherited::build(); 00090 build_(); 00091 } 00092 00093 00094 00096 // build_ // 00098 void CompactFileVMatrix::build_() 00099 { 00100 // cleanup (close file, free structures, etc.) 00101 cleanup(); 00102 00103 // If no file is given, we can stop here. 00104 if (filename_.isEmpty()) 00105 return; 00106 00107 char header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX + 1]; 00108 header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX] = '\n'; 00109 00110 openCurrentFile(); 00111 00112 00113 #ifdef USE_NSPR_FILE 00114 PR_Read(f,header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX); 00115 #else 00116 fread(header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX,1,f); 00117 #endif 00118 00119 int file_length; 00120 00121 file_length = atoi(header); 00122 00123 if (!file_length) { 00124 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (0)"); 00125 } 00126 this->length_ = file_length; 00127 00128 int i = 0; 00129 while (header[i] >= '0' && header[i] <= '9') 00130 i++; 00131 i++; 00132 00133 info = TVec<GroupInfo>(); 00134 00135 // Here we painfully read the header. 00136 while (1) { 00137 char type = header[i]; 00138 if (type == '\n' || type == ' ') 00139 break; 00140 if (!type || !(type == 'o' || type == 'i' || type == 'u')) 00141 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (bad type: %c)", type); 00142 i++; 00143 00144 if (header[i] != ':') 00145 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (2)"); 00146 i++; 00147 00148 int length = atoi(&(header[i])); 00149 if (!length) 00150 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (3)"); 00151 while (header[i] >= '0' && header[i] <= '9') 00152 i++; 00153 00154 if (header[i] != ':') 00155 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (4)"); 00156 i++; 00157 00158 int max = atoi(&(header[i])); 00159 if (!max) 00160 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (5)"); 00161 while (header[i] >= '0' && header[i] <= '9') 00162 i++; 00163 00164 if (header[i] != ':') 00165 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (6)"); 00166 i++; 00167 00168 int bits_per_value = atoi(&(header[i])); 00169 if (!bits_per_value) 00170 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (7)"); 00171 while (header[i] >= '0' && header[i] <= '9') 00172 i++; 00173 00174 if (bits_per_value > 8) 00175 PLERROR("In CompactFileVMatrix::build_ - each field must span at most 8 bits"); 00176 00177 if (header[i] != ' ' && header[i] != '\n') 00178 PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file"); 00179 00180 info.append(GroupInfo(type, length, max, bits_per_value)); 00181 00182 if (header[i] == '\n') { 00183 i++; 00184 break; 00185 } 00186 i++; 00187 } 00188 00189 header_length = i; 00190 00191 00192 // If there is a non-empty list of active groups, activate only those. 00193 if (active_list_.length()) { 00194 for (int i=0; i<active_list_.length(); i++) { 00195 int next_active = (int)(active_list_[i]); 00196 if (i && active_list_[i-1] >= next_active) 00197 PLERROR("In CompactFileVMatrix - Please give the active fields list in ascending order."); 00198 if (i < 0 || i >= info.length()) 00199 PLERROR("In CompactFileVMatrix - Bad list of active groups: group index out of bounds."); 00200 info[next_active].active = true; 00201 } 00202 } 00203 // else, activate all of them 00204 else { 00205 for(int i=0; i<info.length(); i++) 00206 info[i].active = true; 00207 } 00208 00209 int width = 0; 00210 int file_width = 0; // real width of an entry in the file 00211 for (int i=0; i<info.length(); i++) { 00212 info[i].compact_length = (info[i].length * info[i].bits_per_value + 7) / 8; 00213 if (info[i].active) { 00214 if (info[i].type == 'o') { // one-hot encoding 00215 width += info[i].length * (info[i].max + 1); 00216 } 00217 else if (info[i].type == 'i' || info[i].type == 'u') { // int (if max=1) or float encoding 00218 width += info[i].length; 00219 } 00220 } 00221 file_width += info[i].compact_length; 00222 } 00223 00224 this->width_ = width; // + 1; 00225 this->compact_width_ = file_width; // + 1; 00226 00227 // If the option in_ram is specified, we allocate a buffer 00228 if (in_ram_) { 00229 int sz = length_ * compact_width_; 00230 cache = string(sz, '\0'); 00231 cache_index = TVec<unsigned char>((length_ + 7) / 8); 00232 } 00233 00234 updateMtime(filename_); 00235 loadFieldInfos(); 00236 } 00237 00238 00239 00241 // openCurrentFile // 00243 void CompactFileVMatrix::openCurrentFile() 00244 { 00245 if (f) { 00246 PLERROR("In CompactFileVMatrix::openCurrentFile - File already open (%s)", filename_.c_str()); 00247 } 00248 else if (!isfile(filename_)) { 00249 PLERROR("In CompactFileVMatrix::openCurrentFile - File does not exist (%s)", filename_.c_str()); 00250 } 00251 else { 00252 #ifdef USE_NSPR_FILE 00253 f = PR_Open(filename_.c_str(), PR_RDONLY, 0666); 00254 #else 00255 f = fopen(filename_.c_str(), "rb"); 00256 #endif 00257 if (! f) 00258 PLERROR("CompactFileVMatrix::openCurrentFile: could not open file %s", filename_.c_str()); 00259 } 00260 } 00261 00262 00263 00265 // closeCurrentFile // 00267 void CompactFileVMatrix::closeCurrentFile() 00268 { 00269 if (f) 00270 { 00271 #ifdef USE_NSPR_FILE 00272 PR_Close(f); 00273 #else 00274 fclose(f); 00275 #endif 00276 f = 0; 00277 } 00278 } 00279 00281 // declareOptions // 00283 void CompactFileVMatrix::declareOptions(OptionList & ol) 00284 { 00285 declareOption(ol, "filename", &CompactFileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix"); 00286 declareOption(ol, "active_list", &CompactFileVMatrix::active_list_, OptionBase::buildoption, "comma-separated list of active field groups"); 00287 declareOption(ol, "cache", &CompactFileVMatrix::in_ram_, OptionBase::buildoption, "if set to true, all rows read from the file will be cached"); 00288 00289 inherited::declareOptions(ol); 00290 } 00291 00293 // makeDeepCopyFromShallowCopy // 00295 void CompactFileVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00296 { 00297 inherited::makeDeepCopyFromShallowCopy(copies); 00298 00299 TVec<GroupInfo> temp = info; 00300 info = TVec<GroupInfo>(); 00301 for (int i=0; i<temp.length(); i++) { 00302 info.append(temp[i]); 00303 } 00304 00305 deepCopyField(cache_index, copies); 00306 00307 // We don't want to share the file descriptor 00308 f = 0; 00309 openCurrentFile(); 00310 } 00311 00313 // ~CompactFileVMatrix // 00315 CompactFileVMatrix::~CompactFileVMatrix() 00316 { 00317 CompactFileVMatrix::cleanup(); 00318 } 00319 00321 // nFieldGroups // 00323 00324 void CompactFileVMatrix::cleanup() { 00325 CompactFileVMatrix::closeCurrentFile(); 00326 cache = string(); 00327 cache_index = TVec<unsigned char>(); 00328 } 00329 00330 00332 // nFieldGroups // 00334 00335 int CompactFileVMatrix::nGroups() const 00336 { 00337 return info.length(); 00338 } 00339 00340 00342 // getGroup // 00344 00345 GroupInfo& CompactFileVMatrix::getGroup(int group) const 00346 { 00347 if (group < 0 || group >= info.length()) { 00348 PLERROR("In CompactFileVMatrix::getGroup - group index out of bounds."); 00349 } 00350 return info[group]; 00351 } 00352 00353 00355 // groupEncoding // 00357 00358 char CompactFileVMatrix::groupEncoding(int group) const 00359 { 00360 return getGroup(group).type; 00361 } 00362 00363 00364 00366 // groupNFields // 00368 00369 int CompactFileVMatrix::groupNFields(int group) const 00370 { 00371 return getGroup(group).length; 00372 } 00373 00374 00375 00377 // groupNValues // 00379 00380 int CompactFileVMatrix::groupNValues(int group) const 00381 { 00382 return getGroup(group).max; 00383 } 00384 00385 00386 00388 // groupLength // 00390 00391 int CompactFileVMatrix::groupLength(int group) const 00392 { 00393 GroupInfo& info = getGroup(group); 00394 if (info.type == 'i') 00395 return info.length; 00396 else 00397 return info.length * info.max; 00398 } 00399 00400 00401 00403 // getNewRow // 00405 void CompactFileVMatrix::getNewRow(int i, const Vec& v) const 00406 { 00407 #ifdef BOUNDCHECK 00408 if(v.length()!=width_) 00409 PLERROR("In CompactFileVMatrix::getNewRow - length of v (%d) differs from matrix width (%d)",v.length(),width_); 00410 #endif 00411 00412 unsigned char* buffer; 00413 bool m_flag=false; // to check if we allocated the buffer using malloc 00414 00415 if (in_ram_ && (cache_index[i/8] & (1 << (i%8)))) { 00416 buffer = (unsigned char*)(cache.data() + (i*compact_width_)); 00417 } 00418 else { 00419 buffer = (unsigned char*)malloc(compact_width_); 00420 m_flag = true; 00421 00422 #ifdef USE_NSPR_FILE 00423 moveto(i); 00424 PR_Read(f, buffer, compact_width_); 00425 #else 00426 fseek(f, COMPACTFILEVMATRIX_HEADERLENGTH_MAX+(i*compact_width_), SEEK_SET); 00427 fread(buffer, compact_width_, 1, f); 00428 #endif 00429 if (in_ram_) { 00430 cache_index[i/8] |= 1 << (i%8); 00431 for (int j=0; j<compact_width_; j++) 00432 cache[(i*compact_width_) + j] = buffer[j]; 00433 } 00434 } 00435 00436 int current_b = -1; 00437 int current_v = 0; 00438 int value_b = 0; 00439 00440 for (int g=0; g<info.length(); g++) { 00441 if (!info[g].active) { 00442 current_b += info[g].compact_length; 00443 continue; 00444 } 00445 int nbits = info[g].bits_per_value; 00446 int max = info[g].max; 00447 int length = info[g].length; 00448 char type = info[g].type; 00449 00450 if (type == 'o') { 00451 v.subVec(current_v, length * (max + 1)).clear(); 00452 } 00453 00454 int remainder = 0; 00455 for (int i = 0; i < length; i++) { 00456 if (!remainder) { // we're done with the current byte (this block is entered in the first iteration) 00457 remainder = 8; 00458 current_b++; 00459 PLASSERT(current_b < compact_width_); // just in case 00460 value_b = buffer[current_b]; 00461 } 00462 if (remainder < nbits) { // the value overlaps two bytes 00463 int val = value_b; 00464 int overflow = nbits - remainder; 00465 value_b = buffer[++current_b]; 00466 val |= (value_b & (1<<overflow)-1) << remainder; 00467 value_b >>= overflow; 00468 remainder = 8 - nbits + remainder; 00469 if (type == 'i') 00470 v[current_v] = (real)val / max; 00471 else if (type == 'u') 00472 v[current_v] = val; 00473 else if (type == 'o' && val <= max) 00474 v[current_v + val] = 1; 00475 } 00476 else { // the value is whole in the current byte 00477 int val = (value_b & ((1<<nbits) - 1)); 00478 if (type == 'i') 00479 v[current_v] = (real)val / max; 00480 else if (type == 'u') 00481 v[current_v] = val; 00482 else if (type == 'o' && val <= max) { 00483 v[current_v + val] = 1; 00484 } 00485 value_b >>= nbits; 00486 remainder -= nbits; 00487 } 00488 if (type == 'i' || type == 'u') 00489 current_v++; 00490 else if (type == 'o') 00491 current_v += max + 1; 00492 } 00493 } 00494 00495 if (m_flag) 00496 free(buffer); 00497 } 00498 00500 // putSubRow // 00502 void CompactFileVMatrix::putSubRow(int i, int j, Vec v) 00503 { 00504 PLERROR("In CompactFileVMatrix::putSubRow - not implemented."); 00505 } 00506 00507 void CompactFileVMatrix::moveto(int i, int j) const 00508 { 00509 #ifdef USE_NSPR_FILE 00510 PRInt64 offset = i; 00511 offset *= compact_width_; 00512 offset += j; 00513 offset += header_length; 00514 PR_Seek64(f, offset, PR_SEEK_SET); 00515 #else 00516 fseek(f, header_length+(i*compact_width_+j), SEEK_SET); 00517 #endif 00518 } 00519 00521 // put // 00523 void CompactFileVMatrix::put(int i, int j, real value) 00524 { 00525 PLERROR("In CompactFileVMatrix::put - not implemented."); 00526 } 00527 00529 // appendRow // 00531 void CompactFileVMatrix::appendRow(Vec v) 00532 { 00533 PLERROR("In CompactFileVMatrix::put - not implemented."); 00534 } 00535 00537 // flush // 00539 void CompactFileVMatrix::flush() 00540 { 00541 if(f) 00542 { 00543 #ifdef USE_NSPR_FILE 00544 PR_Sync(f); 00545 #else 00546 fflush(f); 00547 #endif 00548 } 00549 } 00550 00552 // updateHeader // 00554 void CompactFileVMatrix::updateHeader() { 00555 PLERROR("In CompactFileVMatrix::put - not implemented."); 00556 } 00557 00558 VMatrixExtensionRegistrar* CompactFileVMatrix::extension_registrar = 00559 new VMatrixExtensionRegistrar( 00560 "cmat", 00561 &CompactFileVMatrix::instantiateFromPPath, 00562 "A compact file matrix"); 00563 00564 } // end of namespace PLearn 00565 00566 00567 /* 00568 Local Variables: 00569 mode:c++ 00570 c-basic-offset:4 00571 c-file-style:"stroustrup" 00572 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00573 indent-tabs-mode:nil 00574 fill-column:79 00575 End: 00576 */ 00577 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :