PLearn 0.1
|
A VMatrix that exists in a .pmat file (native PLearn matrix format, same as for Mat). More...
#include <CompactFileVMatrix.h>
Public Member Functions | |
CompactFileVMatrix () | |
CompactFileVMatrix (const PPath &filename) | |
opens an existing file | |
virtual void | updateHeader () |
Re-write the header with all current field values. | |
virtual int | nGroups () const |
The amount of field groups in the matrix. | |
virtual char | groupEncoding (int group) const |
The encoding of the group-th field group ('i' or 'o') | |
virtual int | groupNFields (int group) const |
The amount of fields in the group-th field group. | |
virtual int | groupNValues (int group) const |
The amount of possible values in the group-th field group. | |
virtual int | groupLength (int group) const |
The length of the group-th field group. | |
virtual void | put (int i, int j, real value) |
This method must be implemented in all subclasses of writable matrices. | |
virtual void | putSubRow (int i, int j, Vec v) |
It is suggested that this method be implemented in subclasses of writable matrices to speed up accesses (default version repeatedly calls put(i,j,value) which may have a significant overhead) | |
virtual void | appendRow (Vec v) |
This method must be implemented for matrices that are allowed to grow. | |
virtual void | flush () |
For matrices stored on disk, this should flush all pending buffered write operations. | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual CompactFileVMatrix * | deepCopy (CopiesMap &copies) const |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transform a shallow copy into a deep copy. | |
virtual | ~CompactFileVMatrix () |
Destructor. | |
Static Public Member Functions | |
static VMat | instantiateFromPPath (const PPath &filename) |
static string | _classname_ () |
CompactFileVMatrix. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
virtual void | getNewRow (int i, const Vec &v) const |
This is the only method requiring implementation in subclasses. | |
virtual void | openCurrentFile () |
Open the current file. | |
virtual void | closeCurrentFile () |
Close the current file. | |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declare this class' options. | |
Protected Attributes | |
PPath | filename_ |
Vec | active_list_ |
bool | in_ram_ |
PRFileDesc * | f |
int | header_length |
int | compact_width_ |
width of a compacted row | |
TVec< GroupInfo > | info |
TVec< unsigned char > | cache_index |
string | cache |
Private Types | |
typedef RowBufferedVMatrix | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
GroupInfo & | getGroup (int group) const |
void | moveto (int i, int j=0) const |
void | cleanup () |
Static Private Attributes | |
static VMatrixExtensionRegistrar * | extension_registrar |
A VMatrix that exists in a .pmat file (native PLearn matrix format, same as for Mat).
Definition at line 90 of file CompactFileVMatrix.h.
typedef RowBufferedVMatrix PLearn::CompactFileVMatrix::inherited [private] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 94 of file CompactFileVMatrix.h.
PLearn::CompactFileVMatrix::CompactFileVMatrix | ( | ) |
Definition at line 66 of file CompactFileVMatrix.cc.
References PLearn::VMatrix::writable.
PLearn::CompactFileVMatrix::CompactFileVMatrix | ( | const PPath & | filename | ) |
PLearn::CompactFileVMatrix::~CompactFileVMatrix | ( | ) | [virtual] |
Destructor.
Definition at line 315 of file CompactFileVMatrix.cc.
References cleanup().
{ CompactFileVMatrix::cleanup(); }
string PLearn::CompactFileVMatrix::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
OptionList & PLearn::CompactFileVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
RemoteMethodMap & PLearn::CompactFileVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
Object * PLearn::CompactFileVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
StaticInitializer CompactFileVMatrix::_static_initializer_ & PLearn::CompactFileVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
void PLearn::CompactFileVMatrix::appendRow | ( | Vec | v | ) | [virtual] |
This method must be implemented for matrices that are allowed to grow.
Reimplemented from PLearn::VMatrix.
Definition at line 531 of file CompactFileVMatrix.cc.
References PLERROR.
{ PLERROR("In CompactFileVMatrix::put - not implemented."); }
void PLearn::CompactFileVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::VMatrix.
Definition at line 87 of file CompactFileVMatrix.cc.
References PLearn::VMatrix::build(), and build_().
{ inherited::build(); build_(); }
void PLearn::CompactFileVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VMatrix.
Definition at line 98 of file CompactFileVMatrix.cc.
References active_list_, cache, cache_index, cleanup(), compact_width_, COMPACTFILEVMATRIX_HEADERLENGTH_MAX, f, filename_, header_length, i, in_ram_, PLearn::Object::info(), PLearn::PPath::isEmpty(), PLearn::TVec< T >::length(), PLearn::VMatrix::length(), PLearn::VMatrix::length_, PLearn::VMatrix::loadFieldInfos(), PLearn::max(), openCurrentFile(), PLERROR, PLearn::VMatrix::updateMtime(), PLearn::VMatrix::width(), and PLearn::VMatrix::width_.
Referenced by build(), and CompactFileVMatrix().
{ // cleanup (close file, free structures, etc.) cleanup(); // If no file is given, we can stop here. if (filename_.isEmpty()) return; char header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX + 1]; header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX] = '\n'; openCurrentFile(); #ifdef USE_NSPR_FILE PR_Read(f,header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX); #else fread(header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX,1,f); #endif int file_length; file_length = atoi(header); if (!file_length) { PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (0)"); } this->length_ = file_length; int i = 0; while (header[i] >= '0' && header[i] <= '9') i++; i++; info = TVec<GroupInfo>(); // Here we painfully read the header. while (1) { char type = header[i]; if (type == '\n' || type == ' ') break; if (!type || !(type == 'o' || type == 'i' || type == 'u')) PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (bad type: %c)", type); i++; if (header[i] != ':') PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (2)"); i++; int length = atoi(&(header[i])); if (!length) PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (3)"); while (header[i] >= '0' && header[i] <= '9') i++; if (header[i] != ':') PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (4)"); i++; int max = atoi(&(header[i])); if (!max) PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (5)"); while (header[i] >= '0' && header[i] <= '9') i++; if (header[i] != ':') PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (6)"); i++; int bits_per_value = atoi(&(header[i])); if (!bits_per_value) PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (7)"); while (header[i] >= '0' && header[i] <= '9') i++; if (bits_per_value > 8) PLERROR("In CompactFileVMatrix::build_ - each field must span at most 8 bits"); if (header[i] != ' ' && header[i] != '\n') PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file"); info.append(GroupInfo(type, length, max, bits_per_value)); if (header[i] == '\n') { i++; break; } i++; } header_length = i; // If there is a non-empty list of active groups, activate only those. if (active_list_.length()) { for (int i=0; i<active_list_.length(); i++) { int next_active = (int)(active_list_[i]); if (i && active_list_[i-1] >= next_active) PLERROR("In CompactFileVMatrix - Please give the active fields list in ascending order."); if (i < 0 || i >= info.length()) PLERROR("In CompactFileVMatrix - Bad list of active groups: group index out of bounds."); info[next_active].active = true; } } // else, activate all of them else { for(int i=0; i<info.length(); i++) info[i].active = true; } int width = 0; int file_width = 0; // real width of an entry in the file for (int i=0; i<info.length(); i++) { info[i].compact_length = (info[i].length * info[i].bits_per_value + 7) / 8; if (info[i].active) { if (info[i].type == 'o') { // one-hot encoding width += info[i].length * (info[i].max + 1); } else if (info[i].type == 'i' || info[i].type == 'u') { // int (if max=1) or float encoding width += info[i].length; } } file_width += info[i].compact_length; } this->width_ = width; // + 1; this->compact_width_ = file_width; // + 1; // If the option in_ram is specified, we allocate a buffer if (in_ram_) { int sz = length_ * compact_width_; cache = string(sz, '\0'); cache_index = TVec<unsigned char>((length_ + 7) / 8); } updateMtime(filename_); loadFieldInfos(); }
string PLearn::CompactFileVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
void PLearn::CompactFileVMatrix::cleanup | ( | ) | [private] |
Definition at line 324 of file CompactFileVMatrix.cc.
References cache, cache_index, and closeCurrentFile().
Referenced by build_(), and ~CompactFileVMatrix().
{ CompactFileVMatrix::closeCurrentFile(); cache = string(); cache_index = TVec<unsigned char>(); }
void PLearn::CompactFileVMatrix::closeCurrentFile | ( | ) | [protected, virtual] |
void PLearn::CompactFileVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declare this class' options.
Reimplemented from PLearn::VMatrix.
Definition at line 283 of file CompactFileVMatrix.cc.
References active_list_, PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::VMatrix::declareOptions(), filename_, and in_ram_.
{ declareOption(ol, "filename", &CompactFileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix"); declareOption(ol, "active_list", &CompactFileVMatrix::active_list_, OptionBase::buildoption, "comma-separated list of active field groups"); declareOption(ol, "cache", &CompactFileVMatrix::in_ram_, OptionBase::buildoption, "if set to true, all rows read from the file will be cached"); inherited::declareOptions(ol); }
static const PPath& PLearn::CompactFileVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 162 of file CompactFileVMatrix.h.
:
CompactFileVMatrix * PLearn::CompactFileVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
void PLearn::CompactFileVMatrix::flush | ( | ) | [virtual] |
For matrices stored on disk, this should flush all pending buffered write operations.
Reimplemented from PLearn::VMatrix.
Definition at line 539 of file CompactFileVMatrix.cc.
References f.
Definition at line 345 of file CompactFileVMatrix.cc.
References PLearn::Object::info(), and PLERROR.
Referenced by groupEncoding(), groupLength(), groupNFields(), and groupNValues().
{ if (group < 0 || group >= info.length()) { PLERROR("In CompactFileVMatrix::getGroup - group index out of bounds."); } return info[group]; }
This is the only method requiring implementation in subclasses.
Implements PLearn::RowBufferedVMatrix.
Definition at line 405 of file CompactFileVMatrix.cc.
References cache, cache_index, PLearn::TVec< T >::clear(), compact_width_, COMPACTFILEVMATRIX_HEADERLENGTH_MAX, f, g, in_ram_, PLearn::Object::info(), j, PLearn::VMatrix::length(), PLearn::TVec< T >::length(), PLearn::max(), moveto(), PLASSERT, PLERROR, PLearn::TVec< T >::subVec(), and PLearn::VMatrix::width_.
{ #ifdef BOUNDCHECK if(v.length()!=width_) PLERROR("In CompactFileVMatrix::getNewRow - length of v (%d) differs from matrix width (%d)",v.length(),width_); #endif unsigned char* buffer; bool m_flag=false; // to check if we allocated the buffer using malloc if (in_ram_ && (cache_index[i/8] & (1 << (i%8)))) { buffer = (unsigned char*)(cache.data() + (i*compact_width_)); } else { buffer = (unsigned char*)malloc(compact_width_); m_flag = true; #ifdef USE_NSPR_FILE moveto(i); PR_Read(f, buffer, compact_width_); #else fseek(f, COMPACTFILEVMATRIX_HEADERLENGTH_MAX+(i*compact_width_), SEEK_SET); fread(buffer, compact_width_, 1, f); #endif if (in_ram_) { cache_index[i/8] |= 1 << (i%8); for (int j=0; j<compact_width_; j++) cache[(i*compact_width_) + j] = buffer[j]; } } int current_b = -1; int current_v = 0; int value_b = 0; for (int g=0; g<info.length(); g++) { if (!info[g].active) { current_b += info[g].compact_length; continue; } int nbits = info[g].bits_per_value; int max = info[g].max; int length = info[g].length; char type = info[g].type; if (type == 'o') { v.subVec(current_v, length * (max + 1)).clear(); } int remainder = 0; for (int i = 0; i < length; i++) { if (!remainder) { // we're done with the current byte (this block is entered in the first iteration) remainder = 8; current_b++; PLASSERT(current_b < compact_width_); // just in case value_b = buffer[current_b]; } if (remainder < nbits) { // the value overlaps two bytes int val = value_b; int overflow = nbits - remainder; value_b = buffer[++current_b]; val |= (value_b & (1<<overflow)-1) << remainder; value_b >>= overflow; remainder = 8 - nbits + remainder; if (type == 'i') v[current_v] = (real)val / max; else if (type == 'u') v[current_v] = val; else if (type == 'o' && val <= max) v[current_v + val] = 1; } else { // the value is whole in the current byte int val = (value_b & ((1<<nbits) - 1)); if (type == 'i') v[current_v] = (real)val / max; else if (type == 'u') v[current_v] = val; else if (type == 'o' && val <= max) { v[current_v + val] = 1; } value_b >>= nbits; remainder -= nbits; } if (type == 'i' || type == 'u') current_v++; else if (type == 'o') current_v += max + 1; } } if (m_flag) free(buffer); }
OptionList & PLearn::CompactFileVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
OptionMap & PLearn::CompactFileVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
RemoteMethodMap & PLearn::CompactFileVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
char PLearn::CompactFileVMatrix::groupEncoding | ( | int | group | ) | const [virtual] |
The encoding of the group-th field group ('i' or 'o')
Definition at line 358 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::type.
{ return getGroup(group).type; }
The length of the group-th field group.
Definition at line 391 of file CompactFileVMatrix.cc.
References getGroup(), PLearn::Object::info(), PLearn::GroupInfo::length, PLearn::GroupInfo::max, and PLearn::GroupInfo::type.
{ GroupInfo& info = getGroup(group); if (info.type == 'i') return info.length; else return info.length * info.max; }
The amount of fields in the group-th field group.
Definition at line 369 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::length.
{ return getGroup(group).length; }
The amount of possible values in the group-th field group.
Definition at line 380 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::max.
{ return getGroup(group).max; }
static VMat PLearn::CompactFileVMatrix::instantiateFromPPath | ( | const PPath & | filename | ) | [inline, static] |
Definition at line 157 of file CompactFileVMatrix.h.
{ return VMat(new CompactFileVMatrix(filename)); }
void PLearn::CompactFileVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transform a shallow copy into a deep copy.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 295 of file CompactFileVMatrix.cc.
References cache_index, PLearn::deepCopyField(), f, i, PLearn::Object::info(), PLearn::TVec< T >::length(), PLearn::RowBufferedVMatrix::makeDeepCopyFromShallowCopy(), and openCurrentFile().
{ inherited::makeDeepCopyFromShallowCopy(copies); TVec<GroupInfo> temp = info; info = TVec<GroupInfo>(); for (int i=0; i<temp.length(); i++) { info.append(temp[i]); } deepCopyField(cache_index, copies); // We don't want to share the file descriptor f = 0; openCurrentFile(); }
Definition at line 507 of file CompactFileVMatrix.cc.
References compact_width_, f, header_length, i, and j.
Referenced by getNewRow().
{ #ifdef USE_NSPR_FILE PRInt64 offset = i; offset *= compact_width_; offset += j; offset += header_length; PR_Seek64(f, offset, PR_SEEK_SET); #else fseek(f, header_length+(i*compact_width_+j), SEEK_SET); #endif }
int PLearn::CompactFileVMatrix::nGroups | ( | ) | const [virtual] |
The amount of field groups in the matrix.
Definition at line 335 of file CompactFileVMatrix.cc.
References PLearn::Object::info().
{ return info.length(); }
void PLearn::CompactFileVMatrix::openCurrentFile | ( | ) | [protected, virtual] |
Open the current file.
Definition at line 243 of file CompactFileVMatrix.cc.
References f, filename_, PLearn::isfile(), and PLERROR.
Referenced by build_(), and makeDeepCopyFromShallowCopy().
{ if (f) { PLERROR("In CompactFileVMatrix::openCurrentFile - File already open (%s)", filename_.c_str()); } else if (!isfile(filename_)) { PLERROR("In CompactFileVMatrix::openCurrentFile - File does not exist (%s)", filename_.c_str()); } else { #ifdef USE_NSPR_FILE f = PR_Open(filename_.c_str(), PR_RDONLY, 0666); #else f = fopen(filename_.c_str(), "rb"); #endif if (! f) PLERROR("CompactFileVMatrix::openCurrentFile: could not open file %s", filename_.c_str()); } }
This method must be implemented in all subclasses of writable matrices.
Sets element (i,j) to value.
Reimplemented from PLearn::VMatrix.
Definition at line 523 of file CompactFileVMatrix.cc.
References PLERROR.
{ PLERROR("In CompactFileVMatrix::put - not implemented."); }
It is suggested that this method be implemented in subclasses of writable matrices to speed up accesses (default version repeatedly calls put(i,j,value) which may have a significant overhead)
Reimplemented from PLearn::VMatrix.
Definition at line 502 of file CompactFileVMatrix.cc.
References PLERROR.
{ PLERROR("In CompactFileVMatrix::putSubRow - not implemented."); }
void PLearn::CompactFileVMatrix::updateHeader | ( | ) | [virtual] |
Re-write the header with all current field values.
Definition at line 554 of file CompactFileVMatrix.cc.
References PLERROR.
{ PLERROR("In CompactFileVMatrix::put - not implemented."); }
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 162 of file CompactFileVMatrix.h.
Vec PLearn::CompactFileVMatrix::active_list_ [protected] |
Definition at line 99 of file CompactFileVMatrix.h.
Referenced by build_(), and declareOptions().
string PLearn::CompactFileVMatrix::cache [mutable, protected] |
Definition at line 113 of file CompactFileVMatrix.h.
Referenced by build_(), cleanup(), and getNewRow().
TVec<unsigned char> PLearn::CompactFileVMatrix::cache_index [protected] |
Definition at line 112 of file CompactFileVMatrix.h.
Referenced by build_(), cleanup(), getNewRow(), and makeDeepCopyFromShallowCopy().
int PLearn::CompactFileVMatrix::compact_width_ [protected] |
width of a compacted row
Definition at line 110 of file CompactFileVMatrix.h.
Referenced by build_(), getNewRow(), and moveto().
VMatrixExtensionRegistrar * PLearn::CompactFileVMatrix::extension_registrar [static, private] |
new VMatrixExtensionRegistrar( "cmat", &CompactFileVMatrix::instantiateFromPPath, "A compact file matrix")
Definition at line 95 of file CompactFileVMatrix.h.
PRFileDesc* PLearn::CompactFileVMatrix::f [protected] |
Definition at line 104 of file CompactFileVMatrix.h.
Referenced by build_(), closeCurrentFile(), flush(), getNewRow(), makeDeepCopyFromShallowCopy(), moveto(), and openCurrentFile().
PPath PLearn::CompactFileVMatrix::filename_ [protected] |
Definition at line 98 of file CompactFileVMatrix.h.
Referenced by build_(), declareOptions(), and openCurrentFile().
int PLearn::CompactFileVMatrix::header_length [protected] |
Definition at line 109 of file CompactFileVMatrix.h.
bool PLearn::CompactFileVMatrix::in_ram_ [protected] |
Definition at line 100 of file CompactFileVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
TVec<GroupInfo> PLearn::CompactFileVMatrix::info [protected] |
Definition at line 111 of file CompactFileVMatrix.h.