|
PLearn 0.1
|
A VMatrix that exists in a .pmat file (native PLearn matrix format, same as for Mat). More...
#include <CompactFileVMatrix.h>


Public Member Functions | |
| CompactFileVMatrix () | |
| CompactFileVMatrix (const PPath &filename) | |
| opens an existing file | |
| virtual void | updateHeader () |
| Re-write the header with all current field values. | |
| virtual int | nGroups () const |
| The amount of field groups in the matrix. | |
| virtual char | groupEncoding (int group) const |
| The encoding of the group-th field group ('i' or 'o') | |
| virtual int | groupNFields (int group) const |
| The amount of fields in the group-th field group. | |
| virtual int | groupNValues (int group) const |
| The amount of possible values in the group-th field group. | |
| virtual int | groupLength (int group) const |
| The length of the group-th field group. | |
| virtual void | put (int i, int j, real value) |
| This method must be implemented in all subclasses of writable matrices. | |
| virtual void | putSubRow (int i, int j, Vec v) |
| It is suggested that this method be implemented in subclasses of writable matrices to speed up accesses (default version repeatedly calls put(i,j,value) which may have a significant overhead) | |
| virtual void | appendRow (Vec v) |
| This method must be implemented for matrices that are allowed to grow. | |
| virtual void | flush () |
| For matrices stored on disk, this should flush all pending buffered write operations. | |
| virtual void | build () |
| Simply calls inherited::build() then build_(). | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual CompactFileVMatrix * | deepCopy (CopiesMap &copies) const |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transform a shallow copy into a deep copy. | |
| virtual | ~CompactFileVMatrix () |
| Destructor. | |
Static Public Member Functions | |
| static VMat | instantiateFromPPath (const PPath &filename) |
| static string | _classname_ () |
| CompactFileVMatrix. | |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
| virtual void | getNewRow (int i, const Vec &v) const |
| This is the only method requiring implementation in subclasses. | |
| virtual void | openCurrentFile () |
| Open the current file. | |
| virtual void | closeCurrentFile () |
| Close the current file. | |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declare this class' options. | |
Protected Attributes | |
| PPath | filename_ |
| Vec | active_list_ |
| bool | in_ram_ |
| PRFileDesc * | f |
| int | header_length |
| int | compact_width_ |
| width of a compacted row | |
| TVec< GroupInfo > | info |
| TVec< unsigned char > | cache_index |
| string | cache |
Private Types | |
| typedef RowBufferedVMatrix | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
| GroupInfo & | getGroup (int group) const |
| void | moveto (int i, int j=0) const |
| void | cleanup () |
Static Private Attributes | |
| static VMatrixExtensionRegistrar * | extension_registrar |
A VMatrix that exists in a .pmat file (native PLearn matrix format, same as for Mat).
Definition at line 90 of file CompactFileVMatrix.h.
typedef RowBufferedVMatrix PLearn::CompactFileVMatrix::inherited [private] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 94 of file CompactFileVMatrix.h.
| PLearn::CompactFileVMatrix::CompactFileVMatrix | ( | ) |
Definition at line 66 of file CompactFileVMatrix.cc.
References PLearn::VMatrix::writable.
| PLearn::CompactFileVMatrix::CompactFileVMatrix | ( | const PPath & | filename | ) |
| PLearn::CompactFileVMatrix::~CompactFileVMatrix | ( | ) | [virtual] |
Destructor.
Definition at line 315 of file CompactFileVMatrix.cc.
References cleanup().
{
CompactFileVMatrix::cleanup();
}

| string PLearn::CompactFileVMatrix::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
| OptionList & PLearn::CompactFileVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
| RemoteMethodMap & PLearn::CompactFileVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
| Object * PLearn::CompactFileVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
| StaticInitializer CompactFileVMatrix::_static_initializer_ & PLearn::CompactFileVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
| void PLearn::CompactFileVMatrix::appendRow | ( | Vec | v | ) | [virtual] |
This method must be implemented for matrices that are allowed to grow.
Reimplemented from PLearn::VMatrix.
Definition at line 531 of file CompactFileVMatrix.cc.
References PLERROR.
{
PLERROR("In CompactFileVMatrix::put - not implemented.");
}
| void PLearn::CompactFileVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::VMatrix.
Definition at line 87 of file CompactFileVMatrix.cc.
References PLearn::VMatrix::build(), and build_().
{
inherited::build();
build_();
}

| void PLearn::CompactFileVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VMatrix.
Definition at line 98 of file CompactFileVMatrix.cc.
References active_list_, cache, cache_index, cleanup(), compact_width_, COMPACTFILEVMATRIX_HEADERLENGTH_MAX, f, filename_, header_length, i, in_ram_, PLearn::Object::info(), PLearn::PPath::isEmpty(), PLearn::TVec< T >::length(), PLearn::VMatrix::length(), PLearn::VMatrix::length_, PLearn::VMatrix::loadFieldInfos(), PLearn::max(), openCurrentFile(), PLERROR, PLearn::VMatrix::updateMtime(), PLearn::VMatrix::width(), and PLearn::VMatrix::width_.
Referenced by build(), and CompactFileVMatrix().
{
// cleanup (close file, free structures, etc.)
cleanup();
// If no file is given, we can stop here.
if (filename_.isEmpty())
return;
char header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX + 1];
header[COMPACTFILEVMATRIX_HEADERLENGTH_MAX] = '\n';
openCurrentFile();
#ifdef USE_NSPR_FILE
PR_Read(f,header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX);
#else
fread(header,COMPACTFILEVMATRIX_HEADERLENGTH_MAX,1,f);
#endif
int file_length;
file_length = atoi(header);
if (!file_length) {
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (0)");
}
this->length_ = file_length;
int i = 0;
while (header[i] >= '0' && header[i] <= '9')
i++;
i++;
info = TVec<GroupInfo>();
// Here we painfully read the header.
while (1) {
char type = header[i];
if (type == '\n' || type == ' ')
break;
if (!type || !(type == 'o' || type == 'i' || type == 'u'))
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (bad type: %c)", type);
i++;
if (header[i] != ':')
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (2)");
i++;
int length = atoi(&(header[i]));
if (!length)
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (3)");
while (header[i] >= '0' && header[i] <= '9')
i++;
if (header[i] != ':')
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (4)");
i++;
int max = atoi(&(header[i]));
if (!max)
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (5)");
while (header[i] >= '0' && header[i] <= '9')
i++;
if (header[i] != ':')
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (6)");
i++;
int bits_per_value = atoi(&(header[i]));
if (!bits_per_value)
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file (7)");
while (header[i] >= '0' && header[i] <= '9')
i++;
if (bits_per_value > 8)
PLERROR("In CompactFileVMatrix::build_ - each field must span at most 8 bits");
if (header[i] != ' ' && header[i] != '\n')
PLERROR("In CompactFileVMatrix::build_ - ill-formed header for the cmat file");
info.append(GroupInfo(type, length, max, bits_per_value));
if (header[i] == '\n') {
i++;
break;
}
i++;
}
header_length = i;
// If there is a non-empty list of active groups, activate only those.
if (active_list_.length()) {
for (int i=0; i<active_list_.length(); i++) {
int next_active = (int)(active_list_[i]);
if (i && active_list_[i-1] >= next_active)
PLERROR("In CompactFileVMatrix - Please give the active fields list in ascending order.");
if (i < 0 || i >= info.length())
PLERROR("In CompactFileVMatrix - Bad list of active groups: group index out of bounds.");
info[next_active].active = true;
}
}
// else, activate all of them
else {
for(int i=0; i<info.length(); i++)
info[i].active = true;
}
int width = 0;
int file_width = 0; // real width of an entry in the file
for (int i=0; i<info.length(); i++) {
info[i].compact_length = (info[i].length * info[i].bits_per_value + 7) / 8;
if (info[i].active) {
if (info[i].type == 'o') { // one-hot encoding
width += info[i].length * (info[i].max + 1);
}
else if (info[i].type == 'i' || info[i].type == 'u') { // int (if max=1) or float encoding
width += info[i].length;
}
}
file_width += info[i].compact_length;
}
this->width_ = width; // + 1;
this->compact_width_ = file_width; // + 1;
// If the option in_ram is specified, we allocate a buffer
if (in_ram_) {
int sz = length_ * compact_width_;
cache = string(sz, '\0');
cache_index = TVec<unsigned char>((length_ + 7) / 8);
}
updateMtime(filename_);
loadFieldInfos();
}


| string PLearn::CompactFileVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
| void PLearn::CompactFileVMatrix::cleanup | ( | ) | [private] |
Definition at line 324 of file CompactFileVMatrix.cc.
References cache, cache_index, and closeCurrentFile().
Referenced by build_(), and ~CompactFileVMatrix().
{
CompactFileVMatrix::closeCurrentFile();
cache = string();
cache_index = TVec<unsigned char>();
}


| void PLearn::CompactFileVMatrix::closeCurrentFile | ( | ) | [protected, virtual] |
| void PLearn::CompactFileVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declare this class' options.
Reimplemented from PLearn::VMatrix.
Definition at line 283 of file CompactFileVMatrix.cc.
References active_list_, PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::VMatrix::declareOptions(), filename_, and in_ram_.
{
declareOption(ol, "filename", &CompactFileVMatrix::filename_, OptionBase::buildoption, "Filename of the matrix");
declareOption(ol, "active_list", &CompactFileVMatrix::active_list_, OptionBase::buildoption, "comma-separated list of active field groups");
declareOption(ol, "cache", &CompactFileVMatrix::in_ram_, OptionBase::buildoption, "if set to true, all rows read from the file will be cached");
inherited::declareOptions(ol);
}

| static const PPath& PLearn::CompactFileVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 162 of file CompactFileVMatrix.h.
:
| CompactFileVMatrix * PLearn::CompactFileVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 61 of file CompactFileVMatrix.cc.
| void PLearn::CompactFileVMatrix::flush | ( | ) | [virtual] |
For matrices stored on disk, this should flush all pending buffered write operations.
Reimplemented from PLearn::VMatrix.
Definition at line 539 of file CompactFileVMatrix.cc.
References f.
Definition at line 345 of file CompactFileVMatrix.cc.
References PLearn::Object::info(), and PLERROR.
Referenced by groupEncoding(), groupLength(), groupNFields(), and groupNValues().
{
if (group < 0 || group >= info.length()) {
PLERROR("In CompactFileVMatrix::getGroup - group index out of bounds.");
}
return info[group];
}


This is the only method requiring implementation in subclasses.
Implements PLearn::RowBufferedVMatrix.
Definition at line 405 of file CompactFileVMatrix.cc.
References cache, cache_index, PLearn::TVec< T >::clear(), compact_width_, COMPACTFILEVMATRIX_HEADERLENGTH_MAX, f, g, in_ram_, PLearn::Object::info(), j, PLearn::VMatrix::length(), PLearn::TVec< T >::length(), PLearn::max(), moveto(), PLASSERT, PLERROR, PLearn::TVec< T >::subVec(), and PLearn::VMatrix::width_.
{
#ifdef BOUNDCHECK
if(v.length()!=width_)
PLERROR("In CompactFileVMatrix::getNewRow - length of v (%d) differs from matrix width (%d)",v.length(),width_);
#endif
unsigned char* buffer;
bool m_flag=false; // to check if we allocated the buffer using malloc
if (in_ram_ && (cache_index[i/8] & (1 << (i%8)))) {
buffer = (unsigned char*)(cache.data() + (i*compact_width_));
}
else {
buffer = (unsigned char*)malloc(compact_width_);
m_flag = true;
#ifdef USE_NSPR_FILE
moveto(i);
PR_Read(f, buffer, compact_width_);
#else
fseek(f, COMPACTFILEVMATRIX_HEADERLENGTH_MAX+(i*compact_width_), SEEK_SET);
fread(buffer, compact_width_, 1, f);
#endif
if (in_ram_) {
cache_index[i/8] |= 1 << (i%8);
for (int j=0; j<compact_width_; j++)
cache[(i*compact_width_) + j] = buffer[j];
}
}
int current_b = -1;
int current_v = 0;
int value_b = 0;
for (int g=0; g<info.length(); g++) {
if (!info[g].active) {
current_b += info[g].compact_length;
continue;
}
int nbits = info[g].bits_per_value;
int max = info[g].max;
int length = info[g].length;
char type = info[g].type;
if (type == 'o') {
v.subVec(current_v, length * (max + 1)).clear();
}
int remainder = 0;
for (int i = 0; i < length; i++) {
if (!remainder) { // we're done with the current byte (this block is entered in the first iteration)
remainder = 8;
current_b++;
PLASSERT(current_b < compact_width_); // just in case
value_b = buffer[current_b];
}
if (remainder < nbits) { // the value overlaps two bytes
int val = value_b;
int overflow = nbits - remainder;
value_b = buffer[++current_b];
val |= (value_b & (1<<overflow)-1) << remainder;
value_b >>= overflow;
remainder = 8 - nbits + remainder;
if (type == 'i')
v[current_v] = (real)val / max;
else if (type == 'u')
v[current_v] = val;
else if (type == 'o' && val <= max)
v[current_v + val] = 1;
}
else { // the value is whole in the current byte
int val = (value_b & ((1<<nbits) - 1));
if (type == 'i')
v[current_v] = (real)val / max;
else if (type == 'u')
v[current_v] = val;
else if (type == 'o' && val <= max) {
v[current_v + val] = 1;
}
value_b >>= nbits;
remainder -= nbits;
}
if (type == 'i' || type == 'u')
current_v++;
else if (type == 'o')
current_v += max + 1;
}
}
if (m_flag)
free(buffer);
}

| OptionList & PLearn::CompactFileVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
| OptionMap & PLearn::CompactFileVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
| RemoteMethodMap & PLearn::CompactFileVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 61 of file CompactFileVMatrix.cc.
| char PLearn::CompactFileVMatrix::groupEncoding | ( | int | group | ) | const [virtual] |
The encoding of the group-th field group ('i' or 'o')
Definition at line 358 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::type.
{
return getGroup(group).type;
}

The length of the group-th field group.
Definition at line 391 of file CompactFileVMatrix.cc.
References getGroup(), PLearn::Object::info(), PLearn::GroupInfo::length, PLearn::GroupInfo::max, and PLearn::GroupInfo::type.
{
GroupInfo& info = getGroup(group);
if (info.type == 'i')
return info.length;
else
return info.length * info.max;
}

The amount of fields in the group-th field group.
Definition at line 369 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::length.
{
return getGroup(group).length;
}

The amount of possible values in the group-th field group.
Definition at line 380 of file CompactFileVMatrix.cc.
References getGroup(), and PLearn::GroupInfo::max.
{
return getGroup(group).max;
}

| static VMat PLearn::CompactFileVMatrix::instantiateFromPPath | ( | const PPath & | filename | ) | [inline, static] |
Definition at line 157 of file CompactFileVMatrix.h.
{
return VMat(new CompactFileVMatrix(filename));
}
| void PLearn::CompactFileVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transform a shallow copy into a deep copy.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 295 of file CompactFileVMatrix.cc.
References cache_index, PLearn::deepCopyField(), f, i, PLearn::Object::info(), PLearn::TVec< T >::length(), PLearn::RowBufferedVMatrix::makeDeepCopyFromShallowCopy(), and openCurrentFile().
{
inherited::makeDeepCopyFromShallowCopy(copies);
TVec<GroupInfo> temp = info;
info = TVec<GroupInfo>();
for (int i=0; i<temp.length(); i++) {
info.append(temp[i]);
}
deepCopyField(cache_index, copies);
// We don't want to share the file descriptor
f = 0;
openCurrentFile();
}

Definition at line 507 of file CompactFileVMatrix.cc.
References compact_width_, f, header_length, i, and j.
Referenced by getNewRow().
{
#ifdef USE_NSPR_FILE
PRInt64 offset = i;
offset *= compact_width_;
offset += j;
offset += header_length;
PR_Seek64(f, offset, PR_SEEK_SET);
#else
fseek(f, header_length+(i*compact_width_+j), SEEK_SET);
#endif
}

| int PLearn::CompactFileVMatrix::nGroups | ( | ) | const [virtual] |
The amount of field groups in the matrix.
Definition at line 335 of file CompactFileVMatrix.cc.
References PLearn::Object::info().
{
return info.length();
}

| void PLearn::CompactFileVMatrix::openCurrentFile | ( | ) | [protected, virtual] |
Open the current file.
Definition at line 243 of file CompactFileVMatrix.cc.
References f, filename_, PLearn::isfile(), and PLERROR.
Referenced by build_(), and makeDeepCopyFromShallowCopy().
{
if (f) {
PLERROR("In CompactFileVMatrix::openCurrentFile - File already open (%s)", filename_.c_str());
}
else if (!isfile(filename_)) {
PLERROR("In CompactFileVMatrix::openCurrentFile - File does not exist (%s)", filename_.c_str());
}
else {
#ifdef USE_NSPR_FILE
f = PR_Open(filename_.c_str(), PR_RDONLY, 0666);
#else
f = fopen(filename_.c_str(), "rb");
#endif
if (! f)
PLERROR("CompactFileVMatrix::openCurrentFile: could not open file %s", filename_.c_str());
}
}


This method must be implemented in all subclasses of writable matrices.
Sets element (i,j) to value.
Reimplemented from PLearn::VMatrix.
Definition at line 523 of file CompactFileVMatrix.cc.
References PLERROR.
{
PLERROR("In CompactFileVMatrix::put - not implemented.");
}
It is suggested that this method be implemented in subclasses of writable matrices to speed up accesses (default version repeatedly calls put(i,j,value) which may have a significant overhead)
Reimplemented from PLearn::VMatrix.
Definition at line 502 of file CompactFileVMatrix.cc.
References PLERROR.
{
PLERROR("In CompactFileVMatrix::putSubRow - not implemented.");
}
| void PLearn::CompactFileVMatrix::updateHeader | ( | ) | [virtual] |
Re-write the header with all current field values.
Definition at line 554 of file CompactFileVMatrix.cc.
References PLERROR.
{
PLERROR("In CompactFileVMatrix::put - not implemented.");
}
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 162 of file CompactFileVMatrix.h.
Vec PLearn::CompactFileVMatrix::active_list_ [protected] |
Definition at line 99 of file CompactFileVMatrix.h.
Referenced by build_(), and declareOptions().
string PLearn::CompactFileVMatrix::cache [mutable, protected] |
Definition at line 113 of file CompactFileVMatrix.h.
Referenced by build_(), cleanup(), and getNewRow().
TVec<unsigned char> PLearn::CompactFileVMatrix::cache_index [protected] |
Definition at line 112 of file CompactFileVMatrix.h.
Referenced by build_(), cleanup(), getNewRow(), and makeDeepCopyFromShallowCopy().
int PLearn::CompactFileVMatrix::compact_width_ [protected] |
width of a compacted row
Definition at line 110 of file CompactFileVMatrix.h.
Referenced by build_(), getNewRow(), and moveto().
VMatrixExtensionRegistrar * PLearn::CompactFileVMatrix::extension_registrar [static, private] |
new VMatrixExtensionRegistrar(
"cmat",
&CompactFileVMatrix::instantiateFromPPath,
"A compact file matrix")
Definition at line 95 of file CompactFileVMatrix.h.
PRFileDesc* PLearn::CompactFileVMatrix::f [protected] |
Definition at line 104 of file CompactFileVMatrix.h.
Referenced by build_(), closeCurrentFile(), flush(), getNewRow(), makeDeepCopyFromShallowCopy(), moveto(), and openCurrentFile().
PPath PLearn::CompactFileVMatrix::filename_ [protected] |
Definition at line 98 of file CompactFileVMatrix.h.
Referenced by build_(), declareOptions(), and openCurrentFile().
int PLearn::CompactFileVMatrix::header_length [protected] |
Definition at line 109 of file CompactFileVMatrix.h.
bool PLearn::CompactFileVMatrix::in_ram_ [protected] |
Definition at line 100 of file CompactFileVMatrix.h.
Referenced by build_(), declareOptions(), and getNewRow().
TVec<GroupInfo> PLearn::CompactFileVMatrix::info [protected] |
Definition at line 111 of file CompactFileVMatrix.h.
1.7.4