PLearn 0.1
|
#include <VecCompressor.h>
Static Public Member Functions | |
static signed char * | compressVec (const Vec &v, signed char *data) |
static void | uncompressVec (signed char *data, const Vec &v) |
uncompresses the data of a vector compressed with compressVec v must have the correct size. | |
static void | writeCompressedVec (ostream &out, const Vec &v) |
writes v in compressed format to the given stream The written data does not contain size info. | |
static void | readCompressedVec (istream &in, const Vec &v) |
reads data of a compressed vector from the given stream v must have the right size already (this is not checked!) | |
static size_t | worstCaseSize (int n) |
Returns the number of bytes that will be used to encode a vector of length n in the worst case. | |
Static Protected Member Functions | |
static bool | issmallint (real x) |
static bool | is0 (real x) |
static bool | isI (real x) |
static bool | isF (real x) |
It encodes only non zero values, using one byte for small integers, and 4-byte floating points for all other values. This representation should be reasonably good for both sparse matrices, and matrices containing categorical data (represented by small integers) possibly with one hot representations.
Definition at line 58 of file VecCompressor.h.
signed char * PLearn::VecCompressor::compressVec | ( | const Vec & | v, |
signed char * | data | ||
) | [static] |
writes v in a compressed form in the data buffer passed as argument. (make sure enough memory is allocated in the data buffer) returns a pointer to the one-after-last element written in the data block
Definition at line 48 of file VecCompressor.cc.
References PLearn::TVec< T >::data(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), and n.
Referenced by PLearn::CompressedVMatrix::appendRow().
{ real* vdata = v.data(); signed char* ptr = data; // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars) // If mode is '0' abs(count) indicates the number of zeroes, // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to 'I' mode // a 0 count means insert 127 zeros and stay in zero mode // If mode is 'F' abs(count) indicates the number of floats that follow // a positive sign indicates switch to 'I' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 floats and stay in float mode // If mode is 'I' abs(count) indicates the number of small integers that follow // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 small integers and stay in 'I' mode int l = v.length(); int i=0; real val = vdata[i]; signed char mode = 'F'; if(fast_exact_is_equal(val, 0.)) mode = '0'; else if(issmallint(val)) mode = 'I'; // else 'F' int count = 0; int istart = 0; float fval = 0.; signed char* pfval = (signed char*)&fval; *ptr++ = mode; while(i<l) { switch(mode) { case '0': istart = i; while(i<l && is0(vdata[i])) i++; count = i - istart; while(count>127) { *ptr++ = 0; count -= 127; } if(i>=l || issmallint(vdata[i])) { *ptr++ = (signed char)(-count); mode = 'I'; } else { *ptr++ = (signed char)count; mode = 'F'; } break; case 'I': istart = i; while(i<l && isI(vdata[i])) i++; count = i - istart; while(count>127) { *ptr++ = 0; int n = 127; while(n--) *ptr++ = (signed char)vdata[istart++]; count -= 127; } if(i>=l || is0(vdata[i])) { *ptr++ = (signed char)(-count); mode = '0'; } else // next value is a floating point { *ptr++ = (signed char)count; mode = 'F'; } while(count--) *ptr++ = (signed char)vdata[istart++]; break; case 'F': istart = i; val = vdata[i]; while(i<l && isF(vdata[i])) i++; count = i - istart; while(count>127) { *ptr++ = 0; int n = 127; while(n--) { fval = (float)vdata[istart++]; *ptr++ = pfval[0]; *ptr++ = pfval[1]; *ptr++ = pfval[2]; *ptr++ = pfval[3]; } count -= 127; } if(i>=l || is0(vdata[i])) { *ptr++ = (signed char)(-count); mode = '0'; } else { *ptr++ = (signed char)count; mode = 'I'; } while(count--) { fval = (float)vdata[istart++]; *ptr++ = pfval[0]; *ptr++ = pfval[1]; *ptr++ = pfval[2]; *ptr++ = pfval[3]; } } } return ptr; }
Definition at line 65 of file VecCompressor.h.
References PLearn::fast_exact_is_equal().
{ return fast_exact_is_equal(x, 0.); }
Definition at line 71 of file VecCompressor.h.
References PLearn::fast_exact_is_equal().
{ return !fast_exact_is_equal(x, 0.) && !issmallint(x); }
Definition at line 68 of file VecCompressor.h.
References PLearn::fast_exact_is_equal().
{ return !fast_exact_is_equal(x, 0.) && issmallint(x); }
Definition at line 62 of file VecCompressor.h.
References PLearn::fast_exact_is_equal().
{ int intx = int(x); return fast_exact_is_equal(floor(x), x) && intx<=127 && intx>=-127; }
void PLearn::VecCompressor::readCompressedVec | ( | istream & | in, |
const Vec & | v | ||
) | [static] |
reads data of a compressed vector from the given stream v must have the right size already (this is not checked!)
Definition at line 391 of file VecCompressor.cc.
References PLearn::TVec< T >::data(), PLearn::TVec< T >::length(), PLERROR, and PLearn::read_sbyte().
{ // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars) // If mode is '0' abs(count) indicates the number of zeroes, // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to 'I' mode // a 0 count means insert 127 zeros and stay in zero mode // If mode is 'F' abs(count) indicates the number of floats that follow // a positive sign indicates switch to 'I' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 floats and stay in float mode // If mode is 'I' abs(count) indicates the number of small integers that follow // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 small integers and stay in 'I' mode real* vptr = v.data(); real* vptrend = vptr+v.length(); signed char mode = read_sbyte(in); float fval = 0.; signed char count; while(vptr!=vptrend) { count = read_sbyte(in); // cerr << int(count) << ' '; switch(mode) { case '0': if(count<0) { mode = 'I'; count = -count; } else if(count>0) mode = 'F'; else count = 127; while(count--) *vptr++ = 0.; break; case 'I': if(count<0) { mode = '0'; count = -count; } else if(count>0) mode = 'F'; else count = 127; while(count--) *vptr++ = real(read_sbyte(in)); break; case 'F': if(count<0) { mode = '0'; count = -count; } else if(count>0) mode = 'I'; else count = 127; while(count--) { in.read((char*)&fval,4); *vptr++ = real(fval); } break; default: PLERROR("Problem in VecCompressor::readCompressedVec this should not happen!!! (wrong data format?)"); } } // cerr << endl; }
void PLearn::VecCompressor::uncompressVec | ( | signed char * | data, |
const Vec & | v | ||
) | [static] |
uncompresses the data of a vector compressed with compressVec v must have the correct size.
Definition at line 182 of file VecCompressor.cc.
References PLearn::TVec< T >::data(), PLearn::TVec< T >::length(), and PLERROR.
Referenced by PLearn::CompressedVMatrix::getNewRow().
{ // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars) // If mode is '0' abs(count) indicates the number of zeroes, // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to 'I' mode // a 0 count means insert 127 zeros and stay in zero mode // If mode is 'F' abs(count) indicates the number of floats that follow // a positive sign indicates switch to 'I' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 floats and stay in float mode // If mode is 'I' abs(count) indicates the number of small integers that follow // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 small integers and stay in 'I' mode real* vptr = v.data(); real* vptrend = vptr+v.length(); signed char* ptr = data; signed char mode = *ptr++; float fval = 0.; signed char* pfval = (signed char*)&fval; signed char count; while(vptr!=vptrend) { count = *ptr++; switch(mode) { case '0': if(count<0) { mode = 'I'; count = -count; } else if(count>0) mode = 'F'; else count = 127; while(count--) *vptr++ = 0.; break; case 'I': if(count<0) { mode = '0'; count = -count; } else if(count>0) mode = 'F'; else count = 127; while(count--) *vptr++ = real(*ptr++); break; case 'F': if(count<0) { mode = '0'; count = -count; } else if(count>0) mode = 'I'; else count = 127; while(count--) { pfval[0] = *ptr++; pfval[1] = *ptr++; pfval[2] = *ptr++; pfval[3] = *ptr++; *vptr++ = real(fval); } break; default: PLERROR("Problem in VecCompressor::uncompressVec this should not happen!!! (wrong data format?)"); } } }
static size_t PLearn::VecCompressor::worstCaseSize | ( | int | n | ) | [inline, static] |
Returns the number of bytes that will be used to encode a vector of length n in the worst case.
Definition at line 95 of file VecCompressor.h.
Referenced by PLearn::CompressedVMatrix::CompressedVMatrix().
{ return 2+4*n+n/128; }
void PLearn::VecCompressor::writeCompressedVec | ( | ostream & | out, |
const Vec & | v | ||
) | [static] |
writes v in compressed format to the given stream The written data does not contain size info.
Definition at line 267 of file VecCompressor.cc.
References PLearn::TVec< T >::data(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), n, and PLearn::write_sbyte().
{ real* vdata = v.data(); // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars) // If mode is '0' abs(count) indicates the number of zeroes, // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to 'I' mode // a 0 count means insert 127 zeros and stay in zero mode // If mode is 'F' abs(count) indicates the number of floats that follow // a positive sign indicates switch to 'I' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 floats and stay in float mode // If mode is 'I' abs(count) indicates the number of small integers that follow // a positive sign indicates switch to 'F' mode // a negative sign indicates switch to '0' mode // a 0 count means insert 127 small integers and stay in 'I' mode int l = v.length(); int i=0; real val = vdata[i]; signed char mode = 'F'; if(fast_exact_is_equal(val, 0.)) mode = '0'; else if(issmallint(val)) mode = 'I'; // else 'F' int count = 0; int istart = 0; float fval = 0.; write_sbyte(out,mode); while(i<l) { switch(mode) { case '0': istart = i; while(i<l && is0(vdata[i])) i++; count = i - istart; while(count>127) { write_sbyte(out,0); count -= 127; } if(i>=l || issmallint(vdata[i])) { write_sbyte(out,-count); mode = 'I'; } else { write_sbyte(out,count); mode = 'F'; } break; case 'I': istart = i; while(i<l && isI(vdata[i])) i++; count = i - istart; while(count>127) { write_sbyte(out,0); int n = 127; while(n--) write_sbyte(out,(signed char)vdata[istart++]); count -= 127; } if(i>=l || is0(vdata[i])) { write_sbyte(out,-count); mode = '0'; } else // next value is a floating point { write_sbyte(out,count); mode = 'F'; } while(count--) write_sbyte(out,(signed char)vdata[istart++]); break; case 'F': istart = i; while(i<l && isF(vdata[i])) i++; count = i - istart; while(count>127) { write_sbyte(out,0); int n = 127; while(n--) { fval = (float)vdata[istart++]; out.write((char*)&fval,4); } count -= 127; } if(i>=l || is0(vdata[i])) { write_sbyte(out,-count); mode = '0'; } else { write_sbyte(out,count); mode = 'I'; } while(count--) { fval = (float)vdata[istart++]; out.write((char*)&fval,4); } } } }