PLearn 0.1
Static Public Member Functions | Static Protected Member Functions
PLearn::VecCompressor Class Reference

#include <VecCompressor.h>

List of all members.

Static Public Member Functions

static signed char * compressVec (const Vec &v, signed char *data)
static void uncompressVec (signed char *data, const Vec &v)
 uncompresses the data of a vector compressed with compressVec v must have the correct size.
static void writeCompressedVec (ostream &out, const Vec &v)
 writes v in compressed format to the given stream The written data does not contain size info.
static void readCompressedVec (istream &in, const Vec &v)
 reads data of a compressed vector from the given stream v must have the right size already (this is not checked!)
static size_t worstCaseSize (int n)
 Returns the number of bytes that will be used to encode a vector of length n in the worst case.

Static Protected Member Functions

static bool issmallint (real x)
static bool is0 (real x)
static bool isI (real x)
static bool isF (real x)

Detailed Description

It encodes only non zero values, using one byte for small integers, and 4-byte floating points for all other values. This representation should be reasonably good for both sparse matrices, and matrices containing categorical data (represented by small integers) possibly with one hot representations.

Definition at line 58 of file VecCompressor.h.


Member Function Documentation

signed char * PLearn::VecCompressor::compressVec ( const Vec v,
signed char *  data 
) [static]

writes v in a compressed form in the data buffer passed as argument. (make sure enough memory is allocated in the data buffer) returns a pointer to the one-after-last element written in the data block

Definition at line 48 of file VecCompressor.cc.

References PLearn::TVec< T >::data(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), and n.

Referenced by PLearn::CompressedVMatrix::appendRow().

{
    real* vdata = v.data();
    signed char* ptr = data;

    // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
    // If mode is '0' abs(count) indicates the number of zeroes, 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to 'I' mode
    //                a 0 count means insert 127 zeros and stay in zero mode
    // If mode is 'F' abs(count) indicates the number of floats that follow 
    //                a positive sign indicates switch to 'I' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 floats and stay in float mode
    // If mode is 'I' abs(count) indicates the number of small integers that follow 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 small integers and stay in 'I' mode

    int l = v.length();

    int i=0;
    real val = vdata[i];

    signed char mode = 'F';
    if(fast_exact_is_equal(val, 0.))
        mode = '0';
    else if(issmallint(val))
        mode = 'I';
    // else 'F'

    int count = 0;
    int istart = 0;
    float fval = 0.;
    signed char* pfval = (signed char*)&fval;

    *ptr++ = mode;

    while(i<l)
    {
        switch(mode)
        {
        case '0':
            istart = i;
            while(i<l && is0(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                *ptr++ = 0;
                count -= 127;
            }
            if(i>=l || issmallint(vdata[i]))
            {
                *ptr++ = (signed char)(-count);
                mode = 'I';
            }
            else
            {
                *ptr++ = (signed char)count;
                mode = 'F';
            }
            break;

        case 'I':
            istart = i;
            while(i<l && isI(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                *ptr++ = 0;
                int n = 127;
                while(n--)
                    *ptr++ = (signed char)vdata[istart++];
                count -= 127;
            }
            if(i>=l || is0(vdata[i]))
            {
                *ptr++ = (signed char)(-count);
                mode = '0';
            }
            else // next value is a floating point
            {
                *ptr++ = (signed char)count;
                mode = 'F';
            }
            while(count--)
                *ptr++ = (signed char)vdata[istart++];                
            break;

        case 'F':
            istart = i;
            val = vdata[i];
            while(i<l && isF(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                *ptr++ = 0;
                int n = 127;
                while(n--)
                {
                    fval = (float)vdata[istart++];
                    *ptr++ = pfval[0];
                    *ptr++ = pfval[1];
                    *ptr++ = pfval[2];
                    *ptr++ = pfval[3];
                }
                count -= 127;
            }
            if(i>=l || is0(vdata[i]))
            {
                *ptr++ = (signed char)(-count);
                mode = '0';
            }
            else
            {
                *ptr++ = (signed char)count;
                mode = 'I';
            }
            while(count--)
            {
                fval = (float)vdata[istart++];
                *ptr++ = pfval[0];
                *ptr++ = pfval[1];
                *ptr++ = pfval[2];
                *ptr++ = pfval[3];
            }
        }
    }
    return ptr;
}

Here is the call graph for this function:

Here is the caller graph for this function:

static bool PLearn::VecCompressor::is0 ( real  x) [inline, static, protected]

Definition at line 65 of file VecCompressor.h.

References PLearn::fast_exact_is_equal().

    { return fast_exact_is_equal(x, 0.); }

Here is the call graph for this function:

static bool PLearn::VecCompressor::isF ( real  x) [inline, static, protected]

Definition at line 71 of file VecCompressor.h.

References PLearn::fast_exact_is_equal().

    { return !fast_exact_is_equal(x, 0.) && !issmallint(x); }

Here is the call graph for this function:

static bool PLearn::VecCompressor::isI ( real  x) [inline, static, protected]

Definition at line 68 of file VecCompressor.h.

References PLearn::fast_exact_is_equal().

    { return !fast_exact_is_equal(x, 0.) && issmallint(x); }

Here is the call graph for this function:

static bool PLearn::VecCompressor::issmallint ( real  x) [inline, static, protected]

Definition at line 62 of file VecCompressor.h.

References PLearn::fast_exact_is_equal().

    { int intx = int(x); return fast_exact_is_equal(floor(x), x) && intx<=127 && intx>=-127; }

Here is the call graph for this function:

void PLearn::VecCompressor::readCompressedVec ( istream &  in,
const Vec v 
) [static]

reads data of a compressed vector from the given stream v must have the right size already (this is not checked!)

Definition at line 391 of file VecCompressor.cc.

References PLearn::TVec< T >::data(), PLearn::TVec< T >::length(), PLERROR, and PLearn::read_sbyte().

{
    // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
    // If mode is '0' abs(count) indicates the number of zeroes, 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to 'I' mode
    //                a 0 count means insert 127 zeros and stay in zero mode
    // If mode is 'F' abs(count) indicates the number of floats that follow 
    //                a positive sign indicates switch to 'I' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 floats and stay in float mode
    // If mode is 'I' abs(count) indicates the number of small integers that follow 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 small integers and stay in 'I' mode

    real* vptr = v.data();
    real* vptrend = vptr+v.length();
    signed char mode = read_sbyte(in);
    float fval = 0.;
    signed char count;

    while(vptr!=vptrend)
    {
        count = read_sbyte(in);
        // cerr << int(count) <<  ' ';
        switch(mode)
        {
        case '0':
            if(count<0)
            {
                mode = 'I';              
                count = -count;
            }
            else if(count>0)
                mode = 'F';
            else
                count = 127;

            while(count--)
                *vptr++ = 0.;
            break;

        case 'I':
            if(count<0)
            {
                mode = '0';
                count = -count;
            }
            else if(count>0)
                mode = 'F';
            else 
                count = 127;

            while(count--)
                *vptr++ = real(read_sbyte(in));
            break;
          
        case 'F':
            if(count<0)
            {
                mode = '0';
                count = -count;
            }
            else if(count>0)
                mode = 'I';
            else 
                count = 127;

            while(count--)
            {
                in.read((char*)&fval,4);
                *vptr++ = real(fval);
            }
            break;

        default:
            PLERROR("Problem in VecCompressor::readCompressedVec this should not happen!!! (wrong data format?)");
        }
    }
    // cerr << endl;
}

Here is the call graph for this function:

void PLearn::VecCompressor::uncompressVec ( signed char *  data,
const Vec v 
) [static]

uncompresses the data of a vector compressed with compressVec v must have the correct size.

Definition at line 182 of file VecCompressor.cc.

References PLearn::TVec< T >::data(), PLearn::TVec< T >::length(), and PLERROR.

Referenced by PLearn::CompressedVMatrix::getNewRow().

{
    // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
    // If mode is '0' abs(count) indicates the number of zeroes, 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to 'I' mode
    //                a 0 count means insert 127 zeros and stay in zero mode
    // If mode is 'F' abs(count) indicates the number of floats that follow 
    //                a positive sign indicates switch to 'I' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 floats and stay in float mode
    // If mode is 'I' abs(count) indicates the number of small integers that follow 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 small integers and stay in 'I' mode

    real* vptr = v.data();
    real* vptrend = vptr+v.length();
    signed char* ptr = data;
    signed char mode = *ptr++;
    float fval = 0.;
    signed char* pfval = (signed char*)&fval;
    signed char count;
    while(vptr!=vptrend)
    {
        count = *ptr++;
        switch(mode)
        {
        case '0':
            if(count<0)
            {
                mode = 'I';              
                count = -count;
            }
            else if(count>0)
                mode = 'F';
            else
                count = 127;

            while(count--)
                *vptr++ = 0.;
            break;

        case 'I':
            if(count<0)
            {
                mode = '0';
                count = -count;
            }
            else if(count>0)
                mode = 'F';
            else 
                count = 127;

            while(count--)
                *vptr++ = real(*ptr++);
            break;
          
        case 'F':
            if(count<0)
            {
                mode = '0';
                count = -count;
            }
            else if(count>0)
                mode = 'I';
            else 
                count = 127;

            while(count--)
            {
                pfval[0] = *ptr++;
                pfval[1] = *ptr++;
                pfval[2] = *ptr++;
                pfval[3] = *ptr++;
                *vptr++ = real(fval);
            }
            break;

        default:
            PLERROR("Problem in VecCompressor::uncompressVec this should not happen!!! (wrong data format?)");
        }
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:

static size_t PLearn::VecCompressor::worstCaseSize ( int  n) [inline, static]

Returns the number of bytes that will be used to encode a vector of length n in the worst case.

Definition at line 95 of file VecCompressor.h.

Referenced by PLearn::CompressedVMatrix::CompressedVMatrix().

    { return 2+4*n+n/128; }

Here is the caller graph for this function:

void PLearn::VecCompressor::writeCompressedVec ( ostream &  out,
const Vec v 
) [static]

writes v in compressed format to the given stream The written data does not contain size info.

Definition at line 267 of file VecCompressor.cc.

References PLearn::TVec< T >::data(), PLearn::fast_exact_is_equal(), i, PLearn::TVec< T >::length(), n, and PLearn::write_sbyte().

{
    real* vdata = v.data();

    // mode can be '0' for zeroes, 'F' for floats, 'I' for small integers (signed chars)
    // If mode is '0' abs(count) indicates the number of zeroes, 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to 'I' mode
    //                a 0 count means insert 127 zeros and stay in zero mode
    // If mode is 'F' abs(count) indicates the number of floats that follow 
    //                a positive sign indicates switch to 'I' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 floats and stay in float mode
    // If mode is 'I' abs(count) indicates the number of small integers that follow 
    //                a positive sign indicates switch to 'F' mode
    //                a negative sign indicates switch to '0' mode
    //                a 0 count means insert 127 small integers and stay in 'I' mode

    int l = v.length();

    int i=0;
    real val = vdata[i];

    signed char mode = 'F';
    if(fast_exact_is_equal(val, 0.))
        mode = '0';
    else if(issmallint(val))
        mode = 'I';
    // else 'F'

    int count = 0;
    int istart = 0;
    float fval = 0.;

    write_sbyte(out,mode);

    while(i<l)
    {
        switch(mode)
        {
        case '0':
            istart = i;
            while(i<l && is0(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                write_sbyte(out,0);
                count -= 127;
            }
            if(i>=l || issmallint(vdata[i]))
            {
                write_sbyte(out,-count);
                mode = 'I';
            }
            else
            {
                write_sbyte(out,count);
                mode = 'F';
            }
            break;

        case 'I':
            istart = i;
            while(i<l && isI(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                write_sbyte(out,0);
                int n = 127;
                while(n--)
                    write_sbyte(out,(signed char)vdata[istart++]);
                count -= 127;
            }
            if(i>=l || is0(vdata[i]))
            {
                write_sbyte(out,-count);
                mode = '0';
            }
            else // next value is a floating point
            {
                write_sbyte(out,count);
                mode = 'F';
            }
            while(count--)
                write_sbyte(out,(signed char)vdata[istart++]);                
            break;

        case 'F':
            istart = i;
            while(i<l && isF(vdata[i]))
                i++;
            count = i - istart;
            while(count>127)
            {
                write_sbyte(out,0);
                int n = 127;
                while(n--)
                {
                    fval = (float)vdata[istart++];
                    out.write((char*)&fval,4);
                }
                count -= 127;
            }
            if(i>=l || is0(vdata[i]))
            {
                write_sbyte(out,-count);
                mode = '0';
            }
            else
            {
                write_sbyte(out,count);
                mode = 'I';
            }
            while(count--)
            {
                fval = (float)vdata[istart++];
                out.write((char*)&fval,4);
            }
        }
    }
}

Here is the call graph for this function:


The documentation for this class was generated from the following files:
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines