PLearn 0.1
CompactVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 //
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 //
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 //
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 //
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 //
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 /* *******************************************************
00038  * $Id: CompactVMatrix.cc 5557 2006-05-10 20:36:58Z lamblin $
00039  ******************************************************* */
00040 
00041 #include "CompactVMatrix.h"
00042 #include <plearn/math/TMat_maths.h>
00043 #include <plearn/math/random.h>
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00048 union short_and_twobytes
00049 {
00050     unsigned short us;
00051     unsigned char twobytes[2];
00052 };
00053 
00056 // norman: added static "inizialization"
00057 unsigned char CompactVMatrix::n_bits_in_byte[256];
00058 
00059 void CompactVMatrix::set_n_bits_in_byte()
00060 {
00061     if (n_bits_in_byte[255]!=8)
00062     {
00063         for (int i=0;i<256;i++)
00064         {
00065             int n=0;
00066             unsigned char byte=i;
00067             for (int j=0;j<8;j++)
00068             {
00069                 n += byte & 1;
00070                 byte >>= 1;
00071             }
00072             n_bits_in_byte[i]=n;
00073         }
00074     }
00075 }
00076 
00077 PLEARN_IMPLEMENT_OBJECT(CompactVMatrix, "ONE LINE DESCR", "NO HELP");
00078 
00079 CompactVMatrix::CompactVMatrix()
00080     : n_symbols(0), n_fixedpoint(0), n_variables(0), one_hot_encoding(true), n_symbol_values(0),
00081       fixedpoint_min(0), fixedpoint_max(0), delta(0), variables_permutation(0)
00082 {
00083 }
00084 
00085 CompactVMatrix::CompactVMatrix(int the_length, int nvariables, int n_binary,
00086                                int n_nonbinary_discrete,
00087                                int n_fixed_point, TVec<int>& n_symbolvalues,
00088                                Vec& fixed_point_min, Vec& fixed_point_max,
00089                                bool onehot_encoding)
00090     : inherited(the_length,nvariables), n_bits(n_binary),
00091       n_symbols(n_nonbinary_discrete), n_fixedpoint(n_fixed_point),
00092       n_variables(nvariables), one_hot_encoding(onehot_encoding),
00093       n_symbol_values(n_symbolvalues),
00094       fixedpoint_min(fixed_point_min), fixedpoint_max(fixed_point_max),
00095       delta(n_fixed_point), variables_permutation(n_variables)
00096 {
00097     normal_width=n_bits+n_fixed_point;
00098     for (int i=0;i<n_symbols;i++)
00099         normal_width += n_symbol_values[i];
00100     setOneHotMode(one_hot_encoding);
00101     for (int i=0;i<n_variables;i++) variables_permutation[i]=i;
00102     for (int i=0;i<n_symbols;i++)
00103         delta[i]=(fixedpoint_max[i]-fixedpoint_min[i])/USHRT_MAX;
00104     symbols_offset = (int)ceil(n_bits/8.0);
00105     fixedpoint_offset = symbols_offset + n_symbols;
00106     row_n_bytes =  fixedpoint_offset + int(sizeof(unsigned short))*n_fixed_point;
00107     data.resize(length_ * row_n_bytes);
00108     set_n_bits_in_byte();
00109 }
00110 
00111 CompactVMatrix::CompactVMatrix(VMat m, int keep_last_variables_last, bool onehot_encoding)
00112     : inherited(m->length(),m->width()), one_hot_encoding(onehot_encoding),
00113       n_symbol_values(m->width()), variables_permutation(m->width())
00114 {
00115     if (!m->hasStats())
00116     {
00117         cout << "CompactVMatrix(VMat, int): VMat did not have stats. Computing them." << endl;
00118         m->computeStats();
00119     }
00120     // determine which variables are binary discrete, multi-class discrete, or continuous
00121     n_bits = n_symbols = n_fixedpoint = 0;
00122     TVec<int> bits_position(m->width());
00123     TVec<int> symbols_position(m->width());
00124     TVec<int> fp_position(m->width());
00125     fixedpoint_min.resize(m->width());
00126     fixedpoint_max.resize(m->width());
00127     delta.resize(m->width());
00128     for (int i=0;i<m->width();i++)
00129     {
00130         VMFieldStat& stat = m->fieldstats[i];
00131         int n_values = (int)stat.counts.size(); // 0 means "continuous"
00132         bool counts_look_continuous = !isMapKeysAreInt(stat.counts);
00133         if (n_values == 0 || counts_look_continuous || i>=m->width()-keep_last_variables_last)
00134         {
00135             fixedpoint_min[n_fixedpoint]=stat.min();
00136             fixedpoint_max[n_fixedpoint]=stat.max();
00137             delta[n_fixedpoint]=(stat.max()-stat.min())/USHRT_MAX;
00138             fp_position[n_fixedpoint++]=i;
00139         }
00140         else
00141         {
00142             if (!fast_exact_is_equal(stat.min(), 0) ||
00143                 !fast_exact_is_equal((stat.max()-stat.min()+1),
00144                                      stat.counts.size()))
00145                 PLERROR("CompactVMatrix:: variable %d looks discrete but has zero-frequency intermediate values or min!=0",i);
00146             if (n_values==2)
00147                 bits_position[n_bits++]=i;
00148             else if (n_values<=256)
00149             {
00150                 symbols_position[n_symbols]=i;
00151                 n_symbol_values[n_symbols++] = n_values;
00152             }
00153             else
00154             {
00155                 fixedpoint_min[n_fixedpoint]=stat.min();
00156                 fixedpoint_max[n_fixedpoint]=stat.max();
00157                 delta[n_fixedpoint]=(stat.max()-stat.min())/USHRT_MAX;
00158                 fp_position[n_fixedpoint++]=i;
00159             }
00160         }
00161     }
00162     fixedpoint_min.resize(n_fixedpoint);
00163     fixedpoint_max.resize(n_fixedpoint);
00164     delta.resize(n_fixedpoint);
00165     n_symbol_values.resize(n_symbols);
00166     n_variables = n_bits + n_symbols + n_fixedpoint;
00167     int j=0;
00168     for (int i=0;i<n_bits;i++,j++)
00169         variables_permutation[j]=bits_position[i];
00170     for (int i=0;i<n_symbols;i++,j++)
00171         variables_permutation[j]=symbols_position[i];
00172     for (int i=0;i<n_fixedpoint;i++,j++)
00173         variables_permutation[j]=fp_position[i];
00174 
00175     normal_width=n_bits+n_fixedpoint;
00176     for (int i=0;i<n_symbols;i++)
00177         normal_width += n_symbol_values[i];
00178     setOneHotMode(one_hot_encoding);
00179     symbols_offset = (int)ceil(n_bits/8.0);
00180     fixedpoint_offset = symbols_offset + n_symbols;
00181     row_n_bytes =  fixedpoint_offset + int(sizeof(unsigned short))*n_fixedpoint;
00182     data.resize(length_ * row_n_bytes);
00183 
00184     // copy the field infos and stats? not really useful with one-hot encoding
00185     // because of non-binary symbols being spread across many columns.
00186     if (!one_hot_encoding)
00187     {
00188         fieldinfos.resize(width_);
00189         fieldstats.resize(width_);
00190         for (int i=0;i<width_;i++)
00191         {
00192             fieldinfos[i]=m->getFieldInfos()[variables_permutation[i]];
00193             fieldstats[i]=m->fieldstats[variables_permutation[i]];
00194         }
00195     }
00196     else
00197     {
00198         fieldinfos.resize(0);
00199         fieldstats.resize(0);
00200     }
00201 
00202     // copy the data
00203     Vec mrow(m->width());
00204     for (int t=0;t<length_;t++)
00205     {
00206         m->getRow(t,mrow);
00207         encodeAndPutRow(t,mrow);
00208     }
00209     set_n_bits_in_byte();
00210 }
00211 
00212 
00213 CompactVMatrix::CompactVMatrix(const string& filename, int nlast) : RowBufferedVMatrix(0,0)
00214 {
00215     load(filename);
00216     n_last=nlast;
00217     set_n_bits_in_byte();
00218 }
00219 
00220 CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m, bool rescale, bool check)
00221     : inherited(m->length(),m->width())
00222 {
00223     if (cvm->width() != m->width())
00224         PLERROR("CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m,...), args have widths %d!=%d",
00225                 cvm->width(),m->width());
00226     // copy all the ordinary fields
00227     row_n_bytes = cvm->row_n_bytes;
00228     data.resize(length_*row_n_bytes);
00229     n_bits = cvm->n_bits;
00230     n_symbols = cvm->n_symbols;
00231     n_fixedpoint = cvm->n_fixedpoint;
00232     n_variables = cvm->n_variables;
00233     n_symbol_values = cvm->n_symbol_values;
00234     fixedpoint_min = cvm->fixedpoint_min.copy();
00235     fixedpoint_max = cvm->fixedpoint_max.copy();
00236     delta = cvm->delta.copy();
00237     variables_permutation = cvm->variables_permutation;
00238     n_last = cvm->n_last;
00239     normal_width = cvm->normal_width;
00240     symbols_offset = cvm->symbols_offset;
00241     fixedpoint_offset = cvm->fixedpoint_offset;
00242 
00243     setOneHotMode(cvm->one_hot_encoding);
00244     Vec row(width_);
00245     int offs=width_-n_fixedpoint;
00246     if (rescale)
00247     {
00248         for (int i=0;i<length_;i++)
00249         {
00250             m->getRow(i,row);
00251             for (int j=0;j<n_fixedpoint;j++)
00252             {
00253                 real element=row[offs+j];
00254                 if (element<fixedpoint_min[j])
00255                     fixedpoint_min[j]=element;
00256                 if (element>fixedpoint_max[j])
00257                     fixedpoint_max[j]=element;
00258             }
00259         }
00260         for (int j=0;j<n_fixedpoint;j++)
00261             delta[j]=(fixedpoint_max[j]-fixedpoint_min[j])/USHRT_MAX;
00262     }
00263     for (int i=0;i<length_;i++)
00264     {
00265         m->getRow(i,row);
00266         if (!rescale && check) // check that range is OK
00267         {
00268             for (int j=0;j<n_fixedpoint;j++)
00269             {
00270                 real element=row[offs+j];
00271                 if (element<fixedpoint_min[j] ||
00272                     element>fixedpoint_max[j])
00273                     PLERROR("CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m,...) out-of-range element(%d,%d)=%g not in [%g,%g]",
00274                             i,j,element,fixedpoint_min[j],fixedpoint_max[j]);
00275             }
00276         }
00277         putRow(i,row);
00278     }
00279 }
00280 
00281 void CompactVMatrix::setOneHotMode(bool on)
00282 {
00283     one_hot_encoding=on;
00284     if (one_hot_encoding)
00285         width_ = normal_width;
00286     else
00287         width_ = n_variables;
00288 }
00289 
00290 void CompactVMatrix::getNewRow(int i, const Vec& v) const
00291 {
00292 #ifdef BOUNDCHECK
00293     if (i<0 || i>=length_)
00294         PLERROR("CompactVMatrix::getNewRow, row %d out of bounds [0,%d]",i,length_-1);
00295     if (v.length()!=width_)
00296         PLERROR("CompactVMatrix::getNewRow, length of v (%d) should be equal to width of VMat (%d)",v.length(),width());
00297 #endif
00298     unsigned char* encoded_row = &data.data[i*row_n_bytes];
00299     real* vp=v.data();
00300     int c=0;
00301     for (int b=0;b<symbols_offset;b++)
00302     {
00303         unsigned char byte=encoded_row[b];
00304         for (int j=0;j<8 && c<n_bits;j++,c++)
00305         {
00306             int bit = byte & 1;
00307             byte >>= 1; // shift right once
00308             vp[c]=bit;
00309         }
00310     }
00311     for (int b=0;b<n_symbols;b++)
00312     {
00313         int byte = encoded_row[symbols_offset+b];
00314         if (one_hot_encoding)
00315         {
00316             int n=n_symbol_values[b];
00317             for (int j=0;j<n;j++) vp[c+j]=0;
00318             vp[c+byte]=1;
00319             c+=n;
00320         }
00321         else vp[c++]=byte;
00322     }
00323     unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset];
00324     for (int j=0;j<n_fixedpoint;j++,c++)
00325     {
00326         unsigned char *uc = &fixed_point_numbers[2*j];
00327         short_and_twobytes u;
00328         u.twobytes[0]=uc[0];
00329         u.twobytes[1]=uc[1];
00330         real decoded = u.us*delta[j]+fixedpoint_min[j];
00331         // correct rounding errors for integers, due to fixed-point low precision
00332         real rounded_decoded = rint(decoded);
00333         if (fabs(rounded_decoded-decoded)<1e-4)
00334             decoded = rounded_decoded;
00335         vp[c]=decoded;
00336     }
00337 }
00338 
00339 //#define SANITYCHECK_CompactVMatrix
00340 #define SANITYCHECK_CompactVMatrix_PRECISION 1e-5
00341 
00342 real CompactVMatrix::dot(int i, int j, int inputsize) const
00343 {
00344     if(inputsize!=width()-n_last)
00345         PLERROR("In CompactVMatrix::dot, in current implementation inputsize must be equal to width()-n_last");
00346 
00347     unsigned char* encoded_row_i = &data.data[i*row_n_bytes];
00348     unsigned char* encoded_row_j = &data.data[j*row_n_bytes];
00349     real dot_product=0.;
00350     int c=0;
00351     for (int b=0;b<symbols_offset;b++)
00352     {
00353         unsigned char byte_i=encoded_row_i[b];
00354         unsigned char byte_j=encoded_row_j[b];
00355         unsigned char byte_and = byte_i & byte_j;
00356 #ifdef SANITYCHECK_CompactVMatrix
00357         real check=dot_product;
00358 #endif
00359         // Here we want to count the number of ON bits in the byte_and
00360         // instead of looping through the bits, we look-up in a
00361         // pre-computed table (n_bits_in_byte), which has been set-up by set_n_bits_in_byte().
00362         dot_product += n_bits_in_byte[byte_and];
00363 #ifdef SANITYCHECK_CompactVMatrix
00364         for (int j=0;j<8 && c<n_bits;j++,c++)
00365         {
00366             check += byte_and & 1;
00367             byte_and >>= 1; // shift right once
00368         }
00369         if (check!=dot_product)
00370             PLERROR("logic error in n_bits_in_byte");
00371 #else
00372         c+=8;
00373         if (c>n_bits) c=n_bits;
00374 #endif
00375     }
00376     if (c>width_-n_last)
00377         PLERROR("CompactVMatrix: n_last should be among discrete non-binary or continuous variables");
00378     for (int b=0;b<n_symbols && c<width_-n_last;b++)
00379     {
00380         int byte_i = encoded_row_i[symbols_offset+b];
00381         int byte_j = encoded_row_j[symbols_offset+b];
00382         if (byte_i==byte_j) dot_product++;
00383         if (one_hot_encoding)
00384             c+=n_symbol_values[b];
00385         else
00386             c++;
00387     }
00388     unsigned char* fixed_point_numbers_i = &encoded_row_i[fixedpoint_offset];
00389     unsigned char* fixed_point_numbers_j = &encoded_row_j[fixedpoint_offset];
00390     for (int k=0;k<n_fixedpoint-n_last && c<width_-n_last;k++,c++)
00391     {
00392         unsigned char *uc = &fixed_point_numbers_i[2*k];
00393         short_and_twobytes u;
00394         u.twobytes[0]=uc[0];
00395         u.twobytes[1]=uc[1];
00396         real decoded_i = u.us*delta[k]+fixedpoint_min[k];
00397         uc = &fixed_point_numbers_j[2*k];
00398         u.twobytes[0]=uc[0];
00399         u.twobytes[1]=uc[1];
00400         real decoded_j = u.us*delta[k]+fixedpoint_min[k];
00401 #ifdef SANITYCHECK_CompactVMatrix
00402         real rounded_decoded_i = rint(decoded_i);
00403         if (fabs(rounded_decoded_i-decoded_i)<1e-4)
00404             decoded_i = rounded_decoded_i;
00405         real rounded_decoded_j = rint(decoded_j);
00406         if (fabs(rounded_decoded_j-decoded_j)<1e-4)
00407             decoded_j = rounded_decoded_j;
00408 #endif
00409         dot_product += decoded_i * decoded_j;
00410     }
00411 
00412     return dot_product;
00413 }
00414 
00415 // I used the code for getRow as a basis to implement this call (Pascal)
00416 real CompactVMatrix::dot(int i, const Vec& v) const
00417 {
00418 #ifdef BOUNDCHECK
00419     if (i<0 || i>=length_)
00420         PLERROR("CompactVMatrix::dot, row %d out of bounds [0,%d]",i,length_-1);
00421 #endif
00422 
00423     if(v.length()!=width()-n_last)
00424         PLERROR("In CompactVMatrix::dot, in current implementation v.length() must be equal to width()-n_last");
00425 
00426     real dot_product = 0.;
00427 
00428     unsigned char* encoded_row = &data.data[i*row_n_bytes];
00429     real* vp=v.data();
00430     int c=0;
00431     for (int b=0;b<symbols_offset;b++)
00432     {
00433         unsigned char byte=encoded_row[b];
00434         for (int j=0;j<8 && c<n_bits;j++,c++)
00435         {
00436             int bit = byte & 1;
00437             byte >>= 1; // shift right once
00438             if(bit)
00439                 dot_product += vp[c];
00440         }
00441     }
00442     for (int b=0;b<n_symbols;b++)
00443     {
00444         int byte = encoded_row[symbols_offset+b];
00445         if (one_hot_encoding)
00446         {
00447             int n=n_symbol_values[b];
00448             dot_product += vp[c+byte];
00449             c += n;
00450         }
00451         else
00452             dot_product += vp[c++]*byte;
00453     }
00454     // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4
00455     // ON SOME MACHINES?
00456     unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset];
00457     for (int j=0;j<n_fixedpoint-n_last && c<v.length();j++,c++)
00458     {
00459         unsigned char *uc = &fixed_point_numbers[2*j];
00460         short_and_twobytes u;
00461         u.twobytes[0]=uc[0];
00462         u.twobytes[1]=uc[1];
00463         real decoded = u.us*delta[j]+fixedpoint_min[j];
00464         // correct rounding errors for integers, due to fixed-point low precision
00465         real rounded_decoded = rint(decoded);
00466         if (fabs(rounded_decoded-decoded)<1e-4)
00467             decoded = rounded_decoded;
00468         dot_product += vp[c] * decoded;
00469     }
00470 
00471     // Very Slow SANITY CHECK
00472 #ifdef SANITYCHECK_CompactVMatrix
00473     Vec v_i(v.length());
00474     getRow(i,v_i);
00475     real dot_product2 = PLearn::dot(v_i.subVec(0,v.length()),v);
00476     real diff = fabs(dot_product-dot_product2)/fabs(dot_product2);
00477     if(diff>SANITYCHECK_CompactVMatrix_PRECISION)
00478         PLERROR("IN CompactVMatrix::dot(int i=%d, v) SANITY CHECK FAILED: difference=%g",i,diff);
00479 #endif
00480 
00481     return dot_product;
00482 }
00483 
00484 
00485 real CompactVMatrix::dotProduct(int i, int j) const
00486 { return dot(i,j,width()-n_last); }
00487 
00488 real CompactVMatrix::squareDifference(int i, int j)
00489 {
00490     if (row_norms.length()==0)
00491         row_norms = Vec(length_,-1.0);
00492     real normi = row_norms[i];
00493     if (normi<0) normi=row_norms[i]=dotProduct(i,i);
00494     real normj = row_norms[j];
00495     if (normj<0) normj=row_norms[j]=dotProduct(j,j);
00496     return normi + normj - 2 * dotProduct(i,j);
00497 }
00498 
00499 void CompactVMatrix::encodeAndPutRow(int i, Vec v)
00500 {
00501     unsigned char* encoded_row = &data.data[i*row_n_bytes];
00502     real* vp=v.data();
00503     int* perm=variables_permutation.data();
00504     int c=0;
00505     // 1 vector element ==> 1 bit
00506     for (int b=0;b<symbols_offset;b++)
00507     {
00508         unsigned char byte=0;
00509         for (int j=0;j<8 && c<n_bits;j++,c++)
00510             byte |= int(vp[perm[c]]) << j; // shift to right bit position
00511         encoded_row[b]=byte;
00512     }
00513     //    1 vector element (integer between 0 and n-1) ==> 1 byte
00514     for (int b=0;b<n_symbols;b++,c++)
00515     {
00516         real val = vp[perm[c]];
00517         int s = int(val);
00518         if (!fast_exact_is_equal(s, val))
00519             PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%g not an integer",
00520                     i,int(perm[c]),val);
00521         encoded_row[symbols_offset+b] = s; // ASSUMES THAT v IS NOT ONE-HOT ENCODED
00522         if (s<0 || s>=n_symbol_values[b])
00523             PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%d not in expected range (0,%d)",
00524                     i,int(perm[c]),s,n_symbol_values[b]-1);
00525     }
00526     // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4
00527     // ON SOME MACHINES?
00528     unsigned short* fixed_point_numbers = (unsigned short*)&encoded_row[fixedpoint_offset];
00529     for (int j=0;j<n_fixedpoint;j++,c++)
00530         fixed_point_numbers[j]=(unsigned short)((vp[perm[c]]-fixedpoint_min[j])/delta[j]);
00531 
00532     invalidateBuffer();
00533 }
00534 
00535 void CompactVMatrix::putRow(int i, Vec v)
00536 {
00537     putSubRow(i,0,v);
00538 }
00539 
00540 void CompactVMatrix::putSubRow(int i, int j, Vec v)
00541 {
00542     unsigned char* encoded_row = &data.data[i*row_n_bytes];
00543     real* vp=v.data();
00544     int c=0;
00545     // 1 vector element ==> 1 bit
00546     for (int b=0;b<symbols_offset;b++)
00547     {
00548         unsigned char byte=0;
00549         for (int k=0;k<8 && c<n_bits;k++,c++)
00550             if (c>=j)
00551                 byte |= int(vp[c-j]) << k; // shift to right bit position
00552         encoded_row[b]=byte;
00553     }
00554     // if (one_hot_encoding)
00555     //   n vector elements in one-hot-code ==> 1 byte
00556     // else
00557     //   1 vector element (integer between 0 and n-1) ==> 1 byte
00558     int n=0;
00559     if (one_hot_encoding)
00560         for (int b=0;b<n_symbols;b++,c+=n)
00561         {
00562             n=n_symbol_values[b];
00563             if (c>=j)
00564             {
00565                 int pos=-1;
00566                 for (int k=0;k<n;k++)
00567                 {
00568                     real vk=vp[c+k-j];
00569                     if (!fast_exact_is_equal(vk, 0) &&
00570                         !fast_exact_is_equal(vk, 1))
00571                         PLERROR("CompactVMatrix::putRow(%d,v): v[%d]=%g!=0 or 1 (not one-hot-code)",
00572                                 i,c,vk);
00573                     if (fast_exact_is_equal(vk, 1))
00574                     {
00575                         if (pos<0) pos=k;
00576                         else PLERROR("CompactVMatrix::putRow(%d,v): %d-th symbol not one-hot-encoded",
00577                                      i,b);
00578                     }
00579                 }
00580                 if (pos<0)
00581                     PLERROR("CompactVMatrix::putRow(%d,v): %d-th symbol not one-hot-encoded",
00582                             i,b);
00583                 encoded_row[symbols_offset+b] = pos;
00584             }
00585         }
00586     else
00587         for (int b=0;b<n_symbols;b++,c++)
00588             if (c>=j)
00589             {
00590                 real val = vp[c-j];
00591                 int s = int(val);
00592                 if (!fast_exact_is_equal(s, val))
00593                     PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%g not an integer",
00594                             i,c,val);
00595                 encoded_row[symbols_offset+b] = s; // ASSUMES THAT v IS NOT ONE-HOT ENCODED
00596                 if (s<0 || s>=n_symbol_values[b])
00597                     PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%d not in expected range (0,%d)",
00598                             i,c,s,n_symbol_values[b]-1);
00599             }
00600 
00601     // 1 vector element (real betweeen fixedpoint_min and fixedpoint_max) ==> 2 bytes
00602     //
00603     // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4
00604     // ON SOME MACHINES?
00605     unsigned short* fixed_point_numbers = (unsigned short*)&encoded_row[fixedpoint_offset];
00606     for (int k=0;k<n_fixedpoint;k++,c++)
00607         if (c>=j)
00608             fixed_point_numbers[k]=(unsigned short)((vp[c-j]-fixedpoint_min[k])/delta[k]);
00609 
00610     invalidateBuffer();
00611 }
00612 
00613 void CompactVMatrix::perturb(int i, Vec v, real noise_level, int n_last)
00614 {
00615 #ifdef BOUNDCHECK
00616     if (i<0 || i>=length_)
00617         PLERROR("CompactVMatrix::perturb, row %d out of bounds [0,%d]",i,length_-1);
00618     if (v.length()!=width_)
00619         PLERROR("CompactVMatrix::perturb, length of v (%d) should be equal to width of VMat (%d)",v.length(),width());
00620 #endif
00621     if (fieldstats.size()!=n_variables)
00622         PLERROR("CompactVMatrix::perturb: stats not computed or wrong size");
00623     if (noise_level<0 || noise_level>1)
00624         PLERROR("CompactVMatrix::perturb: noise_level=%g, should be in [0,1]",noise_level);
00625 
00626     unsigned char* encoded_row = &data.data[i*row_n_bytes];
00627     real* vp=v.data();
00628     int c=0;
00629     int var=0;
00630     Vec probs(width_);
00631     for (int b=0;b<symbols_offset;b++)
00632     {
00633         unsigned char byte=encoded_row[b];
00634         for (int j=0;j<8 && c<n_bits;j++,c++,var++)
00635         {
00636             int bit = byte & 1;
00637             byte >>= 1; // shift right once
00638             vp[c]=binomial_sample((1-noise_level)*bit+noise_level*fieldstats[var].prob(1));
00639         }
00640     }
00641     for (int b=0;b<n_symbols;b++,var++)
00642     {
00643         int byte = encoded_row[symbols_offset+b];
00644         int nv=n_symbol_values[b];
00645         probs.resize(nv);
00646         VMFieldStat& stat=fieldstats[var];
00647         for (int val=0;val<nv;val++)
00648             if (val==byte)
00649                 probs[val]=(1-noise_level)+noise_level*stat.prob(val);
00650             else
00651                 probs[val]=noise_level*stat.prob(val);
00652         byte = multinomial_sample(probs);
00653         if (one_hot_encoding)
00654         {
00655             int n=n_symbol_values[b];
00656             for (int j=0;j<n;j++) vp[c+j]=0;
00657             vp[c+byte]=1;
00658             c+=n;
00659         }
00660         else vp[c++]=byte;
00661     }
00662     unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset];
00663     for (int j=0;j<n_fixedpoint;j++,c++,var++)
00664     {
00665         unsigned char *uc = &fixed_point_numbers[2*j];
00666         short_and_twobytes u;
00667         u.twobytes[0]=uc[0];
00668         u.twobytes[1]=uc[1];
00669         real decoded = u.us*delta[j]+fixedpoint_min[j];
00670         // correct rounding errors for integers, due to fixed-point low precision
00671         real rounded_decoded = rint(decoded);
00672         if (fabs(rounded_decoded-decoded)<1e-4)
00673             decoded = rounded_decoded;
00674         if (var<n_variables-n_last)
00675         {
00676             int ntry=0;
00677             do
00678             {
00679                 vp[c]=decoded+noise_level*fieldstats[var].stddev()*normal_sample();
00680                 ntry++;
00681                 if (ntry>=100)
00682                     PLERROR("CompactVMatrix::perturb:Something wrong in resampling, tried 100 times");
00683             }
00684             while (vp[c]<fixedpoint_min[j] || vp[c]>fixedpoint_max[j]);
00685         }
00686         else
00687             vp[c]=decoded;
00688     }
00689 }
00690 /*
00691   void CompactVMatrix::write(ostream& out) const
00692   {
00693   writeHeader(out,"CompactVMatrix");
00694   writeField(out,"length",length_);
00695   writeField(out,"width",normal_width);
00696   writeField(out,"fieldinfos",fieldinfos);
00697   writeField(out,"fieldstats",fieldstats);
00698   writeField(out,"row_n_bytes",row_n_bytes);
00699   writeField(out,"n_bits",n_bits);
00700   writeField(out,"n_symbols",n_symbols);
00701   writeField(out,"n_fixedpoint",n_fixedpoint);
00702   writeField(out,"one_hot_encoding",one_hot_encoding);
00703   writeField(out,"n_symbol_values",n_symbol_values);
00704   writeField(out,"fixedpoint_min",fixedpoint_min);
00705   writeField(out,"fixedpoint_max",fixedpoint_max);
00706   writeField(out,"delta",delta);
00707   writeField(out,"variables_permutation",variables_permutation);
00708   writeField(out,"symbols_offset",symbols_offset);
00709   writeField(out,"fixedpoint_offset",fixedpoint_offset);
00710   out.write((char*)data.data,data.length()*sizeof(unsigned char));
00711   writeFooter(out,"CompactVMatrix");
00712   }
00713 
00714   void CompactVMatrix::oldread(istream& in)
00715   {
00716   readHeader(in,"CompactVMatrix");
00717   readField(in,"length",length_);
00718   readField(in,"width",normal_width);
00719   readField(in,"fieldinfos",fieldinfos);
00720   fieldinfos.resize(0); // to fix current bug in setting fieldinfos
00721   readField(in,"fieldstats",fieldstats);
00722   readField(in,"row_n_bytes",row_n_bytes);
00723   readField(in,"n_bits",n_bits);
00724   readField(in,"n_symbols",n_symbols);
00725   readField(in,"n_fixedpoint",n_fixedpoint);
00726   n_variables = n_bits + n_symbols + n_fixedpoint;
00727   readField(in,"one_hot_encoding",one_hot_encoding);
00728   setOneHotMode(one_hot_encoding);
00729   readField(in,"n_symbol_values",n_symbol_values);
00730   readField(in,"fixedpoint_min",fixedpoint_min);
00731   readField(in,"fixedpoint_max",fixedpoint_max);
00732   readField(in,"delta",delta);
00733   readField(in,"variables_permutation",variables_permutation);
00734   readField(in,"symbols_offset",symbols_offset);
00735   readField(in,"fixedpoint_offset",fixedpoint_offset);
00736   data.resize(row_n_bytes*length_);
00737   in.read((char*)data.data,data.length()*sizeof(unsigned char));
00738   readFooter(in,"CompactVMatrix");
00739   }
00740 */
00741 void CompactVMatrix::append(CompactVMatrix* vm)
00742 {
00743     if (width_!=vm->width())
00744         PLERROR("CompactVMatrix::append, incompatible width %d vs %d",
00745                 width_,vm->width());
00746     if (row_n_bytes!=vm->row_n_bytes)
00747         PLERROR("CompactVMatrix::append, incompatible row_n_bytes %d vs %d",
00748                 row_n_bytes,vm->row_n_bytes);
00749     if (n_bits!=vm->n_bits)
00750         PLERROR("CompactVMatrix::append, incompatible n_bits %d vs %d",
00751                 n_bits,vm->n_bits);
00752     if (n_symbols!=vm->n_symbols)
00753         PLERROR("CompactVMatrix::append, incompatible n_symbols %d vs %d",
00754                 n_symbols,vm->n_symbols);
00755     if (n_fixedpoint!=vm->n_fixedpoint)
00756         PLERROR("CompactVMatrix::append, incompatible n_fixedpoint %d vs %d",
00757                 n_fixedpoint,vm->n_fixedpoint);
00758     if (n_symbol_values!=vm->n_symbol_values)
00759     {
00760         //n_symbol_values.write(cerr); cerr << endl;
00761         //vm->n_symbol_values.write(cerr); cerr << endl;
00762         PLearn::write(cerr, n_symbol_values);
00763         cerr << endl;
00764         PLearn::write(cerr, vm->n_symbol_values);
00765         cerr << endl;
00766         PLERROR("CompactVMatrix::append, incompatible n_symbol_values");
00767     }
00768     bool rescale = false;
00769     for (int j=0;j<n_fixedpoint && !rescale;j++)
00770         if (fixedpoint_min[j]>vm->fixedpoint_min[j] ||
00771             fixedpoint_max[j]<vm->fixedpoint_max[j]) rescale=true;
00772     if (rescale)
00773     {
00774         cout << "The appended VMat has intervals that are wider than the current one." << endl;
00775         cout << "Start rescaling numeric variables fixed point representation." << endl;
00776         Vec new_min = fixedpoint_min.copy();
00777         Vec new_max = fixedpoint_max.copy();
00778         Vec new_delta = delta.copy();
00779         TVec<bool> change(n_fixedpoint);
00780         for (int j=0;j<n_fixedpoint;j++)
00781         {
00782             change[j]=false;
00783             if (fixedpoint_min[j]>vm->fixedpoint_min[j])
00784             {
00785                 change[j]=true;
00786                 new_min[j]=vm->fixedpoint_min[j];
00787             }
00788             if (fixedpoint_max[j]<vm->fixedpoint_max[j])
00789             {
00790                 change[j]=true;
00791                 new_max[j]=vm->fixedpoint_max[j];
00792             }
00793             if (change[j])
00794                 new_delta[j]=(new_max[j]-new_min[j])/USHRT_MAX;
00795         }
00796         for (int r=0;r<length_;r++)
00797         {
00798             unsigned char* encoded_row = &data.data[r*row_n_bytes];
00799             unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset];
00800             for (int j=0;j<n_fixedpoint;j++)
00801                 if (change[j])
00802                 {
00803                     // DECODE using previous min/max
00804                     unsigned char *uc = &fixed_point_numbers[2*j];
00805                     short_and_twobytes u;
00806                     u.twobytes[0]=uc[0];
00807                     u.twobytes[1]=uc[1];
00808                     real decoded = u.us*delta[j]+fixedpoint_min[j];
00809                     // correct rounding errors for integers, due to fixed-point low precision
00810                     real rounded_decoded = rint(decoded);
00811                     if (fabs(rounded_decoded-decoded)<1e-4)
00812                         decoded = rounded_decoded;
00813                     // ENCODE using new min/max
00814                     fixed_point_numbers[j]=(unsigned char)((decoded-new_min[j])/new_delta[j]);
00815                 }
00816         }
00817         cout << "DONE rescaling numeric variables fixed point representation." << endl;
00818         fixedpoint_min << new_min;
00819         fixedpoint_max << new_max;
00820         delta << new_delta;
00821     }
00822     int new_length=length_+vm->length();
00823     data.resize(row_n_bytes*new_length);
00824     // copy the new data
00825     Vec row(width_);
00826     bool old_vm_encoding = vm->one_hot_encoding;
00827     bool old_encoding = one_hot_encoding;
00828     vm->one_hot_encoding=false;
00829     setOneHotMode(false);
00830     int old_length=length_;
00831     length_=new_length;
00832     for (int r=0;r<vm->length();r++)
00833     {
00834         vm->getRow(r,row);
00835         putRow(old_length+r,row);
00836     }
00837     vm->one_hot_encoding=old_vm_encoding;
00838     setOneHotMode(old_encoding);
00839 }
00840 
00841 void CompactVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00842 {
00843     deepCopyField(data, copies);
00844     deepCopyField(n_symbol_values, copies);
00845     deepCopyField(fixedpoint_min, copies);
00846     deepCopyField(fixedpoint_max, copies);
00847     deepCopyField(variables_permutation, copies);
00848 }
00849 
00850 } // end of namespace PLearn
00851 
00852 
00853 /*
00854   Local Variables:
00855   mode:c++
00856   c-basic-offset:4
00857   c-file-style:"stroustrup"
00858   c-file-offsets:((innamespace . 0)(inline-open . 0))
00859   indent-tabs-mode:nil
00860   fill-column:79
00861   End:
00862 */
00863 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines