PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: CompactVMatrix.cc 5557 2006-05-10 20:36:58Z lamblin $ 00039 ******************************************************* */ 00040 00041 #include "CompactVMatrix.h" 00042 #include <plearn/math/TMat_maths.h> 00043 #include <plearn/math/random.h> 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00048 union short_and_twobytes 00049 { 00050 unsigned short us; 00051 unsigned char twobytes[2]; 00052 }; 00053 00056 // norman: added static "inizialization" 00057 unsigned char CompactVMatrix::n_bits_in_byte[256]; 00058 00059 void CompactVMatrix::set_n_bits_in_byte() 00060 { 00061 if (n_bits_in_byte[255]!=8) 00062 { 00063 for (int i=0;i<256;i++) 00064 { 00065 int n=0; 00066 unsigned char byte=i; 00067 for (int j=0;j<8;j++) 00068 { 00069 n += byte & 1; 00070 byte >>= 1; 00071 } 00072 n_bits_in_byte[i]=n; 00073 } 00074 } 00075 } 00076 00077 PLEARN_IMPLEMENT_OBJECT(CompactVMatrix, "ONE LINE DESCR", "NO HELP"); 00078 00079 CompactVMatrix::CompactVMatrix() 00080 : n_symbols(0), n_fixedpoint(0), n_variables(0), one_hot_encoding(true), n_symbol_values(0), 00081 fixedpoint_min(0), fixedpoint_max(0), delta(0), variables_permutation(0) 00082 { 00083 } 00084 00085 CompactVMatrix::CompactVMatrix(int the_length, int nvariables, int n_binary, 00086 int n_nonbinary_discrete, 00087 int n_fixed_point, TVec<int>& n_symbolvalues, 00088 Vec& fixed_point_min, Vec& fixed_point_max, 00089 bool onehot_encoding) 00090 : inherited(the_length,nvariables), n_bits(n_binary), 00091 n_symbols(n_nonbinary_discrete), n_fixedpoint(n_fixed_point), 00092 n_variables(nvariables), one_hot_encoding(onehot_encoding), 00093 n_symbol_values(n_symbolvalues), 00094 fixedpoint_min(fixed_point_min), fixedpoint_max(fixed_point_max), 00095 delta(n_fixed_point), variables_permutation(n_variables) 00096 { 00097 normal_width=n_bits+n_fixed_point; 00098 for (int i=0;i<n_symbols;i++) 00099 normal_width += n_symbol_values[i]; 00100 setOneHotMode(one_hot_encoding); 00101 for (int i=0;i<n_variables;i++) variables_permutation[i]=i; 00102 for (int i=0;i<n_symbols;i++) 00103 delta[i]=(fixedpoint_max[i]-fixedpoint_min[i])/USHRT_MAX; 00104 symbols_offset = (int)ceil(n_bits/8.0); 00105 fixedpoint_offset = symbols_offset + n_symbols; 00106 row_n_bytes = fixedpoint_offset + int(sizeof(unsigned short))*n_fixed_point; 00107 data.resize(length_ * row_n_bytes); 00108 set_n_bits_in_byte(); 00109 } 00110 00111 CompactVMatrix::CompactVMatrix(VMat m, int keep_last_variables_last, bool onehot_encoding) 00112 : inherited(m->length(),m->width()), one_hot_encoding(onehot_encoding), 00113 n_symbol_values(m->width()), variables_permutation(m->width()) 00114 { 00115 if (!m->hasStats()) 00116 { 00117 cout << "CompactVMatrix(VMat, int): VMat did not have stats. Computing them." << endl; 00118 m->computeStats(); 00119 } 00120 // determine which variables are binary discrete, multi-class discrete, or continuous 00121 n_bits = n_symbols = n_fixedpoint = 0; 00122 TVec<int> bits_position(m->width()); 00123 TVec<int> symbols_position(m->width()); 00124 TVec<int> fp_position(m->width()); 00125 fixedpoint_min.resize(m->width()); 00126 fixedpoint_max.resize(m->width()); 00127 delta.resize(m->width()); 00128 for (int i=0;i<m->width();i++) 00129 { 00130 VMFieldStat& stat = m->fieldstats[i]; 00131 int n_values = (int)stat.counts.size(); // 0 means "continuous" 00132 bool counts_look_continuous = !isMapKeysAreInt(stat.counts); 00133 if (n_values == 0 || counts_look_continuous || i>=m->width()-keep_last_variables_last) 00134 { 00135 fixedpoint_min[n_fixedpoint]=stat.min(); 00136 fixedpoint_max[n_fixedpoint]=stat.max(); 00137 delta[n_fixedpoint]=(stat.max()-stat.min())/USHRT_MAX; 00138 fp_position[n_fixedpoint++]=i; 00139 } 00140 else 00141 { 00142 if (!fast_exact_is_equal(stat.min(), 0) || 00143 !fast_exact_is_equal((stat.max()-stat.min()+1), 00144 stat.counts.size())) 00145 PLERROR("CompactVMatrix:: variable %d looks discrete but has zero-frequency intermediate values or min!=0",i); 00146 if (n_values==2) 00147 bits_position[n_bits++]=i; 00148 else if (n_values<=256) 00149 { 00150 symbols_position[n_symbols]=i; 00151 n_symbol_values[n_symbols++] = n_values; 00152 } 00153 else 00154 { 00155 fixedpoint_min[n_fixedpoint]=stat.min(); 00156 fixedpoint_max[n_fixedpoint]=stat.max(); 00157 delta[n_fixedpoint]=(stat.max()-stat.min())/USHRT_MAX; 00158 fp_position[n_fixedpoint++]=i; 00159 } 00160 } 00161 } 00162 fixedpoint_min.resize(n_fixedpoint); 00163 fixedpoint_max.resize(n_fixedpoint); 00164 delta.resize(n_fixedpoint); 00165 n_symbol_values.resize(n_symbols); 00166 n_variables = n_bits + n_symbols + n_fixedpoint; 00167 int j=0; 00168 for (int i=0;i<n_bits;i++,j++) 00169 variables_permutation[j]=bits_position[i]; 00170 for (int i=0;i<n_symbols;i++,j++) 00171 variables_permutation[j]=symbols_position[i]; 00172 for (int i=0;i<n_fixedpoint;i++,j++) 00173 variables_permutation[j]=fp_position[i]; 00174 00175 normal_width=n_bits+n_fixedpoint; 00176 for (int i=0;i<n_symbols;i++) 00177 normal_width += n_symbol_values[i]; 00178 setOneHotMode(one_hot_encoding); 00179 symbols_offset = (int)ceil(n_bits/8.0); 00180 fixedpoint_offset = symbols_offset + n_symbols; 00181 row_n_bytes = fixedpoint_offset + int(sizeof(unsigned short))*n_fixedpoint; 00182 data.resize(length_ * row_n_bytes); 00183 00184 // copy the field infos and stats? not really useful with one-hot encoding 00185 // because of non-binary symbols being spread across many columns. 00186 if (!one_hot_encoding) 00187 { 00188 fieldinfos.resize(width_); 00189 fieldstats.resize(width_); 00190 for (int i=0;i<width_;i++) 00191 { 00192 fieldinfos[i]=m->getFieldInfos()[variables_permutation[i]]; 00193 fieldstats[i]=m->fieldstats[variables_permutation[i]]; 00194 } 00195 } 00196 else 00197 { 00198 fieldinfos.resize(0); 00199 fieldstats.resize(0); 00200 } 00201 00202 // copy the data 00203 Vec mrow(m->width()); 00204 for (int t=0;t<length_;t++) 00205 { 00206 m->getRow(t,mrow); 00207 encodeAndPutRow(t,mrow); 00208 } 00209 set_n_bits_in_byte(); 00210 } 00211 00212 00213 CompactVMatrix::CompactVMatrix(const string& filename, int nlast) : RowBufferedVMatrix(0,0) 00214 { 00215 load(filename); 00216 n_last=nlast; 00217 set_n_bits_in_byte(); 00218 } 00219 00220 CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m, bool rescale, bool check) 00221 : inherited(m->length(),m->width()) 00222 { 00223 if (cvm->width() != m->width()) 00224 PLERROR("CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m,...), args have widths %d!=%d", 00225 cvm->width(),m->width()); 00226 // copy all the ordinary fields 00227 row_n_bytes = cvm->row_n_bytes; 00228 data.resize(length_*row_n_bytes); 00229 n_bits = cvm->n_bits; 00230 n_symbols = cvm->n_symbols; 00231 n_fixedpoint = cvm->n_fixedpoint; 00232 n_variables = cvm->n_variables; 00233 n_symbol_values = cvm->n_symbol_values; 00234 fixedpoint_min = cvm->fixedpoint_min.copy(); 00235 fixedpoint_max = cvm->fixedpoint_max.copy(); 00236 delta = cvm->delta.copy(); 00237 variables_permutation = cvm->variables_permutation; 00238 n_last = cvm->n_last; 00239 normal_width = cvm->normal_width; 00240 symbols_offset = cvm->symbols_offset; 00241 fixedpoint_offset = cvm->fixedpoint_offset; 00242 00243 setOneHotMode(cvm->one_hot_encoding); 00244 Vec row(width_); 00245 int offs=width_-n_fixedpoint; 00246 if (rescale) 00247 { 00248 for (int i=0;i<length_;i++) 00249 { 00250 m->getRow(i,row); 00251 for (int j=0;j<n_fixedpoint;j++) 00252 { 00253 real element=row[offs+j]; 00254 if (element<fixedpoint_min[j]) 00255 fixedpoint_min[j]=element; 00256 if (element>fixedpoint_max[j]) 00257 fixedpoint_max[j]=element; 00258 } 00259 } 00260 for (int j=0;j<n_fixedpoint;j++) 00261 delta[j]=(fixedpoint_max[j]-fixedpoint_min[j])/USHRT_MAX; 00262 } 00263 for (int i=0;i<length_;i++) 00264 { 00265 m->getRow(i,row); 00266 if (!rescale && check) // check that range is OK 00267 { 00268 for (int j=0;j<n_fixedpoint;j++) 00269 { 00270 real element=row[offs+j]; 00271 if (element<fixedpoint_min[j] || 00272 element>fixedpoint_max[j]) 00273 PLERROR("CompactVMatrix::CompactVMatrix(CompactVMatrix* cvm, VMat m,...) out-of-range element(%d,%d)=%g not in [%g,%g]", 00274 i,j,element,fixedpoint_min[j],fixedpoint_max[j]); 00275 } 00276 } 00277 putRow(i,row); 00278 } 00279 } 00280 00281 void CompactVMatrix::setOneHotMode(bool on) 00282 { 00283 one_hot_encoding=on; 00284 if (one_hot_encoding) 00285 width_ = normal_width; 00286 else 00287 width_ = n_variables; 00288 } 00289 00290 void CompactVMatrix::getNewRow(int i, const Vec& v) const 00291 { 00292 #ifdef BOUNDCHECK 00293 if (i<0 || i>=length_) 00294 PLERROR("CompactVMatrix::getNewRow, row %d out of bounds [0,%d]",i,length_-1); 00295 if (v.length()!=width_) 00296 PLERROR("CompactVMatrix::getNewRow, length of v (%d) should be equal to width of VMat (%d)",v.length(),width()); 00297 #endif 00298 unsigned char* encoded_row = &data.data[i*row_n_bytes]; 00299 real* vp=v.data(); 00300 int c=0; 00301 for (int b=0;b<symbols_offset;b++) 00302 { 00303 unsigned char byte=encoded_row[b]; 00304 for (int j=0;j<8 && c<n_bits;j++,c++) 00305 { 00306 int bit = byte & 1; 00307 byte >>= 1; // shift right once 00308 vp[c]=bit; 00309 } 00310 } 00311 for (int b=0;b<n_symbols;b++) 00312 { 00313 int byte = encoded_row[symbols_offset+b]; 00314 if (one_hot_encoding) 00315 { 00316 int n=n_symbol_values[b]; 00317 for (int j=0;j<n;j++) vp[c+j]=0; 00318 vp[c+byte]=1; 00319 c+=n; 00320 } 00321 else vp[c++]=byte; 00322 } 00323 unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset]; 00324 for (int j=0;j<n_fixedpoint;j++,c++) 00325 { 00326 unsigned char *uc = &fixed_point_numbers[2*j]; 00327 short_and_twobytes u; 00328 u.twobytes[0]=uc[0]; 00329 u.twobytes[1]=uc[1]; 00330 real decoded = u.us*delta[j]+fixedpoint_min[j]; 00331 // correct rounding errors for integers, due to fixed-point low precision 00332 real rounded_decoded = rint(decoded); 00333 if (fabs(rounded_decoded-decoded)<1e-4) 00334 decoded = rounded_decoded; 00335 vp[c]=decoded; 00336 } 00337 } 00338 00339 //#define SANITYCHECK_CompactVMatrix 00340 #define SANITYCHECK_CompactVMatrix_PRECISION 1e-5 00341 00342 real CompactVMatrix::dot(int i, int j, int inputsize) const 00343 { 00344 if(inputsize!=width()-n_last) 00345 PLERROR("In CompactVMatrix::dot, in current implementation inputsize must be equal to width()-n_last"); 00346 00347 unsigned char* encoded_row_i = &data.data[i*row_n_bytes]; 00348 unsigned char* encoded_row_j = &data.data[j*row_n_bytes]; 00349 real dot_product=0.; 00350 int c=0; 00351 for (int b=0;b<symbols_offset;b++) 00352 { 00353 unsigned char byte_i=encoded_row_i[b]; 00354 unsigned char byte_j=encoded_row_j[b]; 00355 unsigned char byte_and = byte_i & byte_j; 00356 #ifdef SANITYCHECK_CompactVMatrix 00357 real check=dot_product; 00358 #endif 00359 // Here we want to count the number of ON bits in the byte_and 00360 // instead of looping through the bits, we look-up in a 00361 // pre-computed table (n_bits_in_byte), which has been set-up by set_n_bits_in_byte(). 00362 dot_product += n_bits_in_byte[byte_and]; 00363 #ifdef SANITYCHECK_CompactVMatrix 00364 for (int j=0;j<8 && c<n_bits;j++,c++) 00365 { 00366 check += byte_and & 1; 00367 byte_and >>= 1; // shift right once 00368 } 00369 if (check!=dot_product) 00370 PLERROR("logic error in n_bits_in_byte"); 00371 #else 00372 c+=8; 00373 if (c>n_bits) c=n_bits; 00374 #endif 00375 } 00376 if (c>width_-n_last) 00377 PLERROR("CompactVMatrix: n_last should be among discrete non-binary or continuous variables"); 00378 for (int b=0;b<n_symbols && c<width_-n_last;b++) 00379 { 00380 int byte_i = encoded_row_i[symbols_offset+b]; 00381 int byte_j = encoded_row_j[symbols_offset+b]; 00382 if (byte_i==byte_j) dot_product++; 00383 if (one_hot_encoding) 00384 c+=n_symbol_values[b]; 00385 else 00386 c++; 00387 } 00388 unsigned char* fixed_point_numbers_i = &encoded_row_i[fixedpoint_offset]; 00389 unsigned char* fixed_point_numbers_j = &encoded_row_j[fixedpoint_offset]; 00390 for (int k=0;k<n_fixedpoint-n_last && c<width_-n_last;k++,c++) 00391 { 00392 unsigned char *uc = &fixed_point_numbers_i[2*k]; 00393 short_and_twobytes u; 00394 u.twobytes[0]=uc[0]; 00395 u.twobytes[1]=uc[1]; 00396 real decoded_i = u.us*delta[k]+fixedpoint_min[k]; 00397 uc = &fixed_point_numbers_j[2*k]; 00398 u.twobytes[0]=uc[0]; 00399 u.twobytes[1]=uc[1]; 00400 real decoded_j = u.us*delta[k]+fixedpoint_min[k]; 00401 #ifdef SANITYCHECK_CompactVMatrix 00402 real rounded_decoded_i = rint(decoded_i); 00403 if (fabs(rounded_decoded_i-decoded_i)<1e-4) 00404 decoded_i = rounded_decoded_i; 00405 real rounded_decoded_j = rint(decoded_j); 00406 if (fabs(rounded_decoded_j-decoded_j)<1e-4) 00407 decoded_j = rounded_decoded_j; 00408 #endif 00409 dot_product += decoded_i * decoded_j; 00410 } 00411 00412 return dot_product; 00413 } 00414 00415 // I used the code for getRow as a basis to implement this call (Pascal) 00416 real CompactVMatrix::dot(int i, const Vec& v) const 00417 { 00418 #ifdef BOUNDCHECK 00419 if (i<0 || i>=length_) 00420 PLERROR("CompactVMatrix::dot, row %d out of bounds [0,%d]",i,length_-1); 00421 #endif 00422 00423 if(v.length()!=width()-n_last) 00424 PLERROR("In CompactVMatrix::dot, in current implementation v.length() must be equal to width()-n_last"); 00425 00426 real dot_product = 0.; 00427 00428 unsigned char* encoded_row = &data.data[i*row_n_bytes]; 00429 real* vp=v.data(); 00430 int c=0; 00431 for (int b=0;b<symbols_offset;b++) 00432 { 00433 unsigned char byte=encoded_row[b]; 00434 for (int j=0;j<8 && c<n_bits;j++,c++) 00435 { 00436 int bit = byte & 1; 00437 byte >>= 1; // shift right once 00438 if(bit) 00439 dot_product += vp[c]; 00440 } 00441 } 00442 for (int b=0;b<n_symbols;b++) 00443 { 00444 int byte = encoded_row[symbols_offset+b]; 00445 if (one_hot_encoding) 00446 { 00447 int n=n_symbol_values[b]; 00448 dot_product += vp[c+byte]; 00449 c += n; 00450 } 00451 else 00452 dot_product += vp[c++]*byte; 00453 } 00454 // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4 00455 // ON SOME MACHINES? 00456 unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset]; 00457 for (int j=0;j<n_fixedpoint-n_last && c<v.length();j++,c++) 00458 { 00459 unsigned char *uc = &fixed_point_numbers[2*j]; 00460 short_and_twobytes u; 00461 u.twobytes[0]=uc[0]; 00462 u.twobytes[1]=uc[1]; 00463 real decoded = u.us*delta[j]+fixedpoint_min[j]; 00464 // correct rounding errors for integers, due to fixed-point low precision 00465 real rounded_decoded = rint(decoded); 00466 if (fabs(rounded_decoded-decoded)<1e-4) 00467 decoded = rounded_decoded; 00468 dot_product += vp[c] * decoded; 00469 } 00470 00471 // Very Slow SANITY CHECK 00472 #ifdef SANITYCHECK_CompactVMatrix 00473 Vec v_i(v.length()); 00474 getRow(i,v_i); 00475 real dot_product2 = PLearn::dot(v_i.subVec(0,v.length()),v); 00476 real diff = fabs(dot_product-dot_product2)/fabs(dot_product2); 00477 if(diff>SANITYCHECK_CompactVMatrix_PRECISION) 00478 PLERROR("IN CompactVMatrix::dot(int i=%d, v) SANITY CHECK FAILED: difference=%g",i,diff); 00479 #endif 00480 00481 return dot_product; 00482 } 00483 00484 00485 real CompactVMatrix::dotProduct(int i, int j) const 00486 { return dot(i,j,width()-n_last); } 00487 00488 real CompactVMatrix::squareDifference(int i, int j) 00489 { 00490 if (row_norms.length()==0) 00491 row_norms = Vec(length_,-1.0); 00492 real normi = row_norms[i]; 00493 if (normi<0) normi=row_norms[i]=dotProduct(i,i); 00494 real normj = row_norms[j]; 00495 if (normj<0) normj=row_norms[j]=dotProduct(j,j); 00496 return normi + normj - 2 * dotProduct(i,j); 00497 } 00498 00499 void CompactVMatrix::encodeAndPutRow(int i, Vec v) 00500 { 00501 unsigned char* encoded_row = &data.data[i*row_n_bytes]; 00502 real* vp=v.data(); 00503 int* perm=variables_permutation.data(); 00504 int c=0; 00505 // 1 vector element ==> 1 bit 00506 for (int b=0;b<symbols_offset;b++) 00507 { 00508 unsigned char byte=0; 00509 for (int j=0;j<8 && c<n_bits;j++,c++) 00510 byte |= int(vp[perm[c]]) << j; // shift to right bit position 00511 encoded_row[b]=byte; 00512 } 00513 // 1 vector element (integer between 0 and n-1) ==> 1 byte 00514 for (int b=0;b<n_symbols;b++,c++) 00515 { 00516 real val = vp[perm[c]]; 00517 int s = int(val); 00518 if (!fast_exact_is_equal(s, val)) 00519 PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%g not an integer", 00520 i,int(perm[c]),val); 00521 encoded_row[symbols_offset+b] = s; // ASSUMES THAT v IS NOT ONE-HOT ENCODED 00522 if (s<0 || s>=n_symbol_values[b]) 00523 PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%d not in expected range (0,%d)", 00524 i,int(perm[c]),s,n_symbol_values[b]-1); 00525 } 00526 // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4 00527 // ON SOME MACHINES? 00528 unsigned short* fixed_point_numbers = (unsigned short*)&encoded_row[fixedpoint_offset]; 00529 for (int j=0;j<n_fixedpoint;j++,c++) 00530 fixed_point_numbers[j]=(unsigned short)((vp[perm[c]]-fixedpoint_min[j])/delta[j]); 00531 00532 invalidateBuffer(); 00533 } 00534 00535 void CompactVMatrix::putRow(int i, Vec v) 00536 { 00537 putSubRow(i,0,v); 00538 } 00539 00540 void CompactVMatrix::putSubRow(int i, int j, Vec v) 00541 { 00542 unsigned char* encoded_row = &data.data[i*row_n_bytes]; 00543 real* vp=v.data(); 00544 int c=0; 00545 // 1 vector element ==> 1 bit 00546 for (int b=0;b<symbols_offset;b++) 00547 { 00548 unsigned char byte=0; 00549 for (int k=0;k<8 && c<n_bits;k++,c++) 00550 if (c>=j) 00551 byte |= int(vp[c-j]) << k; // shift to right bit position 00552 encoded_row[b]=byte; 00553 } 00554 // if (one_hot_encoding) 00555 // n vector elements in one-hot-code ==> 1 byte 00556 // else 00557 // 1 vector element (integer between 0 and n-1) ==> 1 byte 00558 int n=0; 00559 if (one_hot_encoding) 00560 for (int b=0;b<n_symbols;b++,c+=n) 00561 { 00562 n=n_symbol_values[b]; 00563 if (c>=j) 00564 { 00565 int pos=-1; 00566 for (int k=0;k<n;k++) 00567 { 00568 real vk=vp[c+k-j]; 00569 if (!fast_exact_is_equal(vk, 0) && 00570 !fast_exact_is_equal(vk, 1)) 00571 PLERROR("CompactVMatrix::putRow(%d,v): v[%d]=%g!=0 or 1 (not one-hot-code)", 00572 i,c,vk); 00573 if (fast_exact_is_equal(vk, 1)) 00574 { 00575 if (pos<0) pos=k; 00576 else PLERROR("CompactVMatrix::putRow(%d,v): %d-th symbol not one-hot-encoded", 00577 i,b); 00578 } 00579 } 00580 if (pos<0) 00581 PLERROR("CompactVMatrix::putRow(%d,v): %d-th symbol not one-hot-encoded", 00582 i,b); 00583 encoded_row[symbols_offset+b] = pos; 00584 } 00585 } 00586 else 00587 for (int b=0;b<n_symbols;b++,c++) 00588 if (c>=j) 00589 { 00590 real val = vp[c-j]; 00591 int s = int(val); 00592 if (!fast_exact_is_equal(s, val)) 00593 PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%g not an integer", 00594 i,c,val); 00595 encoded_row[symbols_offset+b] = s; // ASSUMES THAT v IS NOT ONE-HOT ENCODED 00596 if (s<0 || s>=n_symbol_values[b]) 00597 PLERROR("CompactVMatrix::encodeAndPutRow(%d,v): v[%d]=%d not in expected range (0,%d)", 00598 i,c,s,n_symbol_values[b]-1); 00599 } 00600 00601 // 1 vector element (real betweeen fixedpoint_min and fixedpoint_max) ==> 2 bytes 00602 // 00603 // WARNING: COULD THIS CAUSE PROBLEMS IF fixedpoint_offset IS NOT A MULTIPLE OF 4 00604 // ON SOME MACHINES? 00605 unsigned short* fixed_point_numbers = (unsigned short*)&encoded_row[fixedpoint_offset]; 00606 for (int k=0;k<n_fixedpoint;k++,c++) 00607 if (c>=j) 00608 fixed_point_numbers[k]=(unsigned short)((vp[c-j]-fixedpoint_min[k])/delta[k]); 00609 00610 invalidateBuffer(); 00611 } 00612 00613 void CompactVMatrix::perturb(int i, Vec v, real noise_level, int n_last) 00614 { 00615 #ifdef BOUNDCHECK 00616 if (i<0 || i>=length_) 00617 PLERROR("CompactVMatrix::perturb, row %d out of bounds [0,%d]",i,length_-1); 00618 if (v.length()!=width_) 00619 PLERROR("CompactVMatrix::perturb, length of v (%d) should be equal to width of VMat (%d)",v.length(),width()); 00620 #endif 00621 if (fieldstats.size()!=n_variables) 00622 PLERROR("CompactVMatrix::perturb: stats not computed or wrong size"); 00623 if (noise_level<0 || noise_level>1) 00624 PLERROR("CompactVMatrix::perturb: noise_level=%g, should be in [0,1]",noise_level); 00625 00626 unsigned char* encoded_row = &data.data[i*row_n_bytes]; 00627 real* vp=v.data(); 00628 int c=0; 00629 int var=0; 00630 Vec probs(width_); 00631 for (int b=0;b<symbols_offset;b++) 00632 { 00633 unsigned char byte=encoded_row[b]; 00634 for (int j=0;j<8 && c<n_bits;j++,c++,var++) 00635 { 00636 int bit = byte & 1; 00637 byte >>= 1; // shift right once 00638 vp[c]=binomial_sample((1-noise_level)*bit+noise_level*fieldstats[var].prob(1)); 00639 } 00640 } 00641 for (int b=0;b<n_symbols;b++,var++) 00642 { 00643 int byte = encoded_row[symbols_offset+b]; 00644 int nv=n_symbol_values[b]; 00645 probs.resize(nv); 00646 VMFieldStat& stat=fieldstats[var]; 00647 for (int val=0;val<nv;val++) 00648 if (val==byte) 00649 probs[val]=(1-noise_level)+noise_level*stat.prob(val); 00650 else 00651 probs[val]=noise_level*stat.prob(val); 00652 byte = multinomial_sample(probs); 00653 if (one_hot_encoding) 00654 { 00655 int n=n_symbol_values[b]; 00656 for (int j=0;j<n;j++) vp[c+j]=0; 00657 vp[c+byte]=1; 00658 c+=n; 00659 } 00660 else vp[c++]=byte; 00661 } 00662 unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset]; 00663 for (int j=0;j<n_fixedpoint;j++,c++,var++) 00664 { 00665 unsigned char *uc = &fixed_point_numbers[2*j]; 00666 short_and_twobytes u; 00667 u.twobytes[0]=uc[0]; 00668 u.twobytes[1]=uc[1]; 00669 real decoded = u.us*delta[j]+fixedpoint_min[j]; 00670 // correct rounding errors for integers, due to fixed-point low precision 00671 real rounded_decoded = rint(decoded); 00672 if (fabs(rounded_decoded-decoded)<1e-4) 00673 decoded = rounded_decoded; 00674 if (var<n_variables-n_last) 00675 { 00676 int ntry=0; 00677 do 00678 { 00679 vp[c]=decoded+noise_level*fieldstats[var].stddev()*normal_sample(); 00680 ntry++; 00681 if (ntry>=100) 00682 PLERROR("CompactVMatrix::perturb:Something wrong in resampling, tried 100 times"); 00683 } 00684 while (vp[c]<fixedpoint_min[j] || vp[c]>fixedpoint_max[j]); 00685 } 00686 else 00687 vp[c]=decoded; 00688 } 00689 } 00690 /* 00691 void CompactVMatrix::write(ostream& out) const 00692 { 00693 writeHeader(out,"CompactVMatrix"); 00694 writeField(out,"length",length_); 00695 writeField(out,"width",normal_width); 00696 writeField(out,"fieldinfos",fieldinfos); 00697 writeField(out,"fieldstats",fieldstats); 00698 writeField(out,"row_n_bytes",row_n_bytes); 00699 writeField(out,"n_bits",n_bits); 00700 writeField(out,"n_symbols",n_symbols); 00701 writeField(out,"n_fixedpoint",n_fixedpoint); 00702 writeField(out,"one_hot_encoding",one_hot_encoding); 00703 writeField(out,"n_symbol_values",n_symbol_values); 00704 writeField(out,"fixedpoint_min",fixedpoint_min); 00705 writeField(out,"fixedpoint_max",fixedpoint_max); 00706 writeField(out,"delta",delta); 00707 writeField(out,"variables_permutation",variables_permutation); 00708 writeField(out,"symbols_offset",symbols_offset); 00709 writeField(out,"fixedpoint_offset",fixedpoint_offset); 00710 out.write((char*)data.data,data.length()*sizeof(unsigned char)); 00711 writeFooter(out,"CompactVMatrix"); 00712 } 00713 00714 void CompactVMatrix::oldread(istream& in) 00715 { 00716 readHeader(in,"CompactVMatrix"); 00717 readField(in,"length",length_); 00718 readField(in,"width",normal_width); 00719 readField(in,"fieldinfos",fieldinfos); 00720 fieldinfos.resize(0); // to fix current bug in setting fieldinfos 00721 readField(in,"fieldstats",fieldstats); 00722 readField(in,"row_n_bytes",row_n_bytes); 00723 readField(in,"n_bits",n_bits); 00724 readField(in,"n_symbols",n_symbols); 00725 readField(in,"n_fixedpoint",n_fixedpoint); 00726 n_variables = n_bits + n_symbols + n_fixedpoint; 00727 readField(in,"one_hot_encoding",one_hot_encoding); 00728 setOneHotMode(one_hot_encoding); 00729 readField(in,"n_symbol_values",n_symbol_values); 00730 readField(in,"fixedpoint_min",fixedpoint_min); 00731 readField(in,"fixedpoint_max",fixedpoint_max); 00732 readField(in,"delta",delta); 00733 readField(in,"variables_permutation",variables_permutation); 00734 readField(in,"symbols_offset",symbols_offset); 00735 readField(in,"fixedpoint_offset",fixedpoint_offset); 00736 data.resize(row_n_bytes*length_); 00737 in.read((char*)data.data,data.length()*sizeof(unsigned char)); 00738 readFooter(in,"CompactVMatrix"); 00739 } 00740 */ 00741 void CompactVMatrix::append(CompactVMatrix* vm) 00742 { 00743 if (width_!=vm->width()) 00744 PLERROR("CompactVMatrix::append, incompatible width %d vs %d", 00745 width_,vm->width()); 00746 if (row_n_bytes!=vm->row_n_bytes) 00747 PLERROR("CompactVMatrix::append, incompatible row_n_bytes %d vs %d", 00748 row_n_bytes,vm->row_n_bytes); 00749 if (n_bits!=vm->n_bits) 00750 PLERROR("CompactVMatrix::append, incompatible n_bits %d vs %d", 00751 n_bits,vm->n_bits); 00752 if (n_symbols!=vm->n_symbols) 00753 PLERROR("CompactVMatrix::append, incompatible n_symbols %d vs %d", 00754 n_symbols,vm->n_symbols); 00755 if (n_fixedpoint!=vm->n_fixedpoint) 00756 PLERROR("CompactVMatrix::append, incompatible n_fixedpoint %d vs %d", 00757 n_fixedpoint,vm->n_fixedpoint); 00758 if (n_symbol_values!=vm->n_symbol_values) 00759 { 00760 //n_symbol_values.write(cerr); cerr << endl; 00761 //vm->n_symbol_values.write(cerr); cerr << endl; 00762 PLearn::write(cerr, n_symbol_values); 00763 cerr << endl; 00764 PLearn::write(cerr, vm->n_symbol_values); 00765 cerr << endl; 00766 PLERROR("CompactVMatrix::append, incompatible n_symbol_values"); 00767 } 00768 bool rescale = false; 00769 for (int j=0;j<n_fixedpoint && !rescale;j++) 00770 if (fixedpoint_min[j]>vm->fixedpoint_min[j] || 00771 fixedpoint_max[j]<vm->fixedpoint_max[j]) rescale=true; 00772 if (rescale) 00773 { 00774 cout << "The appended VMat has intervals that are wider than the current one." << endl; 00775 cout << "Start rescaling numeric variables fixed point representation." << endl; 00776 Vec new_min = fixedpoint_min.copy(); 00777 Vec new_max = fixedpoint_max.copy(); 00778 Vec new_delta = delta.copy(); 00779 TVec<bool> change(n_fixedpoint); 00780 for (int j=0;j<n_fixedpoint;j++) 00781 { 00782 change[j]=false; 00783 if (fixedpoint_min[j]>vm->fixedpoint_min[j]) 00784 { 00785 change[j]=true; 00786 new_min[j]=vm->fixedpoint_min[j]; 00787 } 00788 if (fixedpoint_max[j]<vm->fixedpoint_max[j]) 00789 { 00790 change[j]=true; 00791 new_max[j]=vm->fixedpoint_max[j]; 00792 } 00793 if (change[j]) 00794 new_delta[j]=(new_max[j]-new_min[j])/USHRT_MAX; 00795 } 00796 for (int r=0;r<length_;r++) 00797 { 00798 unsigned char* encoded_row = &data.data[r*row_n_bytes]; 00799 unsigned char* fixed_point_numbers = &encoded_row[fixedpoint_offset]; 00800 for (int j=0;j<n_fixedpoint;j++) 00801 if (change[j]) 00802 { 00803 // DECODE using previous min/max 00804 unsigned char *uc = &fixed_point_numbers[2*j]; 00805 short_and_twobytes u; 00806 u.twobytes[0]=uc[0]; 00807 u.twobytes[1]=uc[1]; 00808 real decoded = u.us*delta[j]+fixedpoint_min[j]; 00809 // correct rounding errors for integers, due to fixed-point low precision 00810 real rounded_decoded = rint(decoded); 00811 if (fabs(rounded_decoded-decoded)<1e-4) 00812 decoded = rounded_decoded; 00813 // ENCODE using new min/max 00814 fixed_point_numbers[j]=(unsigned char)((decoded-new_min[j])/new_delta[j]); 00815 } 00816 } 00817 cout << "DONE rescaling numeric variables fixed point representation." << endl; 00818 fixedpoint_min << new_min; 00819 fixedpoint_max << new_max; 00820 delta << new_delta; 00821 } 00822 int new_length=length_+vm->length(); 00823 data.resize(row_n_bytes*new_length); 00824 // copy the new data 00825 Vec row(width_); 00826 bool old_vm_encoding = vm->one_hot_encoding; 00827 bool old_encoding = one_hot_encoding; 00828 vm->one_hot_encoding=false; 00829 setOneHotMode(false); 00830 int old_length=length_; 00831 length_=new_length; 00832 for (int r=0;r<vm->length();r++) 00833 { 00834 vm->getRow(r,row); 00835 putRow(old_length+r,row); 00836 } 00837 vm->one_hot_encoding=old_vm_encoding; 00838 setOneHotMode(old_encoding); 00839 } 00840 00841 void CompactVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00842 { 00843 deepCopyField(data, copies); 00844 deepCopyField(n_symbol_values, copies); 00845 deepCopyField(fixedpoint_min, copies); 00846 deepCopyField(fixedpoint_max, copies); 00847 deepCopyField(variables_permutation, copies); 00848 } 00849 00850 } // end of namespace PLearn 00851 00852 00853 /* 00854 Local Variables: 00855 mode:c++ 00856 c-basic-offset:4 00857 c-file-style:"stroustrup" 00858 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00859 indent-tabs-mode:nil 00860 fill-column:79 00861 End: 00862 */ 00863 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :