PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999,2000 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // Copyright (C) 2002 Frederic Morin, Xavier Saint-Mleux, Pascal Vincent 00007 // 00008 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 00039 00040 /* ******************************************************* 00041 * $Id: pl_io.cc 9142 2008-06-18 18:29:34Z nouiz $ 00042 * AUTHORS: Pascal Vincent 00043 * This file is part of the PLearn library. 00044 ******************************************************* */ 00045 00046 00049 //#include <limits> 00050 #include "pl_io.h" 00051 #include <plearn/base/plerror.h> 00052 //#include "pl_math.h" 00053 #include <plearn/base/byte_order.h> 00054 #include <cstring> 00055 namespace PLearn { 00056 using namespace std; 00057 00058 00059 /* compressed array of float format 00060 All data on disk are to be in little-endian format. 00061 bit patterns described from a reading perspective 00062 00063 Attempt to read sizenum as 1 byte 00064 Read 1 byte. 00065 If the bit pattern is 00000000 (0x00) then the sizenum is the following 2-byte unsigned short : read it 00066 If the bit pattern is 11000000 (0xC0)then the sizenum is the following 4-byte unsigned int : read it 00067 Otherwise the sizenum is the unsigned byte just read 00068 00069 The two most significant bits of the sizenum are the mode bits, the remaining bits give a size 00070 Mode 00 (decimal 0) means: insert 'size' zeros 00071 Mode 01 (decimal 1) means: insert 'size' zeros and then a 1 00072 Mode 10 (decimal 2) means: insert 'size' values to be found are in the following 'size' signed bytes 00073 Mode 11 (decimal 3) means: insert 'size' values to be found are in the following 'size' floating points (different implementations for single and for double precision below 00074 */ 00075 00076 inline void write_compr_mode_and_size(ostream& out, unsigned char mode, int size) 00077 { 00078 #ifdef BOUNDCHECK 00079 if(size<0 || size>=(1<<30)) 00080 PLERROR("In write_compr_mode_and_size: size out of bounds"); 00081 #endif 00082 unsigned int imode = (unsigned int) mode; 00083 if(size<(1<<6)) 00084 { 00085 unsigned char sizenum = (unsigned char) size | (unsigned char) (imode<<6); 00086 binwrite(out, sizenum); 00087 } 00088 else if(size<(1<<14)) 00089 { 00090 unsigned short sizenum = (unsigned short) size | (unsigned short) (imode<<14); 00091 unsigned char header = 0x00; 00092 binwrite(out, header); 00093 binwrite(out, sizenum); 00094 } 00095 else 00096 { 00097 unsigned int sizenum = (unsigned int) size | (unsigned int) (imode<<30); 00098 unsigned char header = 0xC0; 00099 binwrite(out, header); 00100 binwrite(out, sizenum); 00101 } 00102 } 00103 00104 inline void read_compr_mode_and_size(istream& in, unsigned char& mode, int& size) 00105 { 00106 unsigned char sizenum_byte; 00107 binread(in, sizenum_byte); 00108 if(sizenum_byte==0x00) // sizenum is an unsigned short 00109 { 00110 unsigned short sizenum; 00111 binread(in, sizenum); 00112 mode = (unsigned char)(sizenum>>14); 00113 size = int(sizenum & (unsigned short)0x3FFF); 00114 } 00115 else if(sizenum_byte==0xC0) // sizenum is an unsigned int 00116 { 00117 unsigned int sizenum; 00118 binread(in, sizenum); 00119 mode = (unsigned char)(sizenum>>30); 00120 size = int(sizenum & (unsigned int)0x3FFFFFFF); 00121 } 00122 else // sizenum is the byte we just read 00123 { 00124 mode = sizenum_byte>>6; 00125 size = int(sizenum_byte & (unsigned char)0x3F); 00126 } 00127 } 00128 00129 void binread_compressed(istream& in, double* data, int l) 00130 { 00131 unsigned char mode; 00132 int n; 00133 double* p = data; 00134 char cval; 00135 while(l>0) 00136 { 00137 read_compr_mode_and_size(in, mode, n); 00138 //cerr << "mode: " << int(mode) << " size: " << n << endl; 00139 l -= n; 00140 switch(mode) 00141 { 00142 case 0: 00143 while(n--) 00144 *p++ = 0; 00145 break; 00146 case 1: 00147 while(n--) 00148 *p++ = 0; 00149 *p++ = 1; 00150 --l; 00151 break; 00152 case 2: 00153 while(n--) 00154 { 00155 binread(in,cval); 00156 *p++ = double(cval); 00157 } 00158 break; 00159 case 3: 00160 binread(in,p,n); 00161 p += n; 00162 break; 00163 default: 00164 PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?"); 00165 } 00166 } 00167 00168 if(l!=0) 00169 PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?"); 00170 } 00171 00172 void binwrite_compressed(ostream& out, const double* data, int l) 00173 { 00174 double val = 0.; 00175 while(l) 00176 { 00177 val = *data; 00178 if(fast_exact_is_equal(val, 0.)) 00179 { 00180 int n=0; 00181 while(l && fast_exact_is_equal(*data, 0.)) 00182 { ++n; ++data; --l; } 00183 if(l && fast_exact_is_equal(*data, 1.)) 00184 { 00185 write_compr_mode_and_size(out, 1, n); 00186 ++data; --l; 00187 } 00188 else 00189 write_compr_mode_and_size(out, 0, n); 00190 } 00191 else if(fast_exact_is_equal(val,1.)) 00192 { 00193 write_compr_mode_and_size(out, 1, 0); 00194 ++data; --l; 00195 } 00196 else if( fast_exact_is_equal(double(char(val)),val) ) 00197 { 00198 const double* start = data; 00199 int n=0; 00200 while(l && fast_exact_is_equal(double(char(val=*data)), val) 00201 && !fast_exact_is_equal(val, 0) 00202 && !fast_exact_is_equal(val, 1)) 00203 { ++n; ++data; --l; } 00204 write_compr_mode_and_size(out, 2, n); 00205 while(n--) 00206 binwrite(out,char(*start++)); 00207 } 00208 else 00209 { 00210 const double* start = data; 00211 int n=0; 00212 while(l && !fast_exact_is_equal((val=*data), 0) 00213 && !fast_exact_is_equal(val, 1) 00214 && !fast_exact_is_equal(double(char(val)), val)) 00215 { ++n; ++data; --l; } 00216 write_compr_mode_and_size(out, 3, n); 00217 binwrite(out,start,n); 00218 } 00219 } 00220 } 00221 void binread_compressed(istream& in, float* data, int l) 00222 { 00223 unsigned char mode; 00224 int n; 00225 float* p = data; 00226 while(l>0) 00227 { 00228 read_compr_mode_and_size(in, mode, n); 00229 //cerr << "mode: " << int(mode) << " size: " << n << endl; 00230 if(mode==0 || mode==1) 00231 { 00232 while(n--) 00233 { *p++ = 0; --l; } 00234 if(mode==1) 00235 { *p++ = 1; --l; } 00236 } 00237 else if(mode==2) 00238 { 00239 char val; 00240 while(n--) 00241 { 00242 binread(in,val); 00243 *p++ = float(val); 00244 --l; 00245 } 00246 } 00247 else if(mode==3) 00248 { 00249 binread(in,p,n); 00250 p += n; 00251 l -= n; 00252 } 00253 else 00254 PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?"); 00255 } 00256 00257 if(l!=0) 00258 PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?"); 00259 } 00260 00261 void binwrite_compressed(ostream& out, const float* data, int l) 00262 { 00263 float val = 0.; 00264 while(l) 00265 { 00266 val = *data; 00267 if(fast_exact_is_equal(val, 0.)) 00268 { 00269 int n=0; 00270 while(l && fast_exact_is_equal(*data, 0.)) 00271 { ++n; ++data; --l; } 00272 if(l && fast_exact_is_equal(*data, 1.)) 00273 { 00274 write_compr_mode_and_size(out, 1, n); 00275 ++data; --l; 00276 } 00277 else 00278 write_compr_mode_and_size(out, 0, n); 00279 } 00280 else if(fast_exact_is_equal(val, 1.)) 00281 { 00282 write_compr_mode_and_size(out, 1, 0); 00283 ++data; --l; 00284 } 00285 else if( fast_exact_is_equal(float(char(val)), val) ) 00286 { 00287 const float* start = data; 00288 int n=0; 00289 while(l && fast_exact_is_equal(float(char(val=*data)), val) 00290 && !fast_exact_is_equal(val, 0) 00291 && !fast_exact_is_equal(val, 1)) 00292 { ++n; ++data; --l; } 00293 write_compr_mode_and_size(out, 2, n); 00294 while(n--) 00295 binwrite(out,char(*start++)); 00296 } 00297 else 00298 { 00299 const float* start = data; 00300 int n=0; 00301 while(l && !fast_exact_is_equal((val=*data), 0) 00302 && !fast_exact_is_equal(val, 1) 00303 && !fast_exact_is_equal(float(char(val)), val)) 00304 { ++n; ++data; --l; } 00305 write_compr_mode_and_size(out, 3, n); 00306 binwrite(out,start,n); 00307 } 00308 } 00309 } 00310 00311 // ******************************************** 00312 // *** compressed vector to and from FILE* *** 00313 // ******************************************** 00314 00315 inline void read_compr_mode_and_size(FILE* in, unsigned char& mode, int& size) 00316 { 00317 unsigned char sizenum_byte; 00318 binread(in, sizenum_byte); 00319 if(sizenum_byte==0x00) // sizenum is an unsigned short 00320 { 00321 unsigned short sizenum; 00322 binread(in, sizenum); 00323 mode = (unsigned char)(sizenum>>14); 00324 size = int(sizenum & (unsigned short)0x3FFF); 00325 } 00326 else if(sizenum_byte==0xC0) // sizenum is an unsigned int 00327 { 00328 unsigned int sizenum; 00329 binread(in, sizenum); 00330 mode = (unsigned char)(sizenum>>30); 00331 size = int(sizenum & (unsigned int)0x3FFFFFFF); 00332 } 00333 else // sizenum is the byte we just read 00334 { 00335 mode = sizenum_byte>>6; 00336 size = int(sizenum_byte & (unsigned char)0x3F); 00337 } 00338 } 00339 00340 void binread_compressed(FILE* in, double* data, int l) 00341 { 00342 unsigned char mode; 00343 int n; 00344 double* p = data; 00345 char cval; 00346 while(l>0) 00347 { 00348 read_compr_mode_and_size(in, mode, n); 00349 //cerr << "mode: " << int(mode) << " size: " << n << endl; 00350 l -= n; 00351 switch(mode) 00352 { 00353 case 0: 00354 while(n--) 00355 *p++ = 0; 00356 break; 00357 case 1: 00358 while(n--) 00359 *p++ = 0; 00360 *p++ = 1; 00361 --l; 00362 break; 00363 case 2: 00364 while(n--) 00365 { 00366 binread(in,cval); 00367 *p++ = double(cval); 00368 } 00369 break; 00370 case 3: 00371 binread(in,p,n); 00372 p += n; 00373 break; 00374 default: 00375 PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?"); 00376 } 00377 } 00378 00379 if(l!=0) 00380 PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?"); 00381 } 00382 00383 void binwrite_compressed(FILE* out, const double* data, int l) 00384 { 00385 PLERROR("Not implemented"); 00386 } 00387 00388 void binread_compressed(FILE* in, float* data, int l) 00389 { 00390 unsigned char mode; 00391 int n; 00392 float* p = data; 00393 while(l>0) 00394 { 00395 read_compr_mode_and_size(in, mode, n); 00396 //cerr << "mode: " << int(mode) << " size: " << n << endl; 00397 if(mode==0 || mode==1) 00398 { 00399 while(n--) 00400 { *p++ = 0; --l; } 00401 if(mode==1) 00402 { *p++ = 1; --l; } 00403 } 00404 else if(mode==2) 00405 { 00406 char val; 00407 while(n--) 00408 { 00409 binread(in,val); 00410 *p++ = float(val); 00411 --l; 00412 } 00413 } 00414 else if(mode==3) 00415 { 00416 binread(in,p,n); 00417 p += n; 00418 l -= n; 00419 } 00420 else 00421 PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?"); 00422 } 00423 00424 if(l!=0) 00425 PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?"); 00426 } 00427 00428 void binwrite_compressed(FILE* out, const float* data, int l) 00429 { 00430 PLERROR("Not implemented"); 00431 } 00432 00433 // ******************************************** 00434 // *** compressed vector to and from memory *** 00435 // ******************************************** 00436 00437 inline void write_compr_mode_and_size_ptr(char*& out, unsigned char mode, int size) 00438 { 00439 union {unsigned short s;char cs[2];} unis; 00440 union {unsigned int i;char ci[4];} unii; 00441 00442 if(sizeof(unsigned int)!=4) 00443 PLERROR("Function write_compr_mode_and_size_ptr is not ready for 64 bit."); 00444 00445 #ifdef BOUNDCHECK 00446 if(size<0 || size>=(1<<30)) 00447 PLERROR("In write_compr_mode_and_size: size out of bounds"); 00448 #endif 00449 unsigned int imode = (unsigned int) mode; 00450 if(size<(1<<6)) 00451 { 00452 unsigned char sizenum = (unsigned char) size | (unsigned char) (imode<<6); 00453 (*out++) = sizenum; 00454 } 00455 else if(size<(1<<14)) 00456 { 00457 unis.s = (unsigned short) size | (unsigned short) (imode<<14); 00458 unsigned char header = 0x00; 00459 (*out++) = header; 00460 (*out++) = unis.cs[0]; 00461 (*out++) = unis.cs[1]; 00462 } 00463 else 00464 { 00465 unii.i = (unsigned int) size | (unsigned int) (imode<<30); 00466 unsigned char header = 0xC0; 00467 (*out++) = header; 00468 (*out++) = unii.ci[0]; 00469 (*out++) = unii.ci[1]; 00470 (*out++) = unii.ci[2]; 00471 (*out++) = unii.ci[3]; 00472 } 00473 } 00474 00475 inline void read_compr_mode_and_size_ptr(char*& in, unsigned char& mode, int& size) 00476 { 00477 union {unsigned short s;char cs[2];} unis; 00478 union {unsigned int i;char ci[4];} unii; 00479 00480 unsigned char sizenum_byte; 00481 sizenum_byte = (*in++); 00482 if(sizenum_byte==0x00) // sizenum is an unsigned short 00483 { 00484 unis.cs[0] = (*in++); 00485 unis.cs[1] = (*in++); 00486 mode = (unsigned char)(unis.s>>14); 00487 size = int(unis.s & (unsigned short)0x3FFF); 00488 } 00489 else if(sizenum_byte==0xC0) // sizenum is an unsigned int 00490 { 00491 unii.ci[0] = (*in++); 00492 unii.ci[1] = (*in++); 00493 unii.ci[2] = (*in++); 00494 unii.ci[3] = (*in++); 00495 mode = (unsigned char)(unii.i>>30); 00496 size = int(unii.i & (unsigned int)0x3FFFFFFF); 00497 } 00498 else // sizenum is the byte we just read 00499 { 00500 mode = sizenum_byte>>6; 00501 size = int(sizenum_byte & (unsigned char)0x3F); 00502 } 00503 } 00504 00505 00506 void uncompress_vec(char* comprbuf, double* data, int l, bool double_stored_as_float) 00507 { 00508 unsigned char mode; 00509 int n; 00510 double* p = data; 00511 while(l>0) 00512 { 00513 read_compr_mode_and_size_ptr(comprbuf, mode, n); 00514 //cerr << "mode: " << int(mode) << " size: " << n << endl; 00515 if(mode==0 || mode==1) 00516 { 00517 while(n--) 00518 { *p++ = 0; --l; } 00519 if(mode==1) 00520 { *p++ = 1; --l; } 00521 } 00522 else if(mode==2) 00523 { 00524 char val; 00525 while(n--) 00526 { 00527 val=(*comprbuf++); 00528 *p++ = double(val); 00529 --l; 00530 } 00531 } 00532 else if(mode==3) 00533 { 00534 memcpy(p,comprbuf,sizeof(double)*n); 00535 comprbuf+=sizeof(double)*n; 00536 p += n; 00537 l -= n; 00538 } 00539 else 00540 PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?"); 00541 } 00542 00543 if(l!=0) 00544 PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?"); 00545 } 00546 00547 void compress_vec(char* comprbuf, const double* data, int l, bool double_stored_as_float) 00548 { 00549 // char* comprbufold=comprbuf; 00550 double val = 0.; 00551 while(l) 00552 { 00553 val = *data; 00554 if(fast_exact_is_equal(val, 0.)) 00555 { 00556 int n=0; 00557 while(l && fast_exact_is_equal(*data, 0.)) 00558 { ++n; ++data; --l; } 00559 if(l && fast_exact_is_equal(*data, 1.)) 00560 { 00561 write_compr_mode_and_size_ptr(comprbuf, 1, n); 00562 ++data; --l; 00563 } 00564 else 00565 write_compr_mode_and_size_ptr(comprbuf, 0, n); 00566 } 00567 else if(fast_exact_is_equal(val, 1.)) 00568 { 00569 write_compr_mode_and_size_ptr(comprbuf, 1, 0); 00570 ++data; --l; 00571 } 00572 else if( fast_exact_is_equal(double(char(val)), val) ) 00573 { 00574 const double* start = data; 00575 int n=0; 00576 while(l && fast_exact_is_equal(double(char(val=*data)), val) 00577 && !fast_exact_is_equal(val, 0) 00578 && !fast_exact_is_equal(val, 1)) 00579 { ++n; ++data; --l; } 00580 write_compr_mode_and_size_ptr(comprbuf, 2, n); 00581 while(n--) 00582 (*comprbuf++) = char(*start++); 00583 } 00584 else 00585 { 00586 const double* start = data; 00587 int n=0; 00588 while(l && !fast_exact_is_equal((val=*data), 0) 00589 && !fast_exact_is_equal(val, 1) 00590 && !fast_exact_is_equal(double(char(val)), val)) 00591 { ++n; ++data; --l; } 00592 write_compr_mode_and_size_ptr(comprbuf, 3, n); 00593 memcpy(comprbuf,start,n*sizeof(double)); 00594 comprbuf += n*sizeof(double); 00595 } 00596 } 00597 } 00598 00599 00600 // ******************************************************** 00601 // ******************************************************** 00602 // ********* NEW COMPRESSION FORMAT ********** 00603 // ******************************************************** 00604 // ******************************************************** 00605 00606 /* 00607 00608 Format description: 00609 00610 A succession of [ mode-byte, optionally followed by specificaitons of length N, followed by data ] 00611 00612 The way N is encoded will be explained later. 00613 00614 The bits of the mode-byte are interpreted as follows: 00615 * Most significant bit: 00616 0 : insert the following N values of type T 00617 1 : insert N zeroes and then the following single value of type T 00618 00619 * Next 2 bits indicate the data type T 00620 00 (=0): ones (that's just 1.0, no further data is given to provide the value) 00621 01 (=1): small 1 byte signed integers in the range [-127, +127], or missing values (indicated by -128: bit pattern 0x80) 00622 10 (=2): 4 byte float 00623 11 (=3): 8 byte double 00624 00625 In all but the 00 case, 1 or N corresponding values of type T are 00626 to be read in the stream (after possibly reading N) 00627 00628 * Next 5 bits (values between 0 .. 31) indicate how to get the number N, 00629 00630 1..29: N is that particular value (between 1 and 29) 00631 0 : N is given in the following 1 byte unsigned char 00632 30: N is given in the following 2 byte unsigned short 00633 31: N is given in the following 4 byte unsigned int 00634 00635 00636 */ 00637 00638 00639 size_t new_read_compressed(FILE* in, real* vec, int l, bool swap_endians) 00640 { 00641 size_t nbytes = 0; // number of bytes read 00642 unsigned char mode; // the mode byte 00643 unsigned int N = 0; // N (number of 0s or values to insert) 00644 00645 while(l) 00646 { 00647 if(l<0) 00648 PLERROR("Big problem in new_read_compressed: l=%d", l); 00649 int i = getc(in); 00650 if(i == EOF) 00651 PLERROR("Got EOF while expecting more data in the file!"); 00652 mode = (unsigned char)(i); 00653 00654 ++nbytes; 00655 unsigned char N1 = (mode & 0x1F); 00656 switch(N1) 00657 { 00658 case 0: // N is the 1 byte to follow 00659 N1 = (unsigned char)(getc(in)); 00660 ++nbytes; 00661 N = N1; 00662 break; 00663 case 30: // N is the 2 bytes to follow 00664 unsigned short N2; 00665 fread(&N2,2,1,in); 00666 if(swap_endians) 00667 endianswap(&N2); 00668 nbytes += 2; 00669 N = N2; 00670 break; 00671 case 31: // N is the 4 bytes to follow 00672 fread(&N,4,1,in); 00673 if(swap_endians) 00674 endianswap(&N); 00675 nbytes += 4; 00676 break; 00677 default: // N is N1 00678 N = N1; 00679 } 00680 00681 if(mode & (unsigned char)(0x80)) // most significant bit is on 00682 { // insert N zeros 00683 l -= N; 00684 while(N--) 00685 *vec++ = 0; 00686 N = 1; 00687 } 00688 00689 if(!l) // vec ends with zeroes, so there's no extra single value to append. We're done! 00690 break; 00691 00692 l -= N; 00693 mode = ((mode & ~0x80) >> 5); // get the 2 bits we're interested in 00694 switch(mode) 00695 { 00696 case 0: // type ones 00697 { 00698 while(N--) 00699 *vec++ = 1; 00700 } 00701 break; 00702 case 1: // type signed char (or missing value if -128) 00703 { 00704 signed char val; 00705 nbytes += N; 00706 while(N--) 00707 { 00708 val = (signed char)(getc(in)); 00709 if(val==-128) 00710 *vec++ = MISSING_VALUE; 00711 else 00712 *vec++ = val; 00713 } 00714 } 00715 break; 00716 case 2: // type float 00717 { 00718 float val; 00719 nbytes += N<<2; 00720 while(N--) 00721 { 00722 fread(&val,sizeof(float),1,in); 00723 if(swap_endians) 00724 endianswap(&val); 00725 *vec++ = val; 00726 } 00727 } 00728 break; 00729 case 3: // type double 00730 { 00731 nbytes += N<<3; 00732 fread(vec,sizeof(double),N,in); 00733 if(swap_endians) 00734 endianswap(vec,N); 00735 vec += N; 00736 } 00737 } 00738 } 00739 return nbytes; 00740 } 00741 00742 unsigned char new_get_compr_data_type(double x, double tolerance) 00743 { 00744 if(is_missing(x)) 00745 return 1; 00746 else if(fast_exact_is_equal(x, 1.)) 00747 return 0; 00748 else if(fast_exact_is_equal(double(char(x)), x) && 00749 !fast_exact_is_equal(x, -128)) // -128 codes for missing value 00750 return 1; 00751 else if(fabs(double(float(x))-x)<=tolerance) 00752 return 2; 00753 return 3; 00754 } 00755 00756 unsigned char new_get_compr_data_type(float x) 00757 { 00758 if(is_missing(x)) 00759 return 1; 00760 else if(fast_exact_is_equal(x, 1.)) 00761 return 0; 00762 else if(fast_exact_is_equal(float(char(x)), x) && 00763 !fast_exact_is_equal(x, -128)) // -128 codes for missing value 00764 return 1; 00765 return 2; 00766 } 00767 00769 size_t new_write_mode_and_size(FILE* out, bool insert_zeroes, unsigned int N, unsigned char data_type) 00770 { 00771 size_t nbytes = 0; // nbytes written 00772 unsigned char mode = data_type<<5; 00773 if(insert_zeroes) 00774 mode |= (unsigned char)0x80; 00775 if(N<30) 00776 { 00777 mode |= (unsigned char)N; 00778 putc(mode,out); 00779 nbytes = 1; 00780 } 00781 else if(N<=UCHAR_MAX) 00782 { 00783 putc(mode,out); 00784 putc((unsigned char)N,out); 00785 nbytes = 2; 00786 } 00787 else if(N<=USHRT_MAX) 00788 { 00789 mode |= (unsigned char)30; 00790 putc(mode,out); 00791 unsigned short N2 = (unsigned short)N; 00792 fwrite(&N2,sizeof(unsigned short),1,out); 00793 nbytes = 3; 00794 } 00795 else // (N<=UINT_MAX) 00796 { 00797 mode |= (unsigned char)31; 00798 putc(mode,out); 00799 unsigned int N4 = (unsigned int)N; 00800 fwrite(&N4,sizeof(unsigned int),1,out); 00801 nbytes = 5; 00802 } 00803 return nbytes; 00804 } 00805 00806 size_t new_write_raw_data_as(FILE* out, real *vec, int l, unsigned char data_type) 00807 { 00808 size_t nbytes = 0; // nbytes written 00809 switch(data_type) 00810 { 00811 case 1: 00812 nbytes = l; 00813 while(l--) 00814 { 00815 real val = *vec++; 00816 if(is_missing(val)) 00817 putc(0x80,out); 00818 else 00819 putc((unsigned char)static_cast<signed char>(val),out); 00820 } 00821 break; 00822 case 2: 00823 nbytes = l*sizeof(float); 00824 while(l--) 00825 { 00826 float val = static_cast<float>(*vec++); 00827 fwrite(&val,sizeof(float),1,out); 00828 } 00829 break; 00830 case 3: 00831 nbytes = l*sizeof(double); 00832 while(l--) 00833 { 00834 double val = static_cast<double>(*vec++); 00835 fwrite(&val,sizeof(double),1,out); 00836 } 00837 break; 00838 } 00839 return nbytes; 00840 } 00841 00842 // Warning: this is low-level code written for efficiency 00843 size_t new_write_compressed(FILE* out, real* vec, int l, double tolerance, bool swap_endians) 00844 { 00845 if(swap_endians) 00846 PLERROR("swap_endians in new_write_compressed not yet supported (currently only supported by new_read_compresed"); 00847 00848 size_t nbytes = 0; // number of bytes written 00849 00850 while(l) 00851 { 00852 int nzeroes = 0; 00853 while(l && fast_exact_is_equal(*vec, 0.)) 00854 { 00855 ++nzeroes; 00856 ++vec; 00857 --l; 00858 } 00859 00860 int nvals = 0; 00861 unsigned char data_type = 0; 00862 if(l) 00863 { 00864 real* ptr = vec; 00865 data_type = new_get_compr_data_type(*ptr, tolerance); 00866 ++nvals; 00867 ++ptr; 00868 --l; 00869 while(l && !fast_exact_is_equal(*ptr, 0.) && new_get_compr_data_type(*ptr, tolerance)==data_type) 00870 { 00871 ++nvals; 00872 ++ptr; 00873 --l; 00874 } 00875 } 00876 00877 // Now we know nzeroes, nvals, and data_type 00878 // So let's encode it: 00879 00880 if(nzeroes) // we have zeroes 00881 { 00882 // write the code for zeroes followed by a single value 00883 nbytes += new_write_mode_and_size(out, true, nzeroes, data_type); 00884 if(nvals) // write the following single value 00885 { 00886 nbytes += new_write_raw_data_as(out, vec, 1, data_type); 00887 ++vec; 00888 --nvals; 00889 } 00890 } 00891 00892 if(nvals) // we have some remaining values 00893 { 00894 nbytes += new_write_mode_and_size(out, false, nvals, data_type); 00895 nbytes += new_write_raw_data_as(out, vec, nvals, data_type); 00896 vec += nvals; 00897 } 00898 00899 } // end of for(;;) 00900 return nbytes; 00901 } 00902 00903 } // end of namespace PLearn 00904 00905 00906 /* 00907 Local Variables: 00908 mode:c++ 00909 c-basic-offset:4 00910 c-file-style:"stroustrup" 00911 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00912 indent-tabs-mode:nil 00913 fill-column:79 00914 End: 00915 */ 00916 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :