PLearn 0.1
pl_io.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999,2000 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 // Copyright (C) 2002 Frederic Morin, Xavier Saint-Mleux, Pascal Vincent
00007 //
00008 
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038  
00039 
00040 /* *******************************************************      
00041  * $Id: pl_io.cc 9142 2008-06-18 18:29:34Z nouiz $
00042  * AUTHORS: Pascal Vincent
00043  * This file is part of the PLearn library.
00044  ******************************************************* */
00045 
00046 
00049 //#include <limits>
00050 #include "pl_io.h"
00051 #include <plearn/base/plerror.h>
00052 //#include "pl_math.h"
00053 #include <plearn/base/byte_order.h> 
00054 #include <cstring>
00055 namespace PLearn {
00056 using namespace std;
00057 
00058 
00059 /* compressed array of float format
00060    All data on disk are to be in little-endian format.
00061    bit patterns described from a reading perspective
00062 
00063    Attempt to read sizenum as 1 byte
00064    Read 1 byte.
00065    If the bit pattern is 00000000 (0x00) then the sizenum is the following 2-byte unsigned short : read it
00066    If the bit pattern is 11000000 (0xC0)then the sizenum is the following 4-byte unsigned int : read it
00067    Otherwise the sizenum is the unsigned byte just read
00068 
00069    The two most significant bits of the sizenum are the mode bits, the remaining bits give a size
00070    Mode 00 (decimal 0) means: insert 'size' zeros
00071    Mode 01 (decimal 1) means: insert 'size' zeros and then a 1
00072    Mode 10 (decimal 2) means: insert 'size' values to be found are in the following 'size' signed bytes
00073    Mode 11 (decimal 3) means: insert 'size' values to be found are in the following 'size' floating points (different implementations for single and for double precision below
00074 */
00075 
00076 inline void write_compr_mode_and_size(ostream& out, unsigned char mode, int size)
00077 {
00078 #ifdef BOUNDCHECK
00079     if(size<0 || size>=(1<<30))
00080         PLERROR("In write_compr_mode_and_size: size out of bounds");
00081 #endif
00082     unsigned int imode = (unsigned int) mode;
00083     if(size<(1<<6))
00084     {
00085         unsigned char sizenum = (unsigned char) size | (unsigned char) (imode<<6);
00086         binwrite(out, sizenum);
00087     }
00088     else if(size<(1<<14))
00089     {
00090         unsigned short sizenum = (unsigned short) size | (unsigned short) (imode<<14);
00091         unsigned char header = 0x00;
00092         binwrite(out, header);
00093         binwrite(out, sizenum);
00094     }
00095     else
00096     {
00097         unsigned int sizenum = (unsigned int) size | (unsigned int) (imode<<30);
00098         unsigned char header = 0xC0;
00099         binwrite(out, header);
00100         binwrite(out, sizenum);
00101     }
00102 }
00103 
00104 inline void read_compr_mode_and_size(istream& in, unsigned char& mode, int& size)
00105 {
00106     unsigned char sizenum_byte;
00107     binread(in, sizenum_byte);
00108     if(sizenum_byte==0x00) // sizenum is an unsigned short
00109     {
00110         unsigned short sizenum;
00111         binread(in, sizenum);
00112         mode = (unsigned char)(sizenum>>14);
00113         size = int(sizenum & (unsigned short)0x3FFF);
00114     }
00115     else if(sizenum_byte==0xC0) // sizenum is an unsigned int
00116     {
00117         unsigned int sizenum;
00118         binread(in, sizenum);
00119         mode = (unsigned char)(sizenum>>30);
00120         size = int(sizenum & (unsigned int)0x3FFFFFFF);
00121     }
00122     else // sizenum is the byte we just read
00123     {
00124         mode = sizenum_byte>>6;
00125         size = int(sizenum_byte & (unsigned char)0x3F);
00126     }
00127 }
00128 
00129 void binread_compressed(istream& in, double* data, int l)
00130 {
00131     unsigned char mode;
00132     int n;
00133     double* p = data;
00134     char cval;
00135     while(l>0)
00136     {
00137         read_compr_mode_and_size(in, mode, n);
00138         //cerr << "mode: " << int(mode) << " size: " << n << endl;
00139         l -= n;
00140         switch(mode)
00141         {
00142         case 0:          
00143             while(n--)
00144                 *p++ = 0;
00145             break;
00146         case 1:
00147             while(n--)
00148                 *p++ = 0;
00149             *p++ = 1; 
00150             --l;
00151             break;
00152         case 2:
00153             while(n--)
00154             {
00155                 binread(in,cval);
00156                 *p++ = double(cval);
00157             }
00158             break;
00159         case 3:
00160             binread(in,p,n);
00161             p += n;
00162             break;
00163         default:
00164             PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?");
00165         }
00166     }
00167 
00168     if(l!=0)
00169         PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?");
00170 }
00171 
00172 void binwrite_compressed(ostream& out, const double* data, int l)
00173 {
00174     double val = 0.;
00175     while(l)
00176     {
00177         val = *data;
00178         if(fast_exact_is_equal(val, 0.))
00179         {
00180             int n=0;
00181             while(l && fast_exact_is_equal(*data, 0.))
00182             { ++n; ++data; --l; }
00183             if(l && fast_exact_is_equal(*data, 1.))
00184             {
00185                 write_compr_mode_and_size(out, 1, n);
00186                 ++data; --l;
00187             }
00188             else
00189                 write_compr_mode_and_size(out, 0, n);              
00190         }
00191         else if(fast_exact_is_equal(val,1.))
00192         {
00193             write_compr_mode_and_size(out, 1, 0);
00194             ++data; --l;
00195         }
00196         else if( fast_exact_is_equal(double(char(val)),val) )
00197         {
00198             const double* start = data;
00199             int n=0;
00200             while(l && fast_exact_is_equal(double(char(val=*data)), val)
00201                     && !fast_exact_is_equal(val, 0)
00202                     && !fast_exact_is_equal(val, 1))
00203             { ++n; ++data; --l; }
00204             write_compr_mode_and_size(out, 2, n);
00205             while(n--)
00206                 binwrite(out,char(*start++));
00207         }
00208         else
00209         {
00210             const double* start = data;
00211             int n=0; 
00212             while(l && !fast_exact_is_equal((val=*data), 0)
00213                     && !fast_exact_is_equal(val, 1)
00214                     && !fast_exact_is_equal(double(char(val)), val))
00215             { ++n; ++data; --l; }
00216             write_compr_mode_and_size(out, 3, n);
00217             binwrite(out,start,n);
00218         }
00219     }
00220 }
00221 void binread_compressed(istream& in, float* data, int l)
00222 {
00223     unsigned char mode;
00224     int n;
00225     float* p = data;
00226     while(l>0)
00227     {
00228         read_compr_mode_and_size(in, mode, n);
00229         //cerr << "mode: " << int(mode) << " size: " << n << endl;
00230         if(mode==0 || mode==1)
00231         {
00232             while(n--)
00233             { *p++ = 0; --l; }
00234             if(mode==1)
00235             { *p++ = 1; --l; }
00236         }
00237         else if(mode==2)
00238         {
00239             char val; 
00240             while(n--)
00241             {
00242                 binread(in,val);
00243                 *p++ = float(val);
00244                 --l;
00245             }
00246         }
00247         else if(mode==3)
00248         {
00249             binread(in,p,n);
00250             p += n;
00251             l -= n;
00252         }
00253         else 
00254             PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?");
00255     }
00256 
00257     if(l!=0)
00258         PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?");
00259 }
00260 
00261 void binwrite_compressed(ostream& out, const float* data, int l)
00262 {
00263     float val = 0.;
00264     while(l)
00265     {
00266         val = *data;
00267         if(fast_exact_is_equal(val, 0.))
00268         {
00269             int n=0;
00270             while(l && fast_exact_is_equal(*data, 0.))
00271             { ++n; ++data; --l; }
00272             if(l && fast_exact_is_equal(*data, 1.))
00273             {
00274                 write_compr_mode_and_size(out, 1, n);
00275                 ++data; --l;
00276             }
00277             else
00278                 write_compr_mode_and_size(out, 0, n);              
00279         }
00280         else if(fast_exact_is_equal(val, 1.))
00281         {
00282             write_compr_mode_and_size(out, 1, 0);
00283             ++data; --l;
00284         }
00285         else if( fast_exact_is_equal(float(char(val)), val) )
00286         {
00287             const float* start = data;
00288             int n=0;
00289             while(l && fast_exact_is_equal(float(char(val=*data)), val)
00290                     && !fast_exact_is_equal(val, 0)
00291                     && !fast_exact_is_equal(val, 1))
00292             { ++n; ++data; --l; }
00293             write_compr_mode_and_size(out, 2, n);
00294             while(n--)
00295                 binwrite(out,char(*start++));
00296         }
00297         else
00298         {
00299             const float* start = data;
00300             int n=0; 
00301             while(l && !fast_exact_is_equal((val=*data), 0)
00302                     && !fast_exact_is_equal(val, 1)
00303                     && !fast_exact_is_equal(float(char(val)), val))
00304             { ++n; ++data; --l; }
00305             write_compr_mode_and_size(out, 3, n);
00306             binwrite(out,start,n);
00307         }
00308     }
00309 }
00310 
00311 // ********************************************
00312 // *** compressed vector to and from FILE* ***
00313 // ********************************************
00314 
00315 inline void read_compr_mode_and_size(FILE* in, unsigned char& mode, int& size)
00316 {
00317     unsigned char sizenum_byte;
00318     binread(in, sizenum_byte);
00319     if(sizenum_byte==0x00) // sizenum is an unsigned short
00320     {
00321         unsigned short sizenum;
00322         binread(in, sizenum);
00323         mode = (unsigned char)(sizenum>>14);
00324         size = int(sizenum & (unsigned short)0x3FFF);
00325     }
00326     else if(sizenum_byte==0xC0) // sizenum is an unsigned int
00327     {
00328         unsigned int sizenum;
00329         binread(in, sizenum);
00330         mode = (unsigned char)(sizenum>>30);
00331         size = int(sizenum & (unsigned int)0x3FFFFFFF);
00332     }
00333     else // sizenum is the byte we just read
00334     {
00335         mode = sizenum_byte>>6;
00336         size = int(sizenum_byte & (unsigned char)0x3F);
00337     }
00338 }
00339 
00340 void binread_compressed(FILE* in, double* data, int l)
00341 {
00342     unsigned char mode;
00343     int n;
00344     double* p = data;
00345     char cval;
00346     while(l>0)
00347     {
00348         read_compr_mode_and_size(in, mode, n);
00349         //cerr << "mode: " << int(mode) << " size: " << n << endl;
00350         l -= n;
00351         switch(mode)
00352         {
00353         case 0:          
00354             while(n--)
00355                 *p++ = 0;
00356             break;
00357         case 1:
00358             while(n--)
00359                 *p++ = 0;
00360             *p++ = 1; 
00361             --l;
00362             break;
00363         case 2:
00364             while(n--)
00365             {
00366                 binread(in,cval);
00367                 *p++ = double(cval);
00368             }
00369             break;
00370         case 3:
00371             binread(in,p,n);
00372             p += n;
00373             break;
00374         default:
00375             PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?");
00376         }
00377     }
00378 
00379     if(l!=0)
00380         PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?");
00381 }
00382 
00383 void binwrite_compressed(FILE* out, const double* data, int l)
00384 {
00385     PLERROR("Not implemented");
00386 }
00387 
00388 void binread_compressed(FILE* in, float* data, int l)
00389 {
00390     unsigned char mode;
00391     int n;
00392     float* p = data;
00393     while(l>0)
00394     {
00395         read_compr_mode_and_size(in, mode, n);
00396         //cerr << "mode: " << int(mode) << " size: " << n << endl;
00397         if(mode==0 || mode==1)
00398         {
00399             while(n--)
00400             { *p++ = 0; --l; }
00401             if(mode==1)
00402             { *p++ = 1; --l; }
00403         }
00404         else if(mode==2)
00405         {
00406             char val; 
00407             while(n--)
00408             {
00409                 binread(in,val);
00410                 *p++ = float(val);
00411                 --l;
00412             }
00413         }
00414         else if(mode==3)
00415         {
00416             binread(in,p,n);
00417             p += n;
00418             l -= n;
00419         }
00420         else 
00421             PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?");
00422     }
00423 
00424     if(l!=0)
00425         PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?");
00426 }
00427 
00428 void binwrite_compressed(FILE* out, const float* data, int l)
00429 {
00430     PLERROR("Not implemented");
00431 }
00432 
00433 // ********************************************
00434 // *** compressed vector to and from memory ***
00435 // ********************************************
00436 
00437 inline void write_compr_mode_and_size_ptr(char*& out, unsigned char mode, int size)
00438 {
00439     union {unsigned short s;char cs[2];} unis;
00440     union {unsigned int i;char ci[4];} unii;
00441 
00442     if(sizeof(unsigned int)!=4)
00443         PLERROR("Function write_compr_mode_and_size_ptr is not ready for 64 bit.");
00444 
00445 #ifdef BOUNDCHECK
00446     if(size<0 || size>=(1<<30))
00447         PLERROR("In write_compr_mode_and_size: size out of bounds");
00448 #endif
00449     unsigned int imode = (unsigned int) mode;
00450     if(size<(1<<6))
00451     {
00452         unsigned char sizenum = (unsigned char) size | (unsigned char) (imode<<6);
00453         (*out++) = sizenum;
00454     }
00455     else if(size<(1<<14))
00456     {
00457         unis.s = (unsigned short) size | (unsigned short) (imode<<14);
00458         unsigned char header = 0x00;
00459         (*out++) = header;
00460         (*out++) = unis.cs[0];
00461         (*out++) = unis.cs[1];
00462     }
00463     else
00464     {
00465         unii.i = (unsigned int) size | (unsigned int) (imode<<30);
00466         unsigned char header = 0xC0;
00467         (*out++) = header;
00468         (*out++) = unii.ci[0];
00469         (*out++) = unii.ci[1];
00470         (*out++) = unii.ci[2];
00471         (*out++) = unii.ci[3];
00472     }
00473 }
00474 
00475 inline void read_compr_mode_and_size_ptr(char*& in, unsigned char& mode, int& size)
00476 {
00477     union {unsigned short s;char cs[2];} unis;
00478     union {unsigned int i;char ci[4];} unii;
00479 
00480     unsigned char sizenum_byte;
00481     sizenum_byte = (*in++);
00482     if(sizenum_byte==0x00) // sizenum is an unsigned short
00483     {
00484         unis.cs[0] = (*in++);
00485         unis.cs[1] = (*in++);
00486         mode = (unsigned char)(unis.s>>14);
00487         size = int(unis.s & (unsigned short)0x3FFF);
00488     }
00489     else if(sizenum_byte==0xC0) // sizenum is an unsigned int
00490     {
00491         unii.ci[0] = (*in++);
00492         unii.ci[1] = (*in++);
00493         unii.ci[2] = (*in++);
00494         unii.ci[3] = (*in++);
00495         mode = (unsigned char)(unii.i>>30);
00496         size = int(unii.i & (unsigned int)0x3FFFFFFF);
00497     }
00498     else // sizenum is the byte we just read
00499     {
00500         mode = sizenum_byte>>6;
00501         size = int(sizenum_byte & (unsigned char)0x3F);
00502     }
00503 }
00504 
00505 
00506 void uncompress_vec(char* comprbuf, double* data, int l, bool double_stored_as_float)
00507 {
00508     unsigned char mode;
00509     int n;
00510     double* p = data;
00511     while(l>0)
00512     {
00513         read_compr_mode_and_size_ptr(comprbuf, mode, n);
00514         //cerr << "mode: " << int(mode) << " size: " << n << endl;
00515         if(mode==0 || mode==1)
00516         {
00517             while(n--)
00518             { *p++ = 0; --l; }
00519             if(mode==1)
00520             { *p++ = 1; --l; }
00521         }
00522         else if(mode==2)
00523         {
00524             char val; 
00525             while(n--)
00526             {
00527                 val=(*comprbuf++);
00528                 *p++ = double(val);
00529                 --l;
00530             }
00531         }
00532         else if(mode==3)
00533         {
00534             memcpy(p,comprbuf,sizeof(double)*n);
00535             comprbuf+=sizeof(double)*n;
00536             p += n;
00537             l -= n;
00538         }
00539         else 
00540             PLERROR("BUG IN binread_compressed: mode is only 2 bits, so how can it be other than 0,1,2,3 ?");
00541     }
00542 
00543     if(l!=0)
00544         PLERROR("In binread_compressed : l is not 0 at exit of function, wrong data?");
00545 }
00546 
00547 void compress_vec(char* comprbuf, const double* data, int l, bool double_stored_as_float)
00548 {
00549     //  char* comprbufold=comprbuf;
00550     double val = 0.;
00551     while(l)
00552     {
00553         val = *data;
00554         if(fast_exact_is_equal(val, 0.))
00555         {
00556             int n=0;
00557             while(l && fast_exact_is_equal(*data, 0.))
00558             { ++n; ++data; --l; }
00559             if(l && fast_exact_is_equal(*data, 1.))
00560             {
00561                 write_compr_mode_and_size_ptr(comprbuf, 1, n);
00562                 ++data; --l;
00563             }
00564             else
00565                 write_compr_mode_and_size_ptr(comprbuf, 0, n);              
00566         }
00567         else if(fast_exact_is_equal(val, 1.))
00568         {
00569             write_compr_mode_and_size_ptr(comprbuf, 1, 0);
00570             ++data; --l;
00571         }
00572         else if( fast_exact_is_equal(double(char(val)), val) )
00573         {
00574             const double* start = data;
00575             int n=0;
00576             while(l && fast_exact_is_equal(double(char(val=*data)), val)
00577                     && !fast_exact_is_equal(val, 0)
00578                     && !fast_exact_is_equal(val, 1))
00579             { ++n; ++data; --l; }
00580             write_compr_mode_and_size_ptr(comprbuf, 2, n);
00581             while(n--)
00582                 (*comprbuf++) = char(*start++);
00583         }
00584         else
00585         {
00586             const double* start = data;
00587             int n=0; 
00588             while(l && !fast_exact_is_equal((val=*data), 0)
00589                     && !fast_exact_is_equal(val, 1)
00590                     && !fast_exact_is_equal(double(char(val)), val))
00591             { ++n; ++data; --l; }
00592             write_compr_mode_and_size_ptr(comprbuf, 3, n);
00593             memcpy(comprbuf,start,n*sizeof(double));
00594             comprbuf += n*sizeof(double);
00595         }
00596     }
00597 }
00598 
00599 
00600 // ********************************************************
00601 // ********************************************************
00602 // *********       NEW COMPRESSION FORMAT        **********
00603 // ********************************************************
00604 // ********************************************************
00605 
00606 /*
00607 
00608 Format description:
00609 
00610 A succession of [ mode-byte, optionally followed by specificaitons of length N, followed by data ]
00611 
00612 The way N is encoded will be explained later.
00613 
00614 The bits of the mode-byte are interpreted as follows:
00615 * Most significant bit: 
00616 0 : insert the following N values of type T
00617 1 : insert N zeroes and then the following single value of type T
00618 
00619 * Next 2 bits indicate the data type T
00620 00 (=0): ones (that's just 1.0, no further data is given to provide the value) 
00621 01 (=1): small 1 byte signed integers in the range [-127, +127], or missing values (indicated by -128: bit pattern 0x80)
00622 10 (=2): 4 byte float  
00623 11 (=3): 8 byte double
00624 
00625 In all but the 00 case, 1 or N corresponding values of type T are 
00626 to be read in the stream (after possibly reading N)
00627 
00628 * Next 5 bits (values between 0 .. 31) indicate how to get the number N, 
00629 
00630 1..29: N is that particular value (between 1 and 29)
00631 0 : N is given in the following 1 byte unsigned char
00632 30: N is given in the following 2 byte unsigned short
00633 31: N is given in the following 4 byte unsigned int
00634  
00635 
00636 */
00637 
00638 
00639 size_t new_read_compressed(FILE* in, real* vec, int l, bool swap_endians)
00640 {
00641     size_t nbytes = 0; // number of bytes read
00642     unsigned char mode; // the mode byte
00643     unsigned int N = 0; // N (number of 0s or values to insert) 
00644 
00645     while(l)
00646     {
00647         if(l<0)
00648             PLERROR("Big problem in new_read_compressed: l=%d", l);
00649         int i = getc(in);
00650         if(i == EOF)
00651             PLERROR("Got EOF while expecting more data in the file!");
00652         mode = (unsigned char)(i); 
00653             
00654         ++nbytes;
00655         unsigned char N1 = (mode & 0x1F);
00656         switch(N1)
00657         {
00658         case 0:  // N is the 1 byte to follow
00659             N1 = (unsigned char)(getc(in));
00660             ++nbytes;
00661             N = N1;
00662             break;
00663         case 30: // N is the 2 bytes to follow
00664             unsigned short N2;
00665             fread(&N2,2,1,in);
00666             if(swap_endians)
00667                 endianswap(&N2);
00668             nbytes += 2;
00669             N = N2;
00670             break;
00671         case 31: // N is the 4 bytes to follow
00672             fread(&N,4,1,in);
00673             if(swap_endians)
00674                 endianswap(&N);
00675             nbytes += 4;
00676             break;
00677         default: // N is N1
00678             N = N1;
00679         }
00680 
00681         if(mode & (unsigned char)(0x80)) // most significant bit is on
00682         { // insert N zeros
00683             l -= N;
00684             while(N--)            
00685                 *vec++ = 0;
00686             N = 1;
00687         }
00688 
00689         if(!l)  // vec ends with zeroes, so there's no extra single value to append. We're done!
00690             break;
00691 
00692         l -= N;
00693         mode = ((mode & ~0x80) >> 5); // get the 2 bits we're interested in
00694         switch(mode)
00695         {
00696         case 0: // type ones
00697         {
00698             while(N--)
00699                 *vec++ = 1;
00700         }
00701         break;
00702         case 1: // type signed char (or missing value if -128)
00703         {
00704             signed char val;
00705             nbytes += N;
00706             while(N--)
00707             {
00708                 val = (signed char)(getc(in));
00709                 if(val==-128)
00710                     *vec++ = MISSING_VALUE;
00711                 else
00712                     *vec++ = val;
00713             }
00714         }
00715         break;
00716         case 2: // type float
00717         {
00718             float val;
00719             nbytes += N<<2;
00720             while(N--)
00721             {
00722                 fread(&val,sizeof(float),1,in);
00723                 if(swap_endians)
00724                     endianswap(&val);
00725                 *vec++ = val;
00726             }
00727         }
00728         break;
00729         case 3: // type double
00730         {
00731             nbytes += N<<3;
00732             fread(vec,sizeof(double),N,in);
00733             if(swap_endians)
00734                 endianswap(vec,N);
00735             vec += N;
00736         }
00737         } 
00738     }
00739     return nbytes;
00740 }
00741 
00742 unsigned char new_get_compr_data_type(double x, double tolerance)
00743 {
00744     if(is_missing(x))
00745         return 1;
00746     else if(fast_exact_is_equal(x, 1.))
00747         return 0;
00748     else if(fast_exact_is_equal(double(char(x)), x) &&
00749             !fast_exact_is_equal(x, -128)) // -128 codes for missing value
00750         return 1;
00751     else if(fabs(double(float(x))-x)<=tolerance)
00752         return 2;
00753     return 3;
00754 }
00755 
00756 unsigned char new_get_compr_data_type(float x)
00757 {
00758     if(is_missing(x))
00759         return 1;
00760     else if(fast_exact_is_equal(x, 1.))
00761         return 0;
00762     else if(fast_exact_is_equal(float(char(x)), x) && 
00763             !fast_exact_is_equal(x, -128)) // -128 codes for missing value
00764         return 1;
00765     return 2;
00766 }
00767 
00769 size_t new_write_mode_and_size(FILE* out, bool insert_zeroes, unsigned int N, unsigned char data_type)
00770 {
00771     size_t nbytes = 0; // nbytes written
00772     unsigned char mode = data_type<<5;
00773     if(insert_zeroes)
00774         mode |= (unsigned char)0x80;
00775     if(N<30)
00776     {
00777         mode |= (unsigned char)N;
00778         putc(mode,out);
00779         nbytes = 1;
00780     }
00781     else if(N<=UCHAR_MAX)
00782     {
00783         putc(mode,out);
00784         putc((unsigned char)N,out);
00785         nbytes = 2;
00786     }
00787     else if(N<=USHRT_MAX)
00788     {
00789         mode |= (unsigned char)30;
00790         putc(mode,out);
00791         unsigned short N2 = (unsigned short)N;
00792         fwrite(&N2,sizeof(unsigned short),1,out);
00793         nbytes = 3;
00794     }
00795     else // (N<=UINT_MAX)
00796     {
00797         mode |= (unsigned char)31;
00798         putc(mode,out);
00799         unsigned int N4 = (unsigned int)N;
00800         fwrite(&N4,sizeof(unsigned int),1,out);
00801         nbytes = 5;
00802     }
00803     return nbytes;
00804 }
00805 
00806 size_t new_write_raw_data_as(FILE* out, real *vec, int l, unsigned char data_type)
00807 {
00808     size_t nbytes = 0; // nbytes written
00809     switch(data_type)
00810     {
00811     case 1:
00812         nbytes = l;
00813         while(l--)
00814         {
00815             real val = *vec++;
00816             if(is_missing(val))
00817                 putc(0x80,out);
00818             else
00819                 putc((unsigned char)static_cast<signed char>(val),out);
00820         }
00821         break;
00822     case 2:
00823         nbytes = l*sizeof(float);
00824         while(l--)
00825         {
00826             float val = static_cast<float>(*vec++);
00827             fwrite(&val,sizeof(float),1,out);
00828         }
00829         break;      
00830     case 3:
00831         nbytes = l*sizeof(double);
00832         while(l--)
00833         {
00834             double val = static_cast<double>(*vec++);
00835             fwrite(&val,sizeof(double),1,out);
00836         }
00837         break;
00838     }
00839     return nbytes;
00840 }
00841 
00842 // Warning: this is low-level code written for efficiency
00843 size_t new_write_compressed(FILE* out, real* vec, int l, double tolerance, bool swap_endians)
00844 {
00845     if(swap_endians)
00846         PLERROR("swap_endians in new_write_compressed not yet supported (currently only supported by new_read_compresed");
00847 
00848     size_t nbytes = 0; // number of bytes written
00849 
00850     while(l)
00851     {
00852         int nzeroes = 0;
00853         while(l && fast_exact_is_equal(*vec, 0.))
00854         {
00855             ++nzeroes;
00856             ++vec;
00857             --l;
00858         }
00859 
00860         int nvals = 0;
00861         unsigned char data_type = 0;
00862         if(l)
00863         {
00864             real* ptr = vec;
00865             data_type = new_get_compr_data_type(*ptr, tolerance);
00866             ++nvals;
00867             ++ptr;
00868             --l;
00869             while(l && !fast_exact_is_equal(*ptr, 0.) && new_get_compr_data_type(*ptr, tolerance)==data_type)
00870             {
00871                 ++nvals;
00872                 ++ptr;
00873                 --l;
00874             }
00875         }
00876 
00877         // Now we know nzeroes, nvals, and data_type
00878         // So let's encode it:
00879 
00880         if(nzeroes) // we have zeroes
00881         {
00882             // write the code for zeroes followed by a single value
00883             nbytes += new_write_mode_and_size(out, true, nzeroes, data_type);
00884             if(nvals) // write the following single value
00885             {
00886                 nbytes += new_write_raw_data_as(out, vec, 1, data_type);
00887                 ++vec;
00888                 --nvals;
00889             }
00890         }
00891 
00892         if(nvals) // we have some remaining values
00893         {
00894             nbytes += new_write_mode_and_size(out, false, nvals, data_type);
00895             nbytes += new_write_raw_data_as(out, vec, nvals, data_type);
00896             vec += nvals;
00897         }
00898 
00899     } // end of for(;;)
00900     return nbytes;
00901 }
00902 
00903 } // end of namespace PLearn
00904 
00905 
00906 /*
00907   Local Variables:
00908   mode:c++
00909   c-basic-offset:4
00910   c-file-style:"stroustrup"
00911   c-file-offsets:((innamespace . 0)(inline-open . 0))
00912   indent-tabs-mode:nil
00913   fill-column:79
00914   End:
00915 */
00916 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines