PLearn 0.1
MatIO.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 /* *******************************************************      
00038  * $Id: MatIO.cc 10370 2011-09-23 19:05:51Z nouiz $
00039  * This file is part of the PLearn library.
00040  ******************************************************* */
00041 
00042 #include "MatIO.h"
00043 #include <plearn/base/tostring.h>  
00044 #include "fileutils.h"
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 
00050 /* 
00051    A word on the old PLearn native binary file format for matrices and vectors (.pmat .pvec):
00052 
00053    The file starts with a 64 bytes ASCII ascii header, directly followed by
00054    the data in binary format (a simple memory dump, 4 bytes per value for
00055    floats and 8 bytes for doubles).  The ascii header gives the data type, the
00056    dimensions, and the type of binary representation, as shown in the examples
00057    below. The remaining of the 64 bytes is filled with white spaces (' ') and
00058    a final newline ('\n') The 64th byte must always be a newline, this will be
00059    checked by the loading code.
00060 
00061    The data in these files can be memory mapped if desired.  Notice that
00062    memory-mapping requires that the architecture has the same memory
00063    representation (big endian or little endian) that is used in the file, as
00064    specified in the header (see below) Otherwise you'll get an error message
00065    telling you of the problem.  However traditional loading does not require
00066    this; load will 'translate' the representation if necessary.
00067 
00068    Examples of header lines:
00069 
00070    A 100 element vector of floats (4bytes each) in little endian binary representation:
00071    VECTOR 100 FLOAT LITTLE_ENDIAN
00072 
00073    A 100 element vector of doubles (8 bytes each) in big endian binary representation:
00074    VECTOR 100 double BIG_ENDIAN
00075 
00076    A 20x10 matrix of doubles in little endian binary representation
00077    MATRIX 20 10 DOUBLE LITTLE_ENDIAN
00078 
00079    A 20x10 matrix of floats in big endian binary representation
00080    MATRIX 20 10 FLOAT BIG_ENDIAN
00081 
00082    As a convention, we will use the following file extensions: .pmat and .pvec
00083    (p stands for PLearn) (or possibly .lpmat .bpmat and .lpvec .bpvec if you
00084    wish to keep both a little endian and a big endian version of the matrix in
00085    the same place, to allow memory-mapping on different platforms) 
00086 */
00087 
00088 void loadMat(const string& file_name, TMat<float>& mat)
00089 {
00090     string tmp_file_name = file_name;
00091     bool load_remote =
00092         string_begins_with(file_name, "https:") ||
00093         string_begins_with(file_name, "http:") || 
00094         string_begins_with(file_name, "ftp:");
00095     if(load_remote)
00096     {
00097         tmp_file_name = "/tmp/" + extract_filename(file_name);
00098         string command = "curl --silent " + file_name + " > " + tmp_file_name;
00099         system(command.c_str());
00100     }
00101     string ext = extract_extension(tmp_file_name);
00102     // See if we know the extension...
00103     if(ext==".amat")
00104         loadAscii(tmp_file_name, mat);
00105     else if (ext==".pmat" || ext==".lpmat" || ext==".bpmat")
00106         loadPMat(tmp_file_name,mat);
00107     else // try to guess the format from the header
00108     {
00109         ifstream in(tmp_file_name.c_str());
00110         if(!in)
00111             PLERROR("In loadMat: could not open file %s",tmp_file_name.c_str());      
00112         char c = in.get();
00113         in.close();
00114         if(c=='M') // it's most likely a .pmat format
00115             loadPMat(tmp_file_name,mat);
00116         else 
00117             loadAscii(tmp_file_name,mat);
00118     }
00119     if (load_remote)
00120         rm(tmp_file_name);
00121 }
00122 
00123 void loadMat(const string& file_name, TMat<double>& mat)
00124 {
00125     string tmp_file_name = file_name;
00126     bool load_remote =
00127         string_begins_with(file_name, "https:") ||
00128         string_begins_with(file_name, "http:") || 
00129         string_begins_with(file_name, "ftp:");
00130     if(load_remote)
00131     {
00132         tmp_file_name = "/tmp/" + extract_filename(file_name);
00133         string command = "curl --silent " + file_name + " > " + tmp_file_name;
00134         system(command.c_str());
00135     }
00136     string ext = extract_extension(tmp_file_name);
00137     // See if we know the extension...
00138     if(ext==".amat")
00139         loadAscii(tmp_file_name, mat);
00140     else if (ext==".pmat" || ext==".lpmat" || ext==".bpmat")
00141         loadPMat(tmp_file_name,mat);
00142     else // try to guess the format from the header
00143     {
00144         ifstream in(tmp_file_name.c_str());
00145         if(!in)
00146             PLERROR("In loadMat: could not open file %s",tmp_file_name.c_str());      
00147         char c = in.get();
00148         in.close();
00149         if(c=='M') // it's most likely a .pmat format
00150             loadPMat(tmp_file_name,mat);
00151         else 
00152             loadAscii(tmp_file_name,mat);
00153     }
00154     if (load_remote)
00155         rm(tmp_file_name);
00156 }
00157 
00158 
00159 void loadVec(const string& file_name, TVec<float>& vec)
00160 {
00161     const char* filename = file_name.c_str();
00162     const char* suffix = strrchr(filename,'.');
00163     if (!suffix || strcmp(suffix,".avec")==0)
00164         loadAscii(file_name, vec);
00165     else if (strcmp(suffix,".pvec")==0 || strcmp(suffix,".lpvec")==0 || strcmp(suffix,".bpvec")==0)
00166         loadPVec(file_name,vec);
00167     else
00168         PLERROR("In loadVec: unknown file extension");
00169 }
00170 
00171 void loadVec(const string& file_name, TVec<double>& vec)
00172 {
00173     const char* filename = file_name.c_str();
00174     const char* suffix = strrchr(filename,'.');
00175     if (!suffix || strcmp(suffix,".avec")==0)
00176         loadAscii(file_name, vec);
00177     else if (strcmp(suffix,".pvec")==0 || strcmp(suffix,".lpvec")==0 || strcmp(suffix,".bpvec")==0)
00178         loadPVec(file_name,vec);
00179     else
00180         PLERROR("In loadVec: unknown file extension");
00181 }
00182 
00183 // Old native PLearn format (.pvec and .pmat)
00184 // DATAFILE_HEADER_LENGTH is 64 and is defined in general.h
00185 
00186 void savePVec(const string& filename, const TVec<float>& vec)
00187 {
00188     FILE* f = fopen(filename.c_str(),"wb");
00189     if (!f)
00190         PLERROR("In savePVec, could not open file %s for writing",filename.c_str());
00191 
00192     char header[DATAFILE_HEADERLENGTH]; 
00193 
00194 #ifdef LITTLEENDIAN
00195     sprintf(header,"VECTOR %d FLOAT LITTLE_ENDIAN", vec.length());
00196 #endif
00197 #ifdef BIGENDIAN
00198     sprintf(header,"VECTOR %d FLOAT BIG_ENDIAN", vec.length());
00199 #endif
00200 
00201     // Pad the header with whites and terminate it with '\n'
00202     for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++)
00203         header[pos] = ' ';
00204     header[DATAFILE_HEADERLENGTH-1] = '\n';
00205 
00206     // write the header to the file
00207     fwrite(header,DATAFILE_HEADERLENGTH,1,f);
00208 
00209     // write the data to the file
00210     if(0 < vec.length())
00211     {
00212         const float* p = vec.data();
00213         fwrite(p,sizeof(float),vec.length(),f);
00214     }
00215 
00216     fclose(f);
00217 }
00218 
00219 void savePVec(const string& filename, const TVec<double>& vec)
00220 {
00221     FILE* f = fopen(filename.c_str(),"wb");
00222     if (!f)
00223         PLERROR("In savePVec, could not open file %s for writing",filename.c_str());
00224 
00225     char header[DATAFILE_HEADERLENGTH]; 
00226 
00227 #ifdef LITTLEENDIAN
00228     sprintf(header,"VECTOR %d DOUBLE LITTLE_ENDIAN", vec.length());
00229 #endif
00230 #ifdef BIGENDIAN
00231     sprintf(header,"VECTOR %d DOUBLE BIG_ENDIAN", vec.length());
00232 #endif
00233 
00234     // Pad the header with whites and terminate it with '\n'
00235     for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++)
00236         header[pos] = ' ';
00237     header[DATAFILE_HEADERLENGTH-1] = '\n';
00238 
00239     // write the header to the file
00240     fwrite(header,DATAFILE_HEADERLENGTH,1,f);
00241 
00242     // write the data to the file
00243     if(0 < vec.length())
00244     {
00245         const double* p = vec.data();
00246         fwrite(p,sizeof(double),vec.length(),f);
00247     }
00248 
00249     fclose(f);
00250 }
00251 
00252 void loadPVec(const string& filename, TVec<float>& vec)
00253 {
00254     char header[DATAFILE_HEADERLENGTH];
00255     char matorvec[20];
00256     char datatype[20];
00257     char endiantype[20];
00258     int the_length;
00259 
00260     FILE* f = fopen(filename.c_str(),"rb");
00261     if (!f)
00262         PLERROR("In loadPVec, could not open file %s for reading",filename.c_str());
00263     fread(header,DATAFILE_HEADERLENGTH,1,f);
00264     if(header[DATAFILE_HEADERLENGTH-1]!='\n')
00265         PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00266     sscanf(header,"%s%d%s%s",matorvec,&the_length,datatype,endiantype);
00267     if (strcmp(matorvec,"VECTOR")!=0)
00268         PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00269 
00270     vec.resize(the_length);
00271 
00272     bool is_file_bigendian = false;
00273     if (strcmp(endiantype,"LITTLE_ENDIAN")==0)
00274         is_file_bigendian = false;
00275     else if (strcmp(endiantype,"BIG_ENDIAN")==0)
00276         is_file_bigendian = true;
00277     else
00278         PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.");
00279 
00280     if (strcmp(datatype,"FLOAT")==0)
00281     {
00282         float* p = vec.data();
00283         fread_float(f,p,vec.length(),is_file_bigendian);
00284     }
00285 
00286     else if (strcmp(datatype,"DOUBLE")==0)
00287     {
00288         double* buffer = new double[vec.length()];
00289         float* p = vec.data();
00290         fread_double(f,buffer,vec.length(),is_file_bigendian);
00291         for(int j=0; j<vec.length(); j++)
00292             p[j] = float(buffer[j]);
00293         delete[] buffer;
00294     }
00295 
00296     else
00297         PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.");
00298 
00299     fclose(f);
00300 }
00301 
00302 void loadPVec(const string& filename, TVec<double>& vec)
00303 {
00304     char header[DATAFILE_HEADERLENGTH];
00305     char matorvec[20];
00306     char datatype[20];
00307     char endiantype[20];
00308     int the_length;
00309 
00310     FILE* f = fopen(filename.c_str(),"rb");
00311     if (!f)
00312         PLERROR("In loadPVec, could not open file %s for reading",filename.c_str());
00313     fread(header,DATAFILE_HEADERLENGTH,1,f);
00314     if(header[DATAFILE_HEADERLENGTH-1]!='\n')
00315         PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00316     sscanf(header,"%s%d%s%s",matorvec,&the_length,datatype,endiantype);
00317     if (strcmp(matorvec,"VECTOR")!=0)
00318         PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00319 
00320     vec.resize(the_length);
00321 
00322     bool is_file_bigendian = false;
00323     if (strcmp(endiantype,"LITTLE_ENDIAN")==0)
00324         is_file_bigendian = false;
00325     else if (strcmp(endiantype,"BIG_ENDIAN")==0)
00326         is_file_bigendian = true;
00327     else
00328         PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.");
00329 
00330     if (0 < the_length)
00331     {
00332         if (strcmp(datatype,"FLOAT")==0)
00333         {
00334             float* buffer = new float[vec.length()];
00335             double* p = vec.data();
00336             fread_float(f,buffer,vec.length(),is_file_bigendian);
00337             for(int j=0; j<vec.length(); j++)
00338                 p[j] = double(buffer[j]);
00339             delete[] buffer;
00340         }
00341 
00342         else if (strcmp(datatype,"DOUBLE")==0)
00343         {
00344             double* p = vec.data();
00345             fread_double(f,p,vec.length(),is_file_bigendian);
00346         }
00347       
00348         else
00349             PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.");
00350     }
00351     fclose(f);
00352 }
00353 
00354 
00355 void savePMat(const string& filename, const TMat<float>& mat)
00356 {
00357     FILE* f = fopen(filename.c_str(),"wb");
00358     if (!f)
00359         PLERROR("In savePMat, could not open file %s for writing",filename.c_str());
00360 
00361     char header[DATAFILE_HEADERLENGTH]; 
00362 
00363 #ifdef LITTLEENDIAN
00364     sprintf(header,"MATRIX %d %d FLOAT LITTLE_ENDIAN", mat.length(), mat.width());
00365 #endif
00366 #ifdef BIGENDIAN
00367     sprintf(header,"MATRIX %d %d FLOAT BIG_ENDIAN", mat.length(), mat.width());
00368 #endif
00369 
00370     // Pad the header with whites and terminate it with '\n'
00371     for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++)
00372         header[pos] = ' ';
00373     header[DATAFILE_HEADERLENGTH-1] = '\n';
00374 
00375     // write the header to the file
00376     fwrite(header,DATAFILE_HEADERLENGTH,1,f);
00377 
00378     // write the data to the file
00379     for (int i=0; i<mat.length(); i++) 
00380     {
00381         const float* p = mat[i];
00382         fwrite(p,sizeof(float),mat.width(),f);
00383     }
00384     fclose(f);
00385 }
00386 
00387 void savePMat(const string& filename, const TMat<double>& mat)
00388 {
00389     FILE* f = fopen(filename.c_str(),"wb");
00390     if (!f)
00391         PLERROR("In savePMat, could not open file %s for writing",filename.c_str());
00392 
00393     char header[DATAFILE_HEADERLENGTH]; 
00394 
00395 #ifdef LITTLEENDIAN
00396     sprintf(header,"MATRIX %d %d DOUBLE LITTLE_ENDIAN", mat.length(), mat.width());
00397 #endif
00398 #ifdef BIGENDIAN
00399     sprintf(header,"MATRIX %d %d DOUBLE BIG_ENDIAN", mat.length(), mat.width());
00400 #endif
00401 
00402     // Pad the header with whites and terminate it with '\n'
00403     for(size_t pos=strlen(header); pos<DATAFILE_HEADERLENGTH; pos++)
00404         header[pos] = ' ';
00405     header[DATAFILE_HEADERLENGTH-1] = '\n';
00406 
00407     // write the header to the file
00408     fwrite(header,DATAFILE_HEADERLENGTH,1,f);
00409 
00410     // write the data to the file
00411     for (int i=0; i<mat.length(); i++) 
00412     {
00413         const double* p = mat[i];
00414         fwrite(p,sizeof(double),mat.width(),f);
00415     }
00416     fclose(f);
00417 }
00418 
00419 void savePMatFieldnames(const string& pmatfilename, const TVec<string>& fieldnames)
00420 {
00421     string metadatadir = pmatfilename+".metadata";
00422     if(!isdir(metadatadir))
00423         force_mkdir(metadatadir);
00424     string fname = metadatadir+"/fieldnames";
00425     FILE* f = fopen(fname.c_str(),"w");
00426     if(!f)
00427         PLERROR("Could not open file %s for writing",fname.c_str());
00428     for(int i= 0; i < fieldnames.length(); ++i)
00429         fprintf(f,"%s\t0\n",fieldnames[i].c_str());
00430     fclose(f);
00431 }
00432 
00433 
00434 void loadPMat(const string& filename, TMat<float>& mat)
00435 {
00436     char header[DATAFILE_HEADERLENGTH];
00437     char matorvec[20];
00438     char datatype[20];
00439     char endiantype[20];
00440     int the_length;
00441     int the_width;
00442 
00443     FILE* f = fopen(filename.c_str(),"rb");
00444     if (!f)
00445         PLERROR("In loadPMat, could not open file %s for reading",filename.c_str());
00446     fread(header,DATAFILE_HEADERLENGTH,1,f);
00447     if(header[DATAFILE_HEADERLENGTH-1]!='\n')
00448         PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00449     sscanf(header,"%s%d%d%s%s",matorvec,&the_length,&the_width,datatype,endiantype);
00450     if (strcmp(matorvec,"MATRIX")!=0)
00451         PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00452 
00453     mat.resize(the_length, the_width);
00454 
00455     bool is_file_bigendian = true;
00456     if (strcmp(endiantype,"LITTLE_ENDIAN")==0)
00457         is_file_bigendian = false;
00458     else if (strcmp(endiantype,"BIG_ENDIAN")==0)
00459         is_file_bigendian = true;
00460     else
00461         PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.");
00462 
00463     if (strcmp(datatype,"FLOAT")==0)
00464     {
00465         for (int i=0; i<mat.length(); i++)
00466         {
00467             float* p = mat[i];
00468             fread_float(f,p,mat.width(),is_file_bigendian);
00469         }
00470     }
00471     else if (strcmp(datatype,"DOUBLE")==0)
00472     {
00473         double* buffer = new double[mat.width()];
00474         for (int i=0; i<mat.length(); i++)
00475         {
00476             float* p = mat[i];
00477             fread_double(f,buffer,mat.width(),is_file_bigendian);
00478             for(int j=0; j<mat.width(); j++)
00479                 p[j] = float(buffer[j]);
00480         }
00481         delete[] buffer;
00482     }
00483 
00484     else
00485         PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.");
00486 
00487     fclose(f);
00488 }
00489 
00490 void loadPMat(const string& filename, TMat<double>& mat)
00491 {
00492     char header[DATAFILE_HEADERLENGTH];
00493     char matorvec[20];
00494     char datatype[20];
00495     char endiantype[20];
00496     int the_length=0;
00497     int the_width=0;
00498 
00499     FILE* f = fopen(filename.c_str(),"rb");
00500     if (!f)
00501         PLERROR("In loadPMat, could not open file %s for reading",filename.c_str());
00502     fread(header,DATAFILE_HEADERLENGTH,1,f);
00503     if(header[DATAFILE_HEADERLENGTH-1]!='\n')
00504         PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00505     sscanf(header,"%s%d%d%s%s",matorvec,&the_length,&the_width,datatype,endiantype);
00506     if (strcmp(matorvec,"MATRIX")!=0)
00507         PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str());
00508 
00509     mat.resize(the_length, the_width);
00510 
00511     bool is_file_bigendian = true;
00512     if (strcmp(endiantype,"LITTLE_ENDIAN")==0)
00513         is_file_bigendian = false;
00514     else if (strcmp(endiantype,"BIG_ENDIAN")==0)
00515         is_file_bigendian = true;
00516     else
00517         PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.");
00518 
00519     if (strcmp(datatype,"FLOAT")==0)
00520     {
00521         float* buffer = new float[mat.width()];
00522         for (int i=0; i<mat.length(); i++)
00523         {
00524             double* p = mat[i];
00525             fread_float(f,buffer,mat.width(),is_file_bigendian);
00526             for(int j=0; j<mat.width(); j++)
00527                 p[j] = double(buffer[j]);
00528         }
00529         delete[] buffer;
00530     }
00531 
00532     else if (strcmp(datatype,"DOUBLE")==0)
00533     {
00534         for (int i=0; i<mat.length(); i++)
00535         {
00536             double* p = mat[i];
00537             fread_double(f,p,mat.width(),is_file_bigendian);
00538         }
00539     }
00540 
00541     else
00542         PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.");
00543 
00544     fclose(f);
00545 }
00546 
00547 /*
00548   void newLoadAscii(const string& filename, TMat<double>& mat)
00549   {
00550   ifstream in(filename.c_str());
00551   if(!in)
00552   PLERROR("Could not open file %s for reading", filename.c_str());
00553 
00554   int length = -1;
00555   int width = -1;
00556   bool could_be_old_amat=true; // true while there is still a chance that this be an "old" amat format (length and width in first row with no starting ##)
00557   int c = in.get();
00558   while(isspace(c))
00559   c = in.get();
00560 
00561   if(c=='#') // starts with a comment
00562   {
00563   could_be_old_amat = false;
00564 
00565   // If it's followed by another # and only 2 numbers before the end of line, it's a new .amat format
00566   if(in.get()=='#')
00567   {
00568   in >> length;
00569   in >> width;
00570   }
00571   c = in.get();
00572   while(c==' ' || c=='\t')
00573   c = in.get();
00574   if(c!='\n' && c!='\r') // it wasn't a new .amat format after all... 
00575   length = -1;
00576   }
00577 
00578   if(length==-1)  // still looking for size info...
00579   {      
00580   in.unget();
00581   string line;
00582   getNextNonBlankLine(in,line);
00583   int nfields1 = split(line).size();
00584   getNextNonBlankLine(in,line);
00585   int nfields2 = split(line).size();
00586   if(could_be_old_amat && nfields1==2) // could be an old .amat with first 2 numbers being length width
00587   {
00588   in.seekg(0);
00589   real a, b;
00590   in >> a >> b;
00591   if(real(int(a))==a && real(int(b))==b && a>0 && b>0 && int(b)==nfields2) // it's clearly an old .amat
00592   {
00593   length = int(a);
00594   width = int(b);
00595   }
00596   }
00597 
00598   if(length==-1) // still don't know size info...
00599   {
00600   if(nfields1==nfields2) // looks like a plain ascii file
00601   {
00602   length = countNonBlankLinesOfFile(filename);
00603   width = nfields1;
00604   in.seekg(0); 
00605   }
00606   }
00607   }
00608 
00609   if(length==-1)
00610   PLERROR("In loadAscii: couldn't determine file format automatically");
00611 
00612   mat.resize(length,width);
00613   for(int i=0; i<length; i++)
00614   {
00615   real* mat_i = mat[i];
00616   skipBlanks(in);
00617   for(int j=0; j<width; j++)
00618   in >> mat_i[j];
00619   }
00620 
00621   }
00622 */
00623 
00624 // Gnuplot format
00625 void saveGnuplot(const string& filename, const Vec& vec)
00626 {
00627     FILE* f=fopen(filename.c_str(),"w");
00628     if (!f) 
00629         PLERROR("In Vec::saveGnuplot, couldn't open %s for writing",filename.c_str());
00630 
00631     real* p = vec.data();
00632     for (int i=0; i<vec.length(); i++, p++)
00633         fprintf(f,"%e\n", *p);
00634     fclose(f);
00635 }
00636 
00637 // Gnuplot format
00638 void saveGnuplot(const string& filename, const Mat& mat)
00639 {
00640     ofstream out(filename.c_str());
00641     if (!out) 
00642         PLERROR("In saveGnuplot, couldn't open %s for writing.",filename.c_str());
00643     out.flags(ios::left);
00644     for(int i=0; i<mat.length(); i++)
00645     {
00646         const real* m_i = mat[i];
00647         for(int j=0; j<mat.width(); j++)
00648             out << setw(11) << m_i[j] << ' ';
00649         out << "\n";
00650     }
00651     out.flush();
00652 }
00653 
00654 void loadGnuplot(const string& filename, Mat& mat)
00655 {
00656     ifstream in(filename.c_str());
00657     if (!in) 
00658         PLERROR("In loadGnuplot, couldn't open %s for reading.",filename.c_str());
00659 
00660     char buf[10000];
00661     // First pass to count the number of rows and columns
00662     int nrows = 0;
00663     int ncols = 0;
00664     in.getline(buf,sizeof(buf)-1);
00665     while(in)
00666     {
00667         int pos=0;
00668         while(buf[pos]==' ' || buf[pos]=='\t')
00669             pos++;
00670         if(buf[pos]!='#' && buf[pos]!='\n' && buf[pos]!='\r')
00671         {
00672             nrows++;
00673             if(ncols==0)
00674             {
00675                 istringstream inputline(buf);
00676                 real value;
00677                 while(inputline)
00678                 {
00679                     inputline >> value;
00680                     ncols++;
00681                 }
00682                 ncols--; // correct count
00683             }
00684         }
00685         in.getline(buf,sizeof(buf)-1);
00686     }
00687     in.close();
00688     mat.resize(nrows,ncols);
00689     in.open(filename.c_str());
00690     for(int i=0; i<nrows; i++)
00691     {
00692         char firstchar = '#';
00693         while(firstchar == '#' || firstchar == '\n' || firstchar=='\r')
00694         {
00695             in.getline(buf,sizeof(buf)-1);
00696             int pos=0;
00697             while(buf[pos]==' ' || buf[pos]=='\t')
00698                 pos++;
00699             firstchar = buf[pos];
00700         }
00701         istringstream inputline(buf);      
00702         for(int j=0; j<ncols; j++)
00703             inputline >> mat(i,j);
00704     }
00705     in.close();
00706 }
00707 
00708 void matlabSave( const PPath& dir, const string& plot_title, const Vec& data, 
00709                  const Vec& add_col, const Vec& bounds, string legend, bool save_plot)
00710 {
00711     Vec bidon;
00712     Mat mat(data.length(), 1);
00713     mat << data;
00714     TVec<string> legd;
00715     if(legend != "")
00716         legd.append(legend);
00717     matlabSave(dir, plot_title, 
00718                bidon,
00719                mat, add_col, bounds, legd, save_plot);
00720 }
00721 
00722 void matlabSave( const PPath& dir, const string& plot_title, 
00723                  const Vec& xValues, 
00724                  const Vec& yValues, const Vec& add_col, const Vec& bounds, string legend, bool save_plot)
00725 {
00726     Mat mat(yValues.length(), 1);
00727     mat << yValues;
00728     TVec<string> legd;
00729     if(legend != "")
00730         legd.append(legend);
00731     matlabSave(dir, plot_title, 
00732                xValues,
00733                mat, add_col, bounds, legd, save_plot);
00734 }
00735 
00736 void matlabSave( const PPath& dir, const string& plot_title, const Mat& data, 
00737                  const Vec& add_col, const Vec& bounds, TVec<string> legend, bool save_plot)
00738 {
00739     Vec bid;
00740     matlabSave(dir, plot_title, bid, data, add_col, bounds, legend, save_plot);
00741 }
00742 
00751 void matlabSave( const PPath& dir, const string& plot_title, 
00752                  const Vec& xValues,
00753                  const Mat& yValues, const Vec& add_col, const Vec& bounds, TVec<string> legend, bool save_plot)
00754 {
00755     force_mkdir(dir);  
00756     PPath directory = dir.absolute();
00757     force_mkdir(directory / "Images" / "");
00758   
00759     int w = yValues.width();
00760 
00761     ofstream out;
00762     PPath vec_fname = directory / plot_title + ".mmat";
00763     out.open(vec_fname.absolute().c_str(), ofstream::out | ofstream::trunc);
00764 
00765     real startX = 0.0;
00766     int xLen = xValues.length(); 
00767     if(xLen != 0)
00768     {
00769         if(xLen == yValues.length())
00770             startX = MISSING_VALUE;
00771         else if(xLen == 1)
00772             startX = xValues[0];
00773         else
00774             PLERROR("matlabSave:\n"
00775                     "1) If xValues is empty, the yValues are plotted against the row indices.\n"
00776                     "2) If xValues is not empty and its length is not equal to the length of yValues, \n"
00777                     "then its length must be one and the value xValues[0] will be the start index for the xValues.");
00778     }
00779 
00780     for(int d = 0; d < yValues.length(); d++)
00781     {
00782         if(is_missing(startX))
00783             out << xValues[d] << "\t";
00784         else
00785             out << (startX+d) << "\t";
00786     
00787         for(int col=0; col < w; col++)
00788             out << yValues(d, col) << "\t";
00789     
00790         for(int add=0; add < add_col.length(); add++)
00791             out << add_col[add] << "\t";
00792         out << endl;
00793     }
00794     out.close();
00795   
00796     PPath m_fname = directory / plot_title + ".m";
00797     out.open(m_fname.absolute().c_str(), ofstream::out | ofstream::trunc);
00798     out << "load " << vec_fname.absolute() << " -ascii"   << endl
00799         << plot_title << "= sortrows(" << plot_title << ")" << endl
00800         << "h = plot(" 
00801 //--- X Values
00802         << plot_title << "(:,1), "
00804 //--- Y Values
00805         << plot_title << "(:,2:" << (1+w) << "));"  << endl
00807         << "set(h, 'LineWidth', 1.0)" << endl
00808         << "set(gcf, 'Position', [0, 0, 1000, 750])" << endl
00809         << "hold on" << endl;
00810   
00811     if(legend.isNotEmpty())
00812     {
00813         int leg = legend.length();
00814         int wid = yValues.width();
00815         if(leg != wid)
00816         {
00817             if(legend[0] == "Numbers")
00818             {
00819                 legend.resize(wid);
00820                 for(int c=0; c < wid; c++)
00821                     legend[c] = tostring(c);
00822             }
00823             else
00824                 PLERROR("TimeSeriesAnalysis::matlab_save: legend.length() = %d != %d = yValues.width()",
00825                         leg, wid);
00826         }
00827         out << "legend(h";
00828         for(int l=0; l < leg; l++)
00829         {
00830             legend[l] = underscore_to_space(legend[l]);
00831             out << ", '" << legend[l] << "'"; 
00832         }
00833         out << ");" << endl;
00834     }
00835   
00836     for(int add=0; add < add_col.length(); add++)
00837         out << "g = plot(" << plot_title 
00838             << "(:," << (2+w+add) << "));"
00839             << endl
00840             << "set(g, 'Color', [0.5 0.5 0.5])" << endl;
00841   
00842     if(bounds.isNotEmpty())
00843         out << "xlim([" << bounds[0] << ", " << bounds[1] << "])" << endl
00844             << "ylim([" << bounds[2] << ", " << bounds[3] << "])" << endl;
00845   
00846     out << "title('" << underscore_to_space(plot_title) << "')" << endl;
00847   
00848     if(save_plot)
00849         out << "print('-dpsc2', '" 
00850             << (directory / "Images" / "").absolute()
00851             << plot_title << ".eps')" << endl;
00852   
00853     out.close();
00854 }
00855 
00856 // Ascii without size
00857 void saveAsciiWithoutSize(const string& filename, const Vec& vec)
00858 {
00859     FILE *f;
00860     f=fopen(filename.c_str(),"w");
00861     if (!f)
00862         PLERROR("In Vec::saveAscii: could not open file %s for writing",filename.c_str());
00863     int i;
00864     char buffer[100];
00865     real *p= vec.data();
00866     for (i=0;i<vec.length();i++,p++)
00867     {
00868         pretty_print_number(buffer,*p);
00869         fprintf(f,"%s ",buffer);
00870     }
00871     fclose(f);
00872 }
00873 
00874 // Ascii without size (load assumes size is already correctly set)
00875 void loadAsciiWithoutSize(const string& filename, const Vec& vec)
00876 {
00877     FILE *f;
00878     f=fopen(filename.c_str(),"r");
00879     if (!f)
00880         PLERROR("In Vec::loadAsciiWithoutSize could not open file %s for reading",filename.c_str());
00881 
00882     if (vec.length() < 1)
00883         PLERROR("In Vec::loadAsciiWithoutSize, the size of the vector is not defined yet");
00884 
00885     real* p = vec.data();
00886     for (int i=0;i<vec.length();i++,p++)
00887     {
00888 #ifdef USEDOUBLE
00889         fscanf(f,"%lf",p);
00890 #else
00891         fscanf(f,"%f",p);
00892 #endif
00893     }
00894 }
00895 
00896 void saveAsciiWithoutSize(const string& filename, const Mat& mat)
00897 {
00898     FILE *f;
00899     f=fopen(filename.c_str(),"w");
00900     if (!f)
00901         PLERROR("In saveAscii, could not open file %s for writing",filename.c_str());
00902     char buffer[100];
00903     for(int i=0; i<mat.length(); i++) 
00904     {
00905         const real* row_i = mat[i];
00906         for(int j=0; j<mat.width(); j++)
00907         {
00908             pretty_print_number(buffer,row_i[j]);
00909             fprintf(f,"%s ",buffer);
00910         }
00911         fprintf(f,"\n");
00912     }
00913     fclose(f);
00914 }
00915 
00916 void loadAsciiWithoutSize(const string& filename, const Mat& mat)
00917 {
00918 
00919     FILE *f = fopen(filename.c_str(),"r");
00920     if (!f)
00921         PLERROR("In loadAsciiWithoutSize, could not open file %s for reading.",filename.c_str());
00922 
00923     if (mat.length() < 1 || mat.width() < 1)
00924         PLERROR("In loadAsciiWithoutSize, the size of the matrix is not defined yet");
00925 
00926     for(int i=0; i<mat.length(); i++)
00927     {
00928         real* row_i = mat[i];
00929         for(int j=0; j<mat.width(); j++)
00930 #ifdef USEDOUBLE
00931             fscanf(f,"%lf",&row_i[j]);
00932 #else
00933         fscanf(f,"%f",&row_i[j]);
00934 #endif
00935     }
00936 }
00937 
00938 
00939 // Native SN format (Fmat)
00940 void saveSNMat(const string& filename, const Mat& mat)
00941 {
00942     FILE *f=fopen(filename.c_str(),"wb");
00943     int i=0x1e3d4c51L;
00944     int j=0;
00945     fwrite_int(f,&i,1);
00946     i=2; /*  number of dimensions = 2 for a matrix  */
00947     fwrite_int(f,&i,1);
00948     int length = mat.length();
00949     int width = mat.width();
00950     fwrite_int(f,&length,1);
00951     fwrite_int(f,&width,1);
00952     while (i++ < 3) fwrite_int(f,&j,1);
00953     for (i=0;i<length;i++)
00954         fwrite_float(f,mat[i],width);
00955     fclose(f);
00956 }
00957 
00958 Mat loadSNMat(const string& filename)
00959 {
00960     FILE *f;
00961     int d, nd; int i;
00962     int imn;
00963     int length, width;
00964 
00965     f=fopen(filename.c_str(),"rb");
00966     if (!f)
00967         PLERROR("In loadFmat, could not open file %s for reading",filename.c_str());
00968 
00969     fread_int(f,&imn,1);
00970     if (imn!= (0x1e3d4c51L))
00971         PLERROR("in loadFmat, File does not have the right format");
00972 
00973     /*  read ndim  */
00974     fread_int(f,&nd,1);
00975     if (nd<0 || nd>5)
00976         PLERROR("In loadFmat, Corrupted file: Bad number of dimensions");
00977     if (nd != 2)
00978         PLERROR("In loadFmat, ndim is not 2 (not a matrix!)\n");
00979 
00980     /*  read dim  */
00981     d=0;
00982     fread_int(f,&length,1);
00983     d++;
00984     fread_int(f,&width,1);
00985     d++;
00986     while (d++ < 3) 
00987         fread(&i, sizeof(int), 1, f);
00988     Mat mat(length,width);
00989     for(i=0; i<length; i++)
00990         fread_float(f, mat[i], width);
00991     fclose(f);
00992     return mat;
00993 }
00994 
00995 void saveSNVec(const string& filename, const Vec& vec)
00996 {
00997     FILE* f=fopen(filename.c_str(),"wb");
00998     if(!f)
00999         PLERROR("In Vec::loadSNVec could not open file for writing");
01000     int i=0x1e3d4c51L;
01001     int j=0;
01002     fwrite_int(f,&i,1);
01003     i=1; /*  number of dimensions = 1 for a vector  */
01004     fwrite_int(f,&i,1);
01005     int length = vec.length();
01006     fwrite_int(f,&length,1);
01007     while (i++ < 3) 
01008         fwrite_int(f,&j,1);
01009     fwrite_float(f,vec.data(),length);
01010     fclose(f);
01011 }
01012 
01013 Vec loadSNVec(const string& filename)
01014 {
01015     FILE *f;
01016     int d, nd; int i;
01017     int imn;
01018     int size;
01019 
01020     f=fopen(filename.c_str(),"rb");
01021     if (!f)
01022         PLERROR("In Vec::loadSNVec could not open file %s for reading",filename.c_str());
01023 
01024     fread_int(f,&imn,1);
01025     if (imn!= (0x1e3d4c51L))
01026         PLERROR("In Vec::loadSNVec, File does not have the right format");
01027 
01028     /*  read ndim  */
01029     fread_int(f,&nd,1);
01030     if (nd<0 || nd>5)
01031         PLERROR("In Vec::loadSNVec, Corrupted file: Bad number of dimensions");
01032     if (nd != 1)
01033         PLERROR("In Vec::loadSNVec, ndim is not 1 (not a vector!)\n");
01034 
01035     /*  read dim  */
01036     d=0;
01037     fread_int(f,&i,1);
01038     size=i;d++;
01039     while (d++ < 3) 
01040         fread(&i, sizeof(int), 1, f);
01041 
01042     Vec vec(size);
01043     fread_float(f,vec.data(),size);
01044 
01045     fclose(f);
01046     return vec;
01047 }
01048 
01049 
01050 // Native AD format
01051 
01052 Mat loadADMat(const string& filename)
01053 {
01054     FILE *f = fopen(filename.c_str(),"rb");
01055     if (!f)
01056         PLERROR("In loadADMat, could not open file %s for reading",filename.c_str());
01057     int the_length, the_width;
01058     int magic = 0x2345;
01059     int SNidx2fltmagic = 0x0D02;
01060     int m;
01061     fread_int(f,&m,1);
01062     if (m != magic && m != SNidx2fltmagic)
01063         PLERROR("In load, magic number is incorrect: %d != %d",m,magic);
01064     fread_int(f,&the_length,1);
01065     fread_int(f,&the_width,1);
01066     Mat mat(the_length,the_width);
01067     fread_float(f,mat.data(),the_length*the_width);
01068     fclose(f);
01069     return mat;
01070 }
01071 
01072 Vec loadADVec(const string& filename)
01073 {
01074     FILE* f = fopen(filename.c_str(),"rb");
01075     if (!f)
01076         PLERROR("In Vec::loadADMat could not open file %s for reading",filename.c_str());
01077     int thesize;
01078     int magic = 0x3456;
01079     int m;
01080     fread_int(f,&m,1);
01081     if (m != magic)
01082         PLERROR("In new_Vec_from_File_FILE, magic number is incorret: %d != %d",m,magic);
01083     fread_int(f,&thesize,1);
01084     Vec vec(thesize);
01085     fread_float(f,vec.data(),thesize);
01086     fclose(f);
01087     return vec;
01088 }
01089 
01090 // Used for calling qsort
01091 static int compare_string_pointers(const void *ts1, const void *ts2) 
01092 {
01093     return strcmp(*((char **)ts1),*((char **)ts2));
01094 }
01095 
01096 
01097 // UCI machine-learning-database format
01098 Mat loadUCIMLDB(const string& filename, char ****to_symbols, int **to_n_symbols, TVec<int>* the_max_in_col, TVec<string>* header_columns)
01099 {
01100     FILE *f = fopen(filename.c_str(),"r");
01101     int n_rows= -1, n_cols=1, i,j;
01102     char ***symbols;
01103     int *n_symbols;
01104 #define convert_UCIMLDB_BUF_LEN 10000
01105     char buffer[convert_UCIMLDB_BUF_LEN];
01106     char *cp=buffer;
01107     char *word=buffer;
01108     char *cp2;
01109     real *p;
01110     size_t line_len;
01111 
01112     if (!f)
01113         PLERROR("In loadUCIMLDB, could not open file %s for reading",filename.c_str());
01114 
01115     if((to_symbols && !to_n_symbols) || (!to_symbols && to_n_symbols))
01116         PLERROR("In loadUCIMLDB, to_symbols and to_nsymbols must both be provided (non-null), or both be 0");
01117 
01118     /*  first figure out number of columns and number of rows  */
01119     bool skip_header = false;
01120     if (header_columns) {
01121         skip_header = true;
01122     }
01123     while (!feof(f))
01124     {
01125         do {
01126             fgets(buffer,convert_UCIMLDB_BUF_LEN,f);
01127         } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0));
01128         if (skip_header) {
01129             skip_header = false;
01130         } else {
01131             if (n_rows == -1) {
01132                 /*  read number of columns  */
01133                 while ((cp=strchr(cp,',')))
01134                 {
01135                     cp++;
01136                     n_cols++;
01137                 }
01138             }
01139             n_rows++;
01140         }
01141     }
01142 
01143     fclose(f);
01144 
01145     /*  figure out the set of symbols used for each symbolic row, if any  */
01146     symbols = (char ***)calloc(n_cols,sizeof(char **));
01147     n_symbols = (int *)calloc(n_cols,sizeof(int));
01148 
01149     TVec<int>* max_in_col;
01150     if (the_max_in_col) {
01151         max_in_col = the_max_in_col;
01152     } else {
01153         max_in_col = new TVec<int>();
01154     }
01155     max_in_col->resize(n_cols);
01156     max_in_col->fill(-1);
01157     if(to_symbols)
01158     {
01159         *to_symbols = symbols;
01160         *to_n_symbols = n_symbols;
01161     }
01162     f = fopen(filename.c_str(),"r");
01163 
01164   
01165     /* read header columns */
01166     if (header_columns) {
01167         do {
01168             fgets(buffer,convert_UCIMLDB_BUF_LEN,f);
01169         } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0));
01170 
01171         cp=word=buffer;
01172 
01173         while ((cp=strchr(cp,','))) {
01174             *cp=0;
01175             header_columns->append(word);
01176             cp++;
01177             word=cp;
01178         }
01179         header_columns->append(word);
01180     }
01181 
01182     for (i=0;i<n_rows;i++)
01183     {
01184         do {
01185             fgets(buffer,convert_UCIMLDB_BUF_LEN,f);
01186         } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0));
01187 
01188   
01189         // ignore everything after '|'
01190         char *comm = strchr(buffer,'|');
01191         if (comm) *comm = '\n';
01192     
01193         line_len=strlen(buffer);
01194         cp=word=buffer;
01195         for (j=0;j<n_cols;j++)
01196         {
01197             /*  find next end of word  */
01198             while ((*cp!=',' && *cp!='\n') && cp<=buffer+line_len) cp++;
01199             *cp=0;
01200             /*  is this symbolic?  */
01201             cp2=word;
01202             string the_val = word;
01203       
01204             if (!pl_isnumber(word) && *cp2!='?') { 
01205                 /*  yes, non-missing symbolic character was found:  */
01206                 if (symbols[j])
01207                 { 
01208                     /*  we already had found symbols in this column  */
01209                     int w=0;
01210                     while (symbols[j][w] &&  /*  look for this symbol  */
01211                            strcmp(symbols[j][w],word)!=0 &&
01212                            w<n_symbols[j]) w++;
01213                     if (w==n_rows)
01214                         PLERROR("logic error in loadUCIMLDB");
01215                     if (!symbols[j][w])
01216                     {
01217                         /*  new symbol  */
01218                         symbols[j][w] = (char *)calloc(strlen(word)+1,sizeof(char));
01219                         strcpy(symbols[j][w],word);
01220                         n_symbols[j]++;
01221                     }
01222                 }
01223                 else
01224                 {
01225                     /*  it's the first time we find a symbol in this column  */
01226                     symbols[j] = (char **)calloc(n_rows,sizeof(char *));
01227                     symbols[j][0] = (char *)calloc(strlen(word)+1,sizeof(char));
01228                     strcpy(symbols[j][0], word);
01229                     n_symbols[j]=1;
01230                 }
01231             } else {
01232                 // This is a numerical character: we use it to keep track of the
01233                 // maximum in this column.
01234                 real real_val;
01235                 if (the_val != "?" && pl_isnumber(the_val, &real_val)) {
01236                     if (int(real_val) > (*max_in_col)[j]) {
01237                         (*max_in_col)[j] = int(real_val);
01238                     }
01239                 }
01240             }
01241             word = cp+1;
01242         }
01243     }
01244     fclose(f);
01245 
01246     /*  sort the symbols  */
01247     for (j=0;j<n_cols;j++)
01248         if (symbols[j]) /*  it has symbols  */
01249             qsort(symbols[j],n_symbols[j],sizeof(char *),compare_string_pointers);
01250 
01251     Mat mat(n_rows,n_cols);
01252     /*  NOW actually READ the data  */
01253     {
01254         p = mat.data();
01255         f = fopen(filename.c_str(),"r");
01256 
01257         // skip one line if header present
01258         if (header_columns) {
01259             do {
01260                 fgets(buffer,convert_UCIMLDB_BUF_LEN,f);
01261             } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0));
01262         }
01263 
01264 
01265         for (i=0;i<n_rows;i++)
01266         {
01267             /*  read a row  */
01268             do {
01269                 fgets(buffer,convert_UCIMLDB_BUF_LEN,f);
01270             } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0));
01271 
01272       
01273             // ignore everything after '|'
01274             char *comm = strchr(buffer,'|');
01275             if (comm) *comm = '\n';
01276 
01277             line_len=strlen(buffer);
01278             cp=word=buffer;
01279             /*  interpret a row  */
01280             for (j=0;j<n_cols;j++)
01281             {
01282                 /*  find end of word  */
01283                 while ((*cp!=',' && *cp!='\n') && cp<=buffer+line_len) cp++;
01284                 *cp=0; // Make 'word' point to the current field value only.
01285                 // Skip blanks at beginning.
01286                 while (*word == ' ') {
01287                     word++;
01288                     if (word >= cp)
01289                         PLERROR("In loadUCIMLDB - Error while skipping blanks");
01290                 }
01291                 /*  is this missing?  */
01292                 if (*word == '?')
01293                     *p = MISSING_VALUE;
01294                 else {
01295                     /*  is this symbolic?  */
01296                     bool is_symbolic = false;
01297                     if (symbols[j]) {
01298                         /*  Try to read symbolic data  */
01299                         int w=0;
01300                         while (symbols[j][w] && /*  look for this symbol  */
01301                                strcmp(symbols[j][w],word)!=0 &&
01302                                w<n_symbols[j]) w++;
01303                         if (w != n_rows && symbols[j][w]) {
01304                             // The symbol does exist.
01305                             is_symbolic = true;
01306                             *p = w + (*max_in_col)[j] + 1;
01307                         }
01308                     }
01309                     if (!is_symbolic) {
01310                         /*  read numeric data  */
01311 #ifdef USEDOUBLE
01312                         sscanf(word,"%lf",p);
01313 #else
01314                         sscanf(word,"%f",p);
01315 #endif
01316                     }
01317                 }
01318                 word=cp+1;
01319                 p++;
01320             }
01321         }
01322         fclose(f);
01323     }
01324 
01325     if(!to_symbols)
01326     {
01327         for (i=0; i<mat.width(); i++) 
01328         {
01329             for (j=0; j<n_symbols[i]; j++)
01330                 free(symbols[i][j]);
01331             free(symbols[i]);
01332         }
01333         free(symbols);
01334         free(n_symbols);
01335     }
01336     if (!the_max_in_col)
01337         delete max_in_col;
01338 #undef convert_UCIMLDB_BUF_LEN
01339 
01340     return mat;
01341 }
01342 
01343 
01344 // STATLOG machine-learning-database format
01345 Mat loadSTATLOG(const string& filename, char ****to_symbols, int **to_n_symbols) 
01346 {
01347     FILE *f = fopen(filename.c_str(),"r");
01348     int n_rows= -1, n_cols=0, i,j;
01349     char ***symbols;
01350     int *n_symbols;
01351 #define convert_STATLOG_BUF_LEN 10000
01352     char buffer[convert_STATLOG_BUF_LEN];
01353     char *cp=buffer;
01354     char *word=buffer;
01355     char *cp2;
01356     real *p;
01357     size_t line_len;
01358 
01359     if (!f) 
01360         PLERROR("In loadSTATLOG, could not open file %s for reading",filename.c_str());
01361 
01362     if((to_symbols && !to_n_symbols) || (!to_symbols && to_n_symbols))
01363         PLERROR("In loadUCIMLDB, to_symbols and to_nsymbols must both be provided (non-null), or both be 0");
01364 
01365     /*  first figure out number of columns and number of rows  */
01366 
01367     while (!feof(f))
01368     {
01369         fgets(buffer,convert_STATLOG_BUF_LEN,f);
01370         if (n_rows == -1)
01371         {
01372             /*  read number of columns  */
01373             while (*cp == ' ')  
01374                 cp++;   /*  jumping over blancs at the start of a new line  */
01375             while ( *cp!=0 && *cp!='\n' )
01376             {
01377                 while ( *cp != 0 && *cp != '\n' && *cp != ' ')
01378                 {
01379                     cp++; /*  read one colomn  */
01380                 }
01381                 n_cols++;
01382                 while ( *cp != 0 && *cp != '\n' && *cp == ' ')
01383                 {
01384                     cp++;   /*  jumping over blancs separating columns  */  
01385                 }              
01386             }
01387         }
01388         n_rows++;
01389     }
01390     fclose(f);
01391 
01392 
01393     /*  figure out the set of symbols used for each symbolic row, if any  */
01394     symbols = (char ***)calloc(n_cols,sizeof(char **));
01395     n_symbols = (int *)calloc(n_cols,sizeof(int));
01396     if (to_symbols)
01397     {
01398         *to_symbols = symbols;
01399         *to_n_symbols = n_symbols;
01400     }
01401     f = fopen(filename.c_str(),"r");
01402     for (i=0;i<n_rows;i++)
01403     {
01404         fgets(buffer,convert_STATLOG_BUF_LEN,f);
01405         line_len=strlen(buffer);
01406         cp=word=buffer;
01407         for (j=0;j<n_cols;j++)
01408         {
01409             /*  jumping over blancs at the start of a new line  */ 
01410             while (*cp == ' ')
01411             {
01412                 cp++;   
01413                 word++;
01414             } 
01415             /*  find next end of word  */
01416             while ((*cp!=' ' && *cp!='\n') && cp<=buffer+line_len) cp++;
01417             *cp=0;
01418             /*  is this symbolic?  */
01419             cp2=word;
01420             while (!isalpha((int)*cp2) && *cp2!='?' && cp2 < cp) cp2++;
01421             if (isalpha((int)*cp2) && *cp2!='?')
01422             { 
01423                 /*  yes, non-misisng symbolic character was found:  */
01424                 if (symbols[j]) { 
01425                     /*  we already had found symbols in this column  */
01426                     int w=0;
01427                     while (symbols[j][w] &&  /*  look for this symbol  */
01428                            strcmp(symbols[j][w],word)!=0 &&
01429                            w<n_symbols[j]) w++;
01430                     if (w==n_rows)
01431                         PLERROR("logic error in loadSTATLOG");
01432                     if (!symbols[j][w])
01433                     {
01434                         /*  new symbol  */
01435                         symbols[j][w] = (char *)calloc(strlen(word)+1,sizeof(char));
01436                         strcpy(symbols[j][w],word);
01437                         n_symbols[j]++;
01438                     }
01439                 }
01440                 else
01441                 {
01442                     /*  it's the first time we find a symbol in this column  */
01443                     symbols[j] = (char **)calloc(n_rows,sizeof(char *));
01444                     symbols[j][0] = (char *)calloc(strlen(word)+1,sizeof(char));
01445                     strcpy(symbols[j][0], word);
01446                     n_symbols[j]=1;
01447                 }
01448             }
01449             word = cp+1;
01450         }
01451     }
01452     fclose(f);
01453 
01454     /*  sort the symbols  */
01455     for (j=0;j<n_cols;j++)
01456         if (symbols[j]) /*  it has symbols  */
01457             qsort(symbols[j],n_symbols[j],sizeof(char *),compare_string_pointers);
01458 
01459     Mat mat(n_rows, n_cols);
01460     /*  NOW actually READ the data  */
01461     {
01462         p = mat.data();
01463         f = fopen(filename.c_str(),"r");
01464         for (i=0;i<n_rows;i++)
01465         {
01466             /*  read a row  */
01467             fgets(buffer,convert_STATLOG_BUF_LEN,f);
01468             line_len=strlen(buffer);
01469             cp=word=buffer;
01470             /*  interpret a row  */
01471             for (j=0;j<n_cols;j++)
01472             {
01473                 /*  jumping over blancs at the start of a new line  */ 
01474                 while (*cp == ' ')
01475                 {
01476                     cp++;   
01477                     word++;
01478                 } 
01479                 /*  find end of word  */
01480                 while ((*cp!=' ' && *cp!='\n') && cp<=buffer+line_len) cp++;
01481                 *cp=0;
01482                 /*  is this missing?  */
01483                 if (*word == '?')
01484                     *p = MISSING_VALUE;
01485                 else
01486                     /*  is this symbolic?  */
01487                     if (symbols[j]) {
01488                         /*  read symbolic data  */
01489                         int w=0;
01490                         while (symbols[j][w] && /*  look for this symbol  */
01491                                strcmp(symbols[j][w],word)!=0 &&
01492                                w<n_symbols[j]) w++;
01493                         if (w==n_rows || !symbols[j][w])
01494                             PLERROR("logic error in loadSTATLOG");
01495                         *p = w;
01496                     }
01497                     else
01498                     {
01499                         /*  read numeric data  */
01500 #ifdef USEDOUBLE
01501                         sscanf(word,"%lf",p);
01502 #else
01503                         sscanf(word,"%f",p);
01504 #endif
01505                     }
01506                 word=cp+1;
01507                 p++;
01508             }
01509         }
01510         fclose(f);
01511     }
01512 
01513     if(!to_symbols)
01514     {
01515         for (i=0; i<mat.width(); i++) 
01516         {
01517             for (j=0; j<n_symbols[i]; j++)
01518                 free(symbols[i][j]);
01519             free(symbols[i]);
01520         }
01521         free(symbols);
01522         free(n_symbols);
01523     }
01524 #undef convert_STATLOG_BUF_LEN
01525 
01526     return mat;
01527 }
01528 
01529 
01530 // jpeg stuff...
01531 void loadJPEGrgb(const string& jpeg_filename, Mat& rgbmat, int& row_size, int scale)
01532 {
01533     string tmpfile = jpeg_filename + ".pnm";
01534     char command[1000];
01535     sprintf(command,"djpeg -pnm -scale 1/%d %s > %s",
01536             scale,jpeg_filename.c_str(),tmpfile.c_str());
01537     system(command);
01538     FILE* fp = fopen(tmpfile.c_str(),"r");
01539     if (!fp)
01540         PLERROR("reading %s",tmpfile.c_str());
01541     fscanf(fp,"%s",command);
01542     int w,h;
01543     fscanf(fp,"%d %d\n",&w,&h);
01544     fscanf(fp,"%*d\n");
01545     int n=w*h;
01546     rgbmat.resize(n,3);
01547     real *d=rgbmat.data();
01548     for (int i=0;i<n;i++)
01549         for (int k=0;k<3;k++,d++)
01550             *d =(real)(getc(fp));
01551     fclose(fp);
01552     sprintf(command,"rm %s",tmpfile.c_str());
01553     system(command);
01554     row_size = w;
01555 }
01556 
01557 void parseSizeFromRemainingLines(const PPath& filename, PStream& in, bool& could_be_old_amat, int& length, int& width)
01558 {
01559     string line;
01560     getNextNonBlankLine(in,line);
01561     if(line=="") // There are no value lines
01562     {
01563         width=length=0;
01564         could_be_old_amat=false; 
01565         return; 
01566     }
01567 
01568     int nfields1 = int(split(line).size()); 
01569     getNextNonBlankLine(in,line);
01570     if(line=="") // There is only one line
01571     {
01572         length               = 1;
01573         width                = nfields1;
01574         could_be_old_amat    = false; 
01575         return; 
01576     }
01577 
01578     // The number of lines that seems to contain values  
01579     int guesslength = countNonBlankLinesOfFile(filename);  
01580 
01581     int nfields2 = int(split(line).size());                // The width of the second line.
01582     if(nfields1==nfields2) // looks like a plain ascii file
01583     {
01584         length = guesslength;
01585         width  = nfields1;
01586         return;
01587     }
01588 
01589     if(!could_be_old_amat || nfields1!=2) 
01590         return;  // could not be an old .amat with first 2 numbers being length width
01591 
01592     // OK we now suppose that we may have an old-format vmatrix
01593     // PLERROR("In parseSizeFromRemainingLines - This part of the code is not PStream compatible yet");
01594     
01595     // Get to the beginning of the file
01596     PStream rein = openFile(filename, PStream::raw_ascii, "r");
01597 
01598     // Reread the first line as two real numbers
01599     real a = -1.0, b = -1.0;
01600     rein >> a >> b;
01601 
01602     if(guesslength == int(a)+1                   // +1 since the size line was counted in guesslength but should not
01603        && real(int(a))==a && real(int(b))==b     //  Sizes must be integers and
01604        && a>0 && b>0                             //   positive
01605        && int(b)==nfields2 )                     // The first row of values has the expected width
01606     {
01607         // We assume we have an old style .amat
01608         length = int(a);
01609         width = int(b);
01610     }
01611 }
01612 
01613 } // end of namespace PLearn
01614 
01615 
01616 /*
01617   Local Variables:
01618   mode:c++
01619   c-basic-offset:4
01620   c-file-style:"stroustrup"
01621   c-file-offsets:((innamespace . 0)(inline-open . 0))
01622   indent-tabs-mode:nil
01623   fill-column:79
01624   End:
01625 */
01626 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines