PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // 00007 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: MatIO.cc 10370 2011-09-23 19:05:51Z nouiz $ 00039 * This file is part of the PLearn library. 00040 ******************************************************* */ 00041 00042 #include "MatIO.h" 00043 #include <plearn/base/tostring.h> 00044 #include "fileutils.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 00050 /* 00051 A word on the old PLearn native binary file format for matrices and vectors (.pmat .pvec): 00052 00053 The file starts with a 64 bytes ASCII ascii header, directly followed by 00054 the data in binary format (a simple memory dump, 4 bytes per value for 00055 floats and 8 bytes for doubles). The ascii header gives the data type, the 00056 dimensions, and the type of binary representation, as shown in the examples 00057 below. The remaining of the 64 bytes is filled with white spaces (' ') and 00058 a final newline ('\n') The 64th byte must always be a newline, this will be 00059 checked by the loading code. 00060 00061 The data in these files can be memory mapped if desired. Notice that 00062 memory-mapping requires that the architecture has the same memory 00063 representation (big endian or little endian) that is used in the file, as 00064 specified in the header (see below) Otherwise you'll get an error message 00065 telling you of the problem. However traditional loading does not require 00066 this; load will 'translate' the representation if necessary. 00067 00068 Examples of header lines: 00069 00070 A 100 element vector of floats (4bytes each) in little endian binary representation: 00071 VECTOR 100 FLOAT LITTLE_ENDIAN 00072 00073 A 100 element vector of doubles (8 bytes each) in big endian binary representation: 00074 VECTOR 100 double BIG_ENDIAN 00075 00076 A 20x10 matrix of doubles in little endian binary representation 00077 MATRIX 20 10 DOUBLE LITTLE_ENDIAN 00078 00079 A 20x10 matrix of floats in big endian binary representation 00080 MATRIX 20 10 FLOAT BIG_ENDIAN 00081 00082 As a convention, we will use the following file extensions: .pmat and .pvec 00083 (p stands for PLearn) (or possibly .lpmat .bpmat and .lpvec .bpvec if you 00084 wish to keep both a little endian and a big endian version of the matrix in 00085 the same place, to allow memory-mapping on different platforms) 00086 */ 00087 00088 void loadMat(const string& file_name, TMat<float>& mat) 00089 { 00090 string tmp_file_name = file_name; 00091 bool load_remote = 00092 string_begins_with(file_name, "https:") || 00093 string_begins_with(file_name, "http:") || 00094 string_begins_with(file_name, "ftp:"); 00095 if(load_remote) 00096 { 00097 tmp_file_name = "/tmp/" + extract_filename(file_name); 00098 string command = "curl --silent " + file_name + " > " + tmp_file_name; 00099 system(command.c_str()); 00100 } 00101 string ext = extract_extension(tmp_file_name); 00102 // See if we know the extension... 00103 if(ext==".amat") 00104 loadAscii(tmp_file_name, mat); 00105 else if (ext==".pmat" || ext==".lpmat" || ext==".bpmat") 00106 loadPMat(tmp_file_name,mat); 00107 else // try to guess the format from the header 00108 { 00109 ifstream in(tmp_file_name.c_str()); 00110 if(!in) 00111 PLERROR("In loadMat: could not open file %s",tmp_file_name.c_str()); 00112 char c = in.get(); 00113 in.close(); 00114 if(c=='M') // it's most likely a .pmat format 00115 loadPMat(tmp_file_name,mat); 00116 else 00117 loadAscii(tmp_file_name,mat); 00118 } 00119 if (load_remote) 00120 rm(tmp_file_name); 00121 } 00122 00123 void loadMat(const string& file_name, TMat<double>& mat) 00124 { 00125 string tmp_file_name = file_name; 00126 bool load_remote = 00127 string_begins_with(file_name, "https:") || 00128 string_begins_with(file_name, "http:") || 00129 string_begins_with(file_name, "ftp:"); 00130 if(load_remote) 00131 { 00132 tmp_file_name = "/tmp/" + extract_filename(file_name); 00133 string command = "curl --silent " + file_name + " > " + tmp_file_name; 00134 system(command.c_str()); 00135 } 00136 string ext = extract_extension(tmp_file_name); 00137 // See if we know the extension... 00138 if(ext==".amat") 00139 loadAscii(tmp_file_name, mat); 00140 else if (ext==".pmat" || ext==".lpmat" || ext==".bpmat") 00141 loadPMat(tmp_file_name,mat); 00142 else // try to guess the format from the header 00143 { 00144 ifstream in(tmp_file_name.c_str()); 00145 if(!in) 00146 PLERROR("In loadMat: could not open file %s",tmp_file_name.c_str()); 00147 char c = in.get(); 00148 in.close(); 00149 if(c=='M') // it's most likely a .pmat format 00150 loadPMat(tmp_file_name,mat); 00151 else 00152 loadAscii(tmp_file_name,mat); 00153 } 00154 if (load_remote) 00155 rm(tmp_file_name); 00156 } 00157 00158 00159 void loadVec(const string& file_name, TVec<float>& vec) 00160 { 00161 const char* filename = file_name.c_str(); 00162 const char* suffix = strrchr(filename,'.'); 00163 if (!suffix || strcmp(suffix,".avec")==0) 00164 loadAscii(file_name, vec); 00165 else if (strcmp(suffix,".pvec")==0 || strcmp(suffix,".lpvec")==0 || strcmp(suffix,".bpvec")==0) 00166 loadPVec(file_name,vec); 00167 else 00168 PLERROR("In loadVec: unknown file extension"); 00169 } 00170 00171 void loadVec(const string& file_name, TVec<double>& vec) 00172 { 00173 const char* filename = file_name.c_str(); 00174 const char* suffix = strrchr(filename,'.'); 00175 if (!suffix || strcmp(suffix,".avec")==0) 00176 loadAscii(file_name, vec); 00177 else if (strcmp(suffix,".pvec")==0 || strcmp(suffix,".lpvec")==0 || strcmp(suffix,".bpvec")==0) 00178 loadPVec(file_name,vec); 00179 else 00180 PLERROR("In loadVec: unknown file extension"); 00181 } 00182 00183 // Old native PLearn format (.pvec and .pmat) 00184 // DATAFILE_HEADER_LENGTH is 64 and is defined in general.h 00185 00186 void savePVec(const string& filename, const TVec<float>& vec) 00187 { 00188 FILE* f = fopen(filename.c_str(),"wb"); 00189 if (!f) 00190 PLERROR("In savePVec, could not open file %s for writing",filename.c_str()); 00191 00192 char header[DATAFILE_HEADERLENGTH]; 00193 00194 #ifdef LITTLEENDIAN 00195 sprintf(header,"VECTOR %d FLOAT LITTLE_ENDIAN", vec.length()); 00196 #endif 00197 #ifdef BIGENDIAN 00198 sprintf(header,"VECTOR %d FLOAT BIG_ENDIAN", vec.length()); 00199 #endif 00200 00201 // Pad the header with whites and terminate it with '\n' 00202 for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++) 00203 header[pos] = ' '; 00204 header[DATAFILE_HEADERLENGTH-1] = '\n'; 00205 00206 // write the header to the file 00207 fwrite(header,DATAFILE_HEADERLENGTH,1,f); 00208 00209 // write the data to the file 00210 if(0 < vec.length()) 00211 { 00212 const float* p = vec.data(); 00213 fwrite(p,sizeof(float),vec.length(),f); 00214 } 00215 00216 fclose(f); 00217 } 00218 00219 void savePVec(const string& filename, const TVec<double>& vec) 00220 { 00221 FILE* f = fopen(filename.c_str(),"wb"); 00222 if (!f) 00223 PLERROR("In savePVec, could not open file %s for writing",filename.c_str()); 00224 00225 char header[DATAFILE_HEADERLENGTH]; 00226 00227 #ifdef LITTLEENDIAN 00228 sprintf(header,"VECTOR %d DOUBLE LITTLE_ENDIAN", vec.length()); 00229 #endif 00230 #ifdef BIGENDIAN 00231 sprintf(header,"VECTOR %d DOUBLE BIG_ENDIAN", vec.length()); 00232 #endif 00233 00234 // Pad the header with whites and terminate it with '\n' 00235 for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++) 00236 header[pos] = ' '; 00237 header[DATAFILE_HEADERLENGTH-1] = '\n'; 00238 00239 // write the header to the file 00240 fwrite(header,DATAFILE_HEADERLENGTH,1,f); 00241 00242 // write the data to the file 00243 if(0 < vec.length()) 00244 { 00245 const double* p = vec.data(); 00246 fwrite(p,sizeof(double),vec.length(),f); 00247 } 00248 00249 fclose(f); 00250 } 00251 00252 void loadPVec(const string& filename, TVec<float>& vec) 00253 { 00254 char header[DATAFILE_HEADERLENGTH]; 00255 char matorvec[20]; 00256 char datatype[20]; 00257 char endiantype[20]; 00258 int the_length; 00259 00260 FILE* f = fopen(filename.c_str(),"rb"); 00261 if (!f) 00262 PLERROR("In loadPVec, could not open file %s for reading",filename.c_str()); 00263 fread(header,DATAFILE_HEADERLENGTH,1,f); 00264 if(header[DATAFILE_HEADERLENGTH-1]!='\n') 00265 PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00266 sscanf(header,"%s%d%s%s",matorvec,&the_length,datatype,endiantype); 00267 if (strcmp(matorvec,"VECTOR")!=0) 00268 PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00269 00270 vec.resize(the_length); 00271 00272 bool is_file_bigendian = false; 00273 if (strcmp(endiantype,"LITTLE_ENDIAN")==0) 00274 is_file_bigendian = false; 00275 else if (strcmp(endiantype,"BIG_ENDIAN")==0) 00276 is_file_bigendian = true; 00277 else 00278 PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file."); 00279 00280 if (strcmp(datatype,"FLOAT")==0) 00281 { 00282 float* p = vec.data(); 00283 fread_float(f,p,vec.length(),is_file_bigendian); 00284 } 00285 00286 else if (strcmp(datatype,"DOUBLE")==0) 00287 { 00288 double* buffer = new double[vec.length()]; 00289 float* p = vec.data(); 00290 fread_double(f,buffer,vec.length(),is_file_bigendian); 00291 for(int j=0; j<vec.length(); j++) 00292 p[j] = float(buffer[j]); 00293 delete[] buffer; 00294 } 00295 00296 else 00297 PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file."); 00298 00299 fclose(f); 00300 } 00301 00302 void loadPVec(const string& filename, TVec<double>& vec) 00303 { 00304 char header[DATAFILE_HEADERLENGTH]; 00305 char matorvec[20]; 00306 char datatype[20]; 00307 char endiantype[20]; 00308 int the_length; 00309 00310 FILE* f = fopen(filename.c_str(),"rb"); 00311 if (!f) 00312 PLERROR("In loadPVec, could not open file %s for reading",filename.c_str()); 00313 fread(header,DATAFILE_HEADERLENGTH,1,f); 00314 if(header[DATAFILE_HEADERLENGTH-1]!='\n') 00315 PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00316 sscanf(header,"%s%d%s%s",matorvec,&the_length,datatype,endiantype); 00317 if (strcmp(matorvec,"VECTOR")!=0) 00318 PLERROR("In loadPVec(%s), wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00319 00320 vec.resize(the_length); 00321 00322 bool is_file_bigendian = false; 00323 if (strcmp(endiantype,"LITTLE_ENDIAN")==0) 00324 is_file_bigendian = false; 00325 else if (strcmp(endiantype,"BIG_ENDIAN")==0) 00326 is_file_bigendian = true; 00327 else 00328 PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file."); 00329 00330 if (0 < the_length) 00331 { 00332 if (strcmp(datatype,"FLOAT")==0) 00333 { 00334 float* buffer = new float[vec.length()]; 00335 double* p = vec.data(); 00336 fread_float(f,buffer,vec.length(),is_file_bigendian); 00337 for(int j=0; j<vec.length(); j++) 00338 p[j] = double(buffer[j]); 00339 delete[] buffer; 00340 } 00341 00342 else if (strcmp(datatype,"DOUBLE")==0) 00343 { 00344 double* p = vec.data(); 00345 fread_double(f,p,vec.length(),is_file_bigendian); 00346 } 00347 00348 else 00349 PLERROR("In loadPVec, wrong header for PLearn binary vector format. Please use checkheader (in PLearn/Scripts) to check the file."); 00350 } 00351 fclose(f); 00352 } 00353 00354 00355 void savePMat(const string& filename, const TMat<float>& mat) 00356 { 00357 FILE* f = fopen(filename.c_str(),"wb"); 00358 if (!f) 00359 PLERROR("In savePMat, could not open file %s for writing",filename.c_str()); 00360 00361 char header[DATAFILE_HEADERLENGTH]; 00362 00363 #ifdef LITTLEENDIAN 00364 sprintf(header,"MATRIX %d %d FLOAT LITTLE_ENDIAN", mat.length(), mat.width()); 00365 #endif 00366 #ifdef BIGENDIAN 00367 sprintf(header,"MATRIX %d %d FLOAT BIG_ENDIAN", mat.length(), mat.width()); 00368 #endif 00369 00370 // Pad the header with whites and terminate it with '\n' 00371 for(size_t pos = strlen(header); pos<DATAFILE_HEADERLENGTH; pos++) 00372 header[pos] = ' '; 00373 header[DATAFILE_HEADERLENGTH-1] = '\n'; 00374 00375 // write the header to the file 00376 fwrite(header,DATAFILE_HEADERLENGTH,1,f); 00377 00378 // write the data to the file 00379 for (int i=0; i<mat.length(); i++) 00380 { 00381 const float* p = mat[i]; 00382 fwrite(p,sizeof(float),mat.width(),f); 00383 } 00384 fclose(f); 00385 } 00386 00387 void savePMat(const string& filename, const TMat<double>& mat) 00388 { 00389 FILE* f = fopen(filename.c_str(),"wb"); 00390 if (!f) 00391 PLERROR("In savePMat, could not open file %s for writing",filename.c_str()); 00392 00393 char header[DATAFILE_HEADERLENGTH]; 00394 00395 #ifdef LITTLEENDIAN 00396 sprintf(header,"MATRIX %d %d DOUBLE LITTLE_ENDIAN", mat.length(), mat.width()); 00397 #endif 00398 #ifdef BIGENDIAN 00399 sprintf(header,"MATRIX %d %d DOUBLE BIG_ENDIAN", mat.length(), mat.width()); 00400 #endif 00401 00402 // Pad the header with whites and terminate it with '\n' 00403 for(size_t pos=strlen(header); pos<DATAFILE_HEADERLENGTH; pos++) 00404 header[pos] = ' '; 00405 header[DATAFILE_HEADERLENGTH-1] = '\n'; 00406 00407 // write the header to the file 00408 fwrite(header,DATAFILE_HEADERLENGTH,1,f); 00409 00410 // write the data to the file 00411 for (int i=0; i<mat.length(); i++) 00412 { 00413 const double* p = mat[i]; 00414 fwrite(p,sizeof(double),mat.width(),f); 00415 } 00416 fclose(f); 00417 } 00418 00419 void savePMatFieldnames(const string& pmatfilename, const TVec<string>& fieldnames) 00420 { 00421 string metadatadir = pmatfilename+".metadata"; 00422 if(!isdir(metadatadir)) 00423 force_mkdir(metadatadir); 00424 string fname = metadatadir+"/fieldnames"; 00425 FILE* f = fopen(fname.c_str(),"w"); 00426 if(!f) 00427 PLERROR("Could not open file %s for writing",fname.c_str()); 00428 for(int i= 0; i < fieldnames.length(); ++i) 00429 fprintf(f,"%s\t0\n",fieldnames[i].c_str()); 00430 fclose(f); 00431 } 00432 00433 00434 void loadPMat(const string& filename, TMat<float>& mat) 00435 { 00436 char header[DATAFILE_HEADERLENGTH]; 00437 char matorvec[20]; 00438 char datatype[20]; 00439 char endiantype[20]; 00440 int the_length; 00441 int the_width; 00442 00443 FILE* f = fopen(filename.c_str(),"rb"); 00444 if (!f) 00445 PLERROR("In loadPMat, could not open file %s for reading",filename.c_str()); 00446 fread(header,DATAFILE_HEADERLENGTH,1,f); 00447 if(header[DATAFILE_HEADERLENGTH-1]!='\n') 00448 PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00449 sscanf(header,"%s%d%d%s%s",matorvec,&the_length,&the_width,datatype,endiantype); 00450 if (strcmp(matorvec,"MATRIX")!=0) 00451 PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00452 00453 mat.resize(the_length, the_width); 00454 00455 bool is_file_bigendian = true; 00456 if (strcmp(endiantype,"LITTLE_ENDIAN")==0) 00457 is_file_bigendian = false; 00458 else if (strcmp(endiantype,"BIG_ENDIAN")==0) 00459 is_file_bigendian = true; 00460 else 00461 PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file."); 00462 00463 if (strcmp(datatype,"FLOAT")==0) 00464 { 00465 for (int i=0; i<mat.length(); i++) 00466 { 00467 float* p = mat[i]; 00468 fread_float(f,p,mat.width(),is_file_bigendian); 00469 } 00470 } 00471 else if (strcmp(datatype,"DOUBLE")==0) 00472 { 00473 double* buffer = new double[mat.width()]; 00474 for (int i=0; i<mat.length(); i++) 00475 { 00476 float* p = mat[i]; 00477 fread_double(f,buffer,mat.width(),is_file_bigendian); 00478 for(int j=0; j<mat.width(); j++) 00479 p[j] = float(buffer[j]); 00480 } 00481 delete[] buffer; 00482 } 00483 00484 else 00485 PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file."); 00486 00487 fclose(f); 00488 } 00489 00490 void loadPMat(const string& filename, TMat<double>& mat) 00491 { 00492 char header[DATAFILE_HEADERLENGTH]; 00493 char matorvec[20]; 00494 char datatype[20]; 00495 char endiantype[20]; 00496 int the_length=0; 00497 int the_width=0; 00498 00499 FILE* f = fopen(filename.c_str(),"rb"); 00500 if (!f) 00501 PLERROR("In loadPMat, could not open file %s for reading",filename.c_str()); 00502 fread(header,DATAFILE_HEADERLENGTH,1,f); 00503 if(header[DATAFILE_HEADERLENGTH-1]!='\n') 00504 PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00505 sscanf(header,"%s%d%d%s%s",matorvec,&the_length,&the_width,datatype,endiantype); 00506 if (strcmp(matorvec,"MATRIX")!=0) 00507 PLERROR("In loadPMat(%s), wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file.",filename.c_str()); 00508 00509 mat.resize(the_length, the_width); 00510 00511 bool is_file_bigendian = true; 00512 if (strcmp(endiantype,"LITTLE_ENDIAN")==0) 00513 is_file_bigendian = false; 00514 else if (strcmp(endiantype,"BIG_ENDIAN")==0) 00515 is_file_bigendian = true; 00516 else 00517 PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file."); 00518 00519 if (strcmp(datatype,"FLOAT")==0) 00520 { 00521 float* buffer = new float[mat.width()]; 00522 for (int i=0; i<mat.length(); i++) 00523 { 00524 double* p = mat[i]; 00525 fread_float(f,buffer,mat.width(),is_file_bigendian); 00526 for(int j=0; j<mat.width(); j++) 00527 p[j] = double(buffer[j]); 00528 } 00529 delete[] buffer; 00530 } 00531 00532 else if (strcmp(datatype,"DOUBLE")==0) 00533 { 00534 for (int i=0; i<mat.length(); i++) 00535 { 00536 double* p = mat[i]; 00537 fread_double(f,p,mat.width(),is_file_bigendian); 00538 } 00539 } 00540 00541 else 00542 PLERROR("In loadPMat, wrong header for PLearn binary matrix format. Please use checkheader (in PLearn/Scripts) to check the file."); 00543 00544 fclose(f); 00545 } 00546 00547 /* 00548 void newLoadAscii(const string& filename, TMat<double>& mat) 00549 { 00550 ifstream in(filename.c_str()); 00551 if(!in) 00552 PLERROR("Could not open file %s for reading", filename.c_str()); 00553 00554 int length = -1; 00555 int width = -1; 00556 bool could_be_old_amat=true; // true while there is still a chance that this be an "old" amat format (length and width in first row with no starting ##) 00557 int c = in.get(); 00558 while(isspace(c)) 00559 c = in.get(); 00560 00561 if(c=='#') // starts with a comment 00562 { 00563 could_be_old_amat = false; 00564 00565 // If it's followed by another # and only 2 numbers before the end of line, it's a new .amat format 00566 if(in.get()=='#') 00567 { 00568 in >> length; 00569 in >> width; 00570 } 00571 c = in.get(); 00572 while(c==' ' || c=='\t') 00573 c = in.get(); 00574 if(c!='\n' && c!='\r') // it wasn't a new .amat format after all... 00575 length = -1; 00576 } 00577 00578 if(length==-1) // still looking for size info... 00579 { 00580 in.unget(); 00581 string line; 00582 getNextNonBlankLine(in,line); 00583 int nfields1 = split(line).size(); 00584 getNextNonBlankLine(in,line); 00585 int nfields2 = split(line).size(); 00586 if(could_be_old_amat && nfields1==2) // could be an old .amat with first 2 numbers being length width 00587 { 00588 in.seekg(0); 00589 real a, b; 00590 in >> a >> b; 00591 if(real(int(a))==a && real(int(b))==b && a>0 && b>0 && int(b)==nfields2) // it's clearly an old .amat 00592 { 00593 length = int(a); 00594 width = int(b); 00595 } 00596 } 00597 00598 if(length==-1) // still don't know size info... 00599 { 00600 if(nfields1==nfields2) // looks like a plain ascii file 00601 { 00602 length = countNonBlankLinesOfFile(filename); 00603 width = nfields1; 00604 in.seekg(0); 00605 } 00606 } 00607 } 00608 00609 if(length==-1) 00610 PLERROR("In loadAscii: couldn't determine file format automatically"); 00611 00612 mat.resize(length,width); 00613 for(int i=0; i<length; i++) 00614 { 00615 real* mat_i = mat[i]; 00616 skipBlanks(in); 00617 for(int j=0; j<width; j++) 00618 in >> mat_i[j]; 00619 } 00620 00621 } 00622 */ 00623 00624 // Gnuplot format 00625 void saveGnuplot(const string& filename, const Vec& vec) 00626 { 00627 FILE* f=fopen(filename.c_str(),"w"); 00628 if (!f) 00629 PLERROR("In Vec::saveGnuplot, couldn't open %s for writing",filename.c_str()); 00630 00631 real* p = vec.data(); 00632 for (int i=0; i<vec.length(); i++, p++) 00633 fprintf(f,"%e\n", *p); 00634 fclose(f); 00635 } 00636 00637 // Gnuplot format 00638 void saveGnuplot(const string& filename, const Mat& mat) 00639 { 00640 ofstream out(filename.c_str()); 00641 if (!out) 00642 PLERROR("In saveGnuplot, couldn't open %s for writing.",filename.c_str()); 00643 out.flags(ios::left); 00644 for(int i=0; i<mat.length(); i++) 00645 { 00646 const real* m_i = mat[i]; 00647 for(int j=0; j<mat.width(); j++) 00648 out << setw(11) << m_i[j] << ' '; 00649 out << "\n"; 00650 } 00651 out.flush(); 00652 } 00653 00654 void loadGnuplot(const string& filename, Mat& mat) 00655 { 00656 ifstream in(filename.c_str()); 00657 if (!in) 00658 PLERROR("In loadGnuplot, couldn't open %s for reading.",filename.c_str()); 00659 00660 char buf[10000]; 00661 // First pass to count the number of rows and columns 00662 int nrows = 0; 00663 int ncols = 0; 00664 in.getline(buf,sizeof(buf)-1); 00665 while(in) 00666 { 00667 int pos=0; 00668 while(buf[pos]==' ' || buf[pos]=='\t') 00669 pos++; 00670 if(buf[pos]!='#' && buf[pos]!='\n' && buf[pos]!='\r') 00671 { 00672 nrows++; 00673 if(ncols==0) 00674 { 00675 istringstream inputline(buf); 00676 real value; 00677 while(inputline) 00678 { 00679 inputline >> value; 00680 ncols++; 00681 } 00682 ncols--; // correct count 00683 } 00684 } 00685 in.getline(buf,sizeof(buf)-1); 00686 } 00687 in.close(); 00688 mat.resize(nrows,ncols); 00689 in.open(filename.c_str()); 00690 for(int i=0; i<nrows; i++) 00691 { 00692 char firstchar = '#'; 00693 while(firstchar == '#' || firstchar == '\n' || firstchar=='\r') 00694 { 00695 in.getline(buf,sizeof(buf)-1); 00696 int pos=0; 00697 while(buf[pos]==' ' || buf[pos]=='\t') 00698 pos++; 00699 firstchar = buf[pos]; 00700 } 00701 istringstream inputline(buf); 00702 for(int j=0; j<ncols; j++) 00703 inputline >> mat(i,j); 00704 } 00705 in.close(); 00706 } 00707 00708 void matlabSave( const PPath& dir, const string& plot_title, const Vec& data, 00709 const Vec& add_col, const Vec& bounds, string legend, bool save_plot) 00710 { 00711 Vec bidon; 00712 Mat mat(data.length(), 1); 00713 mat << data; 00714 TVec<string> legd; 00715 if(legend != "") 00716 legd.append(legend); 00717 matlabSave(dir, plot_title, 00718 bidon, 00719 mat, add_col, bounds, legd, save_plot); 00720 } 00721 00722 void matlabSave( const PPath& dir, const string& plot_title, 00723 const Vec& xValues, 00724 const Vec& yValues, const Vec& add_col, const Vec& bounds, string legend, bool save_plot) 00725 { 00726 Mat mat(yValues.length(), 1); 00727 mat << yValues; 00728 TVec<string> legd; 00729 if(legend != "") 00730 legd.append(legend); 00731 matlabSave(dir, plot_title, 00732 xValues, 00733 mat, add_col, bounds, legd, save_plot); 00734 } 00735 00736 void matlabSave( const PPath& dir, const string& plot_title, const Mat& data, 00737 const Vec& add_col, const Vec& bounds, TVec<string> legend, bool save_plot) 00738 { 00739 Vec bid; 00740 matlabSave(dir, plot_title, bid, data, add_col, bounds, legend, save_plot); 00741 } 00742 00751 void matlabSave( const PPath& dir, const string& plot_title, 00752 const Vec& xValues, 00753 const Mat& yValues, const Vec& add_col, const Vec& bounds, TVec<string> legend, bool save_plot) 00754 { 00755 force_mkdir(dir); 00756 PPath directory = dir.absolute(); 00757 force_mkdir(directory / "Images" / ""); 00758 00759 int w = yValues.width(); 00760 00761 ofstream out; 00762 PPath vec_fname = directory / plot_title + ".mmat"; 00763 out.open(vec_fname.absolute().c_str(), ofstream::out | ofstream::trunc); 00764 00765 real startX = 0.0; 00766 int xLen = xValues.length(); 00767 if(xLen != 0) 00768 { 00769 if(xLen == yValues.length()) 00770 startX = MISSING_VALUE; 00771 else if(xLen == 1) 00772 startX = xValues[0]; 00773 else 00774 PLERROR("matlabSave:\n" 00775 "1) If xValues is empty, the yValues are plotted against the row indices.\n" 00776 "2) If xValues is not empty and its length is not equal to the length of yValues, \n" 00777 "then its length must be one and the value xValues[0] will be the start index for the xValues."); 00778 } 00779 00780 for(int d = 0; d < yValues.length(); d++) 00781 { 00782 if(is_missing(startX)) 00783 out << xValues[d] << "\t"; 00784 else 00785 out << (startX+d) << "\t"; 00786 00787 for(int col=0; col < w; col++) 00788 out << yValues(d, col) << "\t"; 00789 00790 for(int add=0; add < add_col.length(); add++) 00791 out << add_col[add] << "\t"; 00792 out << endl; 00793 } 00794 out.close(); 00795 00796 PPath m_fname = directory / plot_title + ".m"; 00797 out.open(m_fname.absolute().c_str(), ofstream::out | ofstream::trunc); 00798 out << "load " << vec_fname.absolute() << " -ascii" << endl 00799 << plot_title << "= sortrows(" << plot_title << ")" << endl 00800 << "h = plot(" 00801 //--- X Values 00802 << plot_title << "(:,1), " 00804 //--- Y Values 00805 << plot_title << "(:,2:" << (1+w) << "));" << endl 00807 << "set(h, 'LineWidth', 1.0)" << endl 00808 << "set(gcf, 'Position', [0, 0, 1000, 750])" << endl 00809 << "hold on" << endl; 00810 00811 if(legend.isNotEmpty()) 00812 { 00813 int leg = legend.length(); 00814 int wid = yValues.width(); 00815 if(leg != wid) 00816 { 00817 if(legend[0] == "Numbers") 00818 { 00819 legend.resize(wid); 00820 for(int c=0; c < wid; c++) 00821 legend[c] = tostring(c); 00822 } 00823 else 00824 PLERROR("TimeSeriesAnalysis::matlab_save: legend.length() = %d != %d = yValues.width()", 00825 leg, wid); 00826 } 00827 out << "legend(h"; 00828 for(int l=0; l < leg; l++) 00829 { 00830 legend[l] = underscore_to_space(legend[l]); 00831 out << ", '" << legend[l] << "'"; 00832 } 00833 out << ");" << endl; 00834 } 00835 00836 for(int add=0; add < add_col.length(); add++) 00837 out << "g = plot(" << plot_title 00838 << "(:," << (2+w+add) << "));" 00839 << endl 00840 << "set(g, 'Color', [0.5 0.5 0.5])" << endl; 00841 00842 if(bounds.isNotEmpty()) 00843 out << "xlim([" << bounds[0] << ", " << bounds[1] << "])" << endl 00844 << "ylim([" << bounds[2] << ", " << bounds[3] << "])" << endl; 00845 00846 out << "title('" << underscore_to_space(plot_title) << "')" << endl; 00847 00848 if(save_plot) 00849 out << "print('-dpsc2', '" 00850 << (directory / "Images" / "").absolute() 00851 << plot_title << ".eps')" << endl; 00852 00853 out.close(); 00854 } 00855 00856 // Ascii without size 00857 void saveAsciiWithoutSize(const string& filename, const Vec& vec) 00858 { 00859 FILE *f; 00860 f=fopen(filename.c_str(),"w"); 00861 if (!f) 00862 PLERROR("In Vec::saveAscii: could not open file %s for writing",filename.c_str()); 00863 int i; 00864 char buffer[100]; 00865 real *p= vec.data(); 00866 for (i=0;i<vec.length();i++,p++) 00867 { 00868 pretty_print_number(buffer,*p); 00869 fprintf(f,"%s ",buffer); 00870 } 00871 fclose(f); 00872 } 00873 00874 // Ascii without size (load assumes size is already correctly set) 00875 void loadAsciiWithoutSize(const string& filename, const Vec& vec) 00876 { 00877 FILE *f; 00878 f=fopen(filename.c_str(),"r"); 00879 if (!f) 00880 PLERROR("In Vec::loadAsciiWithoutSize could not open file %s for reading",filename.c_str()); 00881 00882 if (vec.length() < 1) 00883 PLERROR("In Vec::loadAsciiWithoutSize, the size of the vector is not defined yet"); 00884 00885 real* p = vec.data(); 00886 for (int i=0;i<vec.length();i++,p++) 00887 { 00888 #ifdef USEDOUBLE 00889 fscanf(f,"%lf",p); 00890 #else 00891 fscanf(f,"%f",p); 00892 #endif 00893 } 00894 } 00895 00896 void saveAsciiWithoutSize(const string& filename, const Mat& mat) 00897 { 00898 FILE *f; 00899 f=fopen(filename.c_str(),"w"); 00900 if (!f) 00901 PLERROR("In saveAscii, could not open file %s for writing",filename.c_str()); 00902 char buffer[100]; 00903 for(int i=0; i<mat.length(); i++) 00904 { 00905 const real* row_i = mat[i]; 00906 for(int j=0; j<mat.width(); j++) 00907 { 00908 pretty_print_number(buffer,row_i[j]); 00909 fprintf(f,"%s ",buffer); 00910 } 00911 fprintf(f,"\n"); 00912 } 00913 fclose(f); 00914 } 00915 00916 void loadAsciiWithoutSize(const string& filename, const Mat& mat) 00917 { 00918 00919 FILE *f = fopen(filename.c_str(),"r"); 00920 if (!f) 00921 PLERROR("In loadAsciiWithoutSize, could not open file %s for reading.",filename.c_str()); 00922 00923 if (mat.length() < 1 || mat.width() < 1) 00924 PLERROR("In loadAsciiWithoutSize, the size of the matrix is not defined yet"); 00925 00926 for(int i=0; i<mat.length(); i++) 00927 { 00928 real* row_i = mat[i]; 00929 for(int j=0; j<mat.width(); j++) 00930 #ifdef USEDOUBLE 00931 fscanf(f,"%lf",&row_i[j]); 00932 #else 00933 fscanf(f,"%f",&row_i[j]); 00934 #endif 00935 } 00936 } 00937 00938 00939 // Native SN format (Fmat) 00940 void saveSNMat(const string& filename, const Mat& mat) 00941 { 00942 FILE *f=fopen(filename.c_str(),"wb"); 00943 int i=0x1e3d4c51L; 00944 int j=0; 00945 fwrite_int(f,&i,1); 00946 i=2; /* number of dimensions = 2 for a matrix */ 00947 fwrite_int(f,&i,1); 00948 int length = mat.length(); 00949 int width = mat.width(); 00950 fwrite_int(f,&length,1); 00951 fwrite_int(f,&width,1); 00952 while (i++ < 3) fwrite_int(f,&j,1); 00953 for (i=0;i<length;i++) 00954 fwrite_float(f,mat[i],width); 00955 fclose(f); 00956 } 00957 00958 Mat loadSNMat(const string& filename) 00959 { 00960 FILE *f; 00961 int d, nd; int i; 00962 int imn; 00963 int length, width; 00964 00965 f=fopen(filename.c_str(),"rb"); 00966 if (!f) 00967 PLERROR("In loadFmat, could not open file %s for reading",filename.c_str()); 00968 00969 fread_int(f,&imn,1); 00970 if (imn!= (0x1e3d4c51L)) 00971 PLERROR("in loadFmat, File does not have the right format"); 00972 00973 /* read ndim */ 00974 fread_int(f,&nd,1); 00975 if (nd<0 || nd>5) 00976 PLERROR("In loadFmat, Corrupted file: Bad number of dimensions"); 00977 if (nd != 2) 00978 PLERROR("In loadFmat, ndim is not 2 (not a matrix!)\n"); 00979 00980 /* read dim */ 00981 d=0; 00982 fread_int(f,&length,1); 00983 d++; 00984 fread_int(f,&width,1); 00985 d++; 00986 while (d++ < 3) 00987 fread(&i, sizeof(int), 1, f); 00988 Mat mat(length,width); 00989 for(i=0; i<length; i++) 00990 fread_float(f, mat[i], width); 00991 fclose(f); 00992 return mat; 00993 } 00994 00995 void saveSNVec(const string& filename, const Vec& vec) 00996 { 00997 FILE* f=fopen(filename.c_str(),"wb"); 00998 if(!f) 00999 PLERROR("In Vec::loadSNVec could not open file for writing"); 01000 int i=0x1e3d4c51L; 01001 int j=0; 01002 fwrite_int(f,&i,1); 01003 i=1; /* number of dimensions = 1 for a vector */ 01004 fwrite_int(f,&i,1); 01005 int length = vec.length(); 01006 fwrite_int(f,&length,1); 01007 while (i++ < 3) 01008 fwrite_int(f,&j,1); 01009 fwrite_float(f,vec.data(),length); 01010 fclose(f); 01011 } 01012 01013 Vec loadSNVec(const string& filename) 01014 { 01015 FILE *f; 01016 int d, nd; int i; 01017 int imn; 01018 int size; 01019 01020 f=fopen(filename.c_str(),"rb"); 01021 if (!f) 01022 PLERROR("In Vec::loadSNVec could not open file %s for reading",filename.c_str()); 01023 01024 fread_int(f,&imn,1); 01025 if (imn!= (0x1e3d4c51L)) 01026 PLERROR("In Vec::loadSNVec, File does not have the right format"); 01027 01028 /* read ndim */ 01029 fread_int(f,&nd,1); 01030 if (nd<0 || nd>5) 01031 PLERROR("In Vec::loadSNVec, Corrupted file: Bad number of dimensions"); 01032 if (nd != 1) 01033 PLERROR("In Vec::loadSNVec, ndim is not 1 (not a vector!)\n"); 01034 01035 /* read dim */ 01036 d=0; 01037 fread_int(f,&i,1); 01038 size=i;d++; 01039 while (d++ < 3) 01040 fread(&i, sizeof(int), 1, f); 01041 01042 Vec vec(size); 01043 fread_float(f,vec.data(),size); 01044 01045 fclose(f); 01046 return vec; 01047 } 01048 01049 01050 // Native AD format 01051 01052 Mat loadADMat(const string& filename) 01053 { 01054 FILE *f = fopen(filename.c_str(),"rb"); 01055 if (!f) 01056 PLERROR("In loadADMat, could not open file %s for reading",filename.c_str()); 01057 int the_length, the_width; 01058 int magic = 0x2345; 01059 int SNidx2fltmagic = 0x0D02; 01060 int m; 01061 fread_int(f,&m,1); 01062 if (m != magic && m != SNidx2fltmagic) 01063 PLERROR("In load, magic number is incorrect: %d != %d",m,magic); 01064 fread_int(f,&the_length,1); 01065 fread_int(f,&the_width,1); 01066 Mat mat(the_length,the_width); 01067 fread_float(f,mat.data(),the_length*the_width); 01068 fclose(f); 01069 return mat; 01070 } 01071 01072 Vec loadADVec(const string& filename) 01073 { 01074 FILE* f = fopen(filename.c_str(),"rb"); 01075 if (!f) 01076 PLERROR("In Vec::loadADMat could not open file %s for reading",filename.c_str()); 01077 int thesize; 01078 int magic = 0x3456; 01079 int m; 01080 fread_int(f,&m,1); 01081 if (m != magic) 01082 PLERROR("In new_Vec_from_File_FILE, magic number is incorret: %d != %d",m,magic); 01083 fread_int(f,&thesize,1); 01084 Vec vec(thesize); 01085 fread_float(f,vec.data(),thesize); 01086 fclose(f); 01087 return vec; 01088 } 01089 01090 // Used for calling qsort 01091 static int compare_string_pointers(const void *ts1, const void *ts2) 01092 { 01093 return strcmp(*((char **)ts1),*((char **)ts2)); 01094 } 01095 01096 01097 // UCI machine-learning-database format 01098 Mat loadUCIMLDB(const string& filename, char ****to_symbols, int **to_n_symbols, TVec<int>* the_max_in_col, TVec<string>* header_columns) 01099 { 01100 FILE *f = fopen(filename.c_str(),"r"); 01101 int n_rows= -1, n_cols=1, i,j; 01102 char ***symbols; 01103 int *n_symbols; 01104 #define convert_UCIMLDB_BUF_LEN 10000 01105 char buffer[convert_UCIMLDB_BUF_LEN]; 01106 char *cp=buffer; 01107 char *word=buffer; 01108 char *cp2; 01109 real *p; 01110 size_t line_len; 01111 01112 if (!f) 01113 PLERROR("In loadUCIMLDB, could not open file %s for reading",filename.c_str()); 01114 01115 if((to_symbols && !to_n_symbols) || (!to_symbols && to_n_symbols)) 01116 PLERROR("In loadUCIMLDB, to_symbols and to_nsymbols must both be provided (non-null), or both be 0"); 01117 01118 /* first figure out number of columns and number of rows */ 01119 bool skip_header = false; 01120 if (header_columns) { 01121 skip_header = true; 01122 } 01123 while (!feof(f)) 01124 { 01125 do { 01126 fgets(buffer,convert_UCIMLDB_BUF_LEN,f); 01127 } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0)); 01128 if (skip_header) { 01129 skip_header = false; 01130 } else { 01131 if (n_rows == -1) { 01132 /* read number of columns */ 01133 while ((cp=strchr(cp,','))) 01134 { 01135 cp++; 01136 n_cols++; 01137 } 01138 } 01139 n_rows++; 01140 } 01141 } 01142 01143 fclose(f); 01144 01145 /* figure out the set of symbols used for each symbolic row, if any */ 01146 symbols = (char ***)calloc(n_cols,sizeof(char **)); 01147 n_symbols = (int *)calloc(n_cols,sizeof(int)); 01148 01149 TVec<int>* max_in_col; 01150 if (the_max_in_col) { 01151 max_in_col = the_max_in_col; 01152 } else { 01153 max_in_col = new TVec<int>(); 01154 } 01155 max_in_col->resize(n_cols); 01156 max_in_col->fill(-1); 01157 if(to_symbols) 01158 { 01159 *to_symbols = symbols; 01160 *to_n_symbols = n_symbols; 01161 } 01162 f = fopen(filename.c_str(),"r"); 01163 01164 01165 /* read header columns */ 01166 if (header_columns) { 01167 do { 01168 fgets(buffer,convert_UCIMLDB_BUF_LEN,f); 01169 } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0)); 01170 01171 cp=word=buffer; 01172 01173 while ((cp=strchr(cp,','))) { 01174 *cp=0; 01175 header_columns->append(word); 01176 cp++; 01177 word=cp; 01178 } 01179 header_columns->append(word); 01180 } 01181 01182 for (i=0;i<n_rows;i++) 01183 { 01184 do { 01185 fgets(buffer,convert_UCIMLDB_BUF_LEN,f); 01186 } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0)); 01187 01188 01189 // ignore everything after '|' 01190 char *comm = strchr(buffer,'|'); 01191 if (comm) *comm = '\n'; 01192 01193 line_len=strlen(buffer); 01194 cp=word=buffer; 01195 for (j=0;j<n_cols;j++) 01196 { 01197 /* find next end of word */ 01198 while ((*cp!=',' && *cp!='\n') && cp<=buffer+line_len) cp++; 01199 *cp=0; 01200 /* is this symbolic? */ 01201 cp2=word; 01202 string the_val = word; 01203 01204 if (!pl_isnumber(word) && *cp2!='?') { 01205 /* yes, non-missing symbolic character was found: */ 01206 if (symbols[j]) 01207 { 01208 /* we already had found symbols in this column */ 01209 int w=0; 01210 while (symbols[j][w] && /* look for this symbol */ 01211 strcmp(symbols[j][w],word)!=0 && 01212 w<n_symbols[j]) w++; 01213 if (w==n_rows) 01214 PLERROR("logic error in loadUCIMLDB"); 01215 if (!symbols[j][w]) 01216 { 01217 /* new symbol */ 01218 symbols[j][w] = (char *)calloc(strlen(word)+1,sizeof(char)); 01219 strcpy(symbols[j][w],word); 01220 n_symbols[j]++; 01221 } 01222 } 01223 else 01224 { 01225 /* it's the first time we find a symbol in this column */ 01226 symbols[j] = (char **)calloc(n_rows,sizeof(char *)); 01227 symbols[j][0] = (char *)calloc(strlen(word)+1,sizeof(char)); 01228 strcpy(symbols[j][0], word); 01229 n_symbols[j]=1; 01230 } 01231 } else { 01232 // This is a numerical character: we use it to keep track of the 01233 // maximum in this column. 01234 real real_val; 01235 if (the_val != "?" && pl_isnumber(the_val, &real_val)) { 01236 if (int(real_val) > (*max_in_col)[j]) { 01237 (*max_in_col)[j] = int(real_val); 01238 } 01239 } 01240 } 01241 word = cp+1; 01242 } 01243 } 01244 fclose(f); 01245 01246 /* sort the symbols */ 01247 for (j=0;j<n_cols;j++) 01248 if (symbols[j]) /* it has symbols */ 01249 qsort(symbols[j],n_symbols[j],sizeof(char *),compare_string_pointers); 01250 01251 Mat mat(n_rows,n_cols); 01252 /* NOW actually READ the data */ 01253 { 01254 p = mat.data(); 01255 f = fopen(filename.c_str(),"r"); 01256 01257 // skip one line if header present 01258 if (header_columns) { 01259 do { 01260 fgets(buffer,convert_UCIMLDB_BUF_LEN,f); 01261 } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0)); 01262 } 01263 01264 01265 for (i=0;i<n_rows;i++) 01266 { 01267 /* read a row */ 01268 do { 01269 fgets(buffer,convert_UCIMLDB_BUF_LEN,f); 01270 } while (!feof(f) && (strcmp(buffer,"\n")==0 || strncmp(buffer,";;;",3)==0)); 01271 01272 01273 // ignore everything after '|' 01274 char *comm = strchr(buffer,'|'); 01275 if (comm) *comm = '\n'; 01276 01277 line_len=strlen(buffer); 01278 cp=word=buffer; 01279 /* interpret a row */ 01280 for (j=0;j<n_cols;j++) 01281 { 01282 /* find end of word */ 01283 while ((*cp!=',' && *cp!='\n') && cp<=buffer+line_len) cp++; 01284 *cp=0; // Make 'word' point to the current field value only. 01285 // Skip blanks at beginning. 01286 while (*word == ' ') { 01287 word++; 01288 if (word >= cp) 01289 PLERROR("In loadUCIMLDB - Error while skipping blanks"); 01290 } 01291 /* is this missing? */ 01292 if (*word == '?') 01293 *p = MISSING_VALUE; 01294 else { 01295 /* is this symbolic? */ 01296 bool is_symbolic = false; 01297 if (symbols[j]) { 01298 /* Try to read symbolic data */ 01299 int w=0; 01300 while (symbols[j][w] && /* look for this symbol */ 01301 strcmp(symbols[j][w],word)!=0 && 01302 w<n_symbols[j]) w++; 01303 if (w != n_rows && symbols[j][w]) { 01304 // The symbol does exist. 01305 is_symbolic = true; 01306 *p = w + (*max_in_col)[j] + 1; 01307 } 01308 } 01309 if (!is_symbolic) { 01310 /* read numeric data */ 01311 #ifdef USEDOUBLE 01312 sscanf(word,"%lf",p); 01313 #else 01314 sscanf(word,"%f",p); 01315 #endif 01316 } 01317 } 01318 word=cp+1; 01319 p++; 01320 } 01321 } 01322 fclose(f); 01323 } 01324 01325 if(!to_symbols) 01326 { 01327 for (i=0; i<mat.width(); i++) 01328 { 01329 for (j=0; j<n_symbols[i]; j++) 01330 free(symbols[i][j]); 01331 free(symbols[i]); 01332 } 01333 free(symbols); 01334 free(n_symbols); 01335 } 01336 if (!the_max_in_col) 01337 delete max_in_col; 01338 #undef convert_UCIMLDB_BUF_LEN 01339 01340 return mat; 01341 } 01342 01343 01344 // STATLOG machine-learning-database format 01345 Mat loadSTATLOG(const string& filename, char ****to_symbols, int **to_n_symbols) 01346 { 01347 FILE *f = fopen(filename.c_str(),"r"); 01348 int n_rows= -1, n_cols=0, i,j; 01349 char ***symbols; 01350 int *n_symbols; 01351 #define convert_STATLOG_BUF_LEN 10000 01352 char buffer[convert_STATLOG_BUF_LEN]; 01353 char *cp=buffer; 01354 char *word=buffer; 01355 char *cp2; 01356 real *p; 01357 size_t line_len; 01358 01359 if (!f) 01360 PLERROR("In loadSTATLOG, could not open file %s for reading",filename.c_str()); 01361 01362 if((to_symbols && !to_n_symbols) || (!to_symbols && to_n_symbols)) 01363 PLERROR("In loadUCIMLDB, to_symbols and to_nsymbols must both be provided (non-null), or both be 0"); 01364 01365 /* first figure out number of columns and number of rows */ 01366 01367 while (!feof(f)) 01368 { 01369 fgets(buffer,convert_STATLOG_BUF_LEN,f); 01370 if (n_rows == -1) 01371 { 01372 /* read number of columns */ 01373 while (*cp == ' ') 01374 cp++; /* jumping over blancs at the start of a new line */ 01375 while ( *cp!=0 && *cp!='\n' ) 01376 { 01377 while ( *cp != 0 && *cp != '\n' && *cp != ' ') 01378 { 01379 cp++; /* read one colomn */ 01380 } 01381 n_cols++; 01382 while ( *cp != 0 && *cp != '\n' && *cp == ' ') 01383 { 01384 cp++; /* jumping over blancs separating columns */ 01385 } 01386 } 01387 } 01388 n_rows++; 01389 } 01390 fclose(f); 01391 01392 01393 /* figure out the set of symbols used for each symbolic row, if any */ 01394 symbols = (char ***)calloc(n_cols,sizeof(char **)); 01395 n_symbols = (int *)calloc(n_cols,sizeof(int)); 01396 if (to_symbols) 01397 { 01398 *to_symbols = symbols; 01399 *to_n_symbols = n_symbols; 01400 } 01401 f = fopen(filename.c_str(),"r"); 01402 for (i=0;i<n_rows;i++) 01403 { 01404 fgets(buffer,convert_STATLOG_BUF_LEN,f); 01405 line_len=strlen(buffer); 01406 cp=word=buffer; 01407 for (j=0;j<n_cols;j++) 01408 { 01409 /* jumping over blancs at the start of a new line */ 01410 while (*cp == ' ') 01411 { 01412 cp++; 01413 word++; 01414 } 01415 /* find next end of word */ 01416 while ((*cp!=' ' && *cp!='\n') && cp<=buffer+line_len) cp++; 01417 *cp=0; 01418 /* is this symbolic? */ 01419 cp2=word; 01420 while (!isalpha((int)*cp2) && *cp2!='?' && cp2 < cp) cp2++; 01421 if (isalpha((int)*cp2) && *cp2!='?') 01422 { 01423 /* yes, non-misisng symbolic character was found: */ 01424 if (symbols[j]) { 01425 /* we already had found symbols in this column */ 01426 int w=0; 01427 while (symbols[j][w] && /* look for this symbol */ 01428 strcmp(symbols[j][w],word)!=0 && 01429 w<n_symbols[j]) w++; 01430 if (w==n_rows) 01431 PLERROR("logic error in loadSTATLOG"); 01432 if (!symbols[j][w]) 01433 { 01434 /* new symbol */ 01435 symbols[j][w] = (char *)calloc(strlen(word)+1,sizeof(char)); 01436 strcpy(symbols[j][w],word); 01437 n_symbols[j]++; 01438 } 01439 } 01440 else 01441 { 01442 /* it's the first time we find a symbol in this column */ 01443 symbols[j] = (char **)calloc(n_rows,sizeof(char *)); 01444 symbols[j][0] = (char *)calloc(strlen(word)+1,sizeof(char)); 01445 strcpy(symbols[j][0], word); 01446 n_symbols[j]=1; 01447 } 01448 } 01449 word = cp+1; 01450 } 01451 } 01452 fclose(f); 01453 01454 /* sort the symbols */ 01455 for (j=0;j<n_cols;j++) 01456 if (symbols[j]) /* it has symbols */ 01457 qsort(symbols[j],n_symbols[j],sizeof(char *),compare_string_pointers); 01458 01459 Mat mat(n_rows, n_cols); 01460 /* NOW actually READ the data */ 01461 { 01462 p = mat.data(); 01463 f = fopen(filename.c_str(),"r"); 01464 for (i=0;i<n_rows;i++) 01465 { 01466 /* read a row */ 01467 fgets(buffer,convert_STATLOG_BUF_LEN,f); 01468 line_len=strlen(buffer); 01469 cp=word=buffer; 01470 /* interpret a row */ 01471 for (j=0;j<n_cols;j++) 01472 { 01473 /* jumping over blancs at the start of a new line */ 01474 while (*cp == ' ') 01475 { 01476 cp++; 01477 word++; 01478 } 01479 /* find end of word */ 01480 while ((*cp!=' ' && *cp!='\n') && cp<=buffer+line_len) cp++; 01481 *cp=0; 01482 /* is this missing? */ 01483 if (*word == '?') 01484 *p = MISSING_VALUE; 01485 else 01486 /* is this symbolic? */ 01487 if (symbols[j]) { 01488 /* read symbolic data */ 01489 int w=0; 01490 while (symbols[j][w] && /* look for this symbol */ 01491 strcmp(symbols[j][w],word)!=0 && 01492 w<n_symbols[j]) w++; 01493 if (w==n_rows || !symbols[j][w]) 01494 PLERROR("logic error in loadSTATLOG"); 01495 *p = w; 01496 } 01497 else 01498 { 01499 /* read numeric data */ 01500 #ifdef USEDOUBLE 01501 sscanf(word,"%lf",p); 01502 #else 01503 sscanf(word,"%f",p); 01504 #endif 01505 } 01506 word=cp+1; 01507 p++; 01508 } 01509 } 01510 fclose(f); 01511 } 01512 01513 if(!to_symbols) 01514 { 01515 for (i=0; i<mat.width(); i++) 01516 { 01517 for (j=0; j<n_symbols[i]; j++) 01518 free(symbols[i][j]); 01519 free(symbols[i]); 01520 } 01521 free(symbols); 01522 free(n_symbols); 01523 } 01524 #undef convert_STATLOG_BUF_LEN 01525 01526 return mat; 01527 } 01528 01529 01530 // jpeg stuff... 01531 void loadJPEGrgb(const string& jpeg_filename, Mat& rgbmat, int& row_size, int scale) 01532 { 01533 string tmpfile = jpeg_filename + ".pnm"; 01534 char command[1000]; 01535 sprintf(command,"djpeg -pnm -scale 1/%d %s > %s", 01536 scale,jpeg_filename.c_str(),tmpfile.c_str()); 01537 system(command); 01538 FILE* fp = fopen(tmpfile.c_str(),"r"); 01539 if (!fp) 01540 PLERROR("reading %s",tmpfile.c_str()); 01541 fscanf(fp,"%s",command); 01542 int w,h; 01543 fscanf(fp,"%d %d\n",&w,&h); 01544 fscanf(fp,"%*d\n"); 01545 int n=w*h; 01546 rgbmat.resize(n,3); 01547 real *d=rgbmat.data(); 01548 for (int i=0;i<n;i++) 01549 for (int k=0;k<3;k++,d++) 01550 *d =(real)(getc(fp)); 01551 fclose(fp); 01552 sprintf(command,"rm %s",tmpfile.c_str()); 01553 system(command); 01554 row_size = w; 01555 } 01556 01557 void parseSizeFromRemainingLines(const PPath& filename, PStream& in, bool& could_be_old_amat, int& length, int& width) 01558 { 01559 string line; 01560 getNextNonBlankLine(in,line); 01561 if(line=="") // There are no value lines 01562 { 01563 width=length=0; 01564 could_be_old_amat=false; 01565 return; 01566 } 01567 01568 int nfields1 = int(split(line).size()); 01569 getNextNonBlankLine(in,line); 01570 if(line=="") // There is only one line 01571 { 01572 length = 1; 01573 width = nfields1; 01574 could_be_old_amat = false; 01575 return; 01576 } 01577 01578 // The number of lines that seems to contain values 01579 int guesslength = countNonBlankLinesOfFile(filename); 01580 01581 int nfields2 = int(split(line).size()); // The width of the second line. 01582 if(nfields1==nfields2) // looks like a plain ascii file 01583 { 01584 length = guesslength; 01585 width = nfields1; 01586 return; 01587 } 01588 01589 if(!could_be_old_amat || nfields1!=2) 01590 return; // could not be an old .amat with first 2 numbers being length width 01591 01592 // OK we now suppose that we may have an old-format vmatrix 01593 // PLERROR("In parseSizeFromRemainingLines - This part of the code is not PStream compatible yet"); 01594 01595 // Get to the beginning of the file 01596 PStream rein = openFile(filename, PStream::raw_ascii, "r"); 01597 01598 // Reread the first line as two real numbers 01599 real a = -1.0, b = -1.0; 01600 rein >> a >> b; 01601 01602 if(guesslength == int(a)+1 // +1 since the size line was counted in guesslength but should not 01603 && real(int(a))==a && real(int(b))==b // Sizes must be integers and 01604 && a>0 && b>0 // positive 01605 && int(b)==nfields2 ) // The first row of values has the expected width 01606 { 01607 // We assume we have an old style .amat 01608 length = int(a); 01609 width = int(b); 01610 } 01611 } 01612 01613 } // end of namespace PLearn 01614 01615 01616 /* 01617 Local Variables: 01618 mode:c++ 01619 c-basic-offset:4 01620 c-file-style:"stroustrup" 01621 c-file-offsets:((innamespace . 0)(inline-open . 0)) 01622 indent-tabs-mode:nil 01623 fill-column:79 01624 End: 01625 */ 01626 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :