PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // 00007 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 00038 00039 /* ******************************************************* 00040 * $Id: MatIO.h 9436 2008-09-04 18:48:55Z nouiz $ 00041 * This file is part of the PLearn library. 00042 ******************************************************* */ 00043 00044 #ifndef MatIO_INC 00045 #define MatIO_INC 00046 00047 #include <plearn/math/TMat.h> 00048 #include "fileutils.h" 00049 #include <plearn/base/stringutils.h> 00050 #include <plearn/base/lexical_cast.h> 00051 #include <plearn/base/tostring.h> 00052 #include <plearn/io/openFile.h> 00053 00054 namespace PLearn { 00055 using namespace std; 00056 00057 00060 inline Mat makeMat(int length, int width, const string& values) 00061 { Mat m(length,width); m << values; return m; } 00062 00063 inline Vec makeVec(int length, const string& values) 00064 { Vec v(length); v << values; return v; } 00065 00068 void loadMat(const string& filename, TMat<float>& mat); 00069 void loadMat(const string& filename, TMat<double>& mat); 00070 void loadVec(const string& filename, TVec<float>& vec); 00071 void loadVec(const string& filename, TVec<double>& vec); 00072 00074 void savePVec(const string& filename, const TVec<float>& vec); 00075 void savePVec(const string& filename, const TVec<double>& vec); 00076 void loadPVec(const string& filename, TVec<float>& vec); 00077 void loadPVec(const string& filename, TVec<double>& vec); 00078 void savePMat(const string& filename, const TMat<float>& mat); 00079 void savePMat(const string& filename, const TMat<double>& mat); 00081 void savePMatFieldnames(const string& pmatfilename, const TVec<string>& fieldnames); 00082 void loadPMat(const string& filename, TMat<float>& mat); 00083 void loadPMat(const string& filename, TMat<double>& mat); 00084 00086 00088 template<class T> 00089 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, TVec<map<string,real> >* map_sr=0); 00090 template<class T> void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, TVec<map<string,real> >* map_sr = 0); 00091 template<class T> void loadAscii(const PPath& filename, TMat<T>& mat); 00092 00093 void parseSizeFromRemainingLines(const PPath& filename, PStream& in, bool& could_be_old_amat, int& length, int& width); 00094 00095 // norman: added another function to solve the internal compiler error of .NET when using 00096 // default parameter with templates. See old declaration: 00097 //template<class T> void saveAscii(const string& filename, const TMat<T>& mat, 00098 // const TVec<string>& fieldnames = TVec<string>() ); 00099 template<class T> void saveAscii(const string& filename, const TMat<T>& mat, 00100 const TVec<string>& fieldnames, 00101 int inputsize=-1, int targetsize=-1, int weightsize=-1, int extrasize=0); 00102 template<class T> void saveAscii(const string& filename, const TMat<T>& mat); 00103 00105 template<class T> void saveAscii(const string& filename, const TVec<T>& vec); 00106 template<class T> void loadAscii(const PPath& filename, TVec<T>& vec); 00107 00109 void loadGnuplot(const string& filename, Mat& mat); 00110 void saveGnuplot(const string& filename, const Vec& vec); 00111 void saveGnuplot(const string& filename, const Mat& mat); 00112 00114 // Format readable by matlab 00115 00120 void matlabSave( const PPath& dir, const string& plot_title, const Vec& data, 00121 const Vec& add_col, const Vec& bounds, string lengend="", bool save_plot=true); 00122 void matlabSave( const PPath& dir, const string& plot_title, 00123 const Vec& xValues, 00124 const Vec& yValues, const Vec& add_col, const Vec& bounds, string lengend="", bool save_plot=true); 00125 00127 void matlabSave( const PPath& dir, const string& plot_title, const Mat& data, 00128 const Vec& add_col, const Vec& bounds, TVec<string> legend=TVec<string>(), bool save_plot=true); 00129 00138 void matlabSave( const PPath& dir, const string& plot_title, 00139 const Vec& xValues, 00140 const Mat& yValues, const Vec& add_col, const Vec& bounds, TVec<string> legend=TVec<string>(), bool save_plot=true); 00142 00144 void loadAsciiWithoutSize(const string& filename, const Vec& vec); 00145 void saveAsciiWithoutSize(const string& filename, const Vec& vec); 00146 void loadAsciiWithoutSize(const string& filename, const Mat& mat); 00147 void saveAsciiWithoutSize(const string& filename, const Mat& mat); 00148 00150 Mat loadSNMat(const string& filename); 00151 Vec loadSNVec(const string& filename); 00152 void saveSNMat(const string& filename, const Mat& mat); 00153 void saveSNVec(const string& filename, const Vec& vec); 00154 00156 Mat loadADMat(const string& filename); 00157 Vec loadADVec(const string& filename); 00158 00174 Mat loadUCIMLDB(const string& filename, char ****to_symbols=0, int **to_n_symbols=0, TVec<int>* max_in_col = 0, TVec<string>* header_columns = 0); 00175 00186 Mat loadSTATLOG(const string& filename, char ****to_symbols=0, int **to_n_symbols=0); 00187 00196 void loadJPEGrgb(const string& jpeg_filename, Mat& rgbmat, int& row_size, int scale = 1); 00197 00198 00199 template<class T> 00200 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, TVec<map<string,real> >* map_sr) 00201 { 00202 int inputsize=-1, targetsize=-1, weightsize=-1, extrasize=-1; 00203 loadAscii(filename, mat, fieldnames, inputsize, targetsize, weightsize, extrasize, map_sr); 00204 } 00205 00206 template<class T> 00207 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, TVec<map<string,real> >* map_sr) 00208 { 00209 int extrasize = -1; 00210 loadAscii(filename, mat, fieldnames, inputsize, targetsize, weightsize, extrasize, map_sr); 00211 } 00212 00213 // Intelligent function that will load a file in almost all ascii formats that ever existed in this lab. 00214 // Additionally, if 'map_sr' is provided, it will fill it with the string -> real mappings encountered. 00215 template<class T> 00216 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, int& extrasize, TVec<map<string,real> >* map_sr) 00217 { 00218 PStream in = openFile(filename, PStream::raw_ascii, "r"); 00219 00220 int length = -1; 00221 int width = -1; 00222 inputsize = -1; 00223 targetsize = -1; 00224 weightsize = 0; 00225 bool could_be_old_amat=true; // true while there is still a chance that this be an "old" amat format (length and width in first row with no starting ##) 00226 00227 in >> ws; 00228 string line; 00229 00230 while(in.peek()=='#') 00231 { 00232 in.getline(line); 00233 could_be_old_amat = false; 00234 00235 size_t pos=line.find(":"); 00236 if(pos!=string::npos) 00237 { 00238 string sub=line.substr(0,pos); 00239 if(sub=="#size") // we've found the dimension specification line 00240 { 00241 string siz=removeblanks((line.substr(pos)).substr(1)); 00242 vector<string> dim = split(siz," "); 00243 if(dim.size()!=2) PLERROR("I need exactly 2 dimensions for matrix"); 00244 length = toint(dim[0]); 00245 width = toint(dim[1]); 00246 } 00247 else if(sub=="#sizes") // we've found inputsize targetsize weightsize specification 00248 { 00249 string siz=removeblanks((line.substr(pos)).substr(1)); 00250 vector<string> dim = split(siz," "); 00251 if(dim.size()!=3 && dim.size()!=4) 00252 PLERROR("I need 3 or 4 numbers after #sizes: inputsize targetsize weightsize extrasize"); 00253 inputsize = toint(dim[0]); 00254 targetsize = toint(dim[1]); 00255 weightsize = toint(dim[2]); 00256 if(dim.size()==4) 00257 extrasize = toint(dim[3]); 00258 else 00259 extrasize = 0; 00260 } 00261 else if(sub=="#") // we've found the fieldnames specification line 00262 { 00263 string fnl=line.substr(pos).substr(1); 00264 fieldnames = split(fnl," "); 00265 width=fieldnames.size(); 00266 } 00267 } 00268 in >> ws; 00269 } 00270 00271 if(length==-1) // still looking for size info... 00272 parseSizeFromRemainingLines(filename, in, could_be_old_amat, length, width); 00273 00274 if(length==-1) 00275 PLERROR("In loadAscii: trying to load but couldn't determine file format automatically for %s",filename.absolute().c_str()); 00276 00277 if(width != -1 && width != fieldnames.length()) 00278 { 00279 if (fieldnames.length() != 0) 00280 PLWARNING("In loadAscii: Number of fieldnames (%d) and width (%d) mismatch in file %s. " 00281 "Replacing fieldnames by 'Field-0', 'Field-1', ...", 00282 fieldnames.length(), width, filename.absolute().c_str()); 00283 fieldnames.resize(width); 00284 for(int i= 0; i < width; ++i) 00285 fieldnames[i]= string("Field-") + PLearn::tostring(i); 00286 } 00287 00288 // We are now more careful about the possibility of the stream being in a 00289 // bad state. The sequel in.seekg(0); in.clear(); did not seem to do the job. 00290 in = PStream(); // Close file. 00291 PStream loadmat = openFile(filename, PStream::raw_ascii, "r"); 00292 00293 mat.resize(length,width); 00294 TVec<int> current_map(width); 00295 current_map.fill(1001); // The value of the string mapping we start with. 00296 TVec<T> current_max(width); // The max of the numerical values in each column. 00297 current_max.clear(); 00298 // Initialize the mappings to empty mappings. 00299 if (map_sr) 00300 map_sr->resize(width); 00301 00302 string inp_element; 00303 for(int i=0; i<length; i++) 00304 { 00305 T* mat_i = mat[i]; 00306 skipBlanksAndComments(loadmat); 00307 for(int j=0; j<width; j++) 00308 { 00309 if (loadmat.good()) 00310 { 00311 loadmat >> inp_element; 00312 if (pl_isnumber(inp_element)) 00313 { 00314 mat_i[j] = T(pl_strtod(inp_element.c_str(), 0)); 00315 if (map_sr) 00316 { 00317 T val = mat_i[j]; 00318 // We need to make sure that this number does not conflict 00319 // with a string mapping. 00320 if (val > current_max[j]) 00321 current_max[j] = val; 00322 if (current_max[j] >= current_map[j]) 00323 current_map[j] = int(current_max[j] + 1); 00324 map<string,real>& m = (*map_sr)[j]; 00325 for (map<string,real>::iterator it = m.begin(); it != m.end(); it++) 00326 { 00327 if (fast_exact_is_equal(it->second, val)) 00328 { 00329 // We're screwed, there is currently a mapping between a string 00330 // and this numeric value. We have to change it. 00331 // We pick either the next string mapping value, or the current 00332 // max in the column (+ 1) if it is larger. 00333 int cur_max_plus_one = int(real(current_max[j]) + 1); 00334 if (cur_max_plus_one > current_map[j]) 00335 { 00336 it->second = cur_max_plus_one; 00337 current_map[j] = cur_max_plus_one; 00338 } 00339 else 00340 it->second = current_map[j]; 00341 current_map[j]++; 00342 // In addition, we have to modify all previous data, which sucks. 00343 for (int k = 0; k < i; k++) 00344 { 00345 if (fast_exact_is_equal(mat(k, j), val)) 00346 mat(k, j) = T(it->second); 00347 } 00348 } 00349 } 00350 } 00351 } 00352 else 00353 { 00354 // This is a string! 00355 if (map_sr) // Already encountered ? 00356 { 00357 map<string,real>& m = (*map_sr)[j]; 00358 map<string,real>::iterator it = m.find(inp_element); 00359 if(it != m.end()) // It already exists in the map. 00360 { 00361 mat_i[j] = T(it->second); 00362 } 00363 else // We need to add it. 00364 { 00365 (*map_sr)[j][inp_element] = current_map[j]; 00366 mat_i[j] = T(current_map[j]); 00367 current_map[j]++; 00368 } 00369 } 00370 else 00371 PLERROR("In loadAscii - You need to provide 'map_sr' if you want to load an ASCII file with strings"); 00372 } 00373 } 00374 else 00375 { 00376 PLERROR("In loadAscii - Not enough elements in row %d: found " 00377 "%d, but expected %d", i, j, width); 00378 } 00379 } 00380 } 00381 } 00382 00389 template<class T> 00390 void loadAsciiSingleBinaryDescriptor(const PPath& filename, TMat<T>& mat) 00391 { 00392 PStream in = openFile(filename, PStream::raw_ascii, "r"); 00393 00394 int length = -1; 00395 int width = -1; 00396 00397 in >> ws; 00398 string line; 00399 00400 while(in.peek()=='#') 00401 { 00402 in.getline(line); 00403 00404 size_t pos=line.find(":"); 00405 if(pos!=string::npos) 00406 { 00407 string sub=line.substr(0,pos); 00408 if(sub=="#size") // we've found the dimension specification line 00409 { 00410 string siz=removeblanks((line.substr(pos)).substr(1)); 00411 vector<string> dim = split(siz," "); 00412 if(dim.size()!=2) PLERROR("In loadAsciiSingleBinaryDescriptor: I need exactly 2 dimensions for matrix"); 00413 length = toint(dim[0]); 00414 width = toint(dim[1]); 00415 } 00416 } 00417 in >> ws; 00418 } 00419 00420 if(length==-1) // still looking for size info... 00421 { 00422 PLERROR("In loadAsciiSingleBinaryDescriptor: Be nice and specify a width and length"); 00423 } 00424 00425 // We are now more careful about the possibility of the stream being in a 00426 // bad state. 00427 mat.resize(length,width); 00428 string inp_element; 00429 for(int i=0; i<length; i++) 00430 { 00431 T* mat_i = mat[i]; 00432 skipBlanksAndComments(in); 00433 in >> inp_element; // Read the entry name. 00434 in >> inp_element; // Read the binary descriptor. 00435 if (inp_element.length() != (unsigned int) width) { 00436 PLERROR("In loadAsciiSingleBinaryDescriptor, a descriptor isn't the right size"); 00437 } 00438 for(int j=0; j<width; j++) { 00439 mat_i[j] = pl_strtod(inp_element.substr(j,1).c_str(), 0); 00440 } 00441 } 00442 } 00443 00444 template<class T> 00445 void loadAscii(const PPath& filename, TVec<T>& vec) 00446 { 00447 ifstream in(filename.absolute().c_str()); 00448 if(!in) 00449 PLERROR("In loadAscii could not open file %s for reading",filename.c_str()); 00450 00451 int size = -1; 00452 in >> size; 00453 if (size<0 || size>1e10) 00454 PLERROR("In Vec::loadAscii the file is probably not in the right format: size=%d", size); 00455 vec.resize(size); 00456 typename TVec<T>::iterator it = vec.begin(); 00457 typename TVec<T>::iterator itend = vec.end(); 00458 00459 // We are now more careful about the possibility of the stream being in a 00460 // bad state 00461 string inp_element; 00462 for(; it!=itend; ++it) { 00463 if (in) { 00464 in >> inp_element; 00465 *it = T(pl_strtod(inp_element.c_str(), 0)); 00466 } 00467 if (!in) { 00468 in.clear(); 00469 *it = MISSING_VALUE; 00470 } 00471 } 00472 } 00473 00474 // norman: very stupid function to solve compiler error of VS .NET (see declaration) 00475 template<class T> 00476 void saveAscii(const string& filename, const TMat<T>& mat) 00477 { 00478 saveAscii(filename, mat, TVec<string>()); 00479 } 00480 00481 template<class T> 00482 void saveAscii(const string& filename, const TMat<T>& mat, const TVec<string>& fieldnames, 00483 int inputsize, int targetsize, int weightsize, int extrasize) 00484 { 00485 ofstream out(filename.c_str()); 00486 if (!out) 00487 PLERROR("In saveAscii could not open file %s for writing",filename.c_str()); 00488 00489 out << "#size: "<< mat.length() << ' ' << mat.width() << endl; 00490 out.precision(15); 00491 if(fieldnames.size()>0) 00492 { 00493 out << "#: "; 00494 for(int k=0; k<fieldnames.size(); k++) 00495 //there must not be any space in a field name... 00496 out << space_to_underscore(fieldnames[k]) << ' '; 00497 out << endl; 00498 } 00499 if(inputsize>=0) 00500 out << "#sizes: " << inputsize << ' ' << targetsize << ' ' << weightsize << ' ' << extrasize << endl; 00501 00502 for(int i=0; i<mat.length(); i++) 00503 { 00504 const T* row_i = mat[i]; 00505 for(int j=0; j<mat.width(); j++) 00506 out << row_i[j] << ' '; 00507 out << '\n'; 00508 } 00509 } 00510 00511 template<class T> 00512 void saveAscii(const string& filename, const TVec<T>& vec) 00513 { 00514 ofstream out(filename.c_str()); 00515 if (!out) 00516 PLERROR("In saveAscii: could not open file %s for writing",filename.c_str()); 00517 00518 out << vec.length() << endl; 00519 out.precision(15); 00520 00521 typename TVec<T>::iterator it = vec.begin(); 00522 typename TVec<T>::iterator itend = vec.end(); 00523 for(; it!=itend; ++it) 00524 out << *it << ' '; 00525 out << endl; 00526 } 00527 00528 template<class T> 00529 void loadAscii(const PPath& filename, TMat<T>& mat) 00530 { 00531 TVec<std::string> fn; 00532 loadAscii(filename,mat,fn); 00533 } 00534 00535 } // end of namespace PLearn 00536 00537 00538 #endif 00539 00540 00541 /* 00542 Local Variables: 00543 mode:c++ 00544 c-basic-offset:4 00545 c-file-style:"stroustrup" 00546 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00547 indent-tabs-mode:nil 00548 fill-column:79 00549 End: 00550 */ 00551 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :