PLearn 0.1
MatIO.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037  
00038 
00039 /* *******************************************************      
00040  * $Id: MatIO.h 9436 2008-09-04 18:48:55Z nouiz $
00041  * This file is part of the PLearn library.
00042  ******************************************************* */
00043 
00044 #ifndef MatIO_INC
00045 #define MatIO_INC
00046 
00047 #include <plearn/math/TMat.h>
00048 #include "fileutils.h"                  
00049 #include <plearn/base/stringutils.h>    
00050 #include <plearn/base/lexical_cast.h>   
00051 #include <plearn/base/tostring.h>       
00052 #include <plearn/io/openFile.h>
00053 
00054 namespace PLearn {
00055 using namespace std;
00056 
00057 
00060 inline Mat makeMat(int length, int width, const string& values)
00061 { Mat m(length,width); m << values; return m; } 
00062 
00063 inline Vec makeVec(int length, const string& values)
00064 { Vec v(length); v << values; return v; }
00065 
00068 void loadMat(const string& filename, TMat<float>& mat);
00069 void loadMat(const string& filename, TMat<double>& mat);
00070 void loadVec(const string& filename, TVec<float>& vec);
00071 void loadVec(const string& filename, TVec<double>& vec);
00072 
00074 void savePVec(const string& filename, const TVec<float>& vec);
00075 void savePVec(const string& filename, const TVec<double>& vec);
00076 void loadPVec(const string& filename, TVec<float>& vec);
00077 void loadPVec(const string& filename, TVec<double>& vec);
00078 void savePMat(const string& filename, const TMat<float>& mat);
00079 void savePMat(const string& filename, const TMat<double>& mat);
00081 void savePMatFieldnames(const string& pmatfilename, const TVec<string>& fieldnames);
00082 void loadPMat(const string& filename, TMat<float>& mat);
00083 void loadPMat(const string& filename, TMat<double>& mat);
00084 
00086 
00088 template<class T>
00089 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, TVec<map<string,real> >* map_sr=0);
00090 template<class T> void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, TVec<map<string,real> >* map_sr = 0);
00091 template<class T> void loadAscii(const PPath& filename, TMat<T>& mat);
00092 
00093 void parseSizeFromRemainingLines(const PPath& filename, PStream& in, bool& could_be_old_amat, int& length, int& width);
00094 
00095 // norman: added another function to solve the internal compiler error of .NET when using
00096 // default parameter with templates. See old declaration:
00097 //template<class T> void saveAscii(const string& filename, const TMat<T>& mat, 
00098 //                                const TVec<string>& fieldnames = TVec<string>() );
00099 template<class T> void saveAscii(const string& filename, const TMat<T>& mat, 
00100                                  const TVec<string>& fieldnames, 
00101                                  int inputsize=-1, int targetsize=-1, int weightsize=-1, int extrasize=0);
00102 template<class T> void saveAscii(const string& filename, const TMat<T>& mat);
00103 
00105 template<class T> void saveAscii(const string& filename, const TVec<T>& vec);
00106 template<class T> void loadAscii(const PPath& filename, TVec<T>& vec);
00107 
00109 void loadGnuplot(const string& filename, Mat& mat);
00110 void saveGnuplot(const string& filename, const Vec& vec);
00111 void saveGnuplot(const string& filename, const Mat& mat);
00112 
00114 // Format readable by matlab
00115 
00120 void matlabSave( const PPath& dir, const string& plot_title, const Vec& data, 
00121                  const Vec& add_col, const Vec& bounds, string lengend="", bool save_plot=true);  
00122 void matlabSave( const PPath& dir, const string& plot_title, 
00123                  const Vec& xValues,
00124                  const Vec& yValues, const Vec& add_col, const Vec& bounds, string lengend="", bool save_plot=true);  
00125 
00127 void matlabSave( const PPath& dir, const string& plot_title, const Mat& data, 
00128                  const Vec& add_col, const Vec& bounds, TVec<string> legend=TVec<string>(), bool save_plot=true);
00129 
00138 void matlabSave( const PPath& dir, const string& plot_title, 
00139                  const Vec& xValues,
00140                  const Mat& yValues, const Vec& add_col, const Vec& bounds, TVec<string> legend=TVec<string>(), bool save_plot=true);
00142 
00144 void loadAsciiWithoutSize(const string& filename, const Vec& vec);
00145 void saveAsciiWithoutSize(const string& filename, const Vec& vec);
00146 void loadAsciiWithoutSize(const string& filename, const Mat& mat);
00147 void saveAsciiWithoutSize(const string& filename, const Mat& mat);
00148 
00150 Mat loadSNMat(const string& filename);
00151 Vec loadSNVec(const string& filename);
00152 void saveSNMat(const string& filename, const Mat& mat);
00153 void saveSNVec(const string& filename, const Vec& vec);
00154 
00156 Mat loadADMat(const string& filename);
00157 Vec loadADVec(const string& filename);
00158 
00174 Mat loadUCIMLDB(const string& filename, char ****to_symbols=0, int **to_n_symbols=0, TVec<int>* max_in_col = 0, TVec<string>* header_columns = 0);
00175 
00186 Mat loadSTATLOG(const string& filename, char ****to_symbols=0, int **to_n_symbols=0);
00187 
00196 void loadJPEGrgb(const string& jpeg_filename, Mat& rgbmat, int& row_size, int scale = 1);
00197 
00198 
00199 template<class T>
00200 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, TVec<map<string,real> >* map_sr)
00201 { 
00202     int inputsize=-1, targetsize=-1, weightsize=-1, extrasize=-1;
00203     loadAscii(filename, mat, fieldnames, inputsize, targetsize, weightsize, extrasize, map_sr);
00204 }
00205 
00206 template<class T>
00207 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, TVec<map<string,real> >* map_sr)
00208 {
00209     int extrasize = -1;
00210     loadAscii(filename, mat, fieldnames, inputsize, targetsize, weightsize, extrasize, map_sr);
00211 }
00212 
00213 // Intelligent function that will load a file in almost all ascii formats that ever existed in this lab.
00214 // Additionally, if 'map_sr' is provided, it will fill it with the string -> real mappings encountered.
00215 template<class T>
00216 void loadAscii(const PPath& filename, TMat<T>& mat, TVec<string>& fieldnames, int& inputsize, int& targetsize, int& weightsize, int& extrasize, TVec<map<string,real> >* map_sr)
00217 {
00218     PStream in = openFile(filename, PStream::raw_ascii, "r");
00219   
00220     int length = -1;
00221     int width = -1;
00222     inputsize = -1;
00223     targetsize = -1;
00224     weightsize = 0;
00225     bool could_be_old_amat=true; // true while there is still a chance that this be an "old" amat format (length and width in first row with no starting ##)
00226   
00227     in >> ws;
00228     string line;
00229   
00230     while(in.peek()=='#')
00231     {
00232         in.getline(line);
00233         could_be_old_amat = false;
00234 
00235         size_t pos=line.find(":");
00236         if(pos!=string::npos)
00237         {
00238             string sub=line.substr(0,pos);
00239             if(sub=="#size") // we've found the dimension specification line
00240             {
00241                 string siz=removeblanks((line.substr(pos)).substr(1));
00242                 vector<string> dim = split(siz," ");
00243                 if(dim.size()!=2)  PLERROR("I need exactly 2 dimensions for matrix");
00244                 length = toint(dim[0]);
00245                 width = toint(dim[1]);
00246             }
00247             else if(sub=="#sizes") // we've found inputsize targetsize weightsize specification
00248             {
00249                 string siz=removeblanks((line.substr(pos)).substr(1));
00250                 vector<string> dim = split(siz," ");
00251                 if(dim.size()!=3 && dim.size()!=4)  
00252                     PLERROR("I need 3 or 4 numbers after #sizes: inputsize targetsize weightsize extrasize");
00253                 inputsize = toint(dim[0]);
00254                 targetsize = toint(dim[1]);
00255                 weightsize = toint(dim[2]);
00256                 if(dim.size()==4)
00257                     extrasize = toint(dim[3]);
00258                 else
00259                     extrasize = 0;
00260             }
00261             else if(sub=="#") // we've found the fieldnames specification line
00262             {
00263                 string fnl=line.substr(pos).substr(1);
00264                 fieldnames = split(fnl," ");
00265                 width=fieldnames.size();
00266             }              
00267         }
00268         in >> ws;
00269     }
00270 
00271     if(length==-1)  // still looking for size info...
00272         parseSizeFromRemainingLines(filename, in, could_be_old_amat, length, width);
00273 
00274     if(length==-1)
00275         PLERROR("In loadAscii: trying to load but couldn't determine file format automatically for %s",filename.absolute().c_str());
00276 
00277     if(width != -1 && width != fieldnames.length())
00278     {
00279         if (fieldnames.length() != 0)
00280             PLWARNING("In loadAscii:  Number of fieldnames (%d) and width (%d) mismatch in file %s.  "
00281                       "Replacing fieldnames by 'Field-0', 'Field-1', ...", 
00282                       fieldnames.length(), width, filename.absolute().c_str());
00283         fieldnames.resize(width);
00284         for(int i= 0; i < width; ++i)
00285             fieldnames[i]= string("Field-") + PLearn::tostring(i);
00286     }
00287   
00288     // We are now more careful about the possibility of the stream being in a
00289     // bad state. The sequel in.seekg(0); in.clear(); did not seem to do the job.
00290     in = PStream(); // Close file.
00291     PStream loadmat = openFile(filename, PStream::raw_ascii, "r");
00292   
00293     mat.resize(length,width);
00294     TVec<int> current_map(width);
00295     current_map.fill(1001);   // The value of the string mapping we start with.
00296     TVec<T> current_max(width); // The max of the numerical values in each column.
00297     current_max.clear();
00298     // Initialize the mappings to empty mappings.
00299     if (map_sr) 
00300         map_sr->resize(width);
00301 
00302     string inp_element;
00303     for(int i=0; i<length; i++)
00304     {
00305         T* mat_i = mat[i];
00306         skipBlanksAndComments(loadmat);
00307         for(int j=0; j<width; j++) 
00308         {
00309             if (loadmat.good()) 
00310             {
00311                 loadmat >> inp_element;
00312                 if (pl_isnumber(inp_element)) 
00313                 {
00314                     mat_i[j] = T(pl_strtod(inp_element.c_str(), 0));
00315                     if (map_sr) 
00316                     {
00317                         T val = mat_i[j];
00318                         // We need to make sure that this number does not conflict
00319                         // with a string mapping.
00320                         if (val > current_max[j])
00321                             current_max[j] = val;
00322                         if (current_max[j] >= current_map[j])
00323                             current_map[j] = int(current_max[j] + 1);
00324                         map<string,real>& m = (*map_sr)[j];
00325                         for (map<string,real>::iterator it = m.begin(); it != m.end(); it++) 
00326                         {
00327                             if (fast_exact_is_equal(it->second, val)) 
00328                             {
00329                                 // We're screwed, there is currently a mapping between a string
00330                                 // and this numeric value. We have to change it.
00331                                 // We pick either the next string mapping value, or the current
00332                                 // max in the column (+ 1) if it is larger.
00333                                 int cur_max_plus_one = int(real(current_max[j]) + 1);
00334                                 if (cur_max_plus_one > current_map[j]) 
00335                                 {
00336                                     it->second = cur_max_plus_one;
00337                                     current_map[j] = cur_max_plus_one;
00338                                 } 
00339                                 else
00340                                     it->second = current_map[j];
00341                                 current_map[j]++;
00342                                 // In addition, we have to modify all previous data, which sucks.
00343                                 for (int k = 0; k < i; k++) 
00344                                 {
00345                                     if (fast_exact_is_equal(mat(k, j), val))
00346                                         mat(k, j) = T(it->second);
00347                                 }
00348                             }
00349                         }
00350                     }
00351                 } 
00352                 else 
00353                 {
00354                     // This is a string!
00355                     if (map_sr) // Already encountered ?
00356                     {                        
00357                         map<string,real>& m = (*map_sr)[j];
00358                         map<string,real>::iterator it = m.find(inp_element);
00359                         if(it != m.end()) // It already exists in the map.
00360                         {
00361                             mat_i[j] = T(it->second);
00362                         } 
00363                         else           // We need to add it.
00364                         {                            
00365                             (*map_sr)[j][inp_element] = current_map[j];
00366                             mat_i[j] = T(current_map[j]);
00367                             current_map[j]++;
00368                         }
00369                     } 
00370                     else
00371                         PLERROR("In loadAscii - You need to provide 'map_sr' if you want to load an ASCII file with strings");
00372                 }
00373             }
00374             else 
00375             {
00376                 PLERROR("In loadAscii - Not enough elements in row %d: found "
00377                         "%d, but expected %d", i, j, width);
00378             }
00379         }
00380     }
00381 }
00382 
00389 template<class T>
00390 void loadAsciiSingleBinaryDescriptor(const PPath& filename, TMat<T>& mat)
00391 {
00392     PStream in = openFile(filename, PStream::raw_ascii, "r");
00393 
00394     int length = -1;
00395     int width = -1;
00396   
00397     in >> ws;
00398     string line;
00399 
00400     while(in.peek()=='#')
00401     {
00402         in.getline(line);
00403  
00404         size_t pos=line.find(":");
00405         if(pos!=string::npos)
00406         {
00407             string sub=line.substr(0,pos);
00408             if(sub=="#size") // we've found the dimension specification line
00409             {
00410                 string siz=removeblanks((line.substr(pos)).substr(1));
00411                 vector<string> dim = split(siz," ");
00412                 if(dim.size()!=2)  PLERROR("In loadAsciiSingleBinaryDescriptor: I need exactly 2 dimensions for matrix");
00413                 length = toint(dim[0]);
00414                 width = toint(dim[1]);
00415             }
00416         }
00417         in >> ws;
00418     }
00419 
00420     if(length==-1)  // still looking for size info...
00421     {
00422         PLERROR("In loadAsciiSingleBinaryDescriptor: Be nice and specify a width and length");
00423     }
00424 
00425     // We are now more careful about the possibility of the stream being in a
00426     // bad state.
00427     mat.resize(length,width);
00428     string inp_element;
00429     for(int i=0; i<length; i++)
00430     {
00431         T* mat_i = mat[i];
00432         skipBlanksAndComments(in);
00433         in >> inp_element;  // Read the entry name.
00434         in >> inp_element;  // Read the binary descriptor.
00435         if (inp_element.length() != (unsigned int) width) {
00436             PLERROR("In loadAsciiSingleBinaryDescriptor, a descriptor isn't the right size");
00437         }
00438         for(int j=0; j<width; j++) {
00439             mat_i[j] = pl_strtod(inp_element.substr(j,1).c_str(), 0);
00440         }
00441     }
00442 }
00443 
00444 template<class T>
00445 void loadAscii(const PPath& filename, TVec<T>& vec)
00446 {
00447     ifstream in(filename.absolute().c_str());
00448     if(!in)
00449         PLERROR("In loadAscii could not open file %s for reading",filename.c_str());
00450  
00451     int size = -1;
00452     in >> size;
00453     if (size<0 || size>1e10)
00454         PLERROR("In Vec::loadAscii the file is probably not in the right format: size=%d", size);
00455     vec.resize(size);
00456     typename TVec<T>::iterator it = vec.begin();
00457     typename TVec<T>::iterator itend = vec.end();
00458 
00459     // We are now more careful about the possibility of the stream being in a
00460     // bad state
00461     string inp_element;
00462     for(; it!=itend; ++it) {
00463         if (in) {
00464             in >> inp_element;
00465             *it = T(pl_strtod(inp_element.c_str(), 0));
00466         }
00467         if (!in) {
00468             in.clear();
00469             *it = MISSING_VALUE;
00470         }
00471     }
00472 }
00473 
00474 // norman: very stupid function to solve compiler error of VS .NET (see declaration)
00475 template<class T>
00476 void saveAscii(const string& filename, const TMat<T>& mat)
00477 {
00478     saveAscii(filename, mat, TVec<string>());
00479 }
00480 
00481 template<class T> 
00482 void saveAscii(const string& filename, const TMat<T>& mat, const TVec<string>& fieldnames,
00483                int inputsize, int targetsize, int weightsize, int extrasize)
00484 {
00485     ofstream out(filename.c_str());
00486     if (!out)
00487         PLERROR("In saveAscii could not open file %s for writing",filename.c_str());
00488 
00489     out << "#size: "<< mat.length() << ' ' << mat.width() << endl;
00490     out.precision(15);
00491     if(fieldnames.size()>0)
00492     {
00493         out << "#: ";
00494         for(int k=0; k<fieldnames.size(); k++)
00495             //there must not be any space in a field name...
00496             out << space_to_underscore(fieldnames[k]) << ' ';
00497         out << endl;
00498     }
00499     if(inputsize>=0)
00500         out << "#sizes: " << inputsize << ' ' << targetsize << ' ' << weightsize << ' ' << extrasize << endl;
00501 
00502     for(int i=0; i<mat.length(); i++) 
00503     {
00504         const T* row_i = mat[i];
00505         for(int j=0; j<mat.width(); j++)
00506             out << row_i[j] << ' ';
00507         out << '\n';
00508     }
00509 }
00510   
00511 template<class T> 
00512 void saveAscii(const string& filename, const TVec<T>& vec)
00513 {
00514     ofstream out(filename.c_str());
00515     if (!out)
00516         PLERROR("In saveAscii: could not open file %s for writing",filename.c_str());
00517 
00518     out << vec.length() << endl;
00519     out.precision(15);
00520 
00521     typename TVec<T>::iterator it = vec.begin();
00522     typename TVec<T>::iterator itend = vec.end();
00523     for(; it!=itend; ++it)
00524         out << *it << ' ';
00525     out << endl;
00526 }
00527 
00528 template<class T>
00529 void loadAscii(const PPath& filename, TMat<T>& mat)
00530 {
00531     TVec<std::string> fn;
00532     loadAscii(filename,mat,fn);
00533 }
00534 
00535 } // end of namespace PLearn
00536 
00537 
00538 #endif
00539 
00540 
00541 /*
00542   Local Variables:
00543   mode:c++
00544   c-basic-offset:4
00545   c-file-style:"stroustrup"
00546   c-file-offsets:((innamespace . 0)(inline-open . 0))
00547   indent-tabs-mode:nil
00548   fill-column:79
00549   End:
00550 */
00551 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines