PLearn 0.1
getDataSet.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999,2000 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 
00038 /* *******************************************************      
00039  * $Id: getDataSet.cc 9330 2008-07-31 19:46:01Z chrish $
00040  * AUTHORS: Pascal Vincent
00041  * This file is part of the PLearn library.
00042  ******************************************************* */
00043 
00044 #include <plearn/base/Object.h>
00045 #include "getDataSet.h"
00046 #include <plearn/io/MatIO.h>              
00047 
00048 #include <plearn/base/stringutils.h>      
00049 #include <plearn/io/fileutils.h>          
00050 #include <plearn/io/pl_log.h>
00051 #include <plearn/io/PyPLearnScript.h>
00052 #include <plearn/vmat/DiskVMatrix.h>
00053 #include <plearn/vmat/FileVMatrix.h>
00054 #include <plearn/vmat/VMat.h>
00055 #include <plearn/vmat/VVMatrix.h>
00056 #include <nspr/prtime.h>
00057 
00058 namespace PLearn {
00059 using namespace std;
00060 
00062 // getDataSetDate //
00064 time_t getDataSetDate(const PPath& dataset_path)
00065 {
00066     return getDataSet(dataset_path)->getMtime();
00067 }
00068 
00070 // getDataSet //
00072 VMat getDataSet(const PPath& dataset_path)
00073 {
00074     VMat vm;
00075     // Parse the base file name and the potential parameters.
00076     string dataset_abs;
00077     map<string, string> params;
00078     if (isfile(dataset_path))
00079         dataset_abs = dataset_path;
00080     else
00081         // There may be parameters that need parsing.
00082         parseBaseAndParameters(dataset_path, dataset_abs, params);
00083     PPath dataset(dataset_abs);
00084     bool use_params = false;
00085 
00086     // See getDataSetHelp() for supported formats.
00087     string ext = dataset.extension();
00088     if (isfile(dataset)) {
00089         if (ext == "amat") {
00090             // Check if the extension is ".bin.amat".
00091             // (old deprecated extension)
00092             if (dataset.find(".bin.", ((string::size_type) dataset.size()) - 9) != string::npos) {
00093                 PLERROR("In getDataSet - The '.bin.amat' extension is deprecated, you "
00094                         "must now use the .abmat extension");
00095             } else
00096                 vm = loadAsciiAsVMat(dataset);
00097         } else if (ext == "abmat") {
00098             Mat tempMat;
00099             loadAsciiSingleBinaryDescriptor(dataset, tempMat);
00100             vm = VMat(tempMat);
00101             vm->updateMtime(dataset);
00102         } else if (ext == "pmat") {
00103             vm = new FileVMatrix(dataset);
00104         } 
00105         // else if (ext == "txtmat") {
00106         //    PLERROR("In getDataSet - The old .txtmat files are deprecated, please "
00107         //            "use a standard .vmat or .pymat script"); } 
00108         else if (ext == "vmat" || ext == "txtmat") {
00109             use_params = true;
00110             time_t date = 0;
00111             const string code = readFileAndMacroProcess(dataset, params, date);
00112 
00113             if (removeblanks(code)[0] == '<') {
00114                 // Old XML-like format.
00115                 PLDEPRECATED("In getDataSet - File %s is using the old XML-like VMat format, " 
00116                              "you should switch to a PLearn script (ideally a .pymat file).",
00117                              dataset.c_str());
00118                 vm = new VVMatrix(dataset);
00119             } else {
00120                 vm = dynamic_cast<VMatrix*>(newObject(code));
00121                 if (vm.isNull())
00122                     PLERROR("In getDataSet - Object described in %s is not a VMatrix subclass",
00123                             dataset.c_str());
00124             }
00125             vm->updateMtime(date);
00126         } else if (ext == "pymat" || ext == "py") {
00127             use_params = true;
00128             if (ext == "py")
00129                 PLWARNING("In getDataSet - Note that the Python code in a '.py' file must return a pl.VMatrix");
00130             // Convert 'params' to a vector<string> with elements "paramX=valueX".
00131             map<string, string>::const_iterator it = params.begin();
00132             vector<string> params_vec;
00133             for (; it != params.end(); it++)
00134                 params_vec.push_back(it->first + "=" + it->second);
00135             PP<PyPLearnScript> pyplearn_script = PyPLearnScript::process(dataset, params_vec);
00136             const string code = pyplearn_script->getScript();
00137             vm = dynamic_cast<VMatrix*>(newObject(code));
00138             if (vm.isNull())
00139                 PLERROR("In getDataSet - Object described in %s is not a VMatrix subclass",
00140                         dataset.c_str());
00141             //Their is two case:
00142             //1) params.size()>0, The mtime should be now
00143             //2) params.size()==0 the mtime should be the file mtime
00144             //     But as we can't trust the file mtime as it can
00145             //     have dependency in it that we don't look,
00146             //     we set it to 0 to be safe.
00147             if(params.size()==0)
00148                 vm->updateMtime(0);
00149             else{
00150     // The NSPR PRTime is number of microseconds since the epoch, while
00151     // time_t is the number of seconds since the (same) epoch.
00152     // Translate from the former to the later by dividing by 1e6, using
00153     // NSPR long long manipulation macros to be extra safe.
00154                 PRInt64 time_t_compatible_value;
00155                 PRInt64 one_million = LL_INIT(0, 1000000);
00156                 LL_DIV(time_t_compatible_value, PR_Now(), one_million);
00157                 vm->updateMtime((time_t)time_t_compatible_value);
00158 
00159             }
00160         } else if (VMatrixExtensionRegistrar::VMatrixInstantiator inst =
00161                    VMatrixExtensionRegistrar::getInstantiator(ext))
00162         {
00163             // Support user-added extensions
00164             vm = inst(dataset);
00165             vm->updateMtime(0);
00166         }
00167         else 
00168             PLERROR("In getDataSet - Unknown extension for VMat file: %s", ext.c_str());
00169         if (!use_params && !params.empty())
00170             PLWARNING("In getDataSet - Ignoring parameters when reading file %s",
00171                       dataset.c_str());
00172         // Set default metadata directory if not already set.
00173         if (!vm->hasMetaDataDir())
00174             vm->setMetaDataDir(dataset.dirname() / (dataset.basename() + ".metadata"));
00175     }
00176     else if (isdir(dataset)) {
00177         if (ext == "dmat")
00178             vm = new DiskVMatrix(dataset);
00179         else
00180             PLERROR("In getDataSet - Unknown extension for VMat directory: %s", ext.c_str());
00181     }
00182     else
00183         PLERROR("In getDataSet - cannot open dataset \"%s\"", dataset.c_str());
00184   
00185     vm->loadAllStringMappings();
00186     vm->unduplicateFieldNames();
00187 
00188     // if (vm->inputsize() < 0 && vm->targetsize() < 0 && vm->weightsize() < 0 && vm->extrasize()<=0) 
00189     if (vm->inputsize() < 0 && vm->targetsize() < 0 && vm->weightsize() < 0) 
00190     {
00191         DBG_LOG << "In getDataSet - The loaded VMat has no inputsize, targetsize "
00192                 << "or weightsize specified, setting them to (" << vm->width() << ",0,0)"
00193                 << endl;
00194         vm->defineSizes(vm->width(), 0, 0);
00195     }
00196 
00197     // Set inputsize if it can be deduced from other sizes.
00198     if (vm->inputsize() < 0 && vm->width() >= 0 && vm->targetsize() >= 0 &&
00199             vm->weightsize() >= 0 && vm->extrasize() >= 0)
00200     {
00201         int is = vm->width() - vm->targetsize() - vm->weightsize() -
00202             vm->extrasize();
00203         if (is >= 0)
00204             vm->defineSizes(is, vm->targetsize(), vm->weightsize(),
00205                     vm->extrasize());
00206     }
00207 
00208     // Ensure sizes do not conflict with width.
00209     if (vm->inputsize() >= 0 && vm->targetsize() >= 0 && vm->weightsize() >= 0 &&
00210         vm->width() >= 0 &&  vm->width() < vm->inputsize() + vm->targetsize() + vm->weightsize())
00211         PLERROR("In getDataSet - The matrix width (%d) should not be smaller than inputsize (%d) "
00212                 "+ targetsize (%d) + weightsize (%d)",
00213                 vm->width(), vm->inputsize(), vm->targetsize(), vm->weightsize());
00214 
00215     return vm;
00216 }
00217 
00219 // getDataSetHelp //
00221 string getDataSetHelp() {
00222 
00223     // First make a list of all registered extensions
00224     string exts;
00225     for (VMatrixExtensionRegistrar::ExtensionMap::const_iterator
00226              it  = VMatrixExtensionRegistrar::registeredExtensions().begin(),
00227              end = VMatrixExtensionRegistrar::registeredExtensions().end()
00228              ; it != end ; ++it)
00229     {
00230         exts += "  ." + it->first +
00231             string(max(0, 7-int(it->first.size())), ' ') + ": " +
00232             it->second.documentation() + '\n';
00233     }
00234     
00235     return string(
00236         "Dataset specification must be either:\n"
00237         "- a file with extension:\n"
00238         "  .amat   : ASCII VMatrix\n"
00239         "  .abmat  : ASCII binary (0/1) VMatrix\n"
00240         "  .pmat   : PLearn file VMatrix\n"
00241         "  .vmat   : PLearn script\n"
00242         "  .pymat  : Python script\n")
00243         + exts + string(
00244         "- a directory with extension:\n"
00245         "  .dmat   : Disk VMatrix\n"
00246         "\n"
00247         "Optionally, arguments for scripts can be given with the following syntax:\n"
00248         "  path/file.ext::arg1=val1::arg2=val2::arg3=val3\n");
00249 }
00250 
00251 
00252 //#####  VMatrixExtensionRegistrar  ###########################################
00253 
00254 
00255 VMatrixExtensionRegistrar::VMatrixExtensionRegistrar(
00256     const string& extension, VMatrixInstantiator instantiator, const string& doc)
00257     : m_file_extension(extension),
00258       m_instantiator(instantiator),
00259       m_documentation(doc)
00260 {
00261     registerExtension(*this);
00262 }
00263 
00264 
00265 void VMatrixExtensionRegistrar::registerExtension(const VMatrixExtensionRegistrar& ext)
00266 {
00267     // If inserting same extension a second time, does NOT override first call
00268     if (registeredExtensionsAux().insert(
00269             make_pair(ext.m_file_extension,ext)).second)
00270     {
00271         NAMED_LOG("VMatrixExtensionRegistrar")
00272             << "Registered new VMatrix extension: " << ext.m_file_extension
00273             << " (" << ext.m_documentation << ")"
00274             << endl;
00275     }   
00276 }
00277 
00278 
00279 VMatrixExtensionRegistrar::VMatrixInstantiator
00280 VMatrixExtensionRegistrar::getInstantiator(const string& ext)
00281 {
00282     map<string,VMatrixExtensionRegistrar>::const_iterator found =
00283         registeredExtensions().find(ext);
00284     if (found == registeredExtensions().end())
00285         return 0;
00286     else
00287         return found->second.m_instantiator;
00288 }
00289 
00290 // Static member definition
00291 VMatrixExtensionRegistrar::ExtensionMap&
00292 VMatrixExtensionRegistrar::registeredExtensionsAux()
00293 {
00294     // Sidestep order-or-static-initialization issue across translation units
00295     static ExtensionMap extensions;
00296     return extensions;
00297 }
00298 
00299 
00300 BEGIN_DECLARE_REMOTE_FUNCTIONS
00301 
00302     declareFunction("getDataSet", &getDataSet,
00303                     (BodyDoc("Returns a VMat from a path to the corresponding file.\n"),
00304                      ArgDoc("dataset_path", 
00305                             "the path to the VMat file or directory"),
00306                      RetDoc ("corresponding VMat object")));
00307 
00308 END_DECLARE_REMOTE_FUNCTIONS
00309 
00310 
00311 
00312 } // end of namespace PLearn
00313 
00314 
00315 /*
00316   Local Variables:
00317   mode:c++
00318   c-basic-offset:4
00319   c-file-style:"stroustrup"
00320   c-file-offsets:((innamespace . 0)(inline-open . 0))
00321   indent-tabs-mode:nil
00322   fill-column:79
00323   End:
00324 */
00325 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines