PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999,2000 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // 00007 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 00038 /* ******************************************************* 00039 * $Id: getDataSet.cc 9330 2008-07-31 19:46:01Z chrish $ 00040 * AUTHORS: Pascal Vincent 00041 * This file is part of the PLearn library. 00042 ******************************************************* */ 00043 00044 #include <plearn/base/Object.h> 00045 #include "getDataSet.h" 00046 #include <plearn/io/MatIO.h> 00047 00048 #include <plearn/base/stringutils.h> 00049 #include <plearn/io/fileutils.h> 00050 #include <plearn/io/pl_log.h> 00051 #include <plearn/io/PyPLearnScript.h> 00052 #include <plearn/vmat/DiskVMatrix.h> 00053 #include <plearn/vmat/FileVMatrix.h> 00054 #include <plearn/vmat/VMat.h> 00055 #include <plearn/vmat/VVMatrix.h> 00056 #include <nspr/prtime.h> 00057 00058 namespace PLearn { 00059 using namespace std; 00060 00062 // getDataSetDate // 00064 time_t getDataSetDate(const PPath& dataset_path) 00065 { 00066 return getDataSet(dataset_path)->getMtime(); 00067 } 00068 00070 // getDataSet // 00072 VMat getDataSet(const PPath& dataset_path) 00073 { 00074 VMat vm; 00075 // Parse the base file name and the potential parameters. 00076 string dataset_abs; 00077 map<string, string> params; 00078 if (isfile(dataset_path)) 00079 dataset_abs = dataset_path; 00080 else 00081 // There may be parameters that need parsing. 00082 parseBaseAndParameters(dataset_path, dataset_abs, params); 00083 PPath dataset(dataset_abs); 00084 bool use_params = false; 00085 00086 // See getDataSetHelp() for supported formats. 00087 string ext = dataset.extension(); 00088 if (isfile(dataset)) { 00089 if (ext == "amat") { 00090 // Check if the extension is ".bin.amat". 00091 // (old deprecated extension) 00092 if (dataset.find(".bin.", ((string::size_type) dataset.size()) - 9) != string::npos) { 00093 PLERROR("In getDataSet - The '.bin.amat' extension is deprecated, you " 00094 "must now use the .abmat extension"); 00095 } else 00096 vm = loadAsciiAsVMat(dataset); 00097 } else if (ext == "abmat") { 00098 Mat tempMat; 00099 loadAsciiSingleBinaryDescriptor(dataset, tempMat); 00100 vm = VMat(tempMat); 00101 vm->updateMtime(dataset); 00102 } else if (ext == "pmat") { 00103 vm = new FileVMatrix(dataset); 00104 } 00105 // else if (ext == "txtmat") { 00106 // PLERROR("In getDataSet - The old .txtmat files are deprecated, please " 00107 // "use a standard .vmat or .pymat script"); } 00108 else if (ext == "vmat" || ext == "txtmat") { 00109 use_params = true; 00110 time_t date = 0; 00111 const string code = readFileAndMacroProcess(dataset, params, date); 00112 00113 if (removeblanks(code)[0] == '<') { 00114 // Old XML-like format. 00115 PLDEPRECATED("In getDataSet - File %s is using the old XML-like VMat format, " 00116 "you should switch to a PLearn script (ideally a .pymat file).", 00117 dataset.c_str()); 00118 vm = new VVMatrix(dataset); 00119 } else { 00120 vm = dynamic_cast<VMatrix*>(newObject(code)); 00121 if (vm.isNull()) 00122 PLERROR("In getDataSet - Object described in %s is not a VMatrix subclass", 00123 dataset.c_str()); 00124 } 00125 vm->updateMtime(date); 00126 } else if (ext == "pymat" || ext == "py") { 00127 use_params = true; 00128 if (ext == "py") 00129 PLWARNING("In getDataSet - Note that the Python code in a '.py' file must return a pl.VMatrix"); 00130 // Convert 'params' to a vector<string> with elements "paramX=valueX". 00131 map<string, string>::const_iterator it = params.begin(); 00132 vector<string> params_vec; 00133 for (; it != params.end(); it++) 00134 params_vec.push_back(it->first + "=" + it->second); 00135 PP<PyPLearnScript> pyplearn_script = PyPLearnScript::process(dataset, params_vec); 00136 const string code = pyplearn_script->getScript(); 00137 vm = dynamic_cast<VMatrix*>(newObject(code)); 00138 if (vm.isNull()) 00139 PLERROR("In getDataSet - Object described in %s is not a VMatrix subclass", 00140 dataset.c_str()); 00141 //Their is two case: 00142 //1) params.size()>0, The mtime should be now 00143 //2) params.size()==0 the mtime should be the file mtime 00144 // But as we can't trust the file mtime as it can 00145 // have dependency in it that we don't look, 00146 // we set it to 0 to be safe. 00147 if(params.size()==0) 00148 vm->updateMtime(0); 00149 else{ 00150 // The NSPR PRTime is number of microseconds since the epoch, while 00151 // time_t is the number of seconds since the (same) epoch. 00152 // Translate from the former to the later by dividing by 1e6, using 00153 // NSPR long long manipulation macros to be extra safe. 00154 PRInt64 time_t_compatible_value; 00155 PRInt64 one_million = LL_INIT(0, 1000000); 00156 LL_DIV(time_t_compatible_value, PR_Now(), one_million); 00157 vm->updateMtime((time_t)time_t_compatible_value); 00158 00159 } 00160 } else if (VMatrixExtensionRegistrar::VMatrixInstantiator inst = 00161 VMatrixExtensionRegistrar::getInstantiator(ext)) 00162 { 00163 // Support user-added extensions 00164 vm = inst(dataset); 00165 vm->updateMtime(0); 00166 } 00167 else 00168 PLERROR("In getDataSet - Unknown extension for VMat file: %s", ext.c_str()); 00169 if (!use_params && !params.empty()) 00170 PLWARNING("In getDataSet - Ignoring parameters when reading file %s", 00171 dataset.c_str()); 00172 // Set default metadata directory if not already set. 00173 if (!vm->hasMetaDataDir()) 00174 vm->setMetaDataDir(dataset.dirname() / (dataset.basename() + ".metadata")); 00175 } 00176 else if (isdir(dataset)) { 00177 if (ext == "dmat") 00178 vm = new DiskVMatrix(dataset); 00179 else 00180 PLERROR("In getDataSet - Unknown extension for VMat directory: %s", ext.c_str()); 00181 } 00182 else 00183 PLERROR("In getDataSet - cannot open dataset \"%s\"", dataset.c_str()); 00184 00185 vm->loadAllStringMappings(); 00186 vm->unduplicateFieldNames(); 00187 00188 // if (vm->inputsize() < 0 && vm->targetsize() < 0 && vm->weightsize() < 0 && vm->extrasize()<=0) 00189 if (vm->inputsize() < 0 && vm->targetsize() < 0 && vm->weightsize() < 0) 00190 { 00191 DBG_LOG << "In getDataSet - The loaded VMat has no inputsize, targetsize " 00192 << "or weightsize specified, setting them to (" << vm->width() << ",0,0)" 00193 << endl; 00194 vm->defineSizes(vm->width(), 0, 0); 00195 } 00196 00197 // Set inputsize if it can be deduced from other sizes. 00198 if (vm->inputsize() < 0 && vm->width() >= 0 && vm->targetsize() >= 0 && 00199 vm->weightsize() >= 0 && vm->extrasize() >= 0) 00200 { 00201 int is = vm->width() - vm->targetsize() - vm->weightsize() - 00202 vm->extrasize(); 00203 if (is >= 0) 00204 vm->defineSizes(is, vm->targetsize(), vm->weightsize(), 00205 vm->extrasize()); 00206 } 00207 00208 // Ensure sizes do not conflict with width. 00209 if (vm->inputsize() >= 0 && vm->targetsize() >= 0 && vm->weightsize() >= 0 && 00210 vm->width() >= 0 && vm->width() < vm->inputsize() + vm->targetsize() + vm->weightsize()) 00211 PLERROR("In getDataSet - The matrix width (%d) should not be smaller than inputsize (%d) " 00212 "+ targetsize (%d) + weightsize (%d)", 00213 vm->width(), vm->inputsize(), vm->targetsize(), vm->weightsize()); 00214 00215 return vm; 00216 } 00217 00219 // getDataSetHelp // 00221 string getDataSetHelp() { 00222 00223 // First make a list of all registered extensions 00224 string exts; 00225 for (VMatrixExtensionRegistrar::ExtensionMap::const_iterator 00226 it = VMatrixExtensionRegistrar::registeredExtensions().begin(), 00227 end = VMatrixExtensionRegistrar::registeredExtensions().end() 00228 ; it != end ; ++it) 00229 { 00230 exts += " ." + it->first + 00231 string(max(0, 7-int(it->first.size())), ' ') + ": " + 00232 it->second.documentation() + '\n'; 00233 } 00234 00235 return string( 00236 "Dataset specification must be either:\n" 00237 "- a file with extension:\n" 00238 " .amat : ASCII VMatrix\n" 00239 " .abmat : ASCII binary (0/1) VMatrix\n" 00240 " .pmat : PLearn file VMatrix\n" 00241 " .vmat : PLearn script\n" 00242 " .pymat : Python script\n") 00243 + exts + string( 00244 "- a directory with extension:\n" 00245 " .dmat : Disk VMatrix\n" 00246 "\n" 00247 "Optionally, arguments for scripts can be given with the following syntax:\n" 00248 " path/file.ext::arg1=val1::arg2=val2::arg3=val3\n"); 00249 } 00250 00251 00252 //##### VMatrixExtensionRegistrar ########################################### 00253 00254 00255 VMatrixExtensionRegistrar::VMatrixExtensionRegistrar( 00256 const string& extension, VMatrixInstantiator instantiator, const string& doc) 00257 : m_file_extension(extension), 00258 m_instantiator(instantiator), 00259 m_documentation(doc) 00260 { 00261 registerExtension(*this); 00262 } 00263 00264 00265 void VMatrixExtensionRegistrar::registerExtension(const VMatrixExtensionRegistrar& ext) 00266 { 00267 // If inserting same extension a second time, does NOT override first call 00268 if (registeredExtensionsAux().insert( 00269 make_pair(ext.m_file_extension,ext)).second) 00270 { 00271 NAMED_LOG("VMatrixExtensionRegistrar") 00272 << "Registered new VMatrix extension: " << ext.m_file_extension 00273 << " (" << ext.m_documentation << ")" 00274 << endl; 00275 } 00276 } 00277 00278 00279 VMatrixExtensionRegistrar::VMatrixInstantiator 00280 VMatrixExtensionRegistrar::getInstantiator(const string& ext) 00281 { 00282 map<string,VMatrixExtensionRegistrar>::const_iterator found = 00283 registeredExtensions().find(ext); 00284 if (found == registeredExtensions().end()) 00285 return 0; 00286 else 00287 return found->second.m_instantiator; 00288 } 00289 00290 // Static member definition 00291 VMatrixExtensionRegistrar::ExtensionMap& 00292 VMatrixExtensionRegistrar::registeredExtensionsAux() 00293 { 00294 // Sidestep order-or-static-initialization issue across translation units 00295 static ExtensionMap extensions; 00296 return extensions; 00297 } 00298 00299 00300 BEGIN_DECLARE_REMOTE_FUNCTIONS 00301 00302 declareFunction("getDataSet", &getDataSet, 00303 (BodyDoc("Returns a VMat from a path to the corresponding file.\n"), 00304 ArgDoc("dataset_path", 00305 "the path to the VMat file or directory"), 00306 RetDoc ("corresponding VMat object"))); 00307 00308 END_DECLARE_REMOTE_FUNCTIONS 00309 00310 00311 00312 } // end of namespace PLearn 00313 00314 00315 /* 00316 Local Variables: 00317 mode:c++ 00318 c-basic-offset:4 00319 c-file-style:"stroustrup" 00320 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00321 indent-tabs-mode:nil 00322 fill-column:79 00323 End: 00324 */ 00325 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :