PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PyPLearnScript.cc 00004 // 00005 // Copyright (C) 2005 Christian Dorion 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: PyPLearnScript.cc 10186 2009-05-07 15:43:03Z nouiz $ 00037 ******************************************************* */ 00038 00039 // Authors: Christian Dorion 00040 00044 #include "PyPLearnScript.h" 00045 00046 #include <plearn/sys/Popen.h> 00047 #include <plearn/io/openFile.h> 00048 #include <plearn/io/fileutils.h> 00049 #include <plearn/base/stringutils.h> 00050 #include <plearn/base/pl_repository_revision.h> 00051 #include <plearn/vmat/VMatrix.h> 00052 #include <plearn/io/pl_log.h> 00053 00054 00055 namespace PLearn { 00056 using namespace std; 00057 00058 PP<PyPLearnScript> 00059 PyPLearnScript:: 00060 process( const string& scriptfile, 00061 const vector<string>& args, 00062 const string& drivername ) 00063 { 00064 bool do_help = false; 00065 bool do_dump = false; 00066 00067 // List of arguments passed to the pyplearn_driver.py script. 00068 vector<string> final_args; 00069 // The vars map is for PLearn arguments (i.e. arguments accessed through 00070 // the ${varname} syntax. 00071 map<string, string> vars; 00072 00073 #ifdef WIN32 00074 // Under Windows, we must specify that the driver needs to be run with 00075 // Python, thus the driver becomes an argument. 00076 final_args.push_back(drivername); 00077 #endif 00078 final_args.push_back(scriptfile); 00079 00080 // This option is understood by the pyplearn driver so that is returns a 00081 // plearn representation of a PyPLearnScript object. 00082 final_args.push_back("--PyPLearnScript"); 00083 00084 // For pyplearn files, we support the following command-line 00085 // arguments: --help to print the doc string of the .pyplearn file, 00086 // and --dump to show the result of processing the .pyplearn file 00087 // instead of running it. 00088 if (args.size() >= 1) 00089 { 00090 if (args[0] == "--help") 00091 { 00092 do_help = true; 00093 } 00094 else if (args[0] == "--dump") 00095 { 00096 do_dump = true; 00097 } 00098 } 00099 00100 if (do_help) 00101 { 00102 final_args.push_back("--help"); 00103 } 00104 else 00105 { 00106 // Supply the command-line arguments to the pylearn driver 00107 // script. 00108 for (unsigned int i = 0; i < args.size(); i++) 00109 { 00110 string option = args[i]; 00111 // Skip --foo command-lines options. 00112 if ( option.size() < 2 || option.substr(0, 2) != "--" ) 00113 { 00114 final_args.push_back(args[i]); 00115 } 00116 } 00117 00118 // Add the standard PLearn variables (DATE, DATETIME, etc.) to the 00119 // list of variables used when runing the PLearn preprocessor on the 00120 // output of the .pyplearn file to handle $DATE, etc. 00121 // in PLearn strings. 00122 addFileAndDateVariables(scriptfile, vars, 0); 00123 00124 // Also add these variables (DATE, etc.) to the pyplearn driver 00125 // script so they show up as pyplearn arguments too. 00126 for (map<string, string>::const_iterator it = vars.begin(); 00127 it != vars.end(); ++it) 00128 { 00129 final_args.push_back(it->first + '=' + it->second); 00130 } 00131 00132 // Add the command-line arguments to the list of PLearn arguments. 00133 for (unsigned int i = 0; i < args.size(); i++) 00134 { 00135 string option = args[i]; 00136 // Skip --foo command-lines options. 00137 if (option.size() < 2 || option.substr(0, 2) != "--") 00138 { 00139 pair<string,string> name_val = split_on_first(option, "="); 00140 vars[name_val.first] = name_val.second; 00141 } 00142 } 00143 } 00144 00145 final_args.push_back(string("--enable-logging=") 00146 + join(PL_Log::instance().namedLogging(), ",")); 00147 final_args.push_back(string("--verbosity=") 00148 + tostring(PL_Log::instance().verbosity())) ; 00149 #ifdef WIN32 00150 Popen popen("python.exe", final_args); 00151 #else 00152 Popen popen(drivername, final_args); 00153 #endif 00154 PP<PyPLearnScript> pyplearn_script = new PyPLearnScript(); 00155 00156 popen.in >> *pyplearn_script; 00157 00158 pyplearn_script->vars = vars; 00159 pyplearn_script->do_help = do_help; 00160 pyplearn_script->do_dump = do_dump; 00161 00162 pyplearn_script->build(); 00163 return pyplearn_script; 00164 } 00165 00166 PStream 00167 PyPLearnScript:: 00168 openScriptFile(int argc, char** argv, const std::string& drivername) 00169 { 00170 prgname(argv[0]); // set program name 00171 vector<string> command_line = stringvector(argc-1, argv+1); // neglecting progname 00172 00173 string scriptfile = command_line[0]; 00174 if (!isfile(scriptfile)) 00175 PLERROR("Non-existent script file: %s\n",scriptfile.c_str()); 00176 const string extension = extract_extension(scriptfile); 00177 00178 string script; 00179 if (extension == ".pyplearn") 00180 { 00181 // Make a copy of args with the first argument (the name of the script) 00182 // removed, leaving the first argument to the script at index 0. 00183 vector<string> pyplearn_args(command_line.size()-1); 00184 copy(command_line.begin() + 1, command_line.end(), pyplearn_args.begin()); 00185 00186 PP<PyPLearnScript> pyplearn_script = 00187 PyPLearnScript::process(scriptfile, pyplearn_args, drivername); 00188 script = pyplearn_script->getScript(); 00189 00190 // When we call the pyplearn script with either 00191 // --help or --dump, everything will already have been done by 00192 // the time the PyPLearnScript is built. 00193 if ( script == "" ) 00194 exit(0); 00195 } 00196 else 00197 PLERROR("PyPLearnScript::openScript -- Expecting a .pyplearn file."); 00198 00199 return openString(script, PStream::plearn_ascii); 00200 } 00201 00202 PyPLearnScript::PyPLearnScript() : 00203 plearn_script(""), 00204 vars(), 00205 do_help(false), 00206 do_dump(false), 00207 metainfos(""), 00208 expdir(""), 00209 verbosity(PL_Log::instance().verbosity()), 00210 module_names(PL_Log::instance().namedLogging()) 00211 {} 00212 00213 PLEARN_IMPLEMENT_OBJECT( PyPLearnScript, 00214 "Manage the output of a PyPLearn driver.", 00215 "PyPLearn files are Python scripts used to generate the complex PLearn\n" 00216 "scripts needed for vast experiments. The driver generating the PLearn\n" 00217 "version of the scripts from their PyPLearn counterpart is also written\n" 00218 "in Python. Hence, this class is needed to bridge between the output of\n" 00219 "this driver and the PLearn commands." ); 00220 00221 void PyPLearnScript::run() 00222 { 00223 PL_Log::instance().enableNamedLogging(module_names); 00224 PL_Log::instance().verbosity(verbosity); 00225 00226 PStream in = openString( plearn_script, PStream::plearn_ascii ); 00227 00228 while ( in.good() ) 00229 { 00230 PP<Object> o = readObject(in); 00231 o->run(); 00232 in.skipBlanksAndCommentsAndSeparators(); 00233 } 00234 00235 close(); 00236 } 00237 00238 void PyPLearnScript::declareOptions(OptionList& ol) 00239 { 00240 declareOption( ol, "plearn_script", &PyPLearnScript::plearn_script, 00241 OptionBase::buildoption, 00242 "The plearn_script" ); 00243 00244 declareOption( ol, "vars", &PyPLearnScript::vars, 00245 OptionBase::buildoption, 00246 "Variables set at command line" ); 00247 00248 declareOption( ol, "do_help", &PyPLearnScript::do_help, 00249 OptionBase::buildoption, 00250 "Script is in fact an help message" ); 00251 00252 declareOption( ol, "do_dump", &PyPLearnScript::do_dump, 00253 OptionBase::buildoption, 00254 "Dump the script and forget it" ); 00255 00256 declareOption( ol, "metainfos", &PyPLearnScript::metainfos, 00257 OptionBase::buildoption, 00258 "Informations relative to the experiment settings, to be wrote in an\n" 00259 "expdir file." ); 00260 00261 declareOption( ol, "expdir", &PyPLearnScript::expdir, 00262 OptionBase::buildoption, 00263 "The expdir of the experiment described by the script" ); 00264 00265 declareOption( ol, "verbosity", &PyPLearnScript::verbosity, 00266 OptionBase::buildoption, 00267 "The verbosity level to set for this experiment" ); 00268 00269 declareOption( ol, "module_names", &PyPLearnScript::module_names, 00270 OptionBase::buildoption, 00271 "Modules for which logging should be activated" ); 00272 00273 00274 // Now call the parent class' declareOptions 00275 inherited::declareOptions(ol); 00276 } 00277 00278 void PyPLearnScript::close() 00279 { 00280 if ( !expdir.isEmpty() ) 00281 { 00282 if ( ! isdir( expdir ) ) { 00283 // TODO Why should the expdir be systematically created? 00284 // PLERROR( "The expdir '%s' managed by this PyPLearnScript was not created.", 00285 // expdir.c_str() ); 00286 } else { 00287 00288 PStream out = openFile( expdir / "metainfos.txt", PStream::raw_ascii, "w" ); 00289 out << "__REVISION__ = " << pl_repository_revision() << endl; 00290 out << metainfos << endl; 00291 00292 out = openFile( expdir / "experiment.plearn", PStream::raw_ascii, "w" ); 00293 out << plearn_script << endl; 00294 } 00295 } 00296 } 00297 00298 void PyPLearnScript::build_() 00299 { 00300 if ( do_help || do_dump ) 00301 { 00302 pout << plearn_script << endl; 00303 plearn_script = ""; 00304 expdir = ""; // Disabling the close() method. 00305 } 00306 else 00307 { 00308 PStream is = openString( plearn_script, PStream::raw_ascii ); 00309 time_t latest = 0 ; 00310 plearn_script = readAndMacroProcess( is, vars, latest ); 00311 } 00312 } 00313 00314 // ### Nothing to add here, simply calls build_ 00315 void PyPLearnScript::build() 00316 { 00317 inherited::build(); 00318 build_(); 00319 } 00320 00321 void PyPLearnScript::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00322 { 00323 inherited::makeDeepCopyFromShallowCopy(copies); 00324 } 00325 00326 Object* smartLoadObject(PPath filepath, const vector<string>& args_, time_t& return_date) 00327 { 00328 vector<string> args = args_; 00329 vector<string> args_augmented; 00330 00331 if (!isfile(filepath)) { 00332 // There is no file with this exact name. Maybe there are parameters 00333 // appended to the name? 00334 string base; 00335 map<string, string> params; 00336 parseBaseAndParameters(filepath, base, params); 00337 if (!isfile(base)) 00338 PLERROR("Non-existent script file: %s\n", filepath.c_str()); 00339 // Add new arguments. 00340 args_augmented = args; 00341 map<string, string>::const_iterator it = params.begin(); 00342 for (; it != params.end(); it++) 00343 args_augmented.push_back(it->first + "=" + it->second); 00344 args = args_augmented; 00345 filepath = base; 00346 } 00347 00348 const string extension = extract_extension(filepath); 00349 string script; 00350 time_t date = 0; 00351 00352 PP<PyPLearnScript> pyplearn_script; 00353 PStream in; 00354 00355 if (extension == ".pyplearn" || extension==".pymat") 00356 { 00357 // Make a copy of args with the first argument (the name of the script) 00358 // removed, leaving the first argument to the script at index 0. 00359 vector<string> pyplearn_args(args.size()-1); 00360 copy(args.begin() + 1, args.end(), pyplearn_args.begin()); 00361 00362 pyplearn_script = PyPLearnScript::process(filepath, pyplearn_args); 00363 script = pyplearn_script->getScript(); 00364 00365 // When we call the pyplearn script with either 00366 // --help or --dump, everything will already have been done by 00367 // the time the PyPLearnScript is built. 00368 if ( script == "" ) 00369 PLERROR("Empty script"); 00370 00371 in = openString( script, PStream::plearn_ascii ); 00372 } 00373 else if(extension==".plearn" || extension==".vmat") // perform plearn macro expansion 00374 { 00375 map<string, string> vars; 00376 // populate vars with the arguments passed on the command line 00377 for (unsigned int i=1; i<args.size(); i++) 00378 { 00379 string option = args[i]; 00380 // Skip --foo command-lines options. 00381 if (option.size() < 2 || option.substr(0, 2) != "--") 00382 { 00383 pair<string, string> name_val = split_on_first(option, "="); 00384 vars[name_val.first] = name_val.second; 00385 } 00386 } 00387 script = readFileAndMacroProcess(filepath, vars, date); 00388 in = openString( script, PStream::plearn_ascii ); 00389 } 00390 else if(extension==".psave") // do not perform plearn macro expansion 00391 { 00392 in = openFile(filepath, PStream::plearn_ascii); 00393 date=mtime(filepath); 00394 } 00395 else 00396 PLERROR("In smartLoadObject: unsupported file extension. Must be one of .pyplearn .pymat .plearn .vmat .psave"); 00397 00398 Object* o = readObject(in); 00399 if(extension==".vmat") 00400 ((VMatrix*)o)->updateMtime(date); 00401 return_date=date; 00402 if ( pyplearn_script.isNotNull() ) 00403 pyplearn_script->close(); 00404 00405 return o; 00406 } 00407 00408 00409 } // end of namespace PLearn 00410 00411 00412 /* 00413 Local Variables: 00414 mode:c++ 00415 c-basic-offset:4 00416 c-file-style:"stroustrup" 00417 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00418 indent-tabs-mode:nil 00419 fill-column:79 00420 End: 00421 */ 00422 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :