PLearn 0.1
PyPLearnScript.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PyPLearnScript.cc
00004 //
00005 // Copyright (C) 2005 Christian Dorion 
00006 // 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 // 
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 // 
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 // 
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 // 
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 // 
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************      
00036  * $Id: PyPLearnScript.cc 10186 2009-05-07 15:43:03Z nouiz $ 
00037  ******************************************************* */
00038 
00039 // Authors: Christian Dorion
00040 
00044 #include "PyPLearnScript.h"
00045 
00046 #include <plearn/sys/Popen.h>
00047 #include <plearn/io/openFile.h>
00048 #include <plearn/io/fileutils.h>
00049 #include <plearn/base/stringutils.h>
00050 #include <plearn/base/pl_repository_revision.h>
00051 #include <plearn/vmat/VMatrix.h>
00052 #include <plearn/io/pl_log.h>
00053 
00054 
00055 namespace PLearn {
00056 using namespace std;
00057 
00058 PP<PyPLearnScript>
00059 PyPLearnScript::
00060 process( const string& scriptfile,
00061          const vector<string>& args,
00062          const string& drivername   )
00063 {
00064     bool do_help = false;
00065     bool do_dump = false;
00066   
00067     // List of arguments passed to the pyplearn_driver.py script.
00068     vector<string> final_args;
00069     // The vars map is for PLearn arguments (i.e. arguments accessed through
00070     // the ${varname} syntax.
00071     map<string, string> vars;
00072 
00073 #ifdef WIN32
00074     // Under Windows, we must specify that the driver needs to be run with
00075     // Python, thus the driver becomes an argument.
00076     final_args.push_back(drivername);
00077 #endif
00078     final_args.push_back(scriptfile);
00079   
00080     // This option is understood by the pyplearn driver so that is returns a
00081     // plearn representation of a PyPLearnScript object.
00082     final_args.push_back("--PyPLearnScript");    
00083   
00084     // For pyplearn files, we support the following command-line
00085     // arguments: --help to print the doc string of the .pyplearn file,
00086     // and --dump to show the result of processing the .pyplearn file
00087     // instead of running it.
00088     if (args.size() >= 1)
00089     {
00090         if (args[0] == "--help")
00091         {
00092             do_help = true;
00093         }
00094         else if (args[0] == "--dump")
00095         {
00096             do_dump = true;
00097         }
00098     }
00099       
00100     if (do_help)
00101     {
00102         final_args.push_back("--help");
00103     }
00104     else
00105     {
00106         // Supply the command-line arguments to the pylearn driver
00107         // script.
00108         for (unsigned int i = 0; i < args.size(); i++)
00109         {
00110             string option = args[i];
00111             // Skip --foo command-lines options.
00112             if ( option.size() < 2 || option.substr(0, 2) != "--" )
00113             {
00114                 final_args.push_back(args[i]);
00115             }
00116         }
00117 
00118         // Add the standard PLearn variables (DATE, DATETIME, etc.) to the
00119         // list of variables used when runing the PLearn preprocessor on the
00120         // output of the .pyplearn file to handle $DATE, etc.
00121         // in PLearn strings.
00122         addFileAndDateVariables(scriptfile, vars, 0);
00123 
00124         // Also add these variables (DATE, etc.) to the pyplearn driver
00125         // script so they show up as pyplearn arguments too.
00126         for (map<string, string>::const_iterator it = vars.begin();
00127              it != vars.end(); ++it)
00128         {
00129             final_args.push_back(it->first + '=' + it->second);
00130         }
00131 
00132         // Add the command-line arguments to the list of PLearn arguments.
00133         for (unsigned int i = 0; i < args.size(); i++)
00134         {
00135             string option = args[i];
00136             // Skip --foo command-lines options.
00137             if (option.size() < 2 || option.substr(0, 2) != "--")
00138             {
00139                 pair<string,string> name_val = split_on_first(option, "=");
00140                 vars[name_val.first] = name_val.second;
00141             }
00142         }
00143     }
00144 
00145     final_args.push_back(string("--enable-logging=")
00146                          + join(PL_Log::instance().namedLogging(), ","));
00147     final_args.push_back(string("--verbosity=")
00148                          + tostring(PL_Log::instance().verbosity()))    ;
00149 #ifdef WIN32
00150     Popen popen("python.exe", final_args);
00151 #else
00152     Popen popen(drivername, final_args);
00153 #endif
00154     PP<PyPLearnScript> pyplearn_script = new PyPLearnScript();
00155 
00156     popen.in >> *pyplearn_script;
00157 
00158     pyplearn_script->vars    = vars;
00159     pyplearn_script->do_help = do_help;
00160     pyplearn_script->do_dump = do_dump;
00161 
00162     pyplearn_script->build(); 
00163     return pyplearn_script;  
00164 }
00165 
00166 PStream
00167 PyPLearnScript::
00168 openScriptFile(int argc, char** argv, const std::string& drivername)
00169 {
00170     prgname(argv[0]);   // set program name
00171     vector<string> command_line = stringvector(argc-1, argv+1);   // neglecting progname
00172 
00173     string scriptfile = command_line[0];
00174     if (!isfile(scriptfile))
00175         PLERROR("Non-existent script file: %s\n",scriptfile.c_str());
00176     const string extension = extract_extension(scriptfile);
00177 
00178     string script;
00179     if (extension == ".pyplearn")
00180     {
00181         // Make a copy of args with the first argument (the name of the script)
00182         // removed, leaving the first argument to the script at index 0.
00183         vector<string> pyplearn_args(command_line.size()-1);
00184         copy(command_line.begin() + 1, command_line.end(), pyplearn_args.begin());
00185     
00186         PP<PyPLearnScript> pyplearn_script =
00187             PyPLearnScript::process(scriptfile, pyplearn_args, drivername);
00188         script = pyplearn_script->getScript();
00189         
00190         // When we call the pyplearn script with either
00191         // --help or --dump, everything will already have been done by
00192         // the time the PyPLearnScript is built. 
00193         if ( script == "" )
00194             exit(0);        
00195     }
00196     else
00197         PLERROR("PyPLearnScript::openScript -- Expecting a .pyplearn file.");
00198     
00199     return openString(script, PStream::plearn_ascii);
00200 }
00201 
00202 PyPLearnScript::PyPLearnScript() :
00203     plearn_script(""),
00204     vars(),
00205     do_help(false),
00206     do_dump(false),
00207     metainfos(""),
00208     expdir(""),
00209     verbosity(PL_Log::instance().verbosity()),
00210     module_names(PL_Log::instance().namedLogging())
00211 {}
00212   
00213 PLEARN_IMPLEMENT_OBJECT( PyPLearnScript,
00214                          "Manage the output of a PyPLearn driver.",
00215                          "PyPLearn files are Python scripts used to generate the complex PLearn\n"
00216                          "scripts needed for vast experiments. The driver generating the PLearn\n"
00217                          "version of the scripts from their PyPLearn counterpart is also written\n"
00218                          "in Python. Hence, this class is needed to bridge between the output of\n"
00219                          "this driver and the PLearn commands." );
00220 
00221 void PyPLearnScript::run()
00222 {
00223     PL_Log::instance().enableNamedLogging(module_names);
00224     PL_Log::instance().verbosity(verbosity);
00225 
00226     PStream in = openString( plearn_script, PStream::plearn_ascii );
00227 
00228     while ( in.good() )
00229     {
00230         PP<Object> o = readObject(in);
00231         o->run();
00232         in.skipBlanksAndCommentsAndSeparators();
00233     }
00234 
00235     close();
00236 }
00237 
00238 void PyPLearnScript::declareOptions(OptionList& ol)
00239 {
00240     declareOption( ol, "plearn_script", &PyPLearnScript::plearn_script,
00241                    OptionBase::buildoption,
00242                    "The plearn_script" );
00243 
00244     declareOption( ol, "vars", &PyPLearnScript::vars,
00245                    OptionBase::buildoption,
00246                    "Variables set at command line" );
00247  
00248     declareOption( ol, "do_help", &PyPLearnScript::do_help,
00249                    OptionBase::buildoption,
00250                    "Script is in fact an help message" );
00251 
00252     declareOption( ol, "do_dump", &PyPLearnScript::do_dump,
00253                    OptionBase::buildoption,
00254                    "Dump the script and forget it" );
00255 
00256     declareOption( ol, "metainfos", &PyPLearnScript::metainfos,
00257                    OptionBase::buildoption,
00258                    "Informations relative to the experiment settings, to be wrote in an\n"
00259                    "expdir file." );
00260 
00261     declareOption( ol, "expdir", &PyPLearnScript::expdir,
00262                    OptionBase::buildoption,
00263                    "The expdir of the experiment described by the script" );
00264 
00265     declareOption( ol, "verbosity", &PyPLearnScript::verbosity,
00266                    OptionBase::buildoption,
00267                    "The verbosity level to set for this experiment" );
00268 
00269     declareOption( ol, "module_names", &PyPLearnScript::module_names,
00270                    OptionBase::buildoption,
00271                    "Modules for which logging should be activated" );
00272 
00273   
00274     // Now call the parent class' declareOptions
00275     inherited::declareOptions(ol);
00276 }
00277 
00278 void PyPLearnScript::close()
00279 {
00280     if ( !expdir.isEmpty() )
00281     {
00282         if ( ! isdir( expdir ) ) {
00283             // TODO Why should the expdir be systematically created?
00284         //     PLERROR( "The expdir '%s' managed by this PyPLearnScript was not created.",
00285         //           expdir.c_str() );
00286         } else {
00287 
00288             PStream out = openFile( expdir / "metainfos.txt", PStream::raw_ascii, "w" );
00289             out << "__REVISION__ = " << pl_repository_revision() << endl;
00290             out << metainfos << endl;
00291 
00292             out = openFile( expdir / "experiment.plearn", PStream::raw_ascii, "w" );
00293             out << plearn_script << endl;
00294         }
00295     }
00296 }
00297 
00298 void PyPLearnScript::build_()
00299 {
00300     if ( do_help || do_dump )
00301     {
00302         pout << plearn_script << endl;
00303         plearn_script = "";
00304         expdir        = ""; // Disabling the close() method.
00305     }
00306     else
00307     {
00308         PStream is     = openString( plearn_script, PStream::raw_ascii );
00309         time_t latest = 0 ;
00310         plearn_script  = readAndMacroProcess( is, vars, latest );
00311     }
00312 }
00313 
00314 // ### Nothing to add here, simply calls build_
00315 void PyPLearnScript::build()
00316 {
00317     inherited::build();
00318     build_();
00319 }
00320 
00321 void PyPLearnScript::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00322 {
00323     inherited::makeDeepCopyFromShallowCopy(copies);
00324 }
00325 
00326 Object* smartLoadObject(PPath filepath, const vector<string>& args_, time_t& return_date)
00327 {
00328     vector<string> args = args_;
00329     vector<string> args_augmented;
00330 
00331     if (!isfile(filepath)) {
00332         // There is no file with this exact name. Maybe there are parameters
00333         // appended to the name?
00334         string base;
00335         map<string, string> params;
00336         parseBaseAndParameters(filepath, base, params);
00337         if (!isfile(base))
00338             PLERROR("Non-existent script file: %s\n", filepath.c_str());
00339         // Add new arguments.
00340         args_augmented = args;
00341         map<string, string>::const_iterator it = params.begin();
00342         for (; it != params.end(); it++)
00343             args_augmented.push_back(it->first + "=" + it->second);
00344         args = args_augmented;
00345         filepath = base;
00346     }
00347 
00348     const string extension = extract_extension(filepath);
00349     string script;
00350     time_t date = 0;
00351 
00352     PP<PyPLearnScript> pyplearn_script;
00353     PStream in;
00354 
00355     if (extension == ".pyplearn" || extension==".pymat")
00356     {
00357         // Make a copy of args with the first argument (the name of the script)
00358         // removed, leaving the first argument to the script at index 0.
00359         vector<string> pyplearn_args(args.size()-1);
00360         copy(args.begin() + 1, args.end(), pyplearn_args.begin());
00361     
00362         pyplearn_script = PyPLearnScript::process(filepath, pyplearn_args);
00363         script          = pyplearn_script->getScript();
00364     
00365         // When we call the pyplearn script with either
00366         // --help or --dump, everything will already have been done by
00367         // the time the PyPLearnScript is built. 
00368         if ( script == "" )
00369             PLERROR("Empty script");
00370 
00371         in = openString( script, PStream::plearn_ascii );
00372     }
00373     else if(extension==".plearn" || extension==".vmat")  // perform plearn macro expansion
00374     {
00375         map<string, string> vars;
00376         // populate vars with the arguments passed on the command line
00377         for (unsigned int i=1; i<args.size(); i++)
00378         {
00379             string option = args[i];
00380             // Skip --foo command-lines options.
00381             if (option.size() < 2 || option.substr(0, 2) != "--")
00382             {
00383                 pair<string, string> name_val = split_on_first(option, "=");
00384                 vars[name_val.first] = name_val.second;
00385             }
00386         }
00387         script = readFileAndMacroProcess(filepath, vars, date);
00388         in = openString( script, PStream::plearn_ascii );
00389     }
00390     else if(extension==".psave") // do not perform plearn macro expansion
00391     {
00392         in = openFile(filepath, PStream::plearn_ascii);
00393         date=mtime(filepath);
00394     }
00395     else
00396         PLERROR("In smartLoadObject: unsupported file extension. Must be one of .pyplearn .pymat .plearn .vmat .psave");
00397 
00398     Object* o = readObject(in);
00399     if(extension==".vmat")
00400         ((VMatrix*)o)->updateMtime(date);
00401     return_date=date;
00402     if ( pyplearn_script.isNotNull() )
00403         pyplearn_script->close();
00404 
00405     return o;
00406 }
00407 
00408 
00409 } // end of namespace PLearn
00410 
00411 
00412 /*
00413   Local Variables:
00414   mode:c++
00415   c-basic-offset:4
00416   c-file-style:"stroustrup"
00417   c-file-offsets:((innamespace . 0)(inline-open . 0))
00418   indent-tabs-mode:nil
00419   fill-column:79
00420   End:
00421 */
00422 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines