PLearn 0.1
HyperLearner.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // HyperLearner.cc
00004 // Copyright (C) 2003-2004 ApSTAT Technologies Inc.
00005 //
00006 // Redistribution and use in source and binary forms, with or without
00007 // modification, are permitted provided that the following conditions are met:
00008 //
00009 //  1. Redistributions of source code must retain the above copyright
00010 //     notice, this list of conditions and the following disclaimer.
00011 //
00012 //  2. Redistributions in binary form must reproduce the above copyright
00013 //     notice, this list of conditions and the following disclaimer in the
00014 //     documentation and/or other materials provided with the distribution.
00015 //
00016 //  3. The name of the authors may not be used to endorse or promote
00017 //     products derived from this software without specific prior written
00018 //     permission.
00019 //
00020 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00021 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00022 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00023 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00024 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00025 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00026 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00027 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00028 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00029 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00030 //
00031 // This file is part of the PLearn library. For more information on the PLearn
00032 // library, go to the PLearn Web site at www.plearn.org
00033 
00034 // Author: Pascal Vincent
00035 
00036 /* *******************************************************
00037  * $Id: HyperLearner.cc 10220 2009-05-27 20:40:20Z tihocan $
00038  ******************************************************* */
00039 // Author: Pascal Vincent
00040 
00041 #include "HyperLearner.h"
00042 #include <plearn/base/stringutils.h>
00043 #include <plearn/base/PLearnDiff.h>
00044 #include <plearn/io/load_and_save.h>
00045 #define PL_LOG_MODULE_NAME "HyperLearner"
00046 #include <plearn/io/pl_log.h>
00047 #include <plearn/vmat/FileVMatrix.h>
00048 #include <plearn/sys/Profiler.h>
00049 
00050 namespace PLearn {
00051 
00052 
00053 // ###### HyperLearner ########################################################
00054 
00055 PLEARN_IMPLEMENT_OBJECT(
00056     HyperLearner,
00057     "Learner which optimizes a set of hyper-parameters.",
00058     "Note that the train costs produced by HyperLearner are the cost-stats \n"
00059     "produced by the underlying PTester, and thus have the form: \n"
00060     "for ex: E[train.E[mse]]  \n"
00061     "Consequently if you embed the HyperLearner in a higher level PTester \n"
00062     "the produced cost-stats will be accessible through the form: \n"
00063     "        E[train.E[E[train.E[mse]]]]. \n"
00064     "The test costs are EmbeddedLearner's test costs and have the form:\n"
00065     "        mse\n"
00066     "They are accessible in the higher level PTester as: \n"
00067     "for ex: E[test1.E[mse]]");
00068 
00069 TVec<string> HyperLearner::getTrainCostNames() const
00070 {
00071     if (strategy.size() > 0)
00072         return strategy.lastElement()->getResultNames();
00073     else
00074         return TVec<string>();
00075 }
00076 
00077 HyperLearner::HyperLearner()
00078     : save_mode_(PStream::plearn_ascii),
00079       save_mode("plearn_ascii"),
00080       provide_strategy_expdir(true),
00081       save_final_learner(true),
00082       save_strategy_learner(false),
00083       reloaded(false),
00084       finalize_learner(false)
00085 {
00086     // Forward the 'test' method to the underlying learner.
00087     forward_test = true;
00088     // The default behavior is to let the PTester decide whether or not
00089     // to provide the underlying learner with an experiment directory.
00090     provide_learner_expdir = false;
00091 }
00092 
00093 void
00094 HyperLearner::declareOptions(OptionList &ol)
00095 {
00096     declareOption(ol, "tester", &HyperLearner::tester, OptionBase::buildoption,
00097                   "A model for the kind of train/test to be performed for each \n"
00098                   "combination of hyper-parameters considered. \n"
00099                   "The tester's options are used as follows: \n"
00100                   "  - You don't need to provide tester.learner, it will be set from \n"
00101                   "    the HyperLearner's learner.\n"
00102                   "  - tester.statnames is the list of cost stats to be computed, \n"
00103                   "    reported in the results table, and used as criteria for the hyper-optimization \n"
00104                   "  - tester.splitter is the default splitter to use for validation (although this may be \n"
00105                   "    overridden locally in HyperOptimize, see help on HyperOptimize) \n"
00106                   "  - Similarly, no need to provide tester.learner tester.expdir, \n"
00107                   "    and tester.provide_learner_expdir. \n"
00108                   "    They will be set by HyperOptimize as it deems appropriate\n"
00109                   "    for each hyper-parameter combination it tries. \n"
00110                   "  - *If* HyperOptimize does set a non-empty expdir, then the kind of\n"
00111                   "    report and result files to be generated is taken from the remaining \n"
00112                   "    report_stats and save_... options of the tester \n");
00113 
00114     declareOption(ol, "option_fields", &HyperLearner::option_fields, OptionBase::buildoption,
00115                   "learner option names to be reported in results table");
00116 
00117     declareOption(ol, "dont_restart_upon_change", &HyperLearner::dont_restart_upon_change, OptionBase::buildoption,
00118                   "a list of names of options, used in the strategy, but that do not require \n"
00119                   "calling build() and forget() each time they are modified. \n"
00120                   "Note that this is almost always the case of the 'nstages' option \n"
00121                   "typically modified incrementally in an 'early stopping' fashion... \n");
00122 
00123     declareOption(ol, "strategy", &HyperLearner::strategy, OptionBase::buildoption,
00124                   "The strategy to follow to optimize the hyper-parameters.\n"
00125                   "It's a list of hyper-optimization commands to call"
00126                   " sequentially,\n"
00127                   "(mostly HyperOptimize, HyperSetOption and HyperRetrain"
00128                   " commands).\n");
00129 
00130     declareOption(ol, "provide_strategy_expdir", &HyperLearner::provide_strategy_expdir, OptionBase::buildoption,
00131                   "should each strategy step be provided a directory expdir/Step# to report its results");
00132 
00133     declareOption(ol, "save_final_learner", &HyperLearner::save_final_learner, OptionBase::buildoption,
00134                   "should final learner be saved in expdir/final_learner.psave");
00135 
00136     declareOption(ol, "save_strategy_learner", &HyperLearner::save_strategy_learner, OptionBase::buildoption,
00137                   "should final learner be saved in expdir/Strat#final_learner.psave");
00138 
00139     declareOption(ol, "save_mode", &HyperLearner::save_mode,
00140                   OptionBase::buildoption,
00141                   "The mode to use to save the file.");
00142 
00143     declareOption(
00144         ol, "finalize_learner", &HyperLearner::finalize_learner,
00145         OptionBase::buildoption,
00146         "Default false. If true, will finalize the learner after the training.");
00147 
00148     declareOption(ol, "reloaded", &HyperLearner::reloaded,
00149                   OptionBase::learntoption|OptionBase::nosave,
00150                   "Used to give a warning.");
00151     inherited::declareOptions(ol);
00152 
00153     // Hide some unused options.
00154 
00155     redeclareOption(ol, "seed", &HyperLearner::seed_, OptionBase::nosave,
00156                     "Not used.");
00157 
00158 }
00159 
00160 void HyperLearner::setLearnerOptions(const TVec<string>& option_names,
00161                                      const TVec<string>& option_vals)
00162 {
00163     // This function has been modified to call changeOptions on the learner
00164     // rather than a sequence of setOptions.  Learner options are accumulated
00165     // into a map suitable for calling changeOptions.  TESTER options are
00166     // still handled as a set of discrete setOptions.
00167 
00168     map<string,string> new_learner_options;
00169     int l = option_names.length();
00170     bool do_restart = false;
00171     for(int i=0; i<l; i++) {
00172         string optname = option_names[i];
00173         string newval = option_vals[i];
00174 
00175         // Decide if we should perform build after setting the options
00176         do_restart = do_restart ||
00177             ! dont_restart_upon_change.contains(optname);
00178 
00179         // Here, we incorporate the capability to change an option in the
00180         // tester (rather than the embedded learner) by prefixing the option
00181         // name with the special keyword TESTER.
00182         if (optname.substr(0,7) == "TESTER.") {
00183             // This is the old control flow for banging tester only
00184             optname = optname.substr(7);
00185             if (tester->getOption(optname) != newval)
00186                 tester->setOption(optname, newval);
00187         }
00188         else {
00189             // New control flow: accumulate changes into the map
00190             if (learner_->getOption(optname) != newval)
00191                 new_learner_options[optname] = newval;
00192         }
00193     }
00194     learner_->changeOptions(new_learner_options);
00195     if(do_restart)
00196     {
00197         learner_->build();
00198         learner_->forget();
00199     }
00200 }
00201 
00202 void HyperLearner::setTrainingSet(VMat training_set, bool call_forget)
00203 {
00204 
00205     //will set the training set to the sub learner
00206     inherited::setTrainingSet(training_set, call_forget);
00207 
00208     tester->dataset = training_set;
00209     if(tester->splitter)
00210         tester->splitter->setDataSet(training_set);
00211 
00212     if (call_forget)
00213     {
00214         if(reloaded)
00215             PLWARNING("In HyperLearner::setTrainingSet() - we were asked to"
00216                       " forget after having reloaded a previous version."
00217                       " To avoid doing this, in the PTester that includes this"
00218                       " HyperLearner, set call_forget_in_run = 0.");
00219         build();
00220         forget();
00221     }
00222 }
00223 
00224 void HyperLearner::train()
00225 {
00226     tester->dataset = getTrainingSet();
00227     Vec results;
00228 
00229     if (!train_stats)
00230         setTrainStatsCollector(new VecStatsCollector()); // set a dummy one if
00231     // none is set...
00232 
00233     train_stats->setFieldNames(getTrainCostNames());
00234     
00235 
00236     if(stage==0 && nstages>0)
00237     {
00238         for(int commandnum=0; commandnum<strategy.length(); commandnum++)
00239         {
00240             if(provide_strategy_expdir)
00241             {
00242                 if(expdir!="")
00243                     strategy[commandnum]->setExperimentDirectory( expdir / ("Strat"+tostring(commandnum)) );
00244                 else
00245                     strategy[commandnum]->setExperimentDirectory("");
00246             }
00247 
00248             if(verbosity>0)
00249                 perr<<"HyperLearner: starting the optimization"<<endl;
00250 
00251             results = strategy[commandnum]->optimize();
00252 
00253             if(save_strategy_learner)
00254             {
00255                 PPath strat_expdir=strategy[commandnum]->getExperimentDirectory();
00256                 if(strat_expdir.isEmpty())
00257                     PLERROR("Cannot save the strategy model: no experiment directory has been set");
00258                 if( getLearner().isNull() )
00259                     PLERROR("Cannot save final model: no final learner available");
00260                 PLearn::save(strat_expdir+"final_learner.psave",*getLearner(), save_mode_);
00261             }
00262         }
00263 
00264         train_stats->update(results);
00265 
00266         if(finalize_learner)
00267             learner_->finalize();
00268 
00269         if(save_final_learner)
00270         {
00271             if(expdir=="")
00272                 PLERROR("Cannot save final model: no experiment directory has been set");
00273             if( getLearner().isNull() )
00274                 PLERROR("Cannot save final model: no final learner available");
00275             PLearn::save(expdir+"final_learner.psave",*getLearner(), save_mode_);
00276         }
00277 
00278         stage = 1;
00279     }
00280 }
00281 
00283 // forget //
00285 void HyperLearner::forget()
00286 {
00287     learner_->forget();
00288     stage = 0;
00289 
00290     // Forward the forget to each command of the strategy.
00291     for (int i=0, n=strategy.size() ; i<n ; ++i)
00292         strategy[i]->forget();
00293 }
00294 
00296 // finalize //
00298 void HyperLearner::finalize()
00299 {
00300     inherited::finalize();
00301     learner_->finalize();
00302     if(tester)tester->dataset=NULL;
00303 }
00304 
00306 // build_ //
00308 void HyperLearner::build_()
00309 {
00310     // ### This method should do the real building of the object,
00311     // ### according to set 'options', in *any* situation.
00312     // ### Typical situations include:
00313     // ###  - Initial building of an object from a few user-specified options
00314     // ###  - Building of a "reloaded" object: i.e. from the complete set of all serialised options.
00315     // ###  - Updating or "re-building" of an object after a few "tuning" options have been modified.
00316     // ### You should assume that the parent class' build_() has already been called.
00317 
00318     // Set the Tester's learner to point to the same as the HyperLearner's learner
00319     PLASSERT( tester );
00320     tester->learner = learner_;
00321 
00322     for(int commandnum=0; commandnum<strategy.length(); commandnum++)
00323         strategy[commandnum]->setHyperLearner(this);
00324 
00325     save_mode_ = PStream::parseModeT(save_mode);
00326 }
00327 
00329 // run //
00331 void HyperLearner::run()
00332 {
00333     if(!tester)
00334         PLERROR("No tester specified for HyperLearner.");
00335 
00336     if(!learner_ && !tester->learner)
00337         PLERROR("You must specify the underlying learner, \n"
00338                 "either in the learner option of the HyperLearner \n"
00339                 "or in the learne roption of the HyperLearner's tester \n");
00340 
00341     if(!tester->dataset)
00342         PLERROR("You must specify a dataset field in the tester");
00343 
00344     if(expdir=="")
00345         PLERROR("You should specify an experiment directory with the expdir option.");
00346 
00347     if(!learner_)
00348         learner_ = tester->learner;
00349     setTrainingSet(tester->dataset);
00350     train();    
00351 }
00352 
00353 
00355 // build //
00357 void HyperLearner::build()
00358 {
00359     inherited::build();
00360     build_();
00361 }
00362 
00363 
00364 void HyperLearner::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00365 {
00366     inherited::makeDeepCopyFromShallowCopy(copies);
00367 
00368     deepCopyField(tester, copies);
00369     deepCopyField(option_fields, copies);
00370     deepCopyField(dont_restart_upon_change, copies);
00371     deepCopyField(strategy, copies);
00372 }
00373 
00374 
00376 // auto_save //
00378 void HyperLearner::auto_save()
00379 {
00380     Profiler::pl_profile_start("HyperLearner::auto_save");
00381     if(expdir.isEmpty())
00382         PLERROR("In HyperLearner::auto_save - we can't auto_save as"
00383                 " we don't have any expdir");
00384     PPath f = expdir/"hyper_learner_auto_save.psave";
00385     PPath tmp=f+".tmp";
00386 
00387     if(verbosity>0)
00388         perr << "In HyperLearner::auto_save() - We save the hlearner"
00389              << endl;
00390     PLearn::save(tmp, this, save_mode_);
00391 
00392 #ifdef BOUNDCHECK
00393     HyperLearner *n = new HyperLearner();
00394     PLearn::load(tmp,n);
00395     PLCHECK(PLearn::diff(n,this));
00396     delete n;
00397 #endif
00398 
00399     mvforce(tmp,f);
00400     Profiler::pl_profile_end("HyperLearner::auto_save");
00401 }
00402 
00403 } // end of namespace PLearn
00404 
00405 
00406 /*
00407   Local Variables:
00408   mode:c++
00409   c-basic-offset:4
00410   c-file-style:"stroustrup"
00411   c-file-offsets:((innamespace . 0)(inline-open . 0))
00412   indent-tabs-mode:nil
00413   fill-column:79
00414   End:
00415 */
00416 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines