PLearn 0.1
Experiment.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // Experiment.cc
00004 // 
00005 // Copyright (C) 2002 Pascal Vincent, Frederic Morin
00006 // 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 // 
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 // 
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 // 
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 // 
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 // 
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************      
00036  * $Id: Experiment.cc 3994 2005-08-25 13:35:03Z chapados $ 
00037  ******************************************************* */
00038 
00040 #include "Experiment.h"
00041 //#include "pl_io.h"
00042 #include <plearn/math/VecStatsCollector.h>
00043 #include <plearn/vmat/AsciiVMatrix.h>
00044 #include <plearn/sys/PLMPI.h>
00045 #include <plearn/io/load_and_save.h>
00046 
00047 namespace PLearn {
00048 using namespace std;
00049 
00050 Experiment::Experiment() 
00051     :save_models(true), save_initial_models(false),
00052      save_test_outputs(false), save_test_costs(false)
00053 {}
00054 
00055 PLEARN_IMPLEMENT_OBJECT(Experiment, "DEPRECATED: use PTester instead", "");
00056 
00057 void Experiment::declareOptions(OptionList& ol)
00058 {
00059     declareOption(ol, "expdir", &Experiment::expdir, OptionBase::buildoption,
00060                   "Path of this experiment's directory in which to save all experiment results (will be created if it does not already exist)");
00061     declareOption(ol, "learner", &Experiment::learner, OptionBase::buildoption,
00062                   "The learner to train/test");
00063     declareOption(ol, "dataset", &Experiment::dataset, OptionBase::buildoption,
00064                   "The dataset to use for training/testing (will be split according to what is specified in the testmethod)\n"
00065                   "You may omit this only if your splitter is an ExplicitSplitter");
00066     declareOption(ol, "splitter", &Experiment::splitter, OptionBase::buildoption,
00067                   "The splitter to use to generate one or several train/test pairs from the dataset.");
00068     declareOption(ol, "save_models", &Experiment::save_models, OptionBase::buildoption,
00069                   "If true, the final model#k will be saved in Split#k/final.psave");
00070     declareOption(ol, "save_initial_models", &Experiment::save_initial_models, OptionBase::buildoption,
00071                   "If true, the initial model#k (just after forget() has been called) will be saved in Split#k/initial.psave");
00072     declareOption(ol, "save_test_outputs", &Experiment::save_test_outputs, OptionBase::buildoption,
00073                   "If true, the outputs of the test for split #k will be saved in Split#k/test_outputs.pmat");
00074     declareOption(ol, "save_test_costs", &Experiment::save_test_costs, OptionBase::buildoption,
00075                   "If true, the costs of the test for split #k will be saved in Split#k/test_costs.pmat");
00076     inherited::declareOptions(ol);
00077 }
00078 
00079 void Experiment::build_()
00080 {
00081     splitter->setDataSet(dataset);
00082 }
00083 
00084 // ### Nothing to add here, simply calls build_
00085 void Experiment::build()
00086 {
00087     inherited::build();
00088     build_();
00089 }
00090 
00091 void Experiment::run()
00092 {
00093     if(expdir=="")
00094         PLERROR("No expdir specified for Experiment.");
00095     if(!learner)
00096         PLERROR("No learner specified for Experiment.");
00097     if(!splitter)
00098         PLERROR("No splitter specified for Experiment");
00099 
00100     if(PLMPI::rank==0)
00101     {
00102         if(pathexists(expdir))
00103             PLERROR("Directory (or file) %s already exists. First move it out of the way.",expdir.c_str());
00104 
00105         if(!force_mkdir(expdir))
00106             PLERROR("Could not create experiment directory %s", expdir.c_str());
00107 
00108         // Save this experiment description in the expdir
00109         PLearn::save(append_slash(expdir)+"experiment.psave", *this);
00110     }
00111 
00112     int nsplits = splitter->nsplits();
00113     Array<string> testresnames = learner->testResultsNames();
00114     int ntestres = testresnames.size();
00115     VecStatsCollector teststats;
00116 
00117     // the vmat in which to save results
00118     VMat results;
00119 
00120     if(PLMPI::rank==0)
00121     {
00122         // filename
00123         string fname = append_slash(expdir)+"results.amat";
00124 
00125         // fieldnames
00126         TVec<string> fieldnames(1+ntestres);
00127         fieldnames[0] = "splitnum";
00128         for(int k=0; k<ntestres; k++)
00129             fieldnames[k+1] = testresnames[k];
00130 
00131         /*
00132           cerr << "ntestres: " << ntestres << endl;
00133           cerr << "testresnames: " << testresnames << endl;
00134           cerr << "fieldnames: " << fieldnames << endl;
00135         */
00136 
00137         results = new AsciiVMatrix(fname, 1+ntestres, fieldnames, "# Special values for splitnum are: -1 -> MEAN; -2 -> STDERROR; -3 -> STDDEV");
00138     }
00139 
00140     Vec resultrow(1+ntestres); // will hold splitnum and testresults for that split
00141     Vec resultrow_sub = resultrow.subVec(1,ntestres); // the part containing the test results
00142 
00143     for(int k=0; k<nsplits; k++)
00144     {
00145         TVec<VMat> train_test = splitter->getSplit(k);
00146         if(train_test.size()!=2) 
00147             PLERROR("Splitter returned a split with %d subsets, instead of the expected 2: train&test",train_test.size());
00148         VMat trainset = train_test[0];
00149         VMat testset = train_test[1];
00150         string learner_expdir = append_slash(expdir)+"Split"+tostring(k);
00151         learner->setExperimentDirectory(learner_expdir);
00152 
00153         learner->forget();
00154         if(save_initial_models)
00155             PLearn::save(learner_expdir+slash+"initial.psave",learner);
00156 
00157         learner->setTestDuringTrain(testset);
00158         learner->train(trainset);
00159         if(save_models)
00160             PLearn::save(learner_expdir+slash+"final.psave",learner);
00161 
00162         string test_outputs_fname = learner_expdir+slash+"test_outputs.pmat";
00163         if(save_test_outputs)
00164             test_outputs_fname = learner_expdir+slash+"test_outputs.pmat";
00165         string test_costs_fname = learner_expdir+slash+"test_costs.pmat";
00166         if(save_test_costs)
00167             test_costs_fname = learner_expdir+slash+"test_costs.pmat";
00168         Vec testres = learner->test(testset, test_outputs_fname, test_costs_fname);
00169 
00170         testres.println(cout);
00171 
00172         // PLWARNING("In Experiment: length of Vec returned by test = %d; number of names returned by testResultNames = %d",testres.length(), ntestres);
00173         if(testres.length()!=ntestres)
00174             PLERROR("In Experiment: length of Vec returned by test (%d) differs from number of names returned by testResultNames (%d)",
00175                     testres.length(), ntestres);
00176 
00177         teststats.update(testres);
00178         if(PLMPI::rank==0) // write to file
00179         {
00180             resultrow[0] = k;
00181             resultrow_sub << testres;
00182             results->appendRow(resultrow);
00183         }
00184     }
00185 
00186     if(PLMPI::rank==0) // write global stats to file
00187     {
00188         // MEAN
00189         resultrow[0] = -1;
00190         resultrow_sub << teststats.getMean();
00191         results->appendRow(resultrow);
00192 
00193         // STDERROR
00194         resultrow[0] = -2;
00195         resultrow_sub << teststats.getStdError();
00196         results->appendRow(resultrow);
00197 
00198         // STDDEV
00199         resultrow[0] = -3;
00200         resultrow_sub << teststats.getStdDev();
00201         results->appendRow(resultrow);
00202     }
00203 }
00204 
00205 } // end of namespace PLearn
00206 
00207 
00208 /*
00209   Local Variables:
00210   mode:c++
00211   c-basic-offset:4
00212   c-file-style:"stroustrup"
00213   c-file-offsets:((innamespace . 0)(inline-open . 0))
00214   indent-tabs-mode:nil
00215   fill-column:79
00216   End:
00217 */
00218 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines