PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // Experiment.cc 00004 // 00005 // Copyright (C) 2002 Pascal Vincent, Frederic Morin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: Experiment.cc 3994 2005-08-25 13:35:03Z chapados $ 00037 ******************************************************* */ 00038 00040 #include "Experiment.h" 00041 //#include "pl_io.h" 00042 #include <plearn/math/VecStatsCollector.h> 00043 #include <plearn/vmat/AsciiVMatrix.h> 00044 #include <plearn/sys/PLMPI.h> 00045 #include <plearn/io/load_and_save.h> 00046 00047 namespace PLearn { 00048 using namespace std; 00049 00050 Experiment::Experiment() 00051 :save_models(true), save_initial_models(false), 00052 save_test_outputs(false), save_test_costs(false) 00053 {} 00054 00055 PLEARN_IMPLEMENT_OBJECT(Experiment, "DEPRECATED: use PTester instead", ""); 00056 00057 void Experiment::declareOptions(OptionList& ol) 00058 { 00059 declareOption(ol, "expdir", &Experiment::expdir, OptionBase::buildoption, 00060 "Path of this experiment's directory in which to save all experiment results (will be created if it does not already exist)"); 00061 declareOption(ol, "learner", &Experiment::learner, OptionBase::buildoption, 00062 "The learner to train/test"); 00063 declareOption(ol, "dataset", &Experiment::dataset, OptionBase::buildoption, 00064 "The dataset to use for training/testing (will be split according to what is specified in the testmethod)\n" 00065 "You may omit this only if your splitter is an ExplicitSplitter"); 00066 declareOption(ol, "splitter", &Experiment::splitter, OptionBase::buildoption, 00067 "The splitter to use to generate one or several train/test pairs from the dataset."); 00068 declareOption(ol, "save_models", &Experiment::save_models, OptionBase::buildoption, 00069 "If true, the final model#k will be saved in Split#k/final.psave"); 00070 declareOption(ol, "save_initial_models", &Experiment::save_initial_models, OptionBase::buildoption, 00071 "If true, the initial model#k (just after forget() has been called) will be saved in Split#k/initial.psave"); 00072 declareOption(ol, "save_test_outputs", &Experiment::save_test_outputs, OptionBase::buildoption, 00073 "If true, the outputs of the test for split #k will be saved in Split#k/test_outputs.pmat"); 00074 declareOption(ol, "save_test_costs", &Experiment::save_test_costs, OptionBase::buildoption, 00075 "If true, the costs of the test for split #k will be saved in Split#k/test_costs.pmat"); 00076 inherited::declareOptions(ol); 00077 } 00078 00079 void Experiment::build_() 00080 { 00081 splitter->setDataSet(dataset); 00082 } 00083 00084 // ### Nothing to add here, simply calls build_ 00085 void Experiment::build() 00086 { 00087 inherited::build(); 00088 build_(); 00089 } 00090 00091 void Experiment::run() 00092 { 00093 if(expdir=="") 00094 PLERROR("No expdir specified for Experiment."); 00095 if(!learner) 00096 PLERROR("No learner specified for Experiment."); 00097 if(!splitter) 00098 PLERROR("No splitter specified for Experiment"); 00099 00100 if(PLMPI::rank==0) 00101 { 00102 if(pathexists(expdir)) 00103 PLERROR("Directory (or file) %s already exists. First move it out of the way.",expdir.c_str()); 00104 00105 if(!force_mkdir(expdir)) 00106 PLERROR("Could not create experiment directory %s", expdir.c_str()); 00107 00108 // Save this experiment description in the expdir 00109 PLearn::save(append_slash(expdir)+"experiment.psave", *this); 00110 } 00111 00112 int nsplits = splitter->nsplits(); 00113 Array<string> testresnames = learner->testResultsNames(); 00114 int ntestres = testresnames.size(); 00115 VecStatsCollector teststats; 00116 00117 // the vmat in which to save results 00118 VMat results; 00119 00120 if(PLMPI::rank==0) 00121 { 00122 // filename 00123 string fname = append_slash(expdir)+"results.amat"; 00124 00125 // fieldnames 00126 TVec<string> fieldnames(1+ntestres); 00127 fieldnames[0] = "splitnum"; 00128 for(int k=0; k<ntestres; k++) 00129 fieldnames[k+1] = testresnames[k]; 00130 00131 /* 00132 cerr << "ntestres: " << ntestres << endl; 00133 cerr << "testresnames: " << testresnames << endl; 00134 cerr << "fieldnames: " << fieldnames << endl; 00135 */ 00136 00137 results = new AsciiVMatrix(fname, 1+ntestres, fieldnames, "# Special values for splitnum are: -1 -> MEAN; -2 -> STDERROR; -3 -> STDDEV"); 00138 } 00139 00140 Vec resultrow(1+ntestres); // will hold splitnum and testresults for that split 00141 Vec resultrow_sub = resultrow.subVec(1,ntestres); // the part containing the test results 00142 00143 for(int k=0; k<nsplits; k++) 00144 { 00145 TVec<VMat> train_test = splitter->getSplit(k); 00146 if(train_test.size()!=2) 00147 PLERROR("Splitter returned a split with %d subsets, instead of the expected 2: train&test",train_test.size()); 00148 VMat trainset = train_test[0]; 00149 VMat testset = train_test[1]; 00150 string learner_expdir = append_slash(expdir)+"Split"+tostring(k); 00151 learner->setExperimentDirectory(learner_expdir); 00152 00153 learner->forget(); 00154 if(save_initial_models) 00155 PLearn::save(learner_expdir+slash+"initial.psave",learner); 00156 00157 learner->setTestDuringTrain(testset); 00158 learner->train(trainset); 00159 if(save_models) 00160 PLearn::save(learner_expdir+slash+"final.psave",learner); 00161 00162 string test_outputs_fname = learner_expdir+slash+"test_outputs.pmat"; 00163 if(save_test_outputs) 00164 test_outputs_fname = learner_expdir+slash+"test_outputs.pmat"; 00165 string test_costs_fname = learner_expdir+slash+"test_costs.pmat"; 00166 if(save_test_costs) 00167 test_costs_fname = learner_expdir+slash+"test_costs.pmat"; 00168 Vec testres = learner->test(testset, test_outputs_fname, test_costs_fname); 00169 00170 testres.println(cout); 00171 00172 // PLWARNING("In Experiment: length of Vec returned by test = %d; number of names returned by testResultNames = %d",testres.length(), ntestres); 00173 if(testres.length()!=ntestres) 00174 PLERROR("In Experiment: length of Vec returned by test (%d) differs from number of names returned by testResultNames (%d)", 00175 testres.length(), ntestres); 00176 00177 teststats.update(testres); 00178 if(PLMPI::rank==0) // write to file 00179 { 00180 resultrow[0] = k; 00181 resultrow_sub << testres; 00182 results->appendRow(resultrow); 00183 } 00184 } 00185 00186 if(PLMPI::rank==0) // write global stats to file 00187 { 00188 // MEAN 00189 resultrow[0] = -1; 00190 resultrow_sub << teststats.getMean(); 00191 results->appendRow(resultrow); 00192 00193 // STDERROR 00194 resultrow[0] = -2; 00195 resultrow_sub << teststats.getStdError(); 00196 results->appendRow(resultrow); 00197 00198 // STDDEV 00199 resultrow[0] = -3; 00200 resultrow_sub << teststats.getStdDev(); 00201 results->appendRow(resultrow); 00202 } 00203 } 00204 00205 } // end of namespace PLearn 00206 00207 00208 /* 00209 Local Variables: 00210 mode:c++ 00211 c-basic-offset:4 00212 c-file-style:"stroustrup" 00213 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00214 indent-tabs-mode:nil 00215 fill-column:79 00216 End: 00217 */ 00218 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :