PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // KFoldLogisticClassifier.cc 00004 // 00005 // Copyright (C) 2008 Olivier Delalleau 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Olivier Delalleau 00036 00040 #include "KFoldLogisticClassifier.h" 00041 #include <plearn/opt/ConjGradientOptimizer.h> 00042 #include <plearn/vmat/ExplicitSplitter.h> 00043 #include <plearn/vmat/KFoldSplitter.h> 00044 #include <plearn_learners/generic/NNet.h> 00045 #include <plearn_learners/hyper/EarlyStoppingOracle.h> 00046 #include <plearn_learners/hyper/HyperLearner.h> 00047 #include <plearn_learners/hyper/HyperOptimize.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 PLEARN_IMPLEMENT_OBJECT( 00053 KFoldLogisticClassifier, 00054 "Average of multiple logistic classifiers from K-Fold split of the data.", 00055 "The training set is split into 'kfold' folds, and we train one logistic\n" 00056 "classifier on each fold (whose learning is controlled by early stopping\n" 00057 "based on the validation NLL).\n" 00058 "The output of this classifier is then the average of the outputs of the\n" 00059 "underlying logistic classifiers." 00060 ); 00061 00063 // KFoldLogisticClassifier // 00065 KFoldLogisticClassifier::KFoldLogisticClassifier(): 00066 kfold(5), 00067 max_degraded_steps(20), 00068 max_epochs(500), 00069 step_size(1) 00070 { 00071 } 00072 00074 // declareOptions // 00076 void KFoldLogisticClassifier::declareOptions(OptionList& ol) 00077 { 00078 // Build options. 00079 00080 declareOption(ol, "kfold", &KFoldLogisticClassifier::kfold, 00081 OptionBase::buildoption, 00082 "Number of splits of the data (and of classifiers being trained)."); 00083 00084 declareOption(ol, "max_degraded_steps", 00085 &KFoldLogisticClassifier::max_degraded_steps, 00086 OptionBase::buildoption, 00087 "Maximum number of optimization steps performed after finding a\n" 00088 "candidate for early stopping."); 00089 00090 declareOption(ol, "max_epochs", 00091 &KFoldLogisticClassifier::max_epochs, 00092 OptionBase::buildoption, 00093 "Maximum number of epochs when training logistic classifiers\n"); 00094 00095 declareOption(ol, "step_size", 00096 &KFoldLogisticClassifier::step_size, 00097 OptionBase::buildoption, 00098 "Measure performance every 'step_size' epochs."); 00099 00100 // Learnt options. 00101 00102 declareOption(ol, "log_net", &KFoldLogisticClassifier::log_net, 00103 OptionBase::learntoption, 00104 "Underlying logistic classifiers."); 00105 00106 // Now call the parent class' declareOptions 00107 inherited::declareOptions(ol); 00108 } 00109 00111 // build_ // 00113 void KFoldLogisticClassifier::build_() 00114 { 00115 } 00116 00118 // build // 00120 void KFoldLogisticClassifier::build() 00121 { 00122 inherited::build(); 00123 build_(); 00124 } 00125 00127 // makeDeepCopyFromShallowCopy // 00129 void KFoldLogisticClassifier::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00130 { 00131 inherited::makeDeepCopyFromShallowCopy(copies); 00132 00133 // ### Call deepCopyField on all "pointer-like" fields 00134 // ### that you wish to be deepCopied rather than 00135 // ### shallow-copied. 00136 // ### ex: 00137 // deepCopyField(trainvec, copies); 00138 00139 // ### Remove this line when you have fully implemented this method. 00140 PLERROR("KFoldLogisticClassifier::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00141 } 00142 00144 // outputsize // 00146 int KFoldLogisticClassifier::outputsize() const 00147 { 00148 if (log_net.isEmpty()) 00149 return -1; 00150 else 00151 return log_net[0]->outputsize(); 00152 } 00153 00155 // forget // 00157 void KFoldLogisticClassifier::forget() 00158 { 00162 00169 inherited::forget(); 00170 log_net.resize(0); 00171 } 00172 00174 // train // 00176 void KFoldLogisticClassifier::train() 00177 { 00178 if (!initTrain()) 00179 return; 00180 00181 PLCHECK( stage == 0 ); 00182 00183 // Find out the number of classes in the dataset. 00184 TVec<bool> all_classes; 00185 Vec input, target; 00186 real weight; 00187 PLCHECK(train_set->targetsize() == 1); 00188 for (int i = 0; i < train_set->length(); i++) { 00189 train_set->getExample(i, input, target, weight); 00190 int t = int(round(target[0])); 00191 if (t >= all_classes.length()) { 00192 int n_to_add = t - all_classes.length() + 1; 00193 for (int j = 0; j < n_to_add; j++) 00194 all_classes.append(false); 00195 } 00196 all_classes[t] = true; 00197 } 00198 int n_classes = all_classes.length(); 00199 PLCHECK(n_classes >= 2); 00200 PLCHECK(all_classes.find(false) == -1); 00201 00202 // Split the data. 00203 PP<KFoldSplitter> splitter = new KFoldSplitter(); 00204 splitter->K = this->kfold; 00205 splitter->build(); 00206 splitter->setDataSet(train_set); 00207 00208 // Create logistic regressors. 00209 log_net.resize(0); 00210 string cost_func; 00211 for (int k = 0; k < kfold; k++) { 00212 PP<ConjGradientOptimizer> opt = new ConjGradientOptimizer(); 00213 opt->build(); 00214 PP<NNet> nnet = new NNet(); 00215 nnet->optimizer = opt; 00216 nnet->seed_ = this->seed_; 00217 nnet->report_progress = this->report_progress; 00218 if (n_classes == 2) { 00219 cost_func = "stable_cross_entropy"; 00220 nnet->output_transfer_func = "sigmoid"; 00221 nnet->noutputs = 1; 00222 } else { 00223 cost_func = "NLL"; 00224 nnet->output_transfer_func = "softmax"; 00225 nnet->noutputs = n_classes; 00226 } 00227 nnet->cost_funcs = TVec<string>(1, cost_func); 00228 nnet->batch_size = 0; 00229 nnet->build(); 00230 log_net.append(get_pointer(nnet)); 00231 } 00232 00233 // Train logistic regressors. 00234 for (int k = 0; k < log_net.length(); k++) { 00235 // Initialize the hyper-learning framework for early stopping. 00236 // Splitter. 00237 PP<ExplicitSplitter> hsplitter = new ExplicitSplitter(); 00238 hsplitter->splitsets = TMat<VMat>(1, 2); 00239 hsplitter->splitsets(0) << splitter->getSplit(k); 00240 hsplitter->build(); 00241 // PTester. 00242 PP<PTester> htester = new PTester(); 00243 htester->splitter = hsplitter; 00244 string cost = "E[test.E[" + cost_func + "]]"; 00245 htester->setStatNames(TVec<string>(1, cost), false); 00246 htester->build(); 00247 // Oracle. 00248 PP<EarlyStoppingOracle> early_stop = new EarlyStoppingOracle(); 00249 early_stop->max_degraded_steps = this->max_degraded_steps; 00250 early_stop->range = TVec<double>(3, this->step_size); 00251 early_stop->range[1] = this->max_epochs + 1; 00252 early_stop->option = "nstages"; 00253 early_stop->build(); 00254 // Strategy. 00255 PP<HyperOptimize> strategy = new HyperOptimize(); 00256 strategy->oracle = early_stop; 00257 strategy->which_cost = "0"; 00258 strategy->build(); 00259 // HyperLearner. 00260 PP<HyperLearner> hyper = new HyperLearner(); 00261 hyper->dont_restart_upon_change = TVec<string>(1, "nstages"); 00262 hyper->learner_ = log_net[k]; 00263 hyper->option_fields = hyper->dont_restart_upon_change; 00264 hyper->save_final_learner = false; 00265 hyper->strategy = TVec< PP<HyperCommand> >(1, get_pointer(strategy)); 00266 hyper->tester = htester; 00267 hyper->verbosity = 0; // Get rid of useless output. 00268 hyper->build(); 00269 // Perform training. 00270 hyper->train(); 00271 // Make sure we keep only the best model. 00272 log_net[k] = hyper->getLearner(); 00273 } 00274 this->stage = 1; 00275 } 00276 00278 // computeOutput // 00280 void KFoldLogisticClassifier::computeOutput(const Vec& input, Vec& output) const 00281 { 00282 PLCHECK(!log_net.isEmpty()); 00283 log_net[0]->computeOutput(input, output); 00284 store_output.resize(output.length()); 00285 for (int k = 1; k < log_net.length(); k++) { 00286 log_net[k]->computeOutput(input, store_output); 00287 output += store_output; 00288 } 00289 output /= real(log_net.length()); 00290 } 00291 00293 // computeCostsFromOutputs // 00295 void KFoldLogisticClassifier::computeCostsFromOutputs(const Vec& input, const Vec& output, 00296 const Vec& target, Vec& costs) const 00297 { 00298 int t = int(round(target[0])); 00299 costs.resize(2); 00300 if (output.length() == 1) { 00301 // Binary classification. 00302 PLASSERT(output[0] >= 0 && output[0] <= 1); 00303 if (t == 1) 00304 costs[0] = - pl_log(output[0]); 00305 else { 00306 PLASSERT(t == 0); 00307 costs[0] = - pl_log(1 - output[0]); 00308 } 00309 costs[1] = (output[0] - 0.5) * (2 * target[0] - 1) >= 0 ? 0 : 1; 00310 } else { 00311 // More than two targets. 00312 PLASSERT(is_equal(sum(output), 1)); 00313 costs[0] = - pl_log(output[t]); 00314 costs[1] = argmax(output) == t ? 0 : 1; 00315 } 00316 } 00317 00319 // getTestCostNames // 00321 TVec<string> KFoldLogisticClassifier::getTestCostNames() const 00322 { 00323 static TVec<string> costs; 00324 if (costs.isEmpty()) { 00325 costs.append("nll"); 00326 costs.append("class_error"); 00327 } 00328 return costs; 00329 } 00330 00332 // getTrainCostNames // 00334 TVec<string> KFoldLogisticClassifier::getTrainCostNames() const 00335 { 00336 return TVec<string>(); 00337 } 00338 00339 00340 } // end of namespace PLearn 00341 00342 00343 /* 00344 Local Variables: 00345 mode:c++ 00346 c-basic-offset:4 00347 c-file-style:"stroustrup" 00348 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00349 indent-tabs-mode:nil 00350 fill-column:79 00351 End: 00352 */ 00353 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :