PLearn 0.1
KFoldLogisticClassifier.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // KFoldLogisticClassifier.cc
00004 //
00005 // Copyright (C) 2008 Olivier Delalleau
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Olivier Delalleau
00036 
00040 #include "KFoldLogisticClassifier.h"
00041 #include <plearn/opt/ConjGradientOptimizer.h>
00042 #include <plearn/vmat/ExplicitSplitter.h>
00043 #include <plearn/vmat/KFoldSplitter.h>
00044 #include <plearn_learners/generic/NNet.h>
00045 #include <plearn_learners/hyper/EarlyStoppingOracle.h>
00046 #include <plearn_learners/hyper/HyperLearner.h>
00047 #include <plearn_learners/hyper/HyperOptimize.h>
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 PLEARN_IMPLEMENT_OBJECT(
00053     KFoldLogisticClassifier,
00054     "Average of multiple logistic classifiers from K-Fold split of the data.",
00055     "The training set is split into 'kfold' folds, and we train one logistic\n"
00056     "classifier on each fold (whose learning is controlled by early stopping\n"
00057     "based on the validation NLL).\n"
00058     "The output of this classifier is then the average of the outputs of the\n"
00059     "underlying logistic classifiers."
00060 );
00061 
00063 // KFoldLogisticClassifier //
00065 KFoldLogisticClassifier::KFoldLogisticClassifier():
00066     kfold(5),
00067     max_degraded_steps(20),
00068     max_epochs(500),
00069     step_size(1)
00070 {
00071 }
00072 
00074 // declareOptions //
00076 void KFoldLogisticClassifier::declareOptions(OptionList& ol)
00077 {
00078     // Build options.
00079 
00080     declareOption(ol, "kfold", &KFoldLogisticClassifier::kfold,
00081                   OptionBase::buildoption,
00082         "Number of splits of the data (and of classifiers being trained).");
00083 
00084     declareOption(ol, "max_degraded_steps",
00085                   &KFoldLogisticClassifier::max_degraded_steps,
00086                   OptionBase::buildoption,
00087         "Maximum number of optimization steps performed after finding a\n"
00088         "candidate for early stopping.");
00089 
00090     declareOption(ol, "max_epochs",
00091                   &KFoldLogisticClassifier::max_epochs,
00092                   OptionBase::buildoption,
00093         "Maximum number of epochs when training logistic classifiers\n");
00094 
00095     declareOption(ol, "step_size",
00096                   &KFoldLogisticClassifier::step_size,
00097                   OptionBase::buildoption,
00098         "Measure performance every 'step_size' epochs.");
00099 
00100     // Learnt options.
00101 
00102     declareOption(ol, "log_net", &KFoldLogisticClassifier::log_net,
00103                   OptionBase::learntoption,
00104         "Underlying logistic classifiers.");
00105 
00106     // Now call the parent class' declareOptions
00107     inherited::declareOptions(ol);
00108 }
00109 
00111 // build_ //
00113 void KFoldLogisticClassifier::build_()
00114 {
00115 }
00116 
00118 // build //
00120 void KFoldLogisticClassifier::build()
00121 {
00122     inherited::build();
00123     build_();
00124 }
00125 
00127 // makeDeepCopyFromShallowCopy //
00129 void KFoldLogisticClassifier::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00130 {
00131     inherited::makeDeepCopyFromShallowCopy(copies);
00132 
00133     // ### Call deepCopyField on all "pointer-like" fields
00134     // ### that you wish to be deepCopied rather than
00135     // ### shallow-copied.
00136     // ### ex:
00137     // deepCopyField(trainvec, copies);
00138 
00139     // ### Remove this line when you have fully implemented this method.
00140     PLERROR("KFoldLogisticClassifier::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00141 }
00142 
00144 // outputsize //
00146 int KFoldLogisticClassifier::outputsize() const
00147 {
00148     if (log_net.isEmpty())
00149         return -1;
00150     else
00151         return log_net[0]->outputsize();
00152 }
00153 
00155 // forget //
00157 void KFoldLogisticClassifier::forget()
00158 {
00162 
00169     inherited::forget();
00170     log_net.resize(0);
00171 }
00172 
00174 // train //
00176 void KFoldLogisticClassifier::train()
00177 {
00178     if (!initTrain())
00179         return;
00180 
00181     PLCHECK( stage == 0 );
00182 
00183     // Find out the number of classes in the dataset.
00184     TVec<bool> all_classes;
00185     Vec input, target;
00186     real weight;
00187     PLCHECK(train_set->targetsize() == 1);
00188     for (int i = 0; i < train_set->length(); i++) {
00189         train_set->getExample(i, input, target, weight);
00190         int t = int(round(target[0]));
00191         if (t >= all_classes.length()) {
00192             int n_to_add = t - all_classes.length() + 1;
00193             for (int j = 0; j < n_to_add; j++)
00194                 all_classes.append(false);
00195         }
00196         all_classes[t] = true;
00197     }
00198     int n_classes = all_classes.length();
00199     PLCHECK(n_classes >= 2);
00200     PLCHECK(all_classes.find(false) == -1);
00201 
00202     // Split the data.
00203     PP<KFoldSplitter> splitter = new KFoldSplitter();
00204     splitter->K = this->kfold;
00205     splitter->build();
00206     splitter->setDataSet(train_set);
00207 
00208     // Create logistic regressors.
00209     log_net.resize(0);
00210     string cost_func;
00211     for (int k = 0; k < kfold; k++) {
00212         PP<ConjGradientOptimizer> opt = new ConjGradientOptimizer();
00213         opt->build();
00214         PP<NNet> nnet = new NNet();
00215         nnet->optimizer = opt;
00216         nnet->seed_ = this->seed_;
00217         nnet->report_progress = this->report_progress;
00218         if (n_classes == 2) {
00219             cost_func = "stable_cross_entropy";
00220             nnet->output_transfer_func = "sigmoid";
00221             nnet->noutputs = 1;
00222         } else {
00223             cost_func = "NLL";
00224             nnet->output_transfer_func = "softmax";
00225             nnet->noutputs = n_classes;
00226         }
00227         nnet->cost_funcs = TVec<string>(1, cost_func);
00228         nnet->batch_size = 0;
00229         nnet->build();
00230         log_net.append(get_pointer(nnet));
00231     }
00232 
00233     // Train logistic regressors.
00234     for (int k = 0; k < log_net.length(); k++) {
00235         // Initialize the hyper-learning framework for early stopping.
00236         // Splitter.
00237         PP<ExplicitSplitter> hsplitter = new ExplicitSplitter();
00238         hsplitter->splitsets = TMat<VMat>(1, 2);
00239         hsplitter->splitsets(0) << splitter->getSplit(k);
00240         hsplitter->build();
00241         // PTester.
00242         PP<PTester> htester = new PTester();
00243         htester->splitter = hsplitter;
00244         string cost = "E[test.E[" + cost_func + "]]";
00245         htester->setStatNames(TVec<string>(1, cost), false);
00246         htester->build();
00247         // Oracle.
00248         PP<EarlyStoppingOracle> early_stop = new EarlyStoppingOracle();
00249         early_stop->max_degraded_steps = this->max_degraded_steps;
00250         early_stop->range = TVec<double>(3, this->step_size);
00251         early_stop->range[1] = this->max_epochs + 1;
00252         early_stop->option = "nstages";
00253         early_stop->build();
00254         // Strategy.
00255         PP<HyperOptimize> strategy = new HyperOptimize();
00256         strategy->oracle = early_stop;
00257         strategy->which_cost = "0";
00258         strategy->build();
00259         // HyperLearner.
00260         PP<HyperLearner> hyper = new HyperLearner();
00261         hyper->dont_restart_upon_change = TVec<string>(1, "nstages");
00262         hyper->learner_ = log_net[k];
00263         hyper->option_fields = hyper->dont_restart_upon_change;
00264         hyper->save_final_learner = false;
00265         hyper->strategy = TVec< PP<HyperCommand> >(1, get_pointer(strategy));
00266         hyper->tester = htester;
00267         hyper->verbosity = 0; // Get rid of useless output.
00268         hyper->build();
00269         // Perform training.
00270         hyper->train();
00271         // Make sure we keep only the best model.
00272         log_net[k] = hyper->getLearner();
00273     }
00274     this->stage = 1;
00275 }
00276 
00278 // computeOutput //
00280 void KFoldLogisticClassifier::computeOutput(const Vec& input, Vec& output) const
00281 {
00282     PLCHECK(!log_net.isEmpty());
00283     log_net[0]->computeOutput(input, output);
00284     store_output.resize(output.length());
00285     for (int k = 1; k < log_net.length(); k++) {
00286         log_net[k]->computeOutput(input, store_output);
00287         output += store_output;
00288     }
00289     output /= real(log_net.length());
00290 }
00291 
00293 // computeCostsFromOutputs //
00295 void KFoldLogisticClassifier::computeCostsFromOutputs(const Vec& input, const Vec& output,
00296                                            const Vec& target, Vec& costs) const
00297 {
00298     int t = int(round(target[0]));
00299     costs.resize(2);
00300     if (output.length() == 1) {
00301         // Binary classification.
00302         PLASSERT(output[0] >= 0 && output[0] <= 1);
00303         if (t == 1)
00304             costs[0] = - pl_log(output[0]);
00305         else {
00306             PLASSERT(t == 0);
00307             costs[0] = - pl_log(1 - output[0]);
00308         }
00309         costs[1] = (output[0] - 0.5) * (2 * target[0] - 1) >= 0 ? 0 : 1;
00310     } else {
00311         // More than two targets.
00312         PLASSERT(is_equal(sum(output), 1));
00313         costs[0] = - pl_log(output[t]);
00314         costs[1] = argmax(output) == t ? 0 : 1;
00315     }
00316 }
00317 
00319 // getTestCostNames //
00321 TVec<string> KFoldLogisticClassifier::getTestCostNames() const
00322 {
00323     static TVec<string> costs;
00324     if (costs.isEmpty()) {
00325         costs.append("nll");
00326         costs.append("class_error");
00327     }
00328     return costs;
00329 }
00330 
00332 // getTrainCostNames //
00334 TVec<string> KFoldLogisticClassifier::getTrainCostNames() const
00335 {
00336     return TVec<string>();
00337 }
00338 
00339 
00340 } // end of namespace PLearn
00341 
00342 
00343 /*
00344   Local Variables:
00345   mode:c++
00346   c-basic-offset:4
00347   c-file-style:"stroustrup"
00348   c-file-offsets:((innamespace . 0)(inline-open . 0))
00349   indent-tabs-mode:nil
00350   fill-column:79
00351   End:
00352 */
00353 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines