PLearn 0.1
AutoLinearRegressor.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // AutoLinearRegressor.cc
00004 //
00005 // Copyright (C) 2006 Pascal Vincent
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Vincent
00036 
00040 #include "AutoLinearRegressor.h"
00041 #include "plearn/math/plapack.h"
00042 
00043 namespace PLearn {
00044 using namespace std;
00045 
00046 PLEARN_IMPLEMENT_OBJECT(
00047     AutoLinearRegressor,
00048     "This class performs ridge regression but automatically choosing the weight_decay.",
00049     "The selection of weight decay is done in order to minimize the Generalized \n"
00050     "Cross Validation (GCV) criterion(Craven & Wahba 1979).\n"
00051     "Note: when mature enough, this class might get folded back into LinearRegressor,\n"
00052     "which will require proper handling of the extra bells and whistles of LinearRegressor.\n");
00053 
00054 AutoLinearRegressor::AutoLinearRegressor()
00055     : include_bias(false),
00056       min_weight_decay(1e-6),
00057       weight_decay(0.0)
00058 {
00059 }
00060 
00061 void AutoLinearRegressor::declareOptions(OptionList& ol)
00062 {
00063     // ### Declare all of this object's options here.
00064     // ### For the "flags" of each option, you should typically specify
00065     // ### one of OptionBase::buildoption, OptionBase::learntoption or
00066     // ### OptionBase::tuningoption. If you don't provide one of these three,
00067     // ### this option will be ignored when loading values from a script.
00068     // ### You can also combine flags, for example with OptionBase::nosave:
00069     // ### (OptionBase::buildoption | OptionBase::nosave)
00070 
00071     // ### ex:
00072     // declareOption(ol, "myoption", &AutoLinearRegressor::myoption,
00073     //               OptionBase::buildoption,
00074     //               "Help text describing this option");
00075     // ...
00076 
00077     declareOption(ol, "include_bias", &AutoLinearRegressor::include_bias,
00078                   OptionBase::buildoption,
00079                   "Whether to include a bias term in the regression \n"
00080                   "Note: this is currently ignored.\n");
00081 
00082     declareOption(ol, "min_weight_decay", &AutoLinearRegressor::min_weight_decay,
00083                   OptionBase::buildoption, 
00084                   "The minimum weight decay to try.");
00085 
00086 
00087     declareOption(ol, "weight_decay", &AutoLinearRegressor::weight_decay,
00088                   OptionBase::learntoption, 
00089                   "The weight decay is the factor that multiplies the \n"
00090                   "squared norm of the parameters in the loss function.\n"
00091                   "It is automatically tuned by the algorithm \n");
00092 
00093     declareOption(ol, "weights", &AutoLinearRegressor::weights,
00094                   OptionBase::learntoption, 
00095                   "The weight matrix, which are the parameters computed by "
00096                   "training the regressor.\n");
00097 
00098     declareOption(ol, "mean_target", &AutoLinearRegressor::mean_target,
00099                   OptionBase::learntoption,
00100                   "The mean of the target (used as a default bias).");
00101 
00102     // Now call the parent class' declareOptions
00103     inherited::declareOptions(ol);
00104 }
00105 
00106 void AutoLinearRegressor::build_()
00107 {
00108     // ### This method should do the real building of the object,
00109     // ### according to set 'options', in *any* situation.
00110     // ### Typical situations include:
00111     // ###  - Initial building of an object from a few user-specified options
00112     // ###  - Building of a "reloaded" object: i.e. from the complete set of
00113     // ###    all serialised options.
00114     // ###  - Updating or "re-building" of an object after a few "tuning"
00115     // ###    options have been modified.
00116     // ### You should assume that the parent class' build_() has already been
00117     // ### called.
00118 }
00119 
00120 // ### Nothing to add here, simply calls build_
00121 void AutoLinearRegressor::build()
00122 {
00123     inherited::build();
00124     build_();
00125 }
00126 
00127 
00128 void AutoLinearRegressor::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00129 {
00130     // ### Call deepCopyField on all "pointer-like" fields
00131     // ### that you wish to be deepCopied rather than
00132     // ### shallow-copied.
00133     // ### ex:
00134     // deepCopyField(trainvec, copies);
00135 
00136     // ### Remove this line when you have fully implemented this method.
00137     //PLERROR("AutoLinearRegressor::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00138 
00139     inherited::makeDeepCopyFromShallowCopy(copies);
00140     deepCopyField(weights, copies);
00141     deepCopyField(mean_target, copies);
00142     deepCopyField(extendedinput, copies);
00143 
00144 }
00145 
00146 
00147 int AutoLinearRegressor::outputsize() const
00148 {
00149     return targetsize();
00150 }
00151 
00152 void AutoLinearRegressor::forget()
00153 {
00154     weights.resize(0,0);
00155     stage = 0;
00156     inherited::forget();
00157 }
00158 
00159 void AutoLinearRegressor::train()
00160 {
00161     // The role of the train method is to bring the learner up to
00162     // stage==nstages, updating train_stats with training costs measured
00163     // on-line in the process.
00164 
00165     // This generic PLearner method does a number of standard stuff useful for
00166     // (almost) any learner, and return 'false' if no training should take
00167     // place. See PLearner.h for more details.
00168     if (!initTrain())
00169         return;
00170 
00171     if(stage<1)
00172     {
00173         // clear statistics of previous epoch
00174         train_stats->forget();
00175         
00176         int ninputs = train_set->inputsize();
00177         int ntargets = train_set->targetsize();
00178         int nweights = train_set->weightsize();
00179 
00180         Mat tset = train_set->toMatCopy();
00181         int l = tset.length();
00182         
00183         Mat X = tset.subMatColumns(0,ninputs);
00184         Mat Y = tset.subMatColumns(ninputs, ntargets);
00185         Vec gamma; // the weights
00186 
00187         if (include_bias)
00188         {
00189             Mat col_ones = Mat(l, 1, 1.0);
00190             X = hconcat(col_ones, X);
00191         }
00192 
00193         mean_target.resize(ntargets);
00194         mean_target.fill(0);
00195 
00196         if(nweights!=0)
00197         {
00198             gamma = tset.column(ninputs+ntargets).toVecCopy();
00199             for(int i=0;  i<l; i++)
00200                 multiplyAcc(mean_target, Y(i), gamma[i]);
00201             mean_target /= sum(gamma);
00202         }
00203         else
00204             columnMean(Y, mean_target);
00205         Y -= mean_target;
00206 
00207         int insize = ninputs + (include_bias ? 1 : 0); 
00208         //weights.resize(insize, ntargets);
00209         weights.resize(ntargets, insize);
00210         real best_GCV;
00211       
00212         weight_decay = weightedRidgeRegressionByGCV(X, Y, gamma, weights, best_GCV, min_weight_decay);
00213 
00214         //Mat weights_excluding_biases = weights.subMatRows(include_bias? 1 : 0, ninputs);
00215         Mat weights_excluding_biases = weights.subMatColumns(include_bias? 1 : 0, ninputs);
00216         weights_norm = dot(weights_excluding_biases,weights_excluding_biases);
00217 
00218         //Vec trcosts(1);
00219         Vec trcosts(2);
00220         trcosts[0] = best_GCV;
00221         trcosts[1] = best_GCV;
00222         train_stats->update(trcosts);
00223 
00224         ++stage;
00225         train_stats->finalize(); // finalize statistics for this epoch
00226     }
00227 }
00228 
00229 
00230 void AutoLinearRegressor::computeOutput(const Vec& input, Vec& output) const
00231 {
00232     // Compute the output from the input
00233     int nout = outputsize();
00234     output.resize(nout);
00235     if(!include_bias)        
00236         product(output,weights,input);
00237     else
00238     {   
00239         int nin = inputsize();
00240         extendedinput.resize(1+nin);
00241         extendedinput.subVec(1,nin) << input;
00242         extendedinput[0] = 1.0;
00243         product(output,weights,extendedinput);
00244     }
00245     output += mean_target;
00246 }
00247 
00248 void AutoLinearRegressor::computeCostsFromOutputs(const Vec& input, const Vec& output,
00249                                            const Vec& target, Vec& costs) const
00250 {
00251     // Compute the costs from *already* computed output. 
00252     costs.resize(2);
00253     real squared_loss = powdistance(output,target);
00254     costs[0] = squared_loss + weight_decay*weights_norm;
00255     costs[1] = squared_loss;
00256 }
00257 
00258 TVec<string> AutoLinearRegressor::getTestCostNames() const
00259 {
00260     TVec<string> names;
00261     names.push_back("mse+penalty");
00262     names.push_back("mse");
00263     return names;
00264 }
00265 
00266 TVec<string> AutoLinearRegressor::getTrainCostNames() const
00267 {
00268     TVec<string> names;
00269     names.push_back("GCV_mse");
00270     names.push_back("mse");
00271     return names;
00272 }
00273 
00274 
00275 } // end of namespace PLearn
00276 
00277 
00278 /*
00279   Local Variables:
00280   mode:c++
00281   c-basic-offset:4
00282   c-file-style:"stroustrup"
00283   c-file-offsets:((innamespace . 0)(inline-open . 0))
00284   indent-tabs-mode:nil
00285   fill-column:79
00286   End:
00287 */
00288 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines