PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // AutoLinearRegressor.cc 00004 // 00005 // Copyright (C) 2006 Pascal Vincent 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Vincent 00036 00040 #include "AutoLinearRegressor.h" 00041 #include "plearn/math/plapack.h" 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 PLEARN_IMPLEMENT_OBJECT( 00047 AutoLinearRegressor, 00048 "This class performs ridge regression but automatically choosing the weight_decay.", 00049 "The selection of weight decay is done in order to minimize the Generalized \n" 00050 "Cross Validation (GCV) criterion(Craven & Wahba 1979).\n" 00051 "Note: when mature enough, this class might get folded back into LinearRegressor,\n" 00052 "which will require proper handling of the extra bells and whistles of LinearRegressor.\n"); 00053 00054 AutoLinearRegressor::AutoLinearRegressor() 00055 : include_bias(false), 00056 min_weight_decay(1e-6), 00057 weight_decay(0.0) 00058 { 00059 } 00060 00061 void AutoLinearRegressor::declareOptions(OptionList& ol) 00062 { 00063 // ### Declare all of this object's options here. 00064 // ### For the "flags" of each option, you should typically specify 00065 // ### one of OptionBase::buildoption, OptionBase::learntoption or 00066 // ### OptionBase::tuningoption. If you don't provide one of these three, 00067 // ### this option will be ignored when loading values from a script. 00068 // ### You can also combine flags, for example with OptionBase::nosave: 00069 // ### (OptionBase::buildoption | OptionBase::nosave) 00070 00071 // ### ex: 00072 // declareOption(ol, "myoption", &AutoLinearRegressor::myoption, 00073 // OptionBase::buildoption, 00074 // "Help text describing this option"); 00075 // ... 00076 00077 declareOption(ol, "include_bias", &AutoLinearRegressor::include_bias, 00078 OptionBase::buildoption, 00079 "Whether to include a bias term in the regression \n" 00080 "Note: this is currently ignored.\n"); 00081 00082 declareOption(ol, "min_weight_decay", &AutoLinearRegressor::min_weight_decay, 00083 OptionBase::buildoption, 00084 "The minimum weight decay to try."); 00085 00086 00087 declareOption(ol, "weight_decay", &AutoLinearRegressor::weight_decay, 00088 OptionBase::learntoption, 00089 "The weight decay is the factor that multiplies the \n" 00090 "squared norm of the parameters in the loss function.\n" 00091 "It is automatically tuned by the algorithm \n"); 00092 00093 declareOption(ol, "weights", &AutoLinearRegressor::weights, 00094 OptionBase::learntoption, 00095 "The weight matrix, which are the parameters computed by " 00096 "training the regressor.\n"); 00097 00098 declareOption(ol, "mean_target", &AutoLinearRegressor::mean_target, 00099 OptionBase::learntoption, 00100 "The mean of the target (used as a default bias)."); 00101 00102 // Now call the parent class' declareOptions 00103 inherited::declareOptions(ol); 00104 } 00105 00106 void AutoLinearRegressor::build_() 00107 { 00108 // ### This method should do the real building of the object, 00109 // ### according to set 'options', in *any* situation. 00110 // ### Typical situations include: 00111 // ### - Initial building of an object from a few user-specified options 00112 // ### - Building of a "reloaded" object: i.e. from the complete set of 00113 // ### all serialised options. 00114 // ### - Updating or "re-building" of an object after a few "tuning" 00115 // ### options have been modified. 00116 // ### You should assume that the parent class' build_() has already been 00117 // ### called. 00118 } 00119 00120 // ### Nothing to add here, simply calls build_ 00121 void AutoLinearRegressor::build() 00122 { 00123 inherited::build(); 00124 build_(); 00125 } 00126 00127 00128 void AutoLinearRegressor::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00129 { 00130 // ### Call deepCopyField on all "pointer-like" fields 00131 // ### that you wish to be deepCopied rather than 00132 // ### shallow-copied. 00133 // ### ex: 00134 // deepCopyField(trainvec, copies); 00135 00136 // ### Remove this line when you have fully implemented this method. 00137 //PLERROR("AutoLinearRegressor::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00138 00139 inherited::makeDeepCopyFromShallowCopy(copies); 00140 deepCopyField(weights, copies); 00141 deepCopyField(mean_target, copies); 00142 deepCopyField(extendedinput, copies); 00143 00144 } 00145 00146 00147 int AutoLinearRegressor::outputsize() const 00148 { 00149 return targetsize(); 00150 } 00151 00152 void AutoLinearRegressor::forget() 00153 { 00154 weights.resize(0,0); 00155 stage = 0; 00156 inherited::forget(); 00157 } 00158 00159 void AutoLinearRegressor::train() 00160 { 00161 // The role of the train method is to bring the learner up to 00162 // stage==nstages, updating train_stats with training costs measured 00163 // on-line in the process. 00164 00165 // This generic PLearner method does a number of standard stuff useful for 00166 // (almost) any learner, and return 'false' if no training should take 00167 // place. See PLearner.h for more details. 00168 if (!initTrain()) 00169 return; 00170 00171 if(stage<1) 00172 { 00173 // clear statistics of previous epoch 00174 train_stats->forget(); 00175 00176 int ninputs = train_set->inputsize(); 00177 int ntargets = train_set->targetsize(); 00178 int nweights = train_set->weightsize(); 00179 00180 Mat tset = train_set->toMatCopy(); 00181 int l = tset.length(); 00182 00183 Mat X = tset.subMatColumns(0,ninputs); 00184 Mat Y = tset.subMatColumns(ninputs, ntargets); 00185 Vec gamma; // the weights 00186 00187 if (include_bias) 00188 { 00189 Mat col_ones = Mat(l, 1, 1.0); 00190 X = hconcat(col_ones, X); 00191 } 00192 00193 mean_target.resize(ntargets); 00194 mean_target.fill(0); 00195 00196 if(nweights!=0) 00197 { 00198 gamma = tset.column(ninputs+ntargets).toVecCopy(); 00199 for(int i=0; i<l; i++) 00200 multiplyAcc(mean_target, Y(i), gamma[i]); 00201 mean_target /= sum(gamma); 00202 } 00203 else 00204 columnMean(Y, mean_target); 00205 Y -= mean_target; 00206 00207 int insize = ninputs + (include_bias ? 1 : 0); 00208 //weights.resize(insize, ntargets); 00209 weights.resize(ntargets, insize); 00210 real best_GCV; 00211 00212 weight_decay = weightedRidgeRegressionByGCV(X, Y, gamma, weights, best_GCV, min_weight_decay); 00213 00214 //Mat weights_excluding_biases = weights.subMatRows(include_bias? 1 : 0, ninputs); 00215 Mat weights_excluding_biases = weights.subMatColumns(include_bias? 1 : 0, ninputs); 00216 weights_norm = dot(weights_excluding_biases,weights_excluding_biases); 00217 00218 //Vec trcosts(1); 00219 Vec trcosts(2); 00220 trcosts[0] = best_GCV; 00221 trcosts[1] = best_GCV; 00222 train_stats->update(trcosts); 00223 00224 ++stage; 00225 train_stats->finalize(); // finalize statistics for this epoch 00226 } 00227 } 00228 00229 00230 void AutoLinearRegressor::computeOutput(const Vec& input, Vec& output) const 00231 { 00232 // Compute the output from the input 00233 int nout = outputsize(); 00234 output.resize(nout); 00235 if(!include_bias) 00236 product(output,weights,input); 00237 else 00238 { 00239 int nin = inputsize(); 00240 extendedinput.resize(1+nin); 00241 extendedinput.subVec(1,nin) << input; 00242 extendedinput[0] = 1.0; 00243 product(output,weights,extendedinput); 00244 } 00245 output += mean_target; 00246 } 00247 00248 void AutoLinearRegressor::computeCostsFromOutputs(const Vec& input, const Vec& output, 00249 const Vec& target, Vec& costs) const 00250 { 00251 // Compute the costs from *already* computed output. 00252 costs.resize(2); 00253 real squared_loss = powdistance(output,target); 00254 costs[0] = squared_loss + weight_decay*weights_norm; 00255 costs[1] = squared_loss; 00256 } 00257 00258 TVec<string> AutoLinearRegressor::getTestCostNames() const 00259 { 00260 TVec<string> names; 00261 names.push_back("mse+penalty"); 00262 names.push_back("mse"); 00263 return names; 00264 } 00265 00266 TVec<string> AutoLinearRegressor::getTrainCostNames() const 00267 { 00268 TVec<string> names; 00269 names.push_back("GCV_mse"); 00270 names.push_back("mse"); 00271 return names; 00272 } 00273 00274 00275 } // end of namespace PLearn 00276 00277 00278 /* 00279 Local Variables: 00280 mode:c++ 00281 c-basic-offset:4 00282 c-file-style:"stroustrup" 00283 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00284 indent-tabs-mode:nil 00285 fill-column:79 00286 End: 00287 */ 00288 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :