PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // KernelRidgeRegressor.cc 00004 // 00005 // Copyright (C) 2005 Nicolas Chapados 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $ 00037 ******************************************************* */ 00038 00039 // Authors: Nicolas Chapados 00040 00043 // From PLearn 00044 #include <plearn/vmat/ExtendedVMatrix.h> 00045 00046 #include "KernelRidgeRegressor.h" 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00051 PLEARN_IMPLEMENT_OBJECT( 00052 KernelRidgeRegressor, 00053 "Kernelized version of linear ridge regression.", 00054 "Given a kernel K(x,y) = phi(x)'phi(y), where phi(x) is the projection of a\n" 00055 "vector x into feature space, this class implements a version of the ridge\n" 00056 "estimator, giving the prediction at x as\n" 00057 "\n" 00058 " f(x) = k(x)'(M + lambda I)^-1 y,\n" 00059 "\n" 00060 "where x is the test vector where to estimate the response, k(x) is the\n" 00061 "vector of kernel evaluations between the test vector and the elements of\n" 00062 "the training set, namely\n" 00063 "\n" 00064 " k(x) = (K(x,x1), K(x,x2), ..., K(x,xN))',\n" 00065 "\n" 00066 "M is the Gram Matrix on the elements of the training set, i.e. the matrix\n" 00067 "where the element (i,j) is equal to K(xi, xj), lamdba is the weight decay\n" 00068 "coefficient, and y is the vector of training-set targets.\n" 00069 "\n" 00070 "The disadvantage of this learner is that its training time is O(N^3) in the\n" 00071 "number of training examples (due to the matrix inversion). When saving the\n" 00072 "learner, the training set must be saved, along with an additional vector of\n" 00073 "the length of the training set.\n"); 00074 00075 00076 KernelRidgeRegressor::KernelRidgeRegressor() 00077 : m_include_bias(true), 00078 m_weight_decay(0.0) 00079 { } 00080 00081 00082 void KernelRidgeRegressor::declareOptions(OptionList& ol) 00083 { 00084 declareOption(ol, "kernel", &KernelRidgeRegressor::m_kernel, 00085 OptionBase::buildoption, 00086 "Kernel to use for the computation. This must be a similarity kernel\n" 00087 "(i.e. closer vectors give higher kernel evaluations)."); 00088 00089 declareOption(ol, "weight_decay", &KernelRidgeRegressor::m_weight_decay, 00090 OptionBase::buildoption, 00091 "Weight decay coefficient (default = 0)"); 00092 00093 declareOption(ol, "include_bias", &KernelRidgeRegressor::m_include_bias, 00094 OptionBase::buildoption, 00095 "Whether to include a bias term in the regression (true by default)"); 00096 00097 declareOption(ol, "params", &KernelRidgeRegressor::m_params, 00098 OptionBase::learntoption, 00099 "Vector of learned parameters, determined from the equation\n" 00100 " (M + lambda I)^-1 y"); 00101 00102 declareOption(ol, "training_inputs", &KernelRidgeRegressor::m_training_inputs, 00103 OptionBase::learntoption, 00104 "Saved version of the training set, which must be kept along for\n" 00105 "carrying out kernel evaluations with the test point"); 00106 00107 // Now call the parent class' declareOptions 00108 inherited::declareOptions(ol); 00109 } 00110 00111 void KernelRidgeRegressor::build_() 00112 { 00113 if (! m_kernel) 00114 PLERROR("KernelRidgeRegressor::build_: 'kernel' option must be specified"); 00115 00116 // If we are reloading the model, set the training inputs into the kernel 00117 if (m_training_inputs) 00118 m_kernel->setDataForKernelMatrix(m_training_inputs); 00119 } 00120 00121 // ### Nothing to add here, simply calls build_ 00122 void KernelRidgeRegressor::build() 00123 { 00124 inherited::build(); 00125 build_(); 00126 } 00127 00128 00129 void KernelRidgeRegressor::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00130 { 00131 inherited::makeDeepCopyFromShallowCopy(copies); 00132 00133 deepCopyField(m_kernel, copies); 00134 deepCopyField(m_params, copies); 00135 deepCopyField(m_training_inputs, copies); 00136 } 00137 00138 00139 int KernelRidgeRegressor::outputsize() const 00140 { 00141 return targetsize(); 00142 } 00143 00144 void KernelRidgeRegressor::forget() 00145 { 00146 m_params.resize(0,0); 00147 m_training_inputs = 0; 00148 } 00149 00150 void KernelRidgeRegressor::train() 00151 { 00152 PLASSERT( m_kernel ); 00153 if (! train_set) 00154 PLERROR("KernelRidgeRegressor::train: the training set must be specified"); 00155 int inputsize = train_set->inputsize() ; 00156 int targetsize = train_set->targetsize(); 00157 int weightsize = train_set->weightsize(); 00158 if (inputsize < 0 || targetsize < 0 || weightsize < 0) 00159 PLERROR("KernelRidgeRegressor::train: inconsistent inputsize/targetsize/weightsize " 00160 "(%d/%d/%d) in training set", inputsize, targetsize, weightsize); 00161 if (weightsize > 0) 00162 PLERROR("KernelRidgeRegressor::train: observations weights are not currently supported"); 00163 00164 m_training_inputs = train_set.subMatColumns(0, inputsize).toMat(); 00165 Mat targets = train_set.subMatColumns(inputsize, targetsize).toMat(); 00166 00167 // Compute Gram Matrix and add weight decay to diagonal 00168 m_kernel->setDataForKernelMatrix(m_training_inputs); 00169 Mat gram_mat(m_training_inputs.length(), m_training_inputs.length()); 00170 m_kernel->computeGramMatrix(gram_mat); 00171 addToDiagonal(gram_mat, m_weight_decay); 00172 00173 // Compute parameters 00174 m_params.resize(targets.length(), targets.width()); 00175 solveLinearSystemByCholesky(gram_mat, targets, m_params); 00176 00177 // Compute train error if there is a train_stats_collector. There is 00178 // probably an analytic formula, but ... 00179 if (getTrainStatsCollector()) 00180 test(train_set, getTrainStatsCollector()); 00181 } 00182 00183 00184 void KernelRidgeRegressor::computeOutput(const Vec& input, Vec& output) const 00185 { 00186 PLASSERT( m_kernel && m_params.isNotNull() && m_training_inputs ); 00187 00188 PLASSERT( output.size() == m_params.width() ); 00189 m_kernel_evaluations.resize(m_params.length()); 00190 m_kernel->evaluate_all_x_i(input, m_kernel_evaluations); 00191 00192 // Finally compute k(x,x_i) * (M + \lambda I)^-1 y 00193 product(Mat(1, output.size(), output), 00194 Mat(1, m_kernel_evaluations.size(), m_kernel_evaluations), 00195 m_params); 00196 } 00197 00198 00199 void KernelRidgeRegressor::computeCostsFromOutputs(const Vec& input, const Vec& output, 00200 const Vec& target, Vec& costs) const 00201 { 00202 costs.resize(1); 00203 real squared_loss = powdistance(output,target); 00204 costs[0] = squared_loss; 00205 } 00206 00207 00208 TVec<string> KernelRidgeRegressor::getTestCostNames() const 00209 { 00210 return TVec<string>(1, "mse"); 00211 } 00212 00213 00214 TVec<string> KernelRidgeRegressor::getTrainCostNames() const 00215 { 00216 return TVec<string>(1, "mse"); 00217 } 00218 00219 00220 } // end of namespace PLearn 00221 00222 00223 /* 00224 Local Variables: 00225 mode:c++ 00226 c-basic-offset:4 00227 c-file-style:"stroustrup" 00228 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00229 indent-tabs-mode:nil 00230 fill-column:79 00231 End: 00232 */ 00233 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :