PLearn 0.1
KernelRidgeRegressor.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // KernelRidgeRegressor.cc
00004 //
00005 // Copyright (C) 2005 Nicolas Chapados 
00006 // 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 // 
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 // 
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 // 
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 // 
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 // 
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************      
00036    * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $ 
00037    ******************************************************* */
00038 
00039 // Authors: Nicolas Chapados
00040 
00043 // From PLearn
00044 #include <plearn/vmat/ExtendedVMatrix.h>
00045 
00046 #include "KernelRidgeRegressor.h"
00047 
00048 namespace PLearn {
00049 using namespace std;
00050 
00051 PLEARN_IMPLEMENT_OBJECT(
00052     KernelRidgeRegressor,
00053     "Kernelized version of linear ridge regression.",
00054     "Given a kernel K(x,y) = phi(x)'phi(y), where phi(x) is the projection of a\n"
00055     "vector x into feature space, this class implements a version of the ridge\n"
00056     "estimator, giving the prediction at x as\n"
00057     "\n"
00058     "    f(x) = k(x)'(M + lambda I)^-1 y,\n"
00059     "\n"
00060     "where x is the test vector where to estimate the response, k(x) is the\n"
00061     "vector of kernel evaluations between the test vector and the elements of\n"
00062     "the training set, namely\n"
00063     "\n"
00064     "    k(x) = (K(x,x1), K(x,x2), ..., K(x,xN))',\n"
00065     "\n"
00066     "M is the Gram Matrix on the elements of the training set, i.e. the matrix\n"
00067     "where the element (i,j) is equal to K(xi, xj), lamdba is the weight decay\n"
00068     "coefficient, and y is the vector of training-set targets.\n"
00069     "\n"
00070     "The disadvantage of this learner is that its training time is O(N^3) in the\n"
00071     "number of training examples (due to the matrix inversion).  When saving the\n"
00072     "learner, the training set must be saved, along with an additional vector of\n"
00073     "the length of the training set.\n");
00074 
00075 
00076 KernelRidgeRegressor::KernelRidgeRegressor() 
00077     : m_include_bias(true),
00078       m_weight_decay(0.0)
00079 { }
00080 
00081 
00082 void KernelRidgeRegressor::declareOptions(OptionList& ol)
00083 {
00084     declareOption(ol, "kernel", &KernelRidgeRegressor::m_kernel,
00085                   OptionBase::buildoption,
00086                   "Kernel to use for the computation.  This must be a similarity kernel\n"
00087                   "(i.e. closer vectors give higher kernel evaluations).");
00088 
00089     declareOption(ol, "weight_decay", &KernelRidgeRegressor::m_weight_decay,
00090                   OptionBase::buildoption,
00091                   "Weight decay coefficient (default = 0)");
00092 
00093     declareOption(ol, "include_bias", &KernelRidgeRegressor::m_include_bias,
00094                   OptionBase::buildoption,
00095                   "Whether to include a bias term in the regression (true by default)");
00096 
00097     declareOption(ol, "params", &KernelRidgeRegressor::m_params,
00098                   OptionBase::learntoption,
00099                   "Vector of learned parameters, determined from the equation\n"
00100                   "    (M + lambda I)^-1 y");
00101 
00102     declareOption(ol, "training_inputs", &KernelRidgeRegressor::m_training_inputs,
00103                   OptionBase::learntoption,
00104                   "Saved version of the training set, which must be kept along for\n"
00105                   "carrying out kernel evaluations with the test point");
00106     
00107     // Now call the parent class' declareOptions
00108     inherited::declareOptions(ol);
00109 }
00110 
00111 void KernelRidgeRegressor::build_()
00112 {
00113     if (! m_kernel)
00114         PLERROR("KernelRidgeRegressor::build_: 'kernel' option must be specified");
00115 
00116     // If we are reloading the model, set the training inputs into the kernel
00117     if (m_training_inputs)
00118         m_kernel->setDataForKernelMatrix(m_training_inputs);
00119 }
00120 
00121 // ### Nothing to add here, simply calls build_
00122 void KernelRidgeRegressor::build()
00123 {
00124     inherited::build();
00125     build_();
00126 }
00127 
00128 
00129 void KernelRidgeRegressor::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00130 {
00131     inherited::makeDeepCopyFromShallowCopy(copies);
00132 
00133     deepCopyField(m_kernel,          copies);
00134     deepCopyField(m_params,          copies);
00135     deepCopyField(m_training_inputs, copies);
00136 }
00137 
00138 
00139 int KernelRidgeRegressor::outputsize() const
00140 {
00141     return targetsize();
00142 }
00143 
00144 void KernelRidgeRegressor::forget()
00145 {
00146     m_params.resize(0,0);
00147     m_training_inputs = 0;
00148 }
00149     
00150 void KernelRidgeRegressor::train()
00151 {
00152     PLASSERT( m_kernel );
00153     if (! train_set)
00154         PLERROR("KernelRidgeRegressor::train: the training set must be specified");
00155     int inputsize  = train_set->inputsize() ;
00156     int targetsize = train_set->targetsize();
00157     int weightsize = train_set->weightsize();
00158     if (inputsize  < 0 || targetsize < 0 || weightsize < 0)
00159         PLERROR("KernelRidgeRegressor::train: inconsistent inputsize/targetsize/weightsize "
00160                 "(%d/%d/%d) in training set", inputsize, targetsize, weightsize);
00161     if (weightsize > 0)
00162         PLERROR("KernelRidgeRegressor::train: observations weights are not currently supported");
00163 
00164     m_training_inputs = train_set.subMatColumns(0, inputsize).toMat();
00165     Mat targets       = train_set.subMatColumns(inputsize, targetsize).toMat();
00166     
00167     // Compute Gram Matrix and add weight decay to diagonal
00168     m_kernel->setDataForKernelMatrix(m_training_inputs);
00169     Mat gram_mat(m_training_inputs.length(), m_training_inputs.length());
00170     m_kernel->computeGramMatrix(gram_mat);
00171     addToDiagonal(gram_mat, m_weight_decay);
00172 
00173     // Compute parameters
00174     m_params.resize(targets.length(), targets.width());
00175     solveLinearSystemByCholesky(gram_mat, targets, m_params);
00176 
00177     // Compute train error if there is a train_stats_collector.  There is
00178     // probably an analytic formula, but ...
00179     if (getTrainStatsCollector())
00180         test(train_set, getTrainStatsCollector());
00181 }
00182 
00183 
00184 void KernelRidgeRegressor::computeOutput(const Vec& input, Vec& output) const
00185 {
00186     PLASSERT( m_kernel && m_params.isNotNull() && m_training_inputs );
00187 
00188     PLASSERT( output.size() == m_params.width() );
00189     m_kernel_evaluations.resize(m_params.length());
00190     m_kernel->evaluate_all_x_i(input, m_kernel_evaluations);
00191 
00192     // Finally compute k(x,x_i) * (M + \lambda I)^-1 y
00193     product(Mat(1, output.size(), output),
00194             Mat(1, m_kernel_evaluations.size(), m_kernel_evaluations),
00195             m_params);
00196 }    
00197 
00198 
00199 void KernelRidgeRegressor::computeCostsFromOutputs(const Vec& input, const Vec& output, 
00200                                                    const Vec& target, Vec& costs) const
00201 {
00202     costs.resize(1);
00203     real squared_loss = powdistance(output,target);
00204     costs[0] = squared_loss;
00205 }     
00206 
00207 
00208 TVec<string> KernelRidgeRegressor::getTestCostNames() const
00209 {
00210     return TVec<string>(1, "mse");
00211 }
00212 
00213 
00214 TVec<string> KernelRidgeRegressor::getTrainCostNames() const
00215 {
00216     return TVec<string>(1, "mse");
00217 }
00218 
00219 
00220 } // end of namespace PLearn
00221 
00222 
00223 /*
00224   Local Variables:
00225   mode:c++
00226   c-basic-offset:4
00227   c-file-style:"stroustrup"
00228   c-file-offsets:((innamespace . 0)(inline-open . 0))
00229   indent-tabs-mode:nil
00230   fill-column:79
00231   End:
00232 */
00233 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines