PLearn 0.1
HeterogenuousAffineTransformWeightPenalty.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // HeterogenuousAffineTransformVariable.cc
00004 //
00005 // Copyright (C) 2006 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Hugo Larochelle
00036 
00040 #include "HeterogenuousAffineTransformWeightPenalty.h"
00041 
00042 namespace PLearn {
00043 using namespace std;
00044 
00047 PLEARN_IMPLEMENT_OBJECT(
00048     HeterogenuousAffineTransformWeightPenalty,
00049     "Penalty associated to an affine transform with continuous and discrete input",
00050     "Weight decay penalty associated to an affine transform with continuous and\n"
00051     "discrete inputs. The way the weight decay works with the discrete component\n"
00052     "weights is that a weight decay is applied only to the activated weights, i.e.\n"
00053     "to the rows of the weight matrices corresponding to the discrete components'\n"
00054     "values."
00055     );
00056 
00057 HeterogenuousAffineTransformWeightPenalty::HeterogenuousAffineTransformWeightPenalty()
00058     : weight_decay_(0.0), bias_decay_(0.0), penalty_type_("L2_square")
00059 {}
00060 
00061 // constructors from input variables.
00062 HeterogenuousAffineTransformWeightPenalty::HeterogenuousAffineTransformWeightPenalty(Var input, VarArray weights, TVec<bool> the_input_is_discrete, real weight_decay, real bias_decay, string penalty_type)
00063      : inherited(input & weights, 1, 1), input_is_discrete(the_input_is_discrete), 
00064        weight_decay_(weight_decay),bias_decay_(bias_decay),penalty_type_(penalty_type)
00065  //     : inherited(input & weights, input->length() != 1 ? weights[0]->width() : 1 , input->width() != 1 ? weights[0]->width() : 1)
00066  {
00067      build();
00068  }
00069 
00070 void HeterogenuousAffineTransformWeightPenalty::recomputeSize(int& l, int& w) const
00071 {    
00072     l = w = 1;
00073 }
00074 
00075 void HeterogenuousAffineTransformWeightPenalty::fprop()
00076 {
00077     int n = varray[1]->width();
00078     int l = varray.length()-1;
00079 
00080     if (penalty_type_ == "L1_square")
00081     {
00082         if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = sqrt(fabs(bias_decay_))*sumabs(varray.last()->value);
00083         else valuedata[0] = 0;
00084 
00085         for(int i=1; i<l; i++)
00086         {
00087             if(input_is_discrete[i-1])
00088             {
00089                 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data();
00090                 for(int j=0; j<n; j++)
00091                     valuedata[0] += sqrt(fabs(weight_decay_))*fabs(*row++);
00092             }
00093             else
00094             {
00095                 for(int j=0; j<n; j++)
00096                     valuedata[0] += sqrt(fabs(weight_decay_))*fabs(varray[i]->valuedata[j]);
00097             }
00098         }
00099         valuedata[0] *= valuedata[0];
00100     }
00101     else if (penalty_type_ == "L1")
00102     {
00103         if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = bias_decay_*sumabs(varray.last()->value);
00104         else valuedata[0] = 0;
00105 
00106         for(int i=1; i<l; i++)
00107         {
00108             if(input_is_discrete[i-1])
00109             {
00110                 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data();
00111                 for(int j=0; j<n; j++)
00112                     valuedata[0] += weight_decay_*fabs(*row++);
00113             }
00114             else
00115             {
00116                 for(int j=0; j<n; j++)
00117                     valuedata[0] += weight_decay_*fabs(varray[i]->valuedata[j]);
00118             }
00119         }
00120     }
00121     else if (penalty_type_ == "L2_square")
00122     {
00123         if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = bias_decay_*sumsquare(varray.last()->value);
00124         else valuedata[0] = 0;
00125 
00126         for(int i=1; i<l; i++)
00127         {
00128             if(input_is_discrete[i-1])
00129             {
00130                 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data();
00131                 for(int j=0; j<n; j++)
00132                     valuedata[0] += weight_decay_*square_f(*row++);
00133             }
00134             else
00135             {
00136                 for(int j=0; j<n; j++)
00137                     valuedata[0] += weight_decay_*square_f(varray[i]->valuedata[j]);
00138             }
00139         }
00140     }
00141 
00142 }
00143 
00144 void HeterogenuousAffineTransformWeightPenalty::bprop()
00145 {
00146     int n = varray[1]->width();
00147     int l = varray.length()-1;
00148 
00149     if (penalty_type_ == "L1_square")
00150     {
00151         real delta;
00152         if(!fast_exact_is_equal(bias_decay_, 0))
00153         {
00154             delta = 2*sqrt(valuedata[0]*bias_decay_)*gradientdata[0];
00155             for(int j=0; j<n; j++)
00156                 if(varray.last()->valuedata[j] > 0)
00157                     varray.last()->gradientdata[j] += delta;
00158                 else if(varray.last()->valuedata[j] < 0)
00159                     varray.last()->gradientdata[j] -= delta;
00160         }
00161 
00162         delta = 2*sqrt(valuedata[0]*weight_decay_)*gradientdata[0];
00163         for(int i=1; i<l; i++)
00164         {
00165             if(input_is_discrete[i-1])
00166             {
00167                 int r = (int)varray[0]->valuedata[i-1];
00168                 real* row = varray[i]->matValue.row(r).data();
00169                 real* grow = varray[i]->matGradient.row(r).data();
00170                 for(int j=0; j<n; j++)
00171                 {
00172                     if(row[j] > 0)
00173                         grow[j] += delta;
00174                     else if( row[j] < 0)
00175                         grow[j] -= delta;
00176                 }
00177                 varray[i]->updateRow(r);
00178             }
00179             else
00180             {
00181                 for(int j=0; j<n; j++)
00182                     if(varray[i]->valuedata[j] > 0)
00183                         varray[i]->gradientdata[j] += delta;
00184                     else if(varray[i]->valuedata[j] < 0)
00185                         varray[i]->gradientdata[j] -= delta;
00186             }
00187         }
00188     }
00189     else if (penalty_type_ == "L1")
00190     {
00191         real delta;
00192         if(!fast_exact_is_equal(bias_decay_, 0))
00193         {
00194             delta = bias_decay_*gradientdata[0];
00195             for(int j=0; j<n; j++)
00196                 if(varray.last()->valuedata[j] > 0)
00197                     varray.last()->gradientdata[j] += delta;
00198                 else if(varray.last()->valuedata[j] < 0)
00199                     varray.last()->gradientdata[j] -= delta;
00200         }
00201 
00202         delta = weight_decay_*gradientdata[0];
00203         for(int i=1; i<l; i++)
00204         {
00205             if(input_is_discrete[i-1])
00206             {
00207                 int r = (int)varray[0]->valuedata[i-1];
00208                 real* row = varray[i]->matValue.row(r).data();
00209                 real* grow = varray[i]->matGradient.row(r).data();
00210                 for(int j=0; j<n; j++)
00211                 {
00212                     if(row[j] > 0)
00213                         grow[j] += delta;
00214                     else if( row[j] < 0)
00215                         grow[j] -= delta;
00216                 }
00217                 varray[i]->updateRow(r);
00218             }
00219             else
00220             {
00221                 for(int j=0; j<n; j++)
00222                     if(varray[i]->valuedata[j] > 0)
00223                         varray[i]->gradientdata[j] += delta;
00224                     else if(varray[i]->valuedata[j] < 0)
00225                         varray[i]->gradientdata[j] -= delta;
00226             }
00227         }
00228     }
00229     else if (penalty_type_ == "L2_square")
00230     {
00231         if(!fast_exact_is_equal(bias_decay_, 0))
00232         {
00233             for(int j=0; j<n; j++)
00234                 varray.last()->gradientdata[j] += 2*bias_decay_*varray.last()->valuedata[j]*gradientdata[0];
00235         }
00236 
00237         for(int i=1; i<l; i++)
00238         {
00239             if(input_is_discrete[i-1])
00240             {
00241                 int r = (int)varray[0]->valuedata[i-1];
00242                 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data();
00243                 real* grow = varray[i]->matGradient.row((int)varray[0]->valuedata[i-1]).data();
00244                 for(int j=0; j<n; j++)
00245                 {
00246                     grow[j] += 2*weight_decay_*row[j]*gradientdata[0];
00247                 }
00248                 varray[i]->updateRow(r);
00249             }
00250             else
00251             {
00252                 for(int j=0; j<n; j++)
00253                     varray[i]->gradientdata[j] += 2*weight_decay_*varray[i]->valuedata[j]*gradientdata[0];
00254             }
00255         }
00256     }
00257 
00258 }
00259 
00260 void HeterogenuousAffineTransformWeightPenalty::build()
00261 {
00262     inherited::build();
00263     build_();
00264 }
00265 
00266 void HeterogenuousAffineTransformWeightPenalty::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00267 {
00268     inherited::makeDeepCopyFromShallowCopy(copies);
00269 
00270     deepCopyField(input_is_discrete, copies);
00271     //PLERROR("HeterogenuousAffineTransformWeightPenalty::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00272 }
00273 
00274 void HeterogenuousAffineTransformWeightPenalty::declareOptions(OptionList& ol)
00275 {    
00276     declareOption(ol, "input_is_discrete", &HeterogenuousAffineTransformWeightPenalty::input_is_discrete,
00277                   OptionBase::buildoption,
00278                   "Indication whether each component of the input is discrete or not.");
00279     declareOption(ol, "weight_decay_", &HeterogenuousAffineTransformWeightPenalty::weight_decay_,
00280                   OptionBase::buildoption,
00281                   "Weight decay parameter.");
00282     declareOption(ol, "bias_decay_", &HeterogenuousAffineTransformWeightPenalty::bias_decay_,
00283                   OptionBase::buildoption,
00284                   "Bias decay parameter.");
00285     declareOption(ol, "penalty_type_", &HeterogenuousAffineTransformWeightPenalty::penalty_type_,
00286                   OptionBase::buildoption,
00287                   "Penalty to use on the weights.\n"
00288                   "Can be any of:\n"
00289                   "  - \"L1\": L1 norm,\n"
00290                   "  - \"L1_square\": square of the L1 norm,\n"
00291                   "  - \"L2_square\" (default): square of the L2 norm.\n");
00292 
00293     // Now call the parent class' declareOptions
00294     inherited::declareOptions(ol);
00295 }
00296 
00297 void HeterogenuousAffineTransformWeightPenalty::build_()
00298 {
00299     if(varray[0]->size() != varray.length()-2)
00300         PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): The number of weight variables (%d) and input size (%d) is not the same", varray.length()-2, varray[0]->size());
00301     if(input_is_discrete.length() != varray[0]->size())
00302         PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): input_is_discrete size (%d) and input size (%d) does not match", input_is_discrete.length(), varray[0]->size());
00303     if(!varray[0]->isVec())
00304         PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): input should be a vector");
00305     for(int i=1; i<varray.length(); i++)
00306     {
00307         if(varray[i]->width() != varray[1]->width())
00308             PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): %dth weight matrix has width %d, should be %d", i, varray[i]->width(), size());
00309         if(i<varray.length()-1 && input_is_discrete[i-1])
00310             varray[i-1]->allowPartialUpdates();
00311     }
00312 
00313     string pt = lowerstring( penalty_type_ );
00314     if( pt == "l1" )
00315         penalty_type_ = "L1";
00316     else if( pt == "l1_square" || pt == "l1 square" || pt == "l1square" )
00317         penalty_type_ = "L1_square";
00318     else if( pt == "l2_square" || pt == "l2 square" || pt == "l2square" )
00319         penalty_type_ = "L2_square";
00320     else if( pt == "l2" )
00321     {
00322         PLWARNING("In HeterogenuousAffineTransformWeightPenalty::build_(): L2 penalty not supported, assuming you want L2 square");
00323         penalty_type_ = "L2_square";
00324         }
00325     else
00326         PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): penalty_type_ \"%s\" not supported", penalty_type_.c_str());
00327 }
00328 
00329 
00330 } // end of namespace PLearn
00331 
00332 
00333 /*
00334   Local Variables:
00335   mode:c++
00336   c-basic-offset:4
00337   c-file-style:"stroustrup"
00338   c-file-offsets:((innamespace . 0)(inline-open . 0))
00339   indent-tabs-mode:nil
00340   fill-column:79
00341   End:
00342 */
00343 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines