PLearn 0.1
AffineTransformWeightPenalty.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2001-2002 Nicolas Chapados, Ichiro Takeuchi, Jean-Sebastien Senecal
00007 // Copyright (C) 2002 Xiangdong Wang, Christian Dorion
00008 
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************      
00039  * $Id: AffineTransformWeightPenalty.cc 9239 2008-07-12 21:53:13Z chapados $
00040  * This file is part of the PLearn library.
00041  ******************************************************* */
00042 
00043 #include "AffineTransformWeightPenalty.h"
00044 #include "Var_utils.h"
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 
00050 PLEARN_IMPLEMENT_OBJECT(AffineTransformWeightPenalty,
00051                         "Penalty associated with an affine transformation with weight decay terms",
00052                         "");
00053 
00054 void AffineTransformWeightPenalty::recomputeSize(int& l, int& w) const
00055 { l=1; w=1; }
00056 
00057 void
00058 AffineTransformWeightPenalty::declareOptions(OptionList &ol)
00059 {
00060     declareOption(ol, "weight_decay_", &AffineTransformWeightPenalty::weight_decay_, OptionBase::buildoption, "");
00061     declareOption(ol, "bias_decay_", &AffineTransformWeightPenalty::bias_decay_, OptionBase::buildoption, "");
00062     declareOption(ol, "penalty_type_", &AffineTransformWeightPenalty::penalty_type_, OptionBase::buildoption, "");
00063 }
00064 
00065 void AffineTransformWeightPenalty::fprop()
00066 {
00067     if (penalty_type_ == "L1_square")
00068     {
00069         if (input->length()>1)
00070             valuedata[0] = sqrt(fabs(weight_decay_))*sumabs(input->matValue.subMatRows(1,input->length()-1));
00071         else
00072             valuedata[0] = 0;
00073         if(!fast_exact_is_equal(bias_decay_, 0))
00074             valuedata[0] += sqrt(fabs(bias_decay_))*sumabs(input->matValue(0));
00075 
00076         valuedata[0] *= valuedata[0];
00077     }
00078     else if (penalty_type_ == "L1")
00079     {
00080         if (input->length()>1)
00081             valuedata[0] = weight_decay_*sumabs(input->matValue.subMatRows(1,input->length()-1));
00082         else 
00083             valuedata[0] = 0;
00084         if(!fast_exact_is_equal(bias_decay_, 0))
00085             valuedata[0] += bias_decay_*sumabs(input->matValue(0));
00086     }
00087     else if (penalty_type_ == "L2_square")
00088     {
00089         if (input->length()>1)
00090             valuedata[0] = weight_decay_*sumsquare(input->matValue.subMatRows(1,input->length()-1));
00091         else 
00092             valuedata[0] = 0;
00093         if(!fast_exact_is_equal(bias_decay_, 0))
00094             valuedata[0] += bias_decay_*sumsquare(input->matValue(0));
00095     }
00096 }
00097 
00098 
00099 void AffineTransformWeightPenalty::bprop()
00100 {
00101     int l = input->length() - 1;
00102     if ( penalty_type_ == "L1_square" )
00103     {
00104         if (!input->matGradient.isCompact())
00105             PLERROR("AffineTransformWeightPenalty::bprop, L1_square penalty currently not handling non-compact weight matrix");
00106         int n=input->width();
00107         if (!fast_exact_is_equal(weight_decay_, 0))
00108         {
00109             real delta = 2*sqrt(valuedata[0]*weight_decay_)*gradientdata[0];
00110             real* w = input->matValue[1];
00111             real* d_w = input->matGradient[1];
00112             int tot = l * n; // Number of weights to update.
00113             for (int i = 0; i < tot; i++) {
00114                 if (w[i] > 0)
00115                     d_w[i] += delta;
00116                 else if (w[i] < 0)
00117                     d_w[i] -= delta;
00118             }
00119         }
00120         if(!fast_exact_is_equal(bias_decay_, 0))
00121         {
00122             real delta = 2*sqrt(valuedata[0]*bias_decay_)*gradientdata[0];
00123             real* d_biases = input->matGradient[0];
00124             real* biases = input->matValue[0];
00125             for (int i=0;i<n;i++) {
00126                 if (biases[i]>0)
00127                     d_biases[i] += delta;
00128                 else if (biases[i]<0)
00129                     d_biases[i] -= delta;
00130             }
00131         }
00132     }
00133     else if ( penalty_type_ == "L1")
00134     {
00135         if (!input->matGradient.isCompact())
00136             PLERROR("AffineTransformWeightPenalty::bprop, L1 penalty currently not handling non-compact weight matrix");
00137         int n=input->width();
00138         if (!fast_exact_is_equal(weight_decay_, 0) && l > 0)
00139         {
00140             real delta = weight_decay_ * gradientdata[0];
00141             real* w = input->matValue[1];
00142             real* d_w = input->matGradient[1];
00143             int tot = l * n; // Number of weights to update.
00144             for (int i = 0; i < tot; i++) {
00145                 if (w[i] > 0)
00146                     d_w[i] += delta;
00147                 else if (w[i] < 0)
00148                     d_w[i] -= delta;
00149             }
00150         }
00151         if(!fast_exact_is_equal(bias_decay_, 0) && l >= 0)
00152         {
00153             real delta = bias_decay_ * gradientdata[0];
00154             real* d_biases = input->matGradient[0];
00155             real* biases = input->matValue[0];
00156             for (int i=0;i<n;i++)
00157                 if (biases[i]>0)
00158                     d_biases[i] += delta;
00159                 else if (biases[i]<0)
00160                     d_biases[i] -= delta;
00161         }
00162     }
00163     else if (penalty_type_ == "L2_square" )
00164     {
00165         multiplyAcc(input->matGradient.subMatRows(1,l), input->matValue.subMatRows(1,l), two(weight_decay_)*gradientdata[0]);
00166         if(!fast_exact_is_equal(bias_decay_, 0))
00167             multiplyAcc(input->matGradient(0), input->matValue(0), two(bias_decay_)*gradientdata[0]);
00168     }
00169 }
00170 
00171 
00172 
00173 } // end of namespace PLearn
00174 
00175 
00176 /*
00177   Local Variables:
00178   mode:c++
00179   c-basic-offset:4
00180   c-file-style:"stroustrup"
00181   c-file-offsets:((innamespace . 0)(inline-open . 0))
00182   indent-tabs-mode:nil
00183   fill-column:79
00184   End:
00185 */
00186 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines