PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2001-2002 Nicolas Chapados, Ichiro Takeuchi, Jean-Sebastien Senecal 00007 // Copyright (C) 2002 Xiangdong Wang, Christian Dorion 00008 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ******************************************************* 00039 * $Id: AffineTransformWeightPenalty.cc 9239 2008-07-12 21:53:13Z chapados $ 00040 * This file is part of the PLearn library. 00041 ******************************************************* */ 00042 00043 #include "AffineTransformWeightPenalty.h" 00044 #include "Var_utils.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 00050 PLEARN_IMPLEMENT_OBJECT(AffineTransformWeightPenalty, 00051 "Penalty associated with an affine transformation with weight decay terms", 00052 ""); 00053 00054 void AffineTransformWeightPenalty::recomputeSize(int& l, int& w) const 00055 { l=1; w=1; } 00056 00057 void 00058 AffineTransformWeightPenalty::declareOptions(OptionList &ol) 00059 { 00060 declareOption(ol, "weight_decay_", &AffineTransformWeightPenalty::weight_decay_, OptionBase::buildoption, ""); 00061 declareOption(ol, "bias_decay_", &AffineTransformWeightPenalty::bias_decay_, OptionBase::buildoption, ""); 00062 declareOption(ol, "penalty_type_", &AffineTransformWeightPenalty::penalty_type_, OptionBase::buildoption, ""); 00063 } 00064 00065 void AffineTransformWeightPenalty::fprop() 00066 { 00067 if (penalty_type_ == "L1_square") 00068 { 00069 if (input->length()>1) 00070 valuedata[0] = sqrt(fabs(weight_decay_))*sumabs(input->matValue.subMatRows(1,input->length()-1)); 00071 else 00072 valuedata[0] = 0; 00073 if(!fast_exact_is_equal(bias_decay_, 0)) 00074 valuedata[0] += sqrt(fabs(bias_decay_))*sumabs(input->matValue(0)); 00075 00076 valuedata[0] *= valuedata[0]; 00077 } 00078 else if (penalty_type_ == "L1") 00079 { 00080 if (input->length()>1) 00081 valuedata[0] = weight_decay_*sumabs(input->matValue.subMatRows(1,input->length()-1)); 00082 else 00083 valuedata[0] = 0; 00084 if(!fast_exact_is_equal(bias_decay_, 0)) 00085 valuedata[0] += bias_decay_*sumabs(input->matValue(0)); 00086 } 00087 else if (penalty_type_ == "L2_square") 00088 { 00089 if (input->length()>1) 00090 valuedata[0] = weight_decay_*sumsquare(input->matValue.subMatRows(1,input->length()-1)); 00091 else 00092 valuedata[0] = 0; 00093 if(!fast_exact_is_equal(bias_decay_, 0)) 00094 valuedata[0] += bias_decay_*sumsquare(input->matValue(0)); 00095 } 00096 } 00097 00098 00099 void AffineTransformWeightPenalty::bprop() 00100 { 00101 int l = input->length() - 1; 00102 if ( penalty_type_ == "L1_square" ) 00103 { 00104 if (!input->matGradient.isCompact()) 00105 PLERROR("AffineTransformWeightPenalty::bprop, L1_square penalty currently not handling non-compact weight matrix"); 00106 int n=input->width(); 00107 if (!fast_exact_is_equal(weight_decay_, 0)) 00108 { 00109 real delta = 2*sqrt(valuedata[0]*weight_decay_)*gradientdata[0]; 00110 real* w = input->matValue[1]; 00111 real* d_w = input->matGradient[1]; 00112 int tot = l * n; // Number of weights to update. 00113 for (int i = 0; i < tot; i++) { 00114 if (w[i] > 0) 00115 d_w[i] += delta; 00116 else if (w[i] < 0) 00117 d_w[i] -= delta; 00118 } 00119 } 00120 if(!fast_exact_is_equal(bias_decay_, 0)) 00121 { 00122 real delta = 2*sqrt(valuedata[0]*bias_decay_)*gradientdata[0]; 00123 real* d_biases = input->matGradient[0]; 00124 real* biases = input->matValue[0]; 00125 for (int i=0;i<n;i++) { 00126 if (biases[i]>0) 00127 d_biases[i] += delta; 00128 else if (biases[i]<0) 00129 d_biases[i] -= delta; 00130 } 00131 } 00132 } 00133 else if ( penalty_type_ == "L1") 00134 { 00135 if (!input->matGradient.isCompact()) 00136 PLERROR("AffineTransformWeightPenalty::bprop, L1 penalty currently not handling non-compact weight matrix"); 00137 int n=input->width(); 00138 if (!fast_exact_is_equal(weight_decay_, 0) && l > 0) 00139 { 00140 real delta = weight_decay_ * gradientdata[0]; 00141 real* w = input->matValue[1]; 00142 real* d_w = input->matGradient[1]; 00143 int tot = l * n; // Number of weights to update. 00144 for (int i = 0; i < tot; i++) { 00145 if (w[i] > 0) 00146 d_w[i] += delta; 00147 else if (w[i] < 0) 00148 d_w[i] -= delta; 00149 } 00150 } 00151 if(!fast_exact_is_equal(bias_decay_, 0) && l >= 0) 00152 { 00153 real delta = bias_decay_ * gradientdata[0]; 00154 real* d_biases = input->matGradient[0]; 00155 real* biases = input->matValue[0]; 00156 for (int i=0;i<n;i++) 00157 if (biases[i]>0) 00158 d_biases[i] += delta; 00159 else if (biases[i]<0) 00160 d_biases[i] -= delta; 00161 } 00162 } 00163 else if (penalty_type_ == "L2_square" ) 00164 { 00165 multiplyAcc(input->matGradient.subMatRows(1,l), input->matValue.subMatRows(1,l), two(weight_decay_)*gradientdata[0]); 00166 if(!fast_exact_is_equal(bias_decay_, 0)) 00167 multiplyAcc(input->matGradient(0), input->matValue(0), two(bias_decay_)*gradientdata[0]); 00168 } 00169 } 00170 00171 00172 00173 } // end of namespace PLearn 00174 00175 00176 /* 00177 Local Variables: 00178 mode:c++ 00179 c-basic-offset:4 00180 c-file-style:"stroustrup" 00181 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00182 indent-tabs-mode:nil 00183 fill-column:79 00184 End: 00185 */ 00186 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :