PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // HeterogenuousAffineTransformVariable.cc 00004 // 00005 // Copyright (C) 2006 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Hugo Larochelle 00036 00040 #include "HeterogenuousAffineTransformWeightPenalty.h" 00041 00042 namespace PLearn { 00043 using namespace std; 00044 00047 PLEARN_IMPLEMENT_OBJECT( 00048 HeterogenuousAffineTransformWeightPenalty, 00049 "Penalty associated to an affine transform with continuous and discrete input", 00050 "Weight decay penalty associated to an affine transform with continuous and\n" 00051 "discrete inputs. The way the weight decay works with the discrete component\n" 00052 "weights is that a weight decay is applied only to the activated weights, i.e.\n" 00053 "to the rows of the weight matrices corresponding to the discrete components'\n" 00054 "values." 00055 ); 00056 00057 HeterogenuousAffineTransformWeightPenalty::HeterogenuousAffineTransformWeightPenalty() 00058 : weight_decay_(0.0), bias_decay_(0.0), penalty_type_("L2_square") 00059 {} 00060 00061 // constructors from input variables. 00062 HeterogenuousAffineTransformWeightPenalty::HeterogenuousAffineTransformWeightPenalty(Var input, VarArray weights, TVec<bool> the_input_is_discrete, real weight_decay, real bias_decay, string penalty_type) 00063 : inherited(input & weights, 1, 1), input_is_discrete(the_input_is_discrete), 00064 weight_decay_(weight_decay),bias_decay_(bias_decay),penalty_type_(penalty_type) 00065 // : inherited(input & weights, input->length() != 1 ? weights[0]->width() : 1 , input->width() != 1 ? weights[0]->width() : 1) 00066 { 00067 build(); 00068 } 00069 00070 void HeterogenuousAffineTransformWeightPenalty::recomputeSize(int& l, int& w) const 00071 { 00072 l = w = 1; 00073 } 00074 00075 void HeterogenuousAffineTransformWeightPenalty::fprop() 00076 { 00077 int n = varray[1]->width(); 00078 int l = varray.length()-1; 00079 00080 if (penalty_type_ == "L1_square") 00081 { 00082 if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = sqrt(fabs(bias_decay_))*sumabs(varray.last()->value); 00083 else valuedata[0] = 0; 00084 00085 for(int i=1; i<l; i++) 00086 { 00087 if(input_is_discrete[i-1]) 00088 { 00089 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data(); 00090 for(int j=0; j<n; j++) 00091 valuedata[0] += sqrt(fabs(weight_decay_))*fabs(*row++); 00092 } 00093 else 00094 { 00095 for(int j=0; j<n; j++) 00096 valuedata[0] += sqrt(fabs(weight_decay_))*fabs(varray[i]->valuedata[j]); 00097 } 00098 } 00099 valuedata[0] *= valuedata[0]; 00100 } 00101 else if (penalty_type_ == "L1") 00102 { 00103 if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = bias_decay_*sumabs(varray.last()->value); 00104 else valuedata[0] = 0; 00105 00106 for(int i=1; i<l; i++) 00107 { 00108 if(input_is_discrete[i-1]) 00109 { 00110 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data(); 00111 for(int j=0; j<n; j++) 00112 valuedata[0] += weight_decay_*fabs(*row++); 00113 } 00114 else 00115 { 00116 for(int j=0; j<n; j++) 00117 valuedata[0] += weight_decay_*fabs(varray[i]->valuedata[j]); 00118 } 00119 } 00120 } 00121 else if (penalty_type_ == "L2_square") 00122 { 00123 if(!fast_exact_is_equal(bias_decay_, 0)) valuedata[0] = bias_decay_*sumsquare(varray.last()->value); 00124 else valuedata[0] = 0; 00125 00126 for(int i=1; i<l; i++) 00127 { 00128 if(input_is_discrete[i-1]) 00129 { 00130 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data(); 00131 for(int j=0; j<n; j++) 00132 valuedata[0] += weight_decay_*square_f(*row++); 00133 } 00134 else 00135 { 00136 for(int j=0; j<n; j++) 00137 valuedata[0] += weight_decay_*square_f(varray[i]->valuedata[j]); 00138 } 00139 } 00140 } 00141 00142 } 00143 00144 void HeterogenuousAffineTransformWeightPenalty::bprop() 00145 { 00146 int n = varray[1]->width(); 00147 int l = varray.length()-1; 00148 00149 if (penalty_type_ == "L1_square") 00150 { 00151 real delta; 00152 if(!fast_exact_is_equal(bias_decay_, 0)) 00153 { 00154 delta = 2*sqrt(valuedata[0]*bias_decay_)*gradientdata[0]; 00155 for(int j=0; j<n; j++) 00156 if(varray.last()->valuedata[j] > 0) 00157 varray.last()->gradientdata[j] += delta; 00158 else if(varray.last()->valuedata[j] < 0) 00159 varray.last()->gradientdata[j] -= delta; 00160 } 00161 00162 delta = 2*sqrt(valuedata[0]*weight_decay_)*gradientdata[0]; 00163 for(int i=1; i<l; i++) 00164 { 00165 if(input_is_discrete[i-1]) 00166 { 00167 int r = (int)varray[0]->valuedata[i-1]; 00168 real* row = varray[i]->matValue.row(r).data(); 00169 real* grow = varray[i]->matGradient.row(r).data(); 00170 for(int j=0; j<n; j++) 00171 { 00172 if(row[j] > 0) 00173 grow[j] += delta; 00174 else if( row[j] < 0) 00175 grow[j] -= delta; 00176 } 00177 varray[i]->updateRow(r); 00178 } 00179 else 00180 { 00181 for(int j=0; j<n; j++) 00182 if(varray[i]->valuedata[j] > 0) 00183 varray[i]->gradientdata[j] += delta; 00184 else if(varray[i]->valuedata[j] < 0) 00185 varray[i]->gradientdata[j] -= delta; 00186 } 00187 } 00188 } 00189 else if (penalty_type_ == "L1") 00190 { 00191 real delta; 00192 if(!fast_exact_is_equal(bias_decay_, 0)) 00193 { 00194 delta = bias_decay_*gradientdata[0]; 00195 for(int j=0; j<n; j++) 00196 if(varray.last()->valuedata[j] > 0) 00197 varray.last()->gradientdata[j] += delta; 00198 else if(varray.last()->valuedata[j] < 0) 00199 varray.last()->gradientdata[j] -= delta; 00200 } 00201 00202 delta = weight_decay_*gradientdata[0]; 00203 for(int i=1; i<l; i++) 00204 { 00205 if(input_is_discrete[i-1]) 00206 { 00207 int r = (int)varray[0]->valuedata[i-1]; 00208 real* row = varray[i]->matValue.row(r).data(); 00209 real* grow = varray[i]->matGradient.row(r).data(); 00210 for(int j=0; j<n; j++) 00211 { 00212 if(row[j] > 0) 00213 grow[j] += delta; 00214 else if( row[j] < 0) 00215 grow[j] -= delta; 00216 } 00217 varray[i]->updateRow(r); 00218 } 00219 else 00220 { 00221 for(int j=0; j<n; j++) 00222 if(varray[i]->valuedata[j] > 0) 00223 varray[i]->gradientdata[j] += delta; 00224 else if(varray[i]->valuedata[j] < 0) 00225 varray[i]->gradientdata[j] -= delta; 00226 } 00227 } 00228 } 00229 else if (penalty_type_ == "L2_square") 00230 { 00231 if(!fast_exact_is_equal(bias_decay_, 0)) 00232 { 00233 for(int j=0; j<n; j++) 00234 varray.last()->gradientdata[j] += 2*bias_decay_*varray.last()->valuedata[j]*gradientdata[0]; 00235 } 00236 00237 for(int i=1; i<l; i++) 00238 { 00239 if(input_is_discrete[i-1]) 00240 { 00241 int r = (int)varray[0]->valuedata[i-1]; 00242 real* row = varray[i]->matValue.row((int)varray[0]->valuedata[i-1]).data(); 00243 real* grow = varray[i]->matGradient.row((int)varray[0]->valuedata[i-1]).data(); 00244 for(int j=0; j<n; j++) 00245 { 00246 grow[j] += 2*weight_decay_*row[j]*gradientdata[0]; 00247 } 00248 varray[i]->updateRow(r); 00249 } 00250 else 00251 { 00252 for(int j=0; j<n; j++) 00253 varray[i]->gradientdata[j] += 2*weight_decay_*varray[i]->valuedata[j]*gradientdata[0]; 00254 } 00255 } 00256 } 00257 00258 } 00259 00260 void HeterogenuousAffineTransformWeightPenalty::build() 00261 { 00262 inherited::build(); 00263 build_(); 00264 } 00265 00266 void HeterogenuousAffineTransformWeightPenalty::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00267 { 00268 inherited::makeDeepCopyFromShallowCopy(copies); 00269 00270 deepCopyField(input_is_discrete, copies); 00271 //PLERROR("HeterogenuousAffineTransformWeightPenalty::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00272 } 00273 00274 void HeterogenuousAffineTransformWeightPenalty::declareOptions(OptionList& ol) 00275 { 00276 declareOption(ol, "input_is_discrete", &HeterogenuousAffineTransformWeightPenalty::input_is_discrete, 00277 OptionBase::buildoption, 00278 "Indication whether each component of the input is discrete or not."); 00279 declareOption(ol, "weight_decay_", &HeterogenuousAffineTransformWeightPenalty::weight_decay_, 00280 OptionBase::buildoption, 00281 "Weight decay parameter."); 00282 declareOption(ol, "bias_decay_", &HeterogenuousAffineTransformWeightPenalty::bias_decay_, 00283 OptionBase::buildoption, 00284 "Bias decay parameter."); 00285 declareOption(ol, "penalty_type_", &HeterogenuousAffineTransformWeightPenalty::penalty_type_, 00286 OptionBase::buildoption, 00287 "Penalty to use on the weights.\n" 00288 "Can be any of:\n" 00289 " - \"L1\": L1 norm,\n" 00290 " - \"L1_square\": square of the L1 norm,\n" 00291 " - \"L2_square\" (default): square of the L2 norm.\n"); 00292 00293 // Now call the parent class' declareOptions 00294 inherited::declareOptions(ol); 00295 } 00296 00297 void HeterogenuousAffineTransformWeightPenalty::build_() 00298 { 00299 if(varray[0]->size() != varray.length()-2) 00300 PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): The number of weight variables (%d) and input size (%d) is not the same", varray.length()-2, varray[0]->size()); 00301 if(input_is_discrete.length() != varray[0]->size()) 00302 PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): input_is_discrete size (%d) and input size (%d) does not match", input_is_discrete.length(), varray[0]->size()); 00303 if(!varray[0]->isVec()) 00304 PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): input should be a vector"); 00305 for(int i=1; i<varray.length(); i++) 00306 { 00307 if(varray[i]->width() != varray[1]->width()) 00308 PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): %dth weight matrix has width %d, should be %d", i, varray[i]->width(), size()); 00309 if(i<varray.length()-1 && input_is_discrete[i-1]) 00310 varray[i-1]->allowPartialUpdates(); 00311 } 00312 00313 string pt = lowerstring( penalty_type_ ); 00314 if( pt == "l1" ) 00315 penalty_type_ = "L1"; 00316 else if( pt == "l1_square" || pt == "l1 square" || pt == "l1square" ) 00317 penalty_type_ = "L1_square"; 00318 else if( pt == "l2_square" || pt == "l2 square" || pt == "l2square" ) 00319 penalty_type_ = "L2_square"; 00320 else if( pt == "l2" ) 00321 { 00322 PLWARNING("In HeterogenuousAffineTransformWeightPenalty::build_(): L2 penalty not supported, assuming you want L2 square"); 00323 penalty_type_ = "L2_square"; 00324 } 00325 else 00326 PLERROR("In HeterogenuousAffineTransformWeightPenalty::build_(): penalty_type_ \"%s\" not supported", penalty_type_.c_str()); 00327 } 00328 00329 00330 } // end of namespace PLearn 00331 00332 00333 /* 00334 Local Variables: 00335 mode:c++ 00336 c-basic-offset:4 00337 c-file-style:"stroustrup" 00338 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00339 indent-tabs-mode:nil 00340 fill-column:79 00341 End: 00342 */ 00343 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :