PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // NLLErrModule.cc 00004 // 00005 // Copyright (C) 2005 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: NLLErrModule.cc,v 1.5 2005/12/30 19:53:56 lamblinp Exp $ 00037 ******************************************************* */ 00038 00039 // Authors: Pascal Lamblin 00040 00044 #include "NLLErrModule.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 PLEARN_IMPLEMENT_OBJECT( 00050 NLLErrModule, 00051 "NLLError Module", 00052 "This class computes the Negative Log-Likelihood of the input, given the\n" 00053 "desired 'target'. Also propagates gradient and diagonal of Hessian\n" 00054 "backwards.\n" 00055 "If output_size = 2, the second output is the classification error.\n" 00056 ); 00057 00058 NLLErrModule::NLLErrModule(): 00059 target_size(1) 00060 /* ### Initialize all fields to their default value */ 00061 { 00062 output_size = 1; 00063 } 00064 00065 00066 // retrieve target from input vector 00067 int NLLErrModule::getTarget(const Vec& input) const 00068 { 00069 int t_size = input.size() - input_size; 00070 int target = -1; 00071 00072 // size check 00073 if( t_size == 1 ) 00074 { 00075 target = (int) input[ input_size ]; 00076 } 00077 else if( t_size == input_size ) 00078 { 00079 /* 00080 PLWARNING("NLLErrModule::getTarget: You're giving a target the same\n" 00081 "size as the input, instead of an integer. Checking if\n" 00082 "this is a one-hot vector from this integer.\n"); 00083 */ 00084 00085 Vec the_target = input.subVec( input_size, t_size ); 00086 // get position of '1' 00087 target = argmax( the_target ); 00088 00089 #ifdef BOUNDCHECK 00090 // check if vector matches with a one-hot one 00091 PLASSERT( is_equal( the_target[target], 1. ) ) ; 00092 for( int i=0 ; i<input_size ; i++ ) 00093 PLASSERT( is_equal( the_target[i], 0. ) || i == target ); 00094 #endif 00095 } 00096 else 00097 { 00098 PLERROR("NLLErrModule::getTarget: target.size() is %i,\n" 00099 " but should be 1. 'target' should contain an integer.\n", 00100 t_size); 00101 } 00102 00103 if( target < 0 || target >= input_size ) 00104 PLERROR("NLLErrModule::getTarget: target should be between 0 and" 00105 "input_size (%i).\n", input_size); 00106 00107 return target; 00108 } 00109 00110 // output = error = log(softmax(input))[target] 00111 void NLLErrModule::fprop(const Vec& input, Vec& output) const 00112 { 00113 int target = getTarget( input ); 00114 // size check is done in getTarget() 00115 00116 Vec input_ = input.subVec( 0, input_size ); 00117 output.resize( output_size ); 00118 00119 fp_sm = softmax( input_ ); 00120 output[0] = - pl_log( fp_sm[target] ); 00121 00122 00123 if( output_size > 1 ) 00124 output[1] = ( argmax( input_ ) == target ) ? 0 : 1; 00125 } 00126 00127 // Don't modify class 00128 void NLLErrModule::bpropUpdate(const Vec& input, const Vec& output, 00129 const Vec& output_gradient) 00130 { 00131 int out_size = output.size(); 00132 int og_size = output_gradient.size(); 00133 00134 // for size check 00135 getTarget( input ); 00136 00137 // size check 00138 if( out_size != output_size ) 00139 { 00140 PLWARNING("NLLErrModule::bpropUpdate: output.size()' should be\n" 00141 " equal to 'output_size' (%i != %i)\n", 00142 out_size, output_size); 00143 } 00144 if( og_size != output_size ) 00145 { 00146 PLWARNING("NLLErrModule::bpropUpdate: 'output_gradient.size()'\n" 00147 " should be equal to 'output_size' (%i != %i)\n", 00148 og_size, output_size); 00149 } 00150 } 00151 00152 // We don't care about output_gradient, we consider we're the last variable. 00153 // So we compute the gradient of the error of this variable. 00154 void NLLErrModule::bpropUpdate(const Vec& input, const Vec& output, 00155 Vec& input_gradient, 00156 const Vec& output_gradient) 00157 { 00158 int out_size = output.size(); 00159 int og_size = output_gradient.size(); 00160 int target = getTarget( input ); 00161 bool is_final_cost = false; // if yes, output_gradient is 1 00162 00163 // size check 00164 if( out_size != output_size ) 00165 { 00166 PLERROR("NLLErrModule::bpropUpdate: output.size()' should be\n" 00167 " equal to 'output_size' (%i != %i)\n", 00168 out_size, output_size); 00169 } 00170 if( og_size == 0 ) 00171 { 00172 /* 00173 PLWARNING("NLLErrModule::bpropUpdate: you are not providing" 00174 "output_gradient.\n" 00175 "Assuming this is the final cost, and output_gradient=1.\n"); 00176 */ 00177 is_final_cost = true; 00178 } 00179 else if( og_size != output_size ) 00180 { 00181 PLERROR("NLLErrModule::bpropUpdate: 'output_gradient.size()'\n" 00182 " should be equal to 'output_size' (%i != %i)\n", 00183 og_size, output_size); 00184 } 00185 00186 // input_gradient[i] = output_gradient*( softmax(input)[i] ) if i!=target 00187 // input_gradient[target] = output_gradient*( softmax(input)[target] ) 00188 input_gradient.resize( input_size ); 00189 input_gradient << fp_sm; 00190 00191 input_gradient[target] -= 1; 00192 if( !is_final_cost ) 00193 input_gradient *= output_gradient[0]; 00194 00195 } 00196 00197 // Does nothing (just checks and warns) 00198 void NLLErrModule::bbpropUpdate(const Vec& input, const Vec& output, 00199 const Vec& output_gradient, 00200 const Vec& output_diag_hessian) 00201 { 00202 int odh_size = output_diag_hessian.size(); 00203 if( odh_size != output_size ) 00204 { 00205 PLWARNING("NLLErrModule::bbpropUpdate:" 00206 " 'output_diag_hessian.size()'\n" 00207 " should be equal to 'output_size' (%i != %i)\n", 00208 odh_size, output_size); 00209 } 00210 00211 bpropUpdate( input, output, output_gradient ); 00212 } 00213 00214 // Propagates back output_gradient and output_diag_hessian 00215 void NLLErrModule::bbpropUpdate(const Vec& input, const Vec& output, 00216 Vec& input_gradient, 00217 const Vec& output_gradient, 00218 Vec& input_diag_hessian, 00219 const Vec& output_diag_hessian) 00220 { 00221 int odh_size = output_diag_hessian.size(); 00222 int target = getTarget( input ); 00223 bool is_final_cost = false; // if yes, output_diag_hessian is 0 00224 00225 // size check 00226 // others size checks will be done in bpropUpdate() 00227 if( odh_size == 0 ) 00228 { 00229 PLWARNING("NLLErrModule::bbpropUpdate: you are not providing" 00230 " output_diag_hessian.\n" 00231 "Assuming this is the final cost," 00232 " and output_diag_hessian=0.\n"); 00233 is_final_cost = true; 00234 } 00235 else if( odh_size != output_size ) 00236 { 00237 PLERROR("NLLErrModule::bbpropUpdate:" 00238 " 'output_diag_hessian.size()'\n" 00239 " should be equal to 'output_size' (%i != %i)\n", 00240 odh_size, output_size); 00241 } 00242 00243 bpropUpdate( input, output, input_gradient, output_gradient ); 00244 00245 Vec input_ = input.subVec( 0, input_size ); 00246 input_diag_hessian.resize( input_size ); 00247 Vec softmax_in = softmax( input_ ); 00248 00249 // computation of term dC/dy d²y/dx², 00250 // skipped if estimate_simpler_diag_hessian, unless it is final cost 00251 if( estimate_simpler_diag_hessian && !is_final_cost ) 00252 { 00253 input_diag_hessian.clear(); 00254 } 00255 else 00256 { 00257 for( int i=0 ; i<input_size ; i++ ) 00258 { 00259 real sm_i = softmax_in[i]; 00260 input_diag_hessian[i] = sm_i*( 1-sm_i); 00261 } 00262 00263 if( !is_final_cost ) 00264 input_diag_hessian *= output_gradient[0]; 00265 00266 } 00267 00268 // computation of term d²C/dy² (dy/dx)², 00269 // skipped if it is final cost, because then d²C/dy² == d²C/dC² == 0 00270 if( !is_final_cost ) 00271 { 00272 Vec fprime = softmax_in; 00273 fprime[target] -= 1; 00274 fprime *= fprime; 00275 00276 input_diag_hessian += output_diag_hessian[0] * fprime; 00277 } 00278 00279 } 00280 00281 00282 00283 00284 // 00285 void NLLErrModule::forget() 00286 { 00287 } 00288 00289 00290 // ### Nothing to add here, simply calls build_ 00291 void NLLErrModule::build() 00292 { 00293 inherited::build(); 00294 build_(); 00295 } 00296 00297 void NLLErrModule::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00298 { 00299 inherited::makeDeepCopyFromShallowCopy(copies); 00300 00301 } 00302 00303 void NLLErrModule::declareOptions(OptionList& ol) 00304 { 00305 inherited::declareOptions(ol); 00306 } 00307 00308 void NLLErrModule::build_() 00309 { 00310 if( input_size < 0 ) 00311 { 00312 PLWARNING("NLLErrModule::build_: 'input_size' is < 0.\n" 00313 "You should set it to a positive integer.\n" 00314 "Defaulting to '1' (like a sigmoid function ?)\n"); 00315 input_size = 1; 00316 } 00317 if( output_size != 1 && output_size != 2 ) 00318 { 00319 PLWARNING("NLLErrModule::build_: 'output_size' (%i) should be 1.\n" 00320 "Setting 'output_size' to 1.\n", output_size); 00321 output_size = 1; 00322 } 00323 00324 target_size = 1; 00325 00326 fp_sm.resize(input_size); 00327 } 00328 00329 00330 00331 00332 } // end of namespace PLearn 00333 00334 00335 /* 00336 Local Variables: 00337 mode:c++ 00338 c-basic-offset:4 00339 c-file-style:"stroustrup" 00340 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00341 indent-tabs-mode:nil 00342 fill-column:79 00343 End: 00344 */ 00345 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :