PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // NLLCostModule.cc 00004 // 00005 // Copyright (C) 2006 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin 00036 00041 #include "NLLCostModule.h" 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 PLEARN_IMPLEMENT_OBJECT( 00047 NLLCostModule, 00048 "Computes the NLL, given a probability vector and the true class.", 00049 "If input is the probability vector, and target the index of the true\n" 00050 "class, this module computes cost = -log( input[target] ), and\n" 00051 "back-propagates the gradient and diagonal of Hessian.\n"); 00052 00053 NLLCostModule::NLLCostModule() 00054 { 00055 output_size = 1; 00056 target_size = 1; 00057 } 00058 00059 void NLLCostModule::declareOptions(OptionList& ol) 00060 { 00061 // declareOption(ol, "myoption", &NLLCostModule::myoption, 00062 // OptionBase::buildoption, 00063 // "Help text describing this option"); 00064 00065 // Now call the parent class' declareOptions 00066 inherited::declareOptions(ol); 00067 } 00068 00069 void NLLCostModule::build_() 00070 { 00071 } 00072 00073 // ### Nothing to add here, simply calls build_ 00074 void NLLCostModule::build() 00075 { 00076 inherited::build(); 00077 build_(); 00078 } 00079 00080 00081 void NLLCostModule::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00082 { 00083 inherited::makeDeepCopyFromShallowCopy(copies); 00084 } 00085 00086 00088 // fprop // 00090 void NLLCostModule::fprop(const Vec& input, const Vec& target, Vec& cost) const 00091 { 00092 PLASSERT( input.size() == input_size ); 00093 PLASSERT( target.size() == target_size ); 00094 cost.resize( output_size ); 00095 00096 if( input.hasMissing() ) 00097 cost[0] = MISSING_VALUE; 00098 else 00099 { 00100 PLASSERT_MSG( min(input) >= 0., 00101 "Elements of \"input\" should be positive" ); 00102 PLASSERT_MSG( is_equal( sum(input), 1. ), 00103 "Elements of \"input\" should sum to 1" ); 00104 00105 int the_target = (int) round( target[0] ); 00106 cost[0] = -pl_log( input[ the_target ] ); 00107 } 00108 } 00109 00110 void NLLCostModule::fprop(const Mat& inputs, const Mat& targets, Mat& costs) 00111 const 00112 { 00113 PLASSERT( inputs.width() == input_size ); 00114 PLASSERT( targets.width() == target_size ); 00115 00116 int batch_size = inputs.length(); 00117 PLASSERT( inputs.length() == batch_size ); 00118 PLASSERT( targets.length() == batch_size ); 00119 00120 costs.resize(batch_size, output_size); 00121 00122 for( int k=0; k<batch_size; k++ ) 00123 { 00124 int target_k = (int) round( targets(k, 0) ); 00125 costs(k, 0) = -pl_log( inputs(k, target_k) ); 00126 } 00127 } 00128 00129 void NLLCostModule::fprop(const TVec<Mat*>& ports_value) 00130 { 00131 PLASSERT( ports_value.length() == nPorts() ); 00132 00133 Mat* prediction = ports_value[0]; 00134 Mat* target = ports_value[1]; 00135 Mat* cost = ports_value[2]; 00136 00137 // If we have prediction and target, and we want cost 00138 if( prediction && !prediction->isEmpty() 00139 && target && !target->isEmpty() 00140 && cost && cost->isEmpty() ) 00141 00142 { 00143 PLASSERT( prediction->width() == port_sizes(0, 1) ); 00144 PLASSERT( target->width() == port_sizes(1, 1) ); 00145 00146 int batch_size = prediction->length(); 00147 PLASSERT( target->length() == batch_size ); 00148 00149 cost->resize(batch_size, port_sizes(2, 1)); 00150 00151 00152 for( int i=0; i<batch_size; i++ ) 00153 { 00154 if( (*prediction)(i).hasMissing() || is_missing((*target)(i,0)) ) 00155 (*cost)(i,0) = MISSING_VALUE; 00156 else 00157 { 00158 #ifdef BOUNDCHECK 00159 PLASSERT_MSG( min((*prediction)(i)) >= 0., 00160 "Elements of \"prediction\" should be positive" ); 00161 // Ensure the distribution probabilities sum to 1. We relax a 00162 // bit the default tolerance as probabilities using 00163 // exponentials could suffer numerical imprecisions. 00164 if (!is_equal( sum((*prediction)(i)), 1., 1., 1e-5, 1e-5 )) 00165 PLERROR("In NLLCostModule::fprop - Elements of" 00166 " \"prediction\" should sum to 1" 00167 " (found a sum = %f at row %d)", 00168 sum((*prediction)(i)), i); 00169 #endif 00170 int target_i = (int) round( (*target)(i,0) ); 00171 PLASSERT( is_equal( (*target)(i, 0), target_i ) ); 00172 (*cost)(i,0) = -pl_log( (*prediction)(i, target_i) ); 00173 } 00174 } 00175 } 00176 else if( !prediction && !target && !cost ) 00177 return; 00178 else 00179 PLCHECK_MSG( false, "Unknown port configuration" ); 00180 00181 checkProp(ports_value); 00182 } 00183 00185 // bpropUpdate // 00187 void NLLCostModule::bpropUpdate(const Vec& input, const Vec& target, real cost, 00188 Vec& input_gradient, bool accumulate) 00189 { 00190 PLASSERT( input.size() == input_size ); 00191 PLASSERT( target.size() == target_size ); 00192 00193 if( accumulate ) 00194 { 00195 PLASSERT_MSG( input_gradient.size() == input_size, 00196 "Cannot resize input_gradient AND accumulate into it" ); 00197 } 00198 else 00199 { 00200 input_gradient.resize( input_size ); 00201 input_gradient.clear(); 00202 } 00203 00204 int the_target = (int) round( target[0] ); 00205 // input_gradient[ i ] = 0 if i != t, 00206 // input_gradient[ t ] = -1/x[t] 00207 input_gradient[ the_target ] -= 1. / input[ the_target ]; 00208 } 00209 00210 void NLLCostModule::bpropUpdate(const Mat& inputs, const Mat& targets, 00211 const Vec& costs, Mat& input_gradients, bool accumulate) 00212 { 00213 PLASSERT( inputs.width() == input_size ); 00214 PLASSERT( targets.width() == target_size ); 00215 00216 if( accumulate ) 00217 { 00218 PLASSERT_MSG( input_gradients.width() == input_size && 00219 input_gradients.length() == inputs.length(), 00220 "Cannot resize input_gradients and accumulate into it" ); 00221 } 00222 else 00223 { 00224 input_gradients.resize(inputs.length(), input_size ); 00225 input_gradients.clear(); 00226 } 00227 00228 // input_gradient[ i ] = 0 if i != t, 00229 // input_gradient[ t ] = -1/x[t] 00230 for (int i = 0; i < inputs.length(); i++) { 00231 int the_target = (int) round( targets(i, 0) ); 00232 input_gradients(i, the_target) -= 1. / inputs(i, the_target); 00233 } 00234 } 00235 00236 void NLLCostModule::bpropAccUpdate(const TVec<Mat*>& ports_value, 00237 const TVec<Mat*>& ports_gradient) 00238 { 00239 PLASSERT( ports_value.length() == nPorts() ); 00240 PLASSERT( ports_gradient.length() == nPorts() ); 00241 00242 Mat* prediction = ports_value[0]; 00243 Mat* target = ports_value[1]; 00244 #ifndef NDEBUG 00245 Mat* cost = ports_value[2]; 00246 #endif 00247 Mat* prediction_grad = ports_gradient[0]; 00248 Mat* target_grad = ports_gradient[1]; 00249 Mat* cost_grad = ports_gradient[2]; 00250 00251 // If we have cost_grad and we want prediction_grad 00252 if( prediction_grad && prediction_grad->isEmpty() 00253 && cost_grad && !cost_grad->isEmpty() ) 00254 { 00255 PLASSERT( prediction ); 00256 PLASSERT( target ); 00257 PLASSERT( cost ); 00258 PLASSERT( !target_grad ); 00259 00260 PLASSERT( prediction->width() == port_sizes(0,1) ); 00261 PLASSERT( target->width() == port_sizes(1,1) ); 00262 PLASSERT( cost->width() == port_sizes(2,1) ); 00263 PLASSERT( prediction_grad->width() == port_sizes(0,1) ); 00264 PLASSERT( cost_grad->width() == port_sizes(2,1) ); 00265 PLASSERT( cost_grad->width() == 1 ); 00266 00267 int batch_size = prediction->length(); 00268 PLASSERT( target->length() == batch_size ); 00269 PLASSERT( cost->length() == batch_size ); 00270 PLASSERT( cost_grad->length() == batch_size ); 00271 00272 prediction_grad->resize(batch_size, port_sizes(0,1)); 00273 00274 for( int k=0; k<batch_size; k++ ) 00275 { 00276 // input_gradient[ i ] = 0 if i != t, 00277 // input_gradient[ t ] = -1/x[t] 00278 int target_k = (int) round((*target)(k, 0)); 00279 (*prediction_grad)(k, target_k) -= 00280 (*cost_grad)(k, 0) / (*prediction)(k, target_k); 00281 } 00282 } 00283 else if( !prediction_grad && !target_grad && !cost_grad ) 00284 return; 00285 else if( !cost_grad && prediction_grad && prediction_grad->isEmpty() ) 00286 PLERROR("In NLLCostModule::bpropAccUpdate - cost gradient is NULL,\n" 00287 "cannot compute prediction gradient. Maybe you should set\n" 00288 "\"propagate_gradient = 0\" on the incoming connection.\n"); 00289 else 00290 PLERROR("In OnlineLearningModule::bpropAccUpdate - Port configuration " 00291 "not implemented for class '%s'", classname().c_str()); 00292 00293 checkProp(ports_value); 00294 checkProp(ports_gradient); 00295 } 00296 00297 void NLLCostModule::bbpropUpdate(const Vec& input, const Vec& target, 00298 real cost, 00299 Vec& input_gradient, Vec& input_diag_hessian, 00300 bool accumulate) 00301 { 00302 if( accumulate ) 00303 { 00304 PLASSERT_MSG( input_diag_hessian.size() == input_size, 00305 "Cannot resize input_diag_hessian AND accumulate into it" 00306 ); 00307 } 00308 else 00309 { 00310 input_diag_hessian.resize( input_size ); 00311 input_diag_hessian.clear(); 00312 } 00313 00314 // input_diag_hessian[ i ] = 0 if i!=t 00315 // input_diag_hessian[ t ] = 1/(x[t])^2 00316 int the_target = (int) round( target[0] ); 00317 real input_t = input[ the_target ]; 00318 input_diag_hessian[ the_target ] += 1. / (input_t * input_t); 00319 00320 bpropUpdate( input, target, cost, input_gradient, accumulate ); 00321 } 00322 00323 TVec<string> NLLCostModule::costNames() 00324 { 00325 if (name == "" || name == classname()) 00326 return TVec<string>(1, "NLL"); 00327 else 00328 return TVec<string>(1, name + ".NLL"); 00329 } 00330 00331 } // end of namespace PLearn 00332 00333 00334 /* 00335 Local Variables: 00336 mode:c++ 00337 c-basic-offset:4 00338 c-file-style:"stroustrup" 00339 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00340 indent-tabs-mode:nil 00341 fill-column:79 00342 End: 00343 */ 00344 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :