PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // CombiningCostsModule.cc 00004 // 00005 // Copyright (C) 2006 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin 00036 00041 #include "CombiningCostsModule.h" 00042 #include <plearn/math/TMat_maths.h> 00043 00044 namespace PLearn { 00045 using namespace std; 00046 00047 PLEARN_IMPLEMENT_OBJECT( 00048 CombiningCostsModule, 00049 "Combine several CostModules with the same input and target", 00050 "It is possible to assign a weight on each of the sub_costs, so the\n" 00051 "back-propagated gradient will be a weighted sum of the modules'" 00052 " gradients.\n" 00053 "The first output is the weighted sum of the cost, the following ones\n" 00054 "are the original costs.\n" 00055 ); 00056 00058 // CombiningCostsModule // 00060 CombiningCostsModule::CombiningCostsModule() : 00061 n_sub_costs( 0 ) 00062 { 00063 } 00064 00066 // declareOptions // 00068 void CombiningCostsModule::declareOptions(OptionList& ol) 00069 { 00070 declareOption(ol, "sub_costs", &CombiningCostsModule::sub_costs, 00071 OptionBase::buildoption, 00072 "Vector containing the different sub_costs"); 00073 00074 declareOption(ol, "cost_weights", &CombiningCostsModule::cost_weights, 00075 OptionBase::buildoption, 00076 "The weights associated to each of the sub_costs"); 00077 00078 declareOption(ol, "n_sub_costs", &CombiningCostsModule::n_sub_costs, 00079 OptionBase::learntoption, 00080 "Number of sub_costs"); 00081 00082 // Now call the parent class' declareOptions 00083 inherited::declareOptions(ol); 00084 00085 redeclareOption(ol, "input_size", &CombiningCostsModule::input_size, 00086 OptionBase::learntoption, 00087 "Is set to sub_costs[0]->input_size."); 00088 redeclareOption(ol, "target_size", &CombiningCostsModule::target_size, 00089 OptionBase::learntoption, 00090 "Is set to sub_costs[0]->target_size."); 00091 } 00092 00094 // build_ // 00096 void CombiningCostsModule::build_() 00097 { 00098 n_sub_costs = sub_costs.length(); 00099 if( n_sub_costs == 0 ) 00100 { 00101 //PLWARNING("In CombiningCostsModule::build_ - sub_costs is empty (length 0)"); 00102 return; 00103 } 00104 00105 // Default value: sub_cost[0] has weight 1, the other ones have weight 0. 00106 if( cost_weights.length() == 0 ) 00107 { 00108 cost_weights.resize( n_sub_costs ); 00109 cost_weights.clear(); 00110 cost_weights[0] = 1; 00111 } 00112 00113 if( cost_weights.length() != n_sub_costs ) 00114 PLERROR( "CombiningCostsModule::build_(): cost_weights.length()\n" 00115 "should be equal to n_sub_costs (%d != %d).\n", 00116 cost_weights.length(), n_sub_costs ); 00117 00118 if(sub_costs.length() == 0) 00119 PLERROR( "CombiningCostsModule::build_(): sub_costs.length()\n" 00120 "should be > 0.\n"); 00121 00122 input_size = sub_costs[0]->input_size; 00123 target_size = sub_costs[0]->target_size; 00124 for(int i=1; i<sub_costs.length(); i++) 00125 { 00126 if(sub_costs[i]->input_size != input_size) 00127 PLERROR( "CombiningCostsModule::build_(): sub_costs[%d]->input_size" 00128 " (%d)\n" 00129 "should be equal to %d.\n", 00130 i,sub_costs[i]->input_size, input_size); 00131 00132 if(sub_costs[i]->target_size != target_size) 00133 PLERROR( "CombiningCostsModule::build_(): sub_costs[%d]->target_size" 00134 " (%d)\n" 00135 "should be equal to %d.\n", 00136 i,sub_costs[i]->target_size, target_size); 00137 } 00138 00139 sub_costs_values.resize( n_sub_costs ); 00140 output_size = 1; 00141 for (int i=0; i<n_sub_costs; i++) 00142 output_size += sub_costs[i]->output_size; 00143 00144 // If we have a random_gen and some sub_costs do not, share it with them 00145 if( random_gen ) 00146 for( int i=0; i<n_sub_costs; i++ ) 00147 { 00148 if( !(sub_costs[i]->random_gen) ) 00149 { 00150 sub_costs[i]->random_gen = random_gen; 00151 sub_costs[i]->forget(); 00152 } 00153 } 00154 } 00155 00157 // build // 00159 void CombiningCostsModule::build() 00160 { 00161 inherited::build(); 00162 build_(); 00163 } 00164 00165 00167 // makeDeepCopyFromShallowCopy // 00169 void CombiningCostsModule::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00170 { 00171 inherited::makeDeepCopyFromShallowCopy(copies); 00172 00173 deepCopyField(sub_costs, copies); 00174 deepCopyField(cost_weights, copies); 00175 deepCopyField(sub_costs_values, copies); 00176 deepCopyField(sub_costs_mbatch_values, copies); 00177 deepCopyField(partial_gradient, copies); 00178 deepCopyField(partial_gradients, copies); 00179 deepCopyField(partial_diag_hessian, copies); 00180 } 00181 00182 00184 // fprop // 00186 void CombiningCostsModule::fprop(const Vec& input, const Vec& target, 00187 Vec& cost) const 00188 { 00189 PLASSERT( input.size() == input_size ); 00190 PLASSERT( target.size() == target_size ); 00191 cost.resize( output_size ); 00192 00193 int cost_index = 1; 00194 for( int i=0 ; i<n_sub_costs ; i++ ) 00195 { 00196 Vec sub_costs_val_i_all = cost.subVec(cost_index, 00197 sub_costs[i]->output_size); 00198 sub_costs[i]->fprop(input, target, sub_costs_val_i_all); 00199 sub_costs_values[i] = cost[cost_index]; 00200 cost_index += sub_costs[i]->output_size; 00201 } 00202 00203 cost[0] = dot( cost_weights, sub_costs_values ); 00204 } 00205 00206 void CombiningCostsModule::fprop(const Mat& inputs, const Mat& targets, 00207 Mat& costs) const 00208 { 00209 PLASSERT( inputs.width() == input_size ); 00210 PLASSERT( targets.width() == target_size ); 00211 costs.resize(inputs.length(), output_size); 00212 sub_costs_mbatch_values.resize(n_sub_costs, inputs.length()); 00213 00214 int cost_index = 1; 00215 for (int i=0; i<n_sub_costs; i++) 00216 { 00217 Mat sub_costs_val_i_all = 00218 costs.subMatColumns(cost_index, sub_costs[i]->output_size); 00219 sub_costs[i]->fprop(inputs, targets, sub_costs_val_i_all); 00220 sub_costs_mbatch_values(i) << costs.column(cost_index); 00221 cost_index += sub_costs[i]->output_size; 00222 } 00223 00224 // final_cost = \sum weight_i * cost_i 00225 Mat final_cost = costs.column(0); 00226 Mat m_cost_weights = cost_weights.toMat(n_sub_costs, 1); 00227 transposeProduct(final_cost, sub_costs_mbatch_values, m_cost_weights); 00228 } 00229 00231 // bpropAccUpdate // 00233 void CombiningCostsModule::bpropAccUpdate(const TVec<Mat*>& ports_value, 00234 const TVec<Mat*>& ports_gradient) 00235 { 00236 Mat* inputs = ports_value[0]; 00237 Mat* targets = ports_value[1]; 00238 Mat* costs = ports_value[2]; 00239 PLASSERT( costs && costs->width() == n_sub_costs + 1 ); 00240 Mat* input_gradients = ports_gradient[0]; 00241 PLASSERT( input_gradients && input_gradients->isEmpty() && 00242 input_gradients->width() > 0 ); 00243 input_gradients->resize(inputs->length(), input_gradients->width()); 00244 sub_costs_values.resize(costs->length()); 00245 for( int i=0 ; i<n_sub_costs ; i++ ) 00246 { 00247 sub_costs_values << costs->column(i + 1); 00248 if (fast_exact_is_equal(cost_weights[i], 0)) 00249 { 00250 // Do not compute input_gradients. 00251 sub_costs[i]->bpropUpdate( *inputs, *targets, sub_costs_values); 00252 } 00253 else if (fast_exact_is_equal(cost_weights[i], 1)) 00254 { 00255 // Accumulate directly into input_gradients. 00256 sub_costs[i]->bpropUpdate(*inputs, *targets, sub_costs_values, 00257 *input_gradients, true); 00258 } 00259 else 00260 { 00261 // Put the result into partial_gradients, then accumulate into 00262 // input_gradients with the appropriate weight. 00263 sub_costs[i]->bpropUpdate(*inputs, *targets, sub_costs_values, 00264 partial_gradients, false); 00265 multiplyAcc(*input_gradients, partial_gradients, cost_weights[i]); 00266 } 00267 } 00268 } 00269 00271 // bpropUpdate // 00273 void CombiningCostsModule::bpropUpdate(const Vec& input, const Vec& target, 00274 real cost, Vec& input_gradient, 00275 bool accumulate) 00276 { 00277 PLASSERT( input.size() == input_size ); 00278 PLASSERT( target.size() == target_size ); 00279 00280 if( accumulate ) 00281 { 00282 PLASSERT_MSG( input_gradient.size() == input_size, 00283 "Cannot resize input_gradient AND accumulate into it" ); 00284 } 00285 else 00286 { 00287 input_gradient.resize( input_size ); 00288 input_gradient.clear(); 00289 } 00290 00291 for( int i=0 ; i<n_sub_costs ; i++ ) 00292 { 00293 if( cost_weights[i] == 0. ) 00294 { 00295 // Don't compute input_gradient 00296 sub_costs[i]->bpropUpdate( input, target, sub_costs_values[i] ); 00297 } 00298 else if( cost_weights[i] == 1. ) 00299 { 00300 // Accumulate directly into input_gradient 00301 sub_costs[i]->bpropUpdate( input, target, sub_costs_values[i], 00302 input_gradient, true ); 00303 } 00304 else 00305 { 00306 // Put the result into partial_gradient, then accumulate into 00307 // input_gradient with the appropriate weight 00308 sub_costs[i]->bpropUpdate( input, target, sub_costs_values[i], 00309 partial_gradient, false ); 00310 multiplyAcc( input_gradient, partial_gradient, cost_weights[i] ); 00311 } 00312 } 00313 } 00314 00315 void CombiningCostsModule::bpropUpdate(const Mat& inputs, const Mat& targets, 00316 const Vec& costs, Mat& input_gradients, bool accumulate) 00317 { 00318 PLASSERT( inputs.width() == input_size ); 00319 PLASSERT( targets.width() == target_size ); 00320 00321 if( accumulate ) 00322 { 00323 PLASSERT_MSG( input_gradients.width() == input_size && 00324 input_gradients.length() == inputs.length(), 00325 "Cannot resize input_gradients and accumulate into it" ); 00326 } 00327 else 00328 { 00329 input_gradients.resize(inputs.length(), input_size ); 00330 input_gradients.clear(); 00331 } 00332 00333 00334 Vec sub; 00335 for( int i=0 ; i<n_sub_costs ; i++ ) 00336 { 00337 sub = sub_costs_mbatch_values(i); 00338 if( cost_weights[i] == 0. ) 00339 { 00340 // Do not compute input_gradients. 00341 sub_costs[i]->bpropUpdate( inputs, targets, sub ); 00342 } 00343 else if( cost_weights[i] == 1. ) 00344 { 00345 // Accumulate directly into input_gradients. 00346 00347 sub_costs[i]->bpropUpdate( inputs, targets, sub, input_gradients, 00348 true ); 00349 } 00350 else 00351 { 00352 // Put the result into partial_gradients, then accumulate into 00353 // input_gradients with the appropriate weight. 00354 sub_costs[i]->bpropUpdate( inputs, targets, sub, partial_gradients, 00355 false); 00356 multiplyAcc( input_gradients, partial_gradients, cost_weights[i] ); 00357 } 00358 } 00359 } 00360 00361 00362 void CombiningCostsModule::bpropUpdate(const Vec& input, const Vec& target, 00363 real cost) 00364 { 00365 PLASSERT( input.size() == input_size ); 00366 PLASSERT( target.size() == target_size ); 00367 00368 for( int i=0 ; i<n_sub_costs ; i++ ) 00369 sub_costs[i]->bpropUpdate( input, target, sub_costs_values[i] ); 00370 } 00371 00373 // bbpropUpdate // 00375 void CombiningCostsModule::bbpropUpdate(const Vec& input, const Vec& target, 00376 real cost, 00377 Vec& input_gradient, 00378 Vec& input_diag_hessian, 00379 bool accumulate) 00380 { 00381 PLASSERT( input.size() == input_size ); 00382 PLASSERT( target.size() == target_size ); 00383 00384 if( accumulate ) 00385 { 00386 PLASSERT_MSG( input_gradient.size() == input_size, 00387 "Cannot resize input_gradient AND accumulate into it" ); 00388 PLASSERT_MSG( input_diag_hessian.size() == input_size, 00389 "Cannot resize input_diag_hessian AND accumulate into it" 00390 ); 00391 } 00392 else 00393 { 00394 input_gradient.resize( input_size ); 00395 input_gradient.clear(); 00396 input_diag_hessian.resize( input_size ); 00397 input_diag_hessian.clear(); 00398 } 00399 00400 for( int i=0 ; i<n_sub_costs ; i++ ) 00401 { 00402 if( cost_weights[i] == 0. ) 00403 { 00404 // Don't compute input_gradient nor input_diag_hessian 00405 sub_costs[i]->bbpropUpdate( input, target, sub_costs_values[i] ); 00406 } 00407 else if( cost_weights[i] == 1. ) 00408 { 00409 // Accumulate directly into input_gradient and input_diag_hessian 00410 sub_costs[i]->bbpropUpdate( input, target, sub_costs_values[i], 00411 input_gradient, input_diag_hessian, 00412 true ); 00413 } 00414 else 00415 { 00416 // Put temporary results into partial_*, then multiply and add to 00417 // input_* 00418 sub_costs[i]->bbpropUpdate( input, target, sub_costs_values[i], 00419 partial_gradient, partial_diag_hessian, 00420 false ); 00421 multiplyAcc( input_gradient, partial_gradient, cost_weights[i] ); 00422 multiplyAcc( input_diag_hessian, partial_diag_hessian, 00423 cost_weights[i] ); 00424 } 00425 } 00426 } 00427 00428 void CombiningCostsModule::bbpropUpdate(const Vec& input, const Vec& target, 00429 real cost) 00430 { 00431 PLASSERT( input.size() == input_size ); 00432 PLASSERT( target.size() == target_size ); 00433 00434 for( int i=0 ; i<n_sub_costs ; i++ ) 00435 sub_costs[i]->bbpropUpdate( input, target, sub_costs_values[i] ); 00436 } 00437 00438 00440 // forget // 00442 void CombiningCostsModule::forget() 00443 { 00444 if( !random_gen ) 00445 { 00446 PLWARNING("CombiningCostsModule: cannot forget() without random_gen"); 00447 return; 00448 } 00449 for( int i=0 ; i<n_sub_costs ; i++ ) 00450 { 00451 // Ensure sub_costs[i] can forget 00452 if( !(sub_costs[i]->random_gen) ) 00453 sub_costs[i]->random_gen = random_gen; 00454 sub_costs[i]->forget(); 00455 } 00456 } 00457 00459 // setLearningRate // 00462 void CombiningCostsModule::setLearningRate(real dynamic_learning_rate) 00463 { 00464 for( int i=0 ; i<n_sub_costs ; i++ ) 00465 sub_costs[i]->setLearningRate(dynamic_learning_rate); 00466 } 00467 00469 // finalize // 00473 void CombiningCostsModule::finalize() 00474 { 00475 for( int i=0 ; i<n_sub_costs ; i++ ) 00476 sub_costs[i]->finalize(); 00477 } 00478 00480 bool CombiningCostsModule::bpropDoesNothing() 00481 { 00482 for( int i=0 ; i<n_sub_costs ; i++ ) 00483 if( !(sub_costs[i]->bpropDoesNothing()) ) 00484 return false; 00485 00486 return true; 00487 } 00488 00490 TVec<string> CombiningCostsModule::costNames() 00491 { 00492 TVec<string> names(1, "combined_cost"); 00493 for( int i=0 ; i<n_sub_costs ; i++ ) 00494 names.append( sub_costs[i]->costNames() ); 00495 00496 if (name != "" && name != classname()) 00497 for (int j=0; j<names.length(); j++) 00498 names[j] = name + "." + names[j]; 00499 00500 return names; 00501 } 00502 00503 00504 } // end of namespace PLearn 00505 00506 00507 /* 00508 Local Variables: 00509 mode:c++ 00510 c-basic-offset:4 00511 c-file-style:"stroustrup" 00512 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00513 indent-tabs-mode:nil 00514 fill-column:79 00515 End: 00516 */ 00517 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :