PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // RBMLateralBinomialLayer.cc 00004 // 00005 // Copyright (C) 2006 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Hugo Larochelle 00036 00041 #include "RBMLateralBinomialLayer.h" 00042 #include <plearn/math/TMat_maths.h> 00043 #include "RBMConnection.h" 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00048 PLEARN_IMPLEMENT_OBJECT( 00049 RBMLateralBinomialLayer, 00050 "Layer in an RBM formed with binomial units and lateral connections.", 00051 ""); 00052 00053 RBMLateralBinomialLayer::RBMLateralBinomialLayer( real the_learning_rate ) : 00054 inherited( the_learning_rate ), 00055 n_lateral_connections_passes( 1 ), 00056 dampening_factor( 0. ), 00057 mean_field_precision_threshold( 0. ), 00058 topographic_length( -1 ), 00059 topographic_width( -1 ), 00060 topographic_patch_vradius( 5 ), 00061 topographic_patch_hradius( 5 ), 00062 topographic_lateral_weights_init_value( 0. ), 00063 do_not_learn_topographic_lateral_weights( false ), 00064 use_parametric_mean_field( false ) 00065 { 00066 } 00067 00068 void RBMLateralBinomialLayer::reset() 00069 { 00070 inherited::reset(); 00071 lateral_weights_inc.clear(); 00072 } 00073 00074 void RBMLateralBinomialLayer::clearStats() 00075 { 00076 inherited::clearStats(); 00077 lateral_weights_pos_stats.clear(); 00078 lateral_weights_neg_stats.clear(); 00079 } 00080 00081 void RBMLateralBinomialLayer::forget() 00082 { 00083 inherited::forget(); 00084 //real bu; 00085 //for( int i=0; i<lateral_weights.length(); i++) 00086 // for( int j=0; j<lateral_weights.width(); j++) 00087 // { 00088 // bu = random_gen->bounded_uniform(-1.0/size,1.0/size); 00089 // lateral_weights(i,j) = bu; 00090 // lateral_weights(j,i) = bu; 00091 // } 00092 lateral_weights.clear(); 00093 // Set diagonal to 0 00094 if( lateral_weights.length() != 0 ) 00095 { 00096 real *d = lateral_weights.data(); 00097 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 00098 *d = 0; 00099 } 00100 00101 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 00102 //topographic_lateral_weights[i].clear(); 00103 topographic_lateral_weights[i].fill( topographic_lateral_weights_init_value ); 00104 00105 mean_field_output_weights.clear(); 00106 for( int i=0; i<mean_field_output_weights.length(); i++ ) 00107 mean_field_output_weights(i,i) = 1; 00108 for( int i=0; i<mean_field_output_bias.length(); i++ ) 00109 mean_field_output_bias[i] = -0.5; 00110 00111 } 00112 00114 // generateSample // 00116 void RBMLateralBinomialLayer::generateSample() 00117 { 00118 PLASSERT_MSG(random_gen, 00119 "random_gen should be initialized before generating samples"); 00120 00121 PLCHECK_MSG(expectation_is_up_to_date, "Expectation should be computed " 00122 "before calling generateSample()"); 00123 00124 for( int i=0 ; i<size ; i++ ) 00125 sample[i] = random_gen->binomial_sample( expectation[i] ); 00126 } 00127 00129 // generateSamples // 00131 void RBMLateralBinomialLayer::generateSamples() 00132 { 00133 PLASSERT_MSG(random_gen, 00134 "random_gen should be initialized before generating samples"); 00135 00136 PLCHECK_MSG(expectations_are_up_to_date, "Expectations should be computed " 00137 "before calling generateSamples()"); 00138 00139 PLASSERT( samples.width() == size && samples.length() == batch_size ); 00140 00141 for (int k = 0; k < batch_size; k++) { 00142 for (int i=0 ; i<size ; i++) 00143 samples(k, i) = random_gen->binomial_sample( expectations(k, i) ); 00144 } 00145 } 00146 00148 // computeExpectation // 00150 void RBMLateralBinomialLayer::computeExpectation() 00151 { 00152 if( expectation_is_up_to_date ) 00153 return; 00154 00155 if( use_parametric_mean_field ) 00156 { 00157 if (use_fast_approximations) 00158 for( int i=0 ; i<size ; i++ ) 00159 mean_field_input[i] = fastsigmoid( activation[i] ); 00160 else 00161 for( int i=0 ; i<size ; i++ ) 00162 mean_field_input[i] = sigmoid( activation[i] ); 00163 00164 product(pre_sigmoid_mean_field_output, mean_field_output_weights, mean_field_input); 00165 pre_sigmoid_mean_field_output += mean_field_output_bias; 00166 00167 if (use_fast_approximations) 00168 for( int i=0 ; i<size ; i++ ) 00169 expectation[i] = fastsigmoid( pre_sigmoid_mean_field_output[i] ); 00170 else 00171 for( int i=0 ; i<size ; i++ ) 00172 expectation[i] = sigmoid( pre_sigmoid_mean_field_output[i] ); 00173 00174 // Update mean-field predictor, using KL-divergence gradient: 00175 // dKL/dp_i = -activation[i] - \sum_{j \neq i} p_j + V_i h 00176 // where - V_i is the ith row of mean_field_output_weights 00177 // - h is sigmoid(activation) 00178 00179 real mean_field_i; 00180 product(temp_mean_field_gradient, lateral_weights, expectation); 00181 temp_mean_field_gradient += activation; 00182 for( int i=0 ; i<size ; i++ ) 00183 { 00184 mean_field_i = expectation[i]; 00185 temp_mean_field_gradient[i] = (pre_sigmoid_mean_field_output[i] 00186 - temp_mean_field_gradient[i]) 00187 * mean_field_i * (1 - mean_field_i); 00188 } 00189 00190 externalProductScaleAcc( mean_field_output_weights, temp_mean_field_gradient, 00191 mean_field_input, -learning_rate ); 00192 multiplyScaledAdd( temp_mean_field_gradient, 1.0, -learning_rate, mean_field_output_bias); 00193 } 00194 else 00195 { 00196 if( temp_output.length() != n_lateral_connections_passes+1 ) 00197 { 00198 temp_output.resize(n_lateral_connections_passes+1); 00199 for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ ) 00200 temp_output[i].resize(size); 00201 } 00202 00203 current_temp_output = temp_output[0]; 00204 temp_output.last() = expectation; 00205 00206 if (use_fast_approximations) 00207 for( int i=0 ; i<size ; i++ ) 00208 current_temp_output[i] = fastsigmoid( activation[i] ); 00209 else 00210 for( int i=0 ; i<size ; i++ ) 00211 current_temp_output[i] = sigmoid( activation[i] ); 00212 00213 for( int t=0; t<n_lateral_connections_passes; t++ ) 00214 { 00215 previous_temp_output = current_temp_output; 00216 current_temp_output = temp_output[t+1]; 00217 if( topographic_lateral_weights.length() == 0 ) 00218 product(dampening_expectation, lateral_weights, previous_temp_output); 00219 else 00220 productTopoLateralWeights( dampening_expectation, previous_temp_output ); 00221 dampening_expectation += activation; 00222 if (use_fast_approximations) 00223 { 00224 if( fast_exact_is_equal( dampening_factor, 0) ) 00225 { 00226 for( int i=0 ; i<size ; i++ ) 00227 current_temp_output[i] = fastsigmoid( dampening_expectation[i] ); 00228 } 00229 else 00230 { 00231 for( int i=0 ; i<size ; i++ ) 00232 current_temp_output[i] = 00233 (1-dampening_factor) * fastsigmoid( dampening_expectation[i] ) 00234 + dampening_factor * previous_temp_output[i]; 00235 } 00236 } 00237 else 00238 { 00239 if( fast_exact_is_equal( dampening_factor, 0) ) 00240 { 00241 for( int i=0 ; i<size ; i++ ) 00242 current_temp_output[i] = sigmoid( dampening_expectation[i] ); 00243 } 00244 else 00245 { 00246 for( int i=0 ; i<size ; i++ ) 00247 current_temp_output[i] = 00248 (1-dampening_factor) * sigmoid( dampening_expectation[i] ) 00249 + dampening_factor * previous_temp_output[i]; 00250 } 00251 } 00252 if( !fast_exact_is_equal(mean_field_precision_threshold, 0.) && 00253 dist(current_temp_output, previous_temp_output,2)/size < mean_field_precision_threshold ) 00254 { 00255 expectation << current_temp_output; 00256 break; 00257 } 00258 //cout << sqrt(max(square(current_temp_output-previous_temp_output))) << " "; 00259 //cout << dist(current_temp_output, previous_temp_output,2)/current_temp_output.length() << " "; 00260 } 00261 //cout << endl; 00262 //expectation << current_temp_output; 00263 } 00264 expectation_is_up_to_date = true; 00265 } 00266 00268 // computeExpectations // 00270 void RBMLateralBinomialLayer::computeExpectations() 00271 { 00272 if( expectations_are_up_to_date ) 00273 return; 00274 00275 PLASSERT( expectations.width() == size 00276 && expectations.length() == batch_size ); 00277 00278 if( use_parametric_mean_field ) 00279 { 00280 PLERROR("RBMLateralBinomialLayer::computeExpectations(): use_parametric_mean_field=true " 00281 "not implemented yet."); 00282 } 00283 else 00284 { 00285 dampening_expectations.resize( batch_size, size ); 00286 00287 if( temp_outputs.length() != n_lateral_connections_passes+1 ) 00288 { 00289 temp_outputs.resize(n_lateral_connections_passes+1); 00290 for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ ) 00291 temp_outputs[i].resize( batch_size, size); 00292 } 00293 00294 current_temp_outputs = temp_outputs[0]; 00295 temp_outputs.last() = expectations; 00296 00297 if (use_fast_approximations) 00298 for (int k = 0; k < batch_size; k++) 00299 for (int i = 0 ; i < size ; i++) 00300 current_temp_outputs(k, i) = fastsigmoid(activations(k, i)); 00301 else 00302 for (int k = 0; k < batch_size; k++) 00303 for (int i = 0 ; i < size ; i++) 00304 current_temp_outputs(k, i) = sigmoid(activations(k, i)); 00305 00306 for( int t=0; t<n_lateral_connections_passes; t++ ) 00307 { 00308 previous_temp_outputs = current_temp_outputs; 00309 current_temp_outputs = temp_outputs[t+1]; 00310 if( topographic_lateral_weights.length() == 0 ) 00311 productTranspose(dampening_expectations, previous_temp_outputs, 00312 lateral_weights); 00313 else 00314 for( int b = 0; b<dampening_expectations.length(); b++) 00315 productTopoLateralWeights( dampening_expectations(b), 00316 previous_temp_outputs(b) ); 00317 00318 dampening_expectations += activations; 00319 if (use_fast_approximations) 00320 { 00321 if( fast_exact_is_equal( dampening_factor, 0) ) 00322 { 00323 for(int k = 0; k < batch_size; k++) 00324 for( int i=0 ; i<size ; i++ ) 00325 current_temp_outputs(k, i) = 00326 fastsigmoid( dampening_expectations(k, i) ); 00327 } 00328 else 00329 { 00330 for(int k = 0; k < batch_size; k++) 00331 for( int i=0 ; i<size ; i++ ) 00332 current_temp_outputs(k, i) = (1-dampening_factor) 00333 * fastsigmoid( dampening_expectations(k, i) ) 00334 + dampening_factor * previous_temp_outputs(k, i); 00335 } 00336 } 00337 else 00338 { 00339 if( fast_exact_is_equal( dampening_factor, 0) ) 00340 { 00341 for(int k = 0; k < batch_size; k++) 00342 for( int i=0 ; i<size ; i++ ) 00343 current_temp_outputs(k, i) = 00344 sigmoid( dampening_expectations(k, i) ); 00345 } 00346 else 00347 { 00348 for(int k = 0; k < batch_size; k++) 00349 for( int i=0 ; i<size ; i++ ) 00350 current_temp_outputs(k, i) = (1-dampening_factor) 00351 * sigmoid( dampening_expectations(k, i) ) 00352 + dampening_factor * previous_temp_outputs(k, i); 00353 } 00354 } 00355 } 00356 //expectations << current_temp_outputs; 00357 } 00358 expectations_are_up_to_date = true; 00359 } 00360 00362 // fprop // 00364 void RBMLateralBinomialLayer::fprop( const Vec& input, Vec& output ) const 00365 { 00366 PLASSERT( input.size() == input_size ); 00367 output.resize( output_size ); 00368 00369 add(bias, input, bias_plus_input); 00370 00371 if( use_parametric_mean_field ) 00372 { 00373 if (use_fast_approximations) 00374 for( int i=0 ; i<size ; i++ ) 00375 mean_field_input[i] = fastsigmoid( bias_plus_input[i] ); 00376 else 00377 for( int i=0 ; i<size ; i++ ) 00378 mean_field_input[i] = sigmoid( bias_plus_input[i] ); 00379 00380 product(pre_sigmoid_mean_field_output, mean_field_output_weights, mean_field_input); 00381 pre_sigmoid_mean_field_output += mean_field_output_bias; 00382 00383 if (use_fast_approximations) 00384 for( int i=0 ; i<size ; i++ ) 00385 output[i] = fastsigmoid( pre_sigmoid_mean_field_output[i] ); 00386 else 00387 for( int i=0 ; i<size ; i++ ) 00388 output[i] = sigmoid( pre_sigmoid_mean_field_output[i] ); 00389 } 00390 else 00391 { 00392 00393 if( temp_output.length() != n_lateral_connections_passes+1 ) 00394 { 00395 temp_output.resize(n_lateral_connections_passes+1); 00396 for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ ) 00397 temp_output[i].resize(size); 00398 } 00399 00400 temp_output.last() = output; 00401 current_temp_output = temp_output[0]; 00402 00403 if (use_fast_approximations) 00404 for( int i=0 ; i<size ; i++ ) 00405 current_temp_output[i] = fastsigmoid( bias_plus_input[i] ); 00406 else 00407 for( int i=0 ; i<size ; i++ ) 00408 current_temp_output[i] = sigmoid( bias_plus_input[i] ); 00409 00410 for( int t=0; t<n_lateral_connections_passes; t++ ) 00411 { 00412 previous_temp_output = current_temp_output; 00413 current_temp_output = temp_output[t+1]; 00414 if( topographic_lateral_weights.length() == 0 ) 00415 product(dampening_expectation, lateral_weights, previous_temp_output); 00416 else 00417 productTopoLateralWeights( dampening_expectation, previous_temp_output ); 00418 dampening_expectation += bias_plus_input; 00419 if (use_fast_approximations) 00420 { 00421 if( fast_exact_is_equal( dampening_factor, 0) ) 00422 { 00423 for( int i=0 ; i<size ; i++ ) 00424 current_temp_output[i] = fastsigmoid( dampening_expectation[i] ); 00425 } 00426 else 00427 { 00428 for( int i=0 ; i<size ; i++ ) 00429 current_temp_output[i] = 00430 (1-dampening_factor) * fastsigmoid( dampening_expectation[i] ) 00431 + dampening_factor * previous_temp_output[i]; 00432 } 00433 } 00434 else 00435 { 00436 if( fast_exact_is_equal( dampening_factor, 0) ) 00437 { 00438 for( int i=0 ; i<size ; i++ ) 00439 current_temp_output[i] = sigmoid( dampening_expectation[i] ); 00440 } 00441 else 00442 { 00443 for( int i=0 ; i<size ; i++ ) 00444 current_temp_output[i] = 00445 (1-dampening_factor) * sigmoid( dampening_expectation[i] ) 00446 + dampening_factor * previous_temp_output[i]; 00447 } 00448 } 00449 } 00450 } 00451 } 00452 00453 void RBMLateralBinomialLayer::fprop( const Mat& inputs, Mat& outputs ) 00454 { 00455 int mbatch_size = inputs.length(); 00456 PLASSERT( inputs.width() == size ); 00457 outputs.resize( mbatch_size, size ); 00458 00459 dampening_expectations.resize( mbatch_size, size ); 00460 00461 if( use_parametric_mean_field ) 00462 { 00463 PLERROR("RBMLateralBinomialLayer::fprop: use_parametric_mean_field = true " 00464 "not implemented yet for batch mode."); 00465 } 00466 else 00467 { 00468 if(bias_plus_inputs.length() != inputs.length() || 00469 bias_plus_inputs.width() != inputs.width()) 00470 bias_plus_inputs.resize(inputs.length(), inputs.width()); 00471 bias_plus_inputs << inputs; 00472 bias_plus_inputs += bias; 00473 00474 if( temp_outputs.length() != n_lateral_connections_passes+1 ) 00475 { 00476 temp_outputs.resize(n_lateral_connections_passes+1); 00477 for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ ) 00478 temp_outputs[i].resize(mbatch_size,size); 00479 } 00480 00481 temp_outputs.last() = outputs; 00482 current_temp_outputs = temp_outputs[0]; 00483 00484 if (use_fast_approximations) 00485 for( int k = 0; k < mbatch_size; k++ ) 00486 for( int i = 0; i < size; i++ ) 00487 current_temp_outputs(k,i) = fastsigmoid( bias_plus_inputs(k,i) ); 00488 else 00489 for( int k = 0; k < mbatch_size; k++ ) 00490 for( int i = 0; i < size; i++ ) 00491 current_temp_outputs(k,i) = sigmoid( bias_plus_inputs(k,i) ); 00492 00493 for( int t=0; t<n_lateral_connections_passes; t++ ) 00494 { 00495 previous_temp_outputs = current_temp_outputs; 00496 current_temp_outputs = temp_outputs[t+1]; 00497 if( topographic_lateral_weights.length() == 0 ) 00498 productTranspose(dampening_expectations, previous_temp_outputs, 00499 lateral_weights); 00500 else 00501 for( int b = 0; b<dampening_expectations.length(); b++) 00502 productTopoLateralWeights( dampening_expectations(b), 00503 previous_temp_outputs(b) ); 00504 00505 dampening_expectations += bias_plus_inputs; 00506 if (use_fast_approximations) 00507 { 00508 if( fast_exact_is_equal( dampening_factor, 0) ) 00509 { 00510 for(int k = 0; k < batch_size; k++) 00511 for( int i=0 ; i<size ; i++ ) 00512 current_temp_outputs(k, i) = 00513 fastsigmoid( dampening_expectations(k, i) ); 00514 } 00515 else 00516 { 00517 for(int k = 0; k < batch_size; k++) 00518 for( int i=0 ; i<size ; i++ ) 00519 current_temp_outputs(k, i) = (1-dampening_factor) 00520 * fastsigmoid( dampening_expectations(k, i) ) 00521 + dampening_factor * previous_temp_outputs(k, i); 00522 } 00523 } 00524 else 00525 { 00526 if( fast_exact_is_equal( dampening_factor, 0) ) 00527 { 00528 for(int k = 0; k < batch_size; k++) 00529 for( int i=0 ; i<size ; i++ ) 00530 current_temp_outputs(k, i) = 00531 sigmoid( dampening_expectations(k, i) ); 00532 } 00533 else 00534 { 00535 for(int k = 0; k < batch_size; k++) 00536 for( int i=0 ; i<size ; i++ ) 00537 current_temp_outputs(k, i) = (1-dampening_factor) 00538 * sigmoid( dampening_expectations(k, i) ) 00539 + dampening_factor * previous_temp_outputs(k, i); 00540 } 00541 } 00542 } 00543 } 00544 } 00545 00546 void RBMLateralBinomialLayer::fprop( const Vec& input, const Vec& rbm_bias, 00547 Vec& output ) const 00548 { 00549 PLASSERT( input.size() == input_size ); 00550 PLASSERT( rbm_bias.size() == input_size ); 00551 output.resize( output_size ); 00552 00553 add(rbm_bias, input, bias_plus_input); 00554 00555 if( use_parametric_mean_field ) 00556 { 00557 PLERROR("RBMLateralBinomialLayer::fprop: use_parametric_mean_field = true " 00558 "not implemented yet for rbm_bias input."); 00559 } 00560 else 00561 { 00562 00563 if( temp_output.length() != n_lateral_connections_passes+1 ) 00564 { 00565 temp_output.resize(n_lateral_connections_passes+1); 00566 for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ ) 00567 temp_output[i].resize(size); 00568 } 00569 00570 temp_output.last() = output; 00571 current_temp_output = temp_output[0]; 00572 00573 if (use_fast_approximations) 00574 for( int i=0 ; i<size ; i++ ) 00575 current_temp_output[i] = fastsigmoid( bias_plus_input[i] ); 00576 else 00577 for( int i=0 ; i<size ; i++ ) 00578 current_temp_output[i] = sigmoid( bias_plus_input[i] ); 00579 00580 for( int t=0; t<n_lateral_connections_passes; t++ ) 00581 { 00582 previous_temp_output = current_temp_output; 00583 current_temp_output = temp_output[t+1]; 00584 if( topographic_lateral_weights.length() == 0 ) 00585 product(dampening_expectation, lateral_weights, previous_temp_output); 00586 else 00587 productTopoLateralWeights( dampening_expectation, previous_temp_output ); 00588 dampening_expectation += bias_plus_input; 00589 if (use_fast_approximations) 00590 { 00591 if( fast_exact_is_equal( dampening_factor, 0) ) 00592 { 00593 for( int i=0 ; i<size ; i++ ) 00594 current_temp_output[i] = fastsigmoid( dampening_expectation[i] ); 00595 } 00596 else 00597 { 00598 for( int i=0 ; i<size ; i++ ) 00599 current_temp_output[i] = 00600 (1-dampening_factor) * fastsigmoid( dampening_expectation[i] ) 00601 + dampening_factor * previous_temp_output[i]; 00602 } 00603 } 00604 else 00605 { 00606 if( fast_exact_is_equal( dampening_factor, 0) ) 00607 { 00608 for( int i=0 ; i<size ; i++ ) 00609 current_temp_output[i] = sigmoid( dampening_expectation[i] ); 00610 } 00611 else 00612 { 00613 for( int i=0 ; i<size ; i++ ) 00614 current_temp_output[i] = 00615 (1-dampening_factor) * sigmoid( dampening_expectation[i] ) 00616 + dampening_factor * previous_temp_output[i]; 00617 } 00618 } 00619 } 00620 } 00621 } 00622 00623 // HUGO: NO 0.5! Computes mat[i][j] += 0.5 * (v1[i] * v2[j] + v1[j] * v2[i]) 00624 // Computes mat[i][j] += (v1[i] * v2[j] + v1[j] * v2[i]) 00625 void RBMLateralBinomialLayer::externalSymetricProductAcc(const Mat& mat, const Vec& v1, const Vec& v2) 00626 { 00627 #ifdef BOUNDCHECK 00628 if (v1.length()!=mat.length() || mat.width()!=v2.length() 00629 || v1.length() != v2.length()) 00630 PLERROR("externalSymetricProductAcc(Mat,Vec,Vec), incompatible " 00631 "arguments sizes"); 00632 #endif 00633 00634 real* v_1=v1.data(); 00635 real* v_2=v2.data(); 00636 real* mp = mat.data(); 00637 int l = mat.length(); 00638 int w = mat.width(); 00639 00640 if(mat.isCompact()) 00641 { 00642 real* pv11 = v_1; 00643 real* pv21 = v_2; 00644 for(int i=0; i<l; i++) 00645 { 00646 real* pv22 = v_2; 00647 real* pv12 = v_1; 00648 real val1 = *pv11++; 00649 real val2 = *pv21++; 00650 for(int j=0; j<w; j++) 00651 //*mp++ += 0.5 * (val1 * *pv22++ + val2 * *pv12++) ; 00652 *mp++ += (val1 * *pv22++ + val2 * *pv12++) ; 00653 } 00654 } 00655 else 00656 { 00657 cerr << "!"; 00658 for (int i=0;i<l;i++) 00659 { 00660 real* mi = mat[i]; 00661 real v1i = v_1[i]; 00662 real v2i = v_2[i]; 00663 for (int j=0;j<w;j++) 00664 //mi[j] += 0.5 * ( v1i * v_2[j] + v2i * v_1[j]); 00665 mi[j] += ( v1i * v_2[j] + v2i * v_1[j]); 00666 } 00667 } 00668 } 00669 00670 void RBMLateralBinomialLayer::productTopoLateralWeights(const Vec& result, 00671 const Vec& input ) const 00672 { 00673 // Could be made faster, in terms of memory access 00674 result.clear(); 00675 int connected_neuron; 00676 int wi; 00677 real* current_weights; 00678 int neuron_v, neuron_h; 00679 int vmin, vmax, hmin, hmax; 00680 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 00681 { 00682 neuron_v = i/topographic_width; 00683 neuron_h = i%topographic_width; 00684 wi = 0; 00685 current_weights = topographic_lateral_weights[i].data(); 00686 00687 vmin = neuron_v < topographic_patch_vradius ? 00688 - neuron_v : - topographic_patch_vradius; 00689 vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ? 00690 topographic_length - neuron_v - 1: topographic_patch_vradius; 00691 00692 hmin = neuron_h < topographic_patch_hradius ? 00693 - neuron_h : - topographic_patch_hradius; 00694 hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ? 00695 topographic_width - neuron_h - 1: topographic_patch_hradius; 00696 00697 for( int j = -1 * topographic_patch_vradius; 00698 j <= topographic_patch_vradius ; j++ ) 00699 { 00700 for( int k = -1 * topographic_patch_hradius; 00701 k <= topographic_patch_hradius; k++ ) 00702 { 00703 connected_neuron = (i+j*topographic_width)+k; 00704 if( connected_neuron != i ) 00705 { 00706 if( j >= vmin && j <= vmax && 00707 k >= hmin && k <= hmax ) 00708 result[i] += input[connected_neuron] 00709 * current_weights[wi]; 00710 wi++; 00711 } 00712 } 00713 } 00714 } 00715 } 00716 00717 void RBMLateralBinomialLayer::productTopoLateralWeightsGradients( 00718 const Vec& input, 00719 const Vec& input_gradient, 00720 const Vec& result_gradient, 00721 const TVec< Vec >& weights_gradient 00722 ) 00723 { 00724 // Could be made faster, in terms of memory access 00725 int connected_neuron; 00726 int wi; 00727 real* current_weights; 00728 real* current_weights_gradient; 00729 int neuron_v, neuron_h; 00730 int vmin, vmax, hmin, hmax; 00731 real result_gradient_i; 00732 real input_i; 00733 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 00734 { 00735 neuron_v = i/topographic_width; 00736 neuron_h = i%topographic_width; 00737 wi = 0; 00738 current_weights = topographic_lateral_weights[i].data(); 00739 current_weights_gradient = weights_gradient[i].data(); 00740 00741 vmin = neuron_v < topographic_patch_vradius ? 00742 - neuron_v : - topographic_patch_vradius; 00743 vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ? 00744 topographic_length - neuron_v - 1: topographic_patch_vradius; 00745 00746 hmin = neuron_h < topographic_patch_hradius ? 00747 - neuron_h : - topographic_patch_hradius; 00748 hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ? 00749 topographic_width - neuron_h - 1: topographic_patch_hradius; 00750 00751 result_gradient_i = result_gradient[i]; 00752 input_i = input[i]; 00753 00754 for( int j = -1 * topographic_patch_vradius; 00755 j <= topographic_patch_vradius ; j++ ) 00756 { 00757 for( int k = -1 * topographic_patch_hradius; 00758 k <= topographic_patch_hradius; k++ ) 00759 { 00760 connected_neuron = (i+j*topographic_width)+k; 00761 if( connected_neuron != i ) 00762 { 00763 if( j >= vmin && j <= vmax && 00764 k >= hmin && k <= hmax ) 00765 { 00766 input_gradient[connected_neuron] += 00767 result_gradient_i * current_weights[wi]; 00768 current_weights_gradient[wi] += 00769 //0.5 * ( result_gradient_i * input[connected_neuron] + 00770 ( result_gradient_i * input[connected_neuron] + 00771 input_i * result_gradient[connected_neuron] ); 00772 } 00773 wi++; 00774 } 00775 } 00776 } 00777 } 00778 } 00779 00780 void RBMLateralBinomialLayer::updateTopoLateralWeightsCD( 00781 const Vec& pos_values, 00782 const Vec& neg_values ) 00783 { 00784 if( !do_not_learn_topographic_lateral_weights ) 00785 { 00786 00787 // Could be made faster, in terms of memory access 00788 int connected_neuron; 00789 int wi; 00790 int neuron_v, neuron_h; 00791 int vmin, vmax, hmin, hmax; 00792 real* current_weights; 00793 real pos_values_i; 00794 real neg_values_i; 00795 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 00796 { 00797 neuron_v = i/topographic_width; 00798 neuron_h = i%topographic_width; 00799 wi = 0; 00800 00801 vmin = neuron_v < topographic_patch_vradius ? 00802 - neuron_v : - topographic_patch_vradius; 00803 vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ? 00804 topographic_length - neuron_v - 1: topographic_patch_vradius; 00805 00806 hmin = neuron_h < topographic_patch_hradius ? 00807 - neuron_h : - topographic_patch_hradius; 00808 hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ? 00809 topographic_width - neuron_h - 1: topographic_patch_hradius; 00810 00811 current_weights = topographic_lateral_weights[i].data(); 00812 pos_values_i = pos_values[i]; 00813 neg_values_i = neg_values[i]; 00814 00815 for( int j = - topographic_patch_vradius; 00816 j <= topographic_patch_vradius ; j++ ) 00817 { 00818 for( int k = -topographic_patch_hradius; 00819 k <= topographic_patch_hradius; k++ ) 00820 { 00821 connected_neuron = (i+j*topographic_width)+k; 00822 if( connected_neuron != i ) 00823 { 00824 if( j >= vmin && j <= vmax && 00825 k >= hmin && k <= hmax ) 00826 { 00827 current_weights[wi] += 00828 //learning_rate * 0.5 * ( 00829 learning_rate * ( 00830 pos_values_i * pos_values[connected_neuron] - 00831 neg_values_i * neg_values[connected_neuron] ); 00832 } 00833 wi++; 00834 } 00835 } 00836 } 00837 } 00838 } 00839 } 00840 00842 // bpropUpdate // 00844 void RBMLateralBinomialLayer::bpropUpdate(const Vec& input, const Vec& output, 00845 Vec& input_gradient, 00846 const Vec& output_gradient, 00847 bool accumulate) 00848 { 00849 PLASSERT( input.size() == size ); 00850 PLASSERT( output.size() == size ); 00851 PLASSERT( output_gradient.size() == size ); 00852 00853 if( accumulate ) 00854 PLASSERT_MSG( input_gradient.size() == size, 00855 "Cannot resize input_gradient AND accumulate into it" ); 00856 else 00857 { 00858 input_gradient.resize( size ); 00859 input_gradient.clear(); 00860 } 00861 00862 //if( momentum != 0. ) 00863 // bias_inc.resize( size ); 00864 00865 if( use_parametric_mean_field ) 00866 { 00867 real mean_field_i; 00868 for( int i=0 ; i<size ; i++ ) 00869 { 00870 mean_field_i = output[i]; 00871 temp_mean_field_gradient[i] = output_gradient[i] * mean_field_i * (1 - mean_field_i); 00872 } 00873 00874 transposeProductAcc( input_gradient, mean_field_output_weights, temp_mean_field_gradient ); 00875 00876 externalProductScaleAcc( mean_field_output_weights, temp_mean_field_gradient, 00877 mean_field_input, -learning_rate ); 00878 multiplyScaledAdd( temp_mean_field_gradient, 1.0, -learning_rate, mean_field_output_bias); 00879 00880 real input_mean_field_i; 00881 for( int i=0 ; i<size ; i++ ) 00882 { 00883 input_mean_field_i = mean_field_input[i]; 00884 input_gradient[i] = input_gradient[i] * input_mean_field_i * (1 - input_mean_field_i); 00885 } 00886 } 00887 else 00888 { 00889 temp_input_gradient.clear(); 00890 temp_mean_field_gradient << output_gradient; 00891 current_temp_output = output; 00892 lateral_weights_gradient.clear(); 00893 for( int i=0; i<topographic_lateral_weights_gradient.length(); i++) 00894 topographic_lateral_weights_gradient[i].clear(); 00895 00896 real output_i; 00897 for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- ) 00898 { 00899 for( int i=0 ; i<size ; i++ ) 00900 { 00901 output_i = current_temp_output[i]; 00902 00903 // Contribution from the mean field approximation 00904 temp_mean_field_gradient2[i] = (1-dampening_factor)* 00905 output_i * (1-output_i) * temp_mean_field_gradient[i]; 00906 00907 // Contribution from the dampening 00908 temp_mean_field_gradient[i] *= dampening_factor; 00909 } 00910 00911 // Input gradient contribution 00912 temp_input_gradient += temp_mean_field_gradient2; 00913 00914 // Lateral weights gradient contribution 00915 if( topographic_lateral_weights.length() == 0) 00916 { 00917 externalSymetricProductAcc( lateral_weights_gradient, 00918 temp_mean_field_gradient2, 00919 temp_output[t] ); 00920 00921 transposeProductAcc(temp_mean_field_gradient, lateral_weights, 00922 temp_mean_field_gradient2); 00923 } 00924 else 00925 { 00926 productTopoLateralWeightsGradients( 00927 temp_output[t], 00928 temp_mean_field_gradient, 00929 temp_mean_field_gradient2, 00930 topographic_lateral_weights_gradient); 00931 } 00932 00933 current_temp_output = temp_output[t]; 00934 } 00935 00936 for( int i=0 ; i<size ; i++ ) 00937 { 00938 output_i = current_temp_output[i]; 00939 temp_mean_field_gradient[i] *= output_i * (1-output_i); 00940 } 00941 00942 temp_input_gradient += temp_mean_field_gradient; 00943 00944 input_gradient += temp_input_gradient; 00945 00946 // Update bias 00947 real in_grad_i; 00948 for( int i=0 ; i<size ; i++ ) 00949 { 00950 in_grad_i = temp_input_gradient[i]; 00951 if( momentum == 0. ) 00952 { 00953 // update the bias: bias -= learning_rate * input_gradient 00954 bias[i] -= learning_rate * in_grad_i; 00955 } 00956 else 00957 { 00958 // The update rule becomes: 00959 // bias_inc = momentum * bias_inc - learning_rate * input_gradient 00960 // bias += bias_inc 00961 bias_inc[i] = momentum * bias_inc[i] - learning_rate * in_grad_i; 00962 bias[i] += bias_inc[i]; 00963 } 00964 } 00965 00966 if( topographic_lateral_weights.length() == 0) 00967 { 00968 if( momentum == 0. ) 00969 { 00970 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate, 00971 lateral_weights); 00972 } 00973 else 00974 { 00975 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate, 00976 lateral_weights_inc); 00977 lateral_weights += lateral_weights_inc; 00978 } 00979 } 00980 else 00981 { 00982 if( !do_not_learn_topographic_lateral_weights ) 00983 { 00984 if( momentum == 0. ) 00985 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 00986 multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0, 00987 -learning_rate, 00988 topographic_lateral_weights[i]); 00989 00990 else 00991 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 00992 "topographic weights"); 00993 } 00994 } 00995 00996 // Set diagonal to 0 00997 if( lateral_weights.length() != 0 ) 00998 { 00999 real *d = lateral_weights.data(); 01000 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01001 *d = 0; 01002 } 01003 } 01004 } 01005 01006 void RBMLateralBinomialLayer::bpropUpdate(const Mat& inputs, const Mat& outputs, 01007 Mat& input_gradients, 01008 const Mat& output_gradients, 01009 bool accumulate) 01010 { 01011 PLASSERT( inputs.width() == size ); 01012 PLASSERT( outputs.width() == size ); 01013 PLASSERT( output_gradients.width() == size ); 01014 01015 int mbatch_size = inputs.length(); 01016 PLASSERT( outputs.length() == mbatch_size ); 01017 PLASSERT( output_gradients.length() == mbatch_size ); 01018 01019 if( accumulate ) 01020 { 01021 PLASSERT_MSG( input_gradients.width() == size && 01022 input_gradients.length() == mbatch_size, 01023 "Cannot resize input_gradients and accumulate into it" ); 01024 } 01025 else 01026 { 01027 input_gradients.resize(mbatch_size, size); 01028 input_gradients.clear(); 01029 } 01030 01031 //if( momentum != 0. ) 01032 // bias_inc.resize( size ); 01033 01034 // TODO Can we do this more efficiently? (using BLAS) 01035 01036 // We use the average gradient over the mini-batch. 01037 real avg_lr = learning_rate / inputs.length(); 01038 01039 if( use_parametric_mean_field ) 01040 { 01041 PLERROR("RBMLateralBinomialLayer::bpropUpdate: use_parametric_mean_field=true " 01042 "not implemented yet for batch mode."); 01043 } 01044 else 01045 { 01046 lateral_weights_gradient.clear(); 01047 real output_i; 01048 for (int j = 0; j < mbatch_size; j++) 01049 { 01050 temp_input_gradient.clear(); 01051 temp_mean_field_gradient << output_gradients(j); 01052 current_temp_output = outputs(j); 01053 01054 for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- ) 01055 { 01056 01057 for( int i=0 ; i<size ; i++ ) 01058 { 01059 output_i = current_temp_output[i]; 01060 01061 // Contribution from the mean field approximation 01062 temp_mean_field_gradient2[i] = (1-dampening_factor)* 01063 output_i * (1-output_i) * temp_mean_field_gradient[i]; 01064 01065 // Contribution from the dampening 01066 temp_mean_field_gradient[i] *= dampening_factor; 01067 } 01068 01069 // Input gradient contribution 01070 temp_input_gradient += temp_mean_field_gradient2; 01071 01072 // Lateral weights gradient contribution 01073 if( topographic_lateral_weights.length() == 0) 01074 { 01075 01076 externalSymetricProductAcc( lateral_weights_gradient, 01077 temp_mean_field_gradient2, 01078 temp_outputs[t](j) ); 01079 01080 transposeProductAcc(temp_mean_field_gradient, lateral_weights, 01081 temp_mean_field_gradient2); 01082 } 01083 else 01084 { 01085 productTopoLateralWeightsGradients( 01086 temp_outputs[t](j), 01087 temp_mean_field_gradient, 01088 temp_mean_field_gradient2, 01089 topographic_lateral_weights_gradient); 01090 } 01091 01092 current_temp_output = temp_outputs[t](j); 01093 } 01094 01095 for( int i=0 ; i<size ; i++ ) 01096 { 01097 output_i = current_temp_output[i]; 01098 temp_mean_field_gradient[i] *= output_i * (1-output_i); 01099 } 01100 01101 temp_input_gradient += temp_mean_field_gradient; 01102 01103 input_gradients(j) += temp_input_gradient; 01104 01105 // Update bias 01106 real in_grad_i; 01107 for( int i=0 ; i<size ; i++ ) 01108 { 01109 in_grad_i = temp_input_gradient[i]; 01110 if( momentum == 0. ) 01111 { 01112 // update the bias: bias -= learning_rate * input_gradient 01113 bias[i] -= avg_lr * in_grad_i; 01114 } 01115 else 01116 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 01117 "momentum with mini-batches"); 01118 } 01119 } 01120 01121 if( topographic_lateral_weights.length() == 0) 01122 { 01123 if( momentum == 0. ) 01124 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate, 01125 lateral_weights); 01126 else 01127 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 01128 "momentum with mini-batches"); 01129 } 01130 else 01131 { 01132 if( !do_not_learn_topographic_lateral_weights ) 01133 { 01134 if( momentum == 0. ) 01135 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 01136 multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0, 01137 -learning_rate, 01138 topographic_lateral_weights[i]); 01139 01140 else 01141 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 01142 "topographic weights"); 01143 } 01144 01145 } 01146 01147 // Set diagonal to 0 01148 if( lateral_weights.length() != 0 ) 01149 { 01150 real *d = lateral_weights.data(); 01151 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01152 *d = 0; 01153 } 01154 } 01155 } 01156 01157 01159 void RBMLateralBinomialLayer::bpropUpdate(const Vec& input, const Vec& rbm_bias, 01160 const Vec& output, 01161 Vec& input_gradient, Vec& rbm_bias_gradient, 01162 const Vec& output_gradient) 01163 { 01164 PLASSERT( input.size() == size ); 01165 PLASSERT( rbm_bias.size() == size ); 01166 PLASSERT( output.size() == size ); 01167 PLASSERT( output_gradient.size() == size ); 01168 input_gradient.resize( size ); 01169 rbm_bias_gradient.resize( size ); 01170 01171 if( use_parametric_mean_field ) 01172 { 01173 PLERROR("RBMLateralBinomialLayer::bpropUpdate: use_parametric_mean_field=true " 01174 "not implemented yet for bias input."); 01175 } 01176 else 01177 { 01178 temp_input_gradient.clear(); 01179 temp_mean_field_gradient << output_gradient; 01180 current_temp_output = output; 01181 lateral_weights_gradient.clear(); 01182 01183 real output_i; 01184 for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- ) 01185 { 01186 01187 for( int i=0 ; i<size ; i++ ) 01188 { 01189 output_i = current_temp_output[i]; 01190 01191 // Contribution from the mean field approximation 01192 temp_mean_field_gradient2[i] = (1-dampening_factor)* 01193 output_i * (1-output_i) * temp_mean_field_gradient[i]; 01194 01195 // Contribution from the dampening 01196 temp_mean_field_gradient[i] *= dampening_factor; 01197 } 01198 01199 // Input gradient contribution 01200 temp_input_gradient += temp_mean_field_gradient2; 01201 01202 // Lateral weights gradient contribution 01203 if( topographic_lateral_weights.length() == 0) 01204 { 01205 01206 externalSymetricProductAcc( lateral_weights_gradient, 01207 temp_mean_field_gradient2, 01208 temp_output[t] ); 01209 01210 transposeProductAcc(temp_mean_field_gradient, lateral_weights, 01211 temp_mean_field_gradient2); 01212 } 01213 else 01214 { 01215 productTopoLateralWeightsGradients( 01216 temp_output[t], 01217 temp_mean_field_gradient, 01218 temp_mean_field_gradient2, 01219 topographic_lateral_weights_gradient); 01220 } 01221 01222 current_temp_output = temp_output[t]; 01223 } 01224 01225 for( int i=0 ; i<size ; i++ ) 01226 { 01227 output_i = current_temp_output[i]; 01228 temp_mean_field_gradient[i] *= output_i * (1-output_i); 01229 } 01230 01231 temp_input_gradient += temp_mean_field_gradient; 01232 01233 input_gradient << temp_input_gradient; 01234 rbm_bias_gradient << temp_input_gradient; 01235 01236 if( topographic_lateral_weights.length() == 0) 01237 { 01238 if( momentum == 0. ) 01239 { 01240 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate, 01241 lateral_weights); 01242 } 01243 else 01244 { 01245 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate, 01246 lateral_weights_inc); 01247 lateral_weights += lateral_weights_inc; 01248 } 01249 } 01250 else 01251 { 01252 if( !do_not_learn_topographic_lateral_weights ) 01253 { 01254 if( momentum == 0. ) 01255 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 01256 multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0, 01257 -learning_rate, 01258 topographic_lateral_weights[i]); 01259 01260 else 01261 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 01262 "topographic weights"); 01263 } 01264 } 01265 01266 // Set diagonal to 0 01267 if( lateral_weights.length() != 0 ) 01268 { 01269 real *d = lateral_weights.data(); 01270 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01271 *d = 0; 01272 } 01273 } 01274 } 01275 01276 real RBMLateralBinomialLayer::fpropNLL(const Vec& target) 01277 { 01278 PLASSERT( target.size() == input_size ); 01279 computeExpectation(); 01280 01281 real ret = 0; 01282 real target_i, expectation_i; 01283 for( int i=0 ; i<size ; i++ ) 01284 { 01285 target_i = target[i]; 01286 expectation_i = expectation[i]; 01287 // TODO: implement more numerically stable version 01288 if(!fast_exact_is_equal(target_i,0.0)) 01289 ret -= target_i*safeflog(expectation_i) ; 01290 if(!fast_exact_is_equal(target_i,1.0)) 01291 ret -= (1-target_i)*safeflog(1-expectation_i); 01292 } 01293 return ret; 01294 } 01295 01296 void RBMLateralBinomialLayer::fpropNLL(const Mat& targets, const Mat& costs_column) 01297 { 01298 computeExpectations(); 01299 01300 PLASSERT( targets.width() == input_size ); 01301 PLASSERT( targets.length() == batch_size ); 01302 PLASSERT( costs_column.width() == 1 ); 01303 PLASSERT( costs_column.length() == batch_size ); 01304 01305 for (int k=0;k<batch_size;k++) // loop over minibatch 01306 { 01307 real nll = 0; 01308 real* expectation = expectations[k]; 01309 real* target = targets[k]; 01310 for( int i=0 ; i<size ; i++ ) // loop over outputs 01311 { 01312 // TODO: implement more numerically stable version 01313 if(!fast_exact_is_equal(target[i],0.0)) 01314 nll -= target[i]*safeflog(expectation[i]) ; 01315 if(!fast_exact_is_equal(target[i],1.0)) 01316 nll -= (1-target[i])*safeflog(1-expectation[i]); 01317 } 01318 costs_column(k,0) = nll; 01319 } 01320 } 01321 01322 void RBMLateralBinomialLayer::bpropNLL(const Vec& target, real nll, Vec& bias_gradient) 01323 { 01324 computeExpectation(); 01325 01326 PLASSERT( target.size() == input_size ); 01327 bias_gradient.resize( size ); 01328 bias_gradient.clear(); 01329 01330 if( use_parametric_mean_field ) 01331 { 01332 PLERROR("RBMLateralBinomialLayer::bpropNLL: use_parametric_mean_field=true " 01333 "not implemented yet."); 01334 } 01335 else 01336 { 01337 // bias_gradient = expectation - target 01338 substract(expectation, target, temp_mean_field_gradient); 01339 01340 current_temp_output = expectation; 01341 lateral_weights_gradient.clear(); 01342 01343 real output_i; 01344 for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- ) 01345 { 01346 for( int i=0 ; i<size ; i++ ) 01347 { 01348 output_i = current_temp_output[i]; 01349 01350 // Contribution from the mean field approximation 01351 temp_mean_field_gradient2[i] = (1-dampening_factor)* 01352 output_i * (1-output_i) * temp_mean_field_gradient[i]; 01353 01354 // Contribution from the dampening 01355 temp_mean_field_gradient[i] *= dampening_factor; 01356 } 01357 01358 // Input gradient contribution 01359 bias_gradient += temp_mean_field_gradient2; 01360 01361 // Lateral weights gradient contribution 01362 if( topographic_lateral_weights.length() == 0) 01363 { 01364 externalSymetricProductAcc( lateral_weights_gradient, 01365 temp_mean_field_gradient2, 01366 temp_output[t] ); 01367 01368 transposeProductAcc(temp_mean_field_gradient, lateral_weights, 01369 temp_mean_field_gradient2); 01370 } 01371 else 01372 { 01373 productTopoLateralWeightsGradients( 01374 temp_output[t], 01375 temp_mean_field_gradient, 01376 temp_mean_field_gradient2, 01377 topographic_lateral_weights_gradient); 01378 } 01379 01380 current_temp_output = temp_output[t]; 01381 } 01382 01383 for( int i=0 ; i<size ; i++ ) 01384 { 01385 output_i = current_temp_output[i]; 01386 temp_mean_field_gradient[i] *= output_i * (1-output_i); 01387 } 01388 01389 bias_gradient += temp_mean_field_gradient; 01390 01391 if( topographic_lateral_weights.length() == 0) 01392 { 01393 // Update lateral connections 01394 if( momentum == 0. ) 01395 { 01396 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate, 01397 lateral_weights); 01398 } 01399 else 01400 { 01401 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate, 01402 lateral_weights_inc); 01403 lateral_weights += lateral_weights_inc; 01404 } 01405 } 01406 else 01407 { 01408 if( !do_not_learn_topographic_lateral_weights ) 01409 { 01410 if( momentum == 0. ) 01411 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 01412 multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0, 01413 -learning_rate, 01414 topographic_lateral_weights[i]); 01415 01416 else 01417 PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for " 01418 "topographic weights"); 01419 } 01420 } 01421 // Set diagonal to 0 01422 if( lateral_weights.length() != 0 ) 01423 { 01424 real *d = lateral_weights.data(); 01425 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01426 *d = 0; 01427 } 01428 } 01429 } 01430 01431 void RBMLateralBinomialLayer::bpropNLL(const Mat& targets, const Mat& costs_column, 01432 Mat& bias_gradients) 01433 { 01434 computeExpectations(); 01435 01436 PLASSERT( targets.width() == input_size ); 01437 PLASSERT( targets.length() == batch_size ); 01438 PLASSERT( costs_column.width() == 1 ); 01439 PLASSERT( costs_column.length() == batch_size ); 01440 bias_gradients.resize( batch_size, size ); 01441 bias_gradients.clear(); 01442 01443 01444 // TODO Can we do this more efficiently? (using BLAS) 01445 01446 if( use_parametric_mean_field ) 01447 { 01448 PLERROR("RBMLateralBinomialLayer::bpropNLL: use_parametric_mean_field=true " 01449 "not implemented yet."); 01450 } 01451 else 01452 { 01453 01454 // We use the average gradient over the mini-batch. 01455 lateral_weights_gradient.clear(); 01456 real output_i; 01457 for (int j = 0; j < batch_size; j++) 01458 { 01459 // top_gradient = expectations(j) - targets(j) 01460 substract(expectations(j), targets(j), temp_mean_field_gradient); 01461 current_temp_output = expectations(j); 01462 01463 for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- ) 01464 { 01465 for( int i=0 ; i<size ; i++ ) 01466 { 01467 output_i = current_temp_output[i]; 01468 01469 // Contribution from the mean field approximation 01470 temp_mean_field_gradient2[i] = (1-dampening_factor)* 01471 output_i * (1-output_i) * temp_mean_field_gradient[i]; 01472 01473 // Contribution from the dampening 01474 temp_mean_field_gradient[i] *= dampening_factor; 01475 } 01476 01477 // Input gradient contribution 01478 bias_gradients(j) += temp_mean_field_gradient2; 01479 01480 // Lateral weights gradient contribution 01481 if( topographic_lateral_weights.length() == 0) 01482 { 01483 01484 externalSymetricProductAcc( lateral_weights_gradient, 01485 temp_mean_field_gradient2, 01486 temp_outputs[t](j) ); 01487 01488 transposeProductAcc(temp_mean_field_gradient, lateral_weights, 01489 temp_mean_field_gradient2); 01490 } 01491 else 01492 { 01493 productTopoLateralWeightsGradients( 01494 temp_outputs[t](j), 01495 temp_mean_field_gradient, 01496 temp_mean_field_gradient2, 01497 topographic_lateral_weights_gradient); 01498 } 01499 current_temp_output = temp_outputs[t](j); 01500 } 01501 01502 for( int i=0 ; i<size ; i++ ) 01503 { 01504 output_i = current_temp_output[i]; 01505 temp_mean_field_gradient[i] *= output_i * (1-output_i); 01506 } 01507 01508 bias_gradients(j) += temp_mean_field_gradient; 01509 } 01510 01511 // Update lateral connections 01512 if( topographic_lateral_weights.length() == 0 ) 01513 { 01514 if( momentum == 0. ) 01515 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate, 01516 lateral_weights); 01517 else 01518 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for " 01519 "momentum with mini-batches"); 01520 } 01521 else 01522 { 01523 if( !do_not_learn_topographic_lateral_weights ) 01524 { 01525 if( momentum == 0. ) 01526 for( int i=0; i<topographic_lateral_weights.length(); i++ ) 01527 multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0, 01528 -learning_rate, 01529 topographic_lateral_weights[i]); 01530 01531 else 01532 PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for " 01533 "topographic weights"); 01534 } 01535 } 01536 01537 // Set diagonal to 0 01538 if( lateral_weights.length() != 0 ) 01539 { 01540 real *d = lateral_weights.data(); 01541 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01542 *d = 0; 01543 } 01544 } 01545 } 01546 01547 void RBMLateralBinomialLayer::accumulatePosStats( const Vec& pos_values ) 01548 { 01549 inherited::accumulatePosStats( pos_values); 01550 externalProductAcc(lateral_weights_pos_stats, pos_values, pos_values); 01551 } 01552 01553 void RBMLateralBinomialLayer::accumulatePosStats( const Mat& pos_values ) 01554 { 01555 inherited::accumulatePosStats( pos_values); 01556 transposeProductAcc(lateral_weights_pos_stats, pos_values, pos_values); 01557 } 01558 01559 void RBMLateralBinomialLayer::accumulateNegStats( const Vec& neg_values ) 01560 { 01561 inherited::accumulateNegStats( neg_values); 01562 externalProductAcc(lateral_weights_neg_stats, neg_values, neg_values); 01563 } 01564 01565 void RBMLateralBinomialLayer::accumulateNegStats( const Mat& neg_values ) 01566 { 01567 inherited::accumulateNegStats( neg_values); 01568 transposeProductAcc(lateral_weights_neg_stats, neg_values, neg_values); 01569 } 01570 01571 01572 void RBMLateralBinomialLayer::update() 01573 { 01574 //real pos_factor = 0.5 * learning_rate / pos_count; 01575 //real neg_factor = - 0.5 * learning_rate / neg_count; 01576 real pos_factor = learning_rate / pos_count; 01577 real neg_factor = - learning_rate / neg_count; 01578 01579 if( topographic_lateral_weights.length() != 0 ) 01580 PLERROR("In RBMLateralBinomialLayer:update - Not implemented for " 01581 "topographic weights"); 01582 01583 // Update lateral connections 01584 if( momentum == 0. ) 01585 { 01586 multiplyScaledAdd( lateral_weights_pos_stats, neg_factor, pos_factor, 01587 lateral_weights_neg_stats); 01588 lateral_weights += lateral_weights_neg_stats; 01589 } 01590 else 01591 { 01592 multiplyScaledAdd( lateral_weights_pos_stats, neg_factor, pos_factor, 01593 lateral_weights_neg_stats); 01594 multiplyScaledAdd( lateral_weights_neg_stats, momentum, 1.0, 01595 lateral_weights_inc); 01596 lateral_weights += lateral_weights_inc; 01597 } 01598 01599 // Set diagonal to 0 01600 if( lateral_weights.length() != 0 ) 01601 { 01602 real *d = lateral_weights.data(); 01603 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01604 *d = 0; 01605 } 01606 01607 // Call to update() must be at the end, since update() calls clearStats()! 01608 inherited::update(); 01609 } 01610 01611 void RBMLateralBinomialLayer::update( const Vec& grad) 01612 { 01613 inherited::update( grad ); 01614 PLWARNING("RBMLateralBinomialLayer::update( grad ): does not update the\n" 01615 "lateral connections."); 01616 } 01617 01618 void RBMLateralBinomialLayer::update( const Vec& pos_values, const Vec& neg_values ) 01619 { 01620 // Update lateral connections 01621 if( topographic_lateral_weights.length() == 0 ) 01622 { 01623 if( momentum == 0. ) 01624 { 01625 externalProductScaleAcc(lateral_weights, pos_values, pos_values, 01626 //0.5 * learning_rate); 01627 learning_rate); 01628 externalProductScaleAcc(lateral_weights, neg_values, neg_values, 01629 //- 0.5 * learning_rate); 01630 -learning_rate); 01631 } 01632 else 01633 { 01634 lateral_weights_inc *= momentum; 01635 externalProductScaleAcc(lateral_weights_inc, pos_values, pos_values, 01636 //0.5 * learning_rate); 01637 learning_rate); 01638 externalProductScaleAcc(lateral_weights_inc, neg_values, neg_values, 01639 //- 0.5 * learning_rate); 01640 - learning_rate); 01641 lateral_weights += lateral_weights_inc; 01642 } 01643 01644 // Set diagonal to 0 01645 if( lateral_weights.length() != 0 ) 01646 { 01647 real *d = lateral_weights.data(); 01648 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01649 *d = 0; 01650 } 01651 } 01652 else 01653 { 01654 if( momentum == 0. ) 01655 updateTopoLateralWeightsCD(pos_values, neg_values); 01656 else 01657 PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for " 01658 "topographic weights"); 01659 } 01660 01661 inherited::update( pos_values, neg_values ); 01662 } 01663 01664 void RBMLateralBinomialLayer::update( const Mat& pos_values, const Mat& neg_values ) 01665 { 01666 int n = pos_values.length(); 01667 PLASSERT( neg_values.length() == n ); 01668 01669 // We take the average gradient over the mini-batch. 01670 //real avg_lr = 0.5 * learning_rate / n; 01671 real avg_lr = learning_rate / n; 01672 01673 // Update lateral connections 01674 if( topographic_lateral_weights.length() == 0 ) 01675 { 01676 if( momentum == 0. ) 01677 { 01678 transposeProductScaleAcc(lateral_weights, pos_values, pos_values, 01679 avg_lr, 1); 01680 transposeProductScaleAcc(lateral_weights, neg_values, neg_values, 01681 -avg_lr, 1); 01682 } 01683 else 01684 { 01685 lateral_weights_inc *= momentum; 01686 transposeProductScaleAcc(lateral_weights_inc, pos_values, pos_values, 01687 avg_lr, 1); 01688 transposeProductScaleAcc(lateral_weights_inc, neg_values, neg_values, 01689 -avg_lr, 1); 01690 lateral_weights += lateral_weights_inc; 01691 } 01692 01693 // Set diagonal to 0 01694 if( lateral_weights.length() != 0 ) 01695 { 01696 real *d = lateral_weights.data(); 01697 for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1) 01698 *d = 0; 01699 } 01700 } 01701 else 01702 { 01703 if( momentum == 0. ) 01704 { 01705 for(int b=0; b<pos_values.length(); b++) 01706 updateTopoLateralWeightsCD(pos_values(b), neg_values(b)); 01707 01708 } 01709 else 01710 PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for " 01711 "topographic weights"); 01712 } 01713 01714 inherited::update( pos_values, neg_values ); 01715 } 01716 01717 void RBMLateralBinomialLayer::updateCDandGibbs( const Mat& pos_values, 01718 const Mat& cd_neg_values, 01719 const Mat& gibbs_neg_values, 01720 real background_gibbs_update_ratio ) 01721 { 01722 inherited::updateCDandGibbs( pos_values, cd_neg_values, 01723 gibbs_neg_values, background_gibbs_update_ratio ); 01724 PLERROR("In RBMLateralBinomialLayer::updateCDandGibbs(): not implemented yet."); 01725 } 01726 01727 void RBMLateralBinomialLayer::updateGibbs( const Mat& pos_values, 01728 const Mat& gibbs_neg_values) 01729 { 01730 inherited::updateGibbs( pos_values, gibbs_neg_values ); 01731 PLERROR("In RBMLateralBinomialLayer::updateCDandGibbs(): not implemented yet."); 01732 } 01733 01734 void RBMLateralBinomialLayer::declareOptions(OptionList& ol) 01735 { 01736 declareOption(ol, "n_lateral_connections_passes", 01737 &RBMLateralBinomialLayer::n_lateral_connections_passes, 01738 OptionBase::buildoption, 01739 "Number of passes through the lateral connections.\n"); 01740 01741 declareOption(ol, "dampening_factor", 01742 &RBMLateralBinomialLayer::dampening_factor, 01743 OptionBase::buildoption, 01744 "Dampening factor ( expectation_t = (1-df) * currrent mean field" 01745 " + df * expectation_{t-1}).\n"); 01746 01747 declareOption(ol, "mean_field_precision_threshold", 01748 &RBMLateralBinomialLayer::mean_field_precision_threshold, 01749 OptionBase::buildoption, 01750 "Mean-field precision threshold that, once reached, stops the mean-field\n" 01751 "expectation approximation computation. Used only in computeExpectation().\n" 01752 "Precision is computed as:\n" 01753 " dist(last_mean_field, current_mean_field) / size\n"); 01754 01755 declareOption(ol, "topographic_length", 01756 &RBMLateralBinomialLayer::topographic_length, 01757 OptionBase::buildoption, 01758 "Length of the topographic map.\n"); 01759 01760 declareOption(ol, "topographic_width", 01761 &RBMLateralBinomialLayer::topographic_width, 01762 OptionBase::buildoption, 01763 "Width of the topographic map.\n"); 01764 01765 declareOption(ol, "topographic_patch_vradius", 01766 &RBMLateralBinomialLayer::topographic_patch_vradius, 01767 OptionBase::buildoption, 01768 "Vertical radius of the topographic local weight patches.\n"); 01769 01770 declareOption(ol, "topographic_patch_hradius", 01771 &RBMLateralBinomialLayer::topographic_patch_hradius, 01772 OptionBase::buildoption, 01773 "Horizontal radius of the topographic local weight patches.\n"); 01774 01775 declareOption(ol, "topographic_lateral_weights_init_value", 01776 &RBMLateralBinomialLayer::topographic_lateral_weights_init_value, 01777 OptionBase::buildoption, 01778 "Initial value for the topographic_lateral_weights.\n"); 01779 01780 declareOption(ol, "do_not_learn_topographic_lateral_weights", 01781 &RBMLateralBinomialLayer::do_not_learn_topographic_lateral_weights, 01782 OptionBase::buildoption, 01783 "Indication that the topographic_lateral_weights should\n" 01784 "be fixed at their initial value.\n"); 01785 01786 declareOption(ol, "lateral_weights", 01787 &RBMLateralBinomialLayer::lateral_weights, 01788 OptionBase::learntoption, 01789 "Lateral connections.\n"); 01790 01791 declareOption(ol, "topographic_lateral_weights", 01792 &RBMLateralBinomialLayer::topographic_lateral_weights, 01793 OptionBase::learntoption, 01794 "Local topographic lateral connections.\n"); 01795 01796 declareOption(ol, "use_parametric_mean_field", 01797 &RBMLateralBinomialLayer::use_parametric_mean_field, 01798 OptionBase::buildoption, 01799 "Indication that a parametric predictor of the mean-field\n" 01800 "approximation of the hidden layer conditional distribution.\n"); 01801 01802 declareOption(ol, "mean_field_output_weights", 01803 &RBMLateralBinomialLayer::mean_field_output_weights, 01804 OptionBase::learntoption, 01805 "Output weights of the mean field predictor.\n"); 01806 01807 declareOption(ol, "mean_field_output_bias", 01808 &RBMLateralBinomialLayer::mean_field_output_bias, 01809 OptionBase::learntoption, 01810 "Output bias of the mean field predictor.\n"); 01811 01812 // Now call the parent class' declareOptions 01813 inherited::declareOptions(ol); 01814 } 01815 01816 void RBMLateralBinomialLayer::build_() 01817 { 01818 if( n_lateral_connections_passes == 0 && 01819 !fast_exact_is_equal(dampening_factor, 0) ) 01820 PLERROR("In RBMLateralBinomialLayer::build_(): when not using the lateral\n" 01821 "connections, dampening_factor should be 0."); 01822 01823 if( dampening_factor < 0 || dampening_factor > 1) 01824 PLERROR("In RBMLateralBinomialLayer::build_(): dampening_factor should be\n" 01825 "in [0,1]."); 01826 01827 if( n_lateral_connections_passes < 0 ) 01828 PLERROR("In RBMLateralBinomialLayer::build_(): n_lateral_connections_passes\n" 01829 " should be >= 0."); 01830 01831 if( use_parametric_mean_field && topographic_length > 0 && topographic_width > 0 ) 01832 PLERROR("RBMLateralBinomialLayer::build_(): can't use parametric mean field " 01833 "and topographic lateral connections."); 01834 01835 if( use_parametric_mean_field ) 01836 { 01837 mean_field_output_weights.resize(size,size); 01838 mean_field_output_bias.resize(size); 01839 mean_field_input.resize(size); 01840 pre_sigmoid_mean_field_output.resize(size); 01841 } 01842 01843 if( topographic_length <= 0 || topographic_width <= 0) 01844 { 01845 lateral_weights.resize(size,size); 01846 01847 lateral_weights_gradient.resize(size,size); 01848 lateral_weights_pos_stats.resize(size,size); 01849 lateral_weights_neg_stats.resize(size,size); 01850 if( momentum != 0. ) 01851 { 01852 bias_inc.resize( size ); 01853 lateral_weights_inc.resize(size,size); 01854 } 01855 } 01856 else 01857 { 01858 if( size != topographic_length * topographic_width ) 01859 PLERROR( "In RBMLateralBinomialLayer::build_(): size != " 01860 "topographic_length * topographic_width.\n" ); 01861 01862 if( topographic_length-1 <= 2*topographic_patch_vradius ) 01863 PLERROR( "In RBMLateralBinomialLayer::build_(): " 01864 "topographic_patch_vradius is too large.\n" ); 01865 01866 if( topographic_width-1 <= 2*topographic_patch_hradius ) 01867 PLERROR( "In RBMLateralBinomialLayer::build_(): " 01868 "topographic_patch_hradius is too large.\n" ); 01869 01870 topographic_lateral_weights.resize(size); 01871 topographic_lateral_weights_gradient.resize(size); 01872 for( int i=0; i<size; i++ ) 01873 { 01874 topographic_lateral_weights[i].resize( 01875 ( 2 * topographic_patch_hradius + 1 ) * 01876 ( 2 * topographic_patch_vradius + 1 ) - 1 ); 01877 topographic_lateral_weights_gradient[i].resize( 01878 ( 2 * topographic_patch_hradius + 1 ) * 01879 ( 2 * topographic_patch_vradius + 1 ) - 1 ); 01880 } 01881 01882 // Should probably have separate lateral_weights_*_stats 01883 } 01884 01885 // Resizing temporary variables 01886 dampening_expectation.resize(size); 01887 temp_input_gradient.resize(size); 01888 temp_mean_field_gradient.resize(size); 01889 temp_mean_field_gradient2.resize(size); 01890 } 01891 01892 void RBMLateralBinomialLayer::build() 01893 { 01894 inherited::build(); 01895 build_(); 01896 } 01897 01898 01899 void RBMLateralBinomialLayer::makeDeepCopyFromShallowCopy(CopiesMap& copies) 01900 { 01901 inherited::makeDeepCopyFromShallowCopy(copies); 01902 deepCopyField(lateral_weights,copies); 01903 deepCopyField(topographic_lateral_weights,copies); 01904 deepCopyField(lateral_weights_pos_stats,copies); 01905 deepCopyField(lateral_weights_neg_stats,copies); 01906 deepCopyField(dampening_expectation,copies); 01907 deepCopyField(dampening_expectations,copies); 01908 deepCopyField(mean_field_input,copies); 01909 deepCopyField(pre_sigmoid_mean_field_output,copies); 01910 deepCopyField(temp_output,copies); 01911 deepCopyField(temp_outputs,copies); 01912 deepCopyField(current_temp_output,copies); 01913 deepCopyField(previous_temp_output,copies); 01914 deepCopyField(current_temp_outputs,copies); 01915 deepCopyField(previous_temp_outputs,copies); 01916 deepCopyField(bias_plus_input,copies); 01917 deepCopyField(bias_plus_inputs,copies); 01918 deepCopyField(temp_input_gradient,copies); 01919 deepCopyField(temp_mean_field_gradient,copies); 01920 deepCopyField(temp_mean_field_gradient2,copies); 01921 deepCopyField(lateral_weights_gradient,copies); 01922 deepCopyField(lateral_weights_inc,copies); 01923 deepCopyField(topographic_lateral_weights_gradient,copies); 01924 deepCopyField(mean_field_output_weights,copies); 01925 deepCopyField(mean_field_output_bias,copies); 01926 } 01927 01928 real RBMLateralBinomialLayer::energy(const Vec& unit_values) const 01929 { 01930 if( topographic_lateral_weights.length() == 0 ) 01931 product(dampening_expectation, lateral_weights, unit_values); 01932 else 01933 productTopoLateralWeights( dampening_expectation, unit_values ); 01934 return -dot(unit_values, bias) - 0.5 * dot(unit_values, dampening_expectation); 01935 } 01936 01937 real RBMLateralBinomialLayer::freeEnergyContribution(const Vec& unit_activations) 01938 const 01939 { 01940 PLERROR( 01941 "In RBMLateralBinomialLayer::freeEnergyContribution(): not implemented."); 01942 return -1; 01943 } 01944 01945 int RBMLateralBinomialLayer::getConfigurationCount() 01946 { 01947 return size < 31 ? 1<<size : INFINITE_CONFIGURATIONS; 01948 } 01949 01950 void RBMLateralBinomialLayer::getConfiguration(int conf_index, Vec& output) 01951 { 01952 PLASSERT( output.length() == size ); 01953 PLASSERT( conf_index >= 0 && conf_index < getConfigurationCount() ); 01954 01955 for ( int i = 0; i < size; ++i ) { 01956 output[i] = conf_index & 1; 01957 conf_index >>= 1; 01958 } 01959 } 01960 01961 } // end of namespace PLearn 01962 01963 01964 /* 01965 Local Variables: 01966 mode:c++ 01967 c-basic-offset:4 01968 c-file-style:"stroustrup" 01969 c-file-offsets:((innamespace . 0)(inline-open . 0)) 01970 indent-tabs-mode:nil 01971 fill-column:79 01972 End: 01973 */ 01974 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :