PLearn 0.1
RBMLateralBinomialLayer.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // RBMLateralBinomialLayer.cc
00004 //
00005 // Copyright (C) 2006 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Hugo Larochelle
00036 
00041 #include "RBMLateralBinomialLayer.h"
00042 #include <plearn/math/TMat_maths.h>
00043 #include "RBMConnection.h"
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00048 PLEARN_IMPLEMENT_OBJECT(
00049     RBMLateralBinomialLayer,
00050     "Layer in an RBM formed with binomial units and lateral connections.",
00051     "");
00052 
00053 RBMLateralBinomialLayer::RBMLateralBinomialLayer( real the_learning_rate ) :
00054     inherited( the_learning_rate ),
00055     n_lateral_connections_passes( 1 ),
00056     dampening_factor( 0. ),
00057     mean_field_precision_threshold( 0. ),
00058     topographic_length( -1 ),
00059     topographic_width( -1 ),
00060     topographic_patch_vradius( 5 ),
00061     topographic_patch_hradius( 5 ),
00062     topographic_lateral_weights_init_value( 0. ),
00063     do_not_learn_topographic_lateral_weights( false ),
00064     use_parametric_mean_field( false )
00065 {
00066 }
00067 
00068 void RBMLateralBinomialLayer::reset()
00069 {
00070     inherited::reset();
00071     lateral_weights_inc.clear();
00072 }
00073 
00074 void RBMLateralBinomialLayer::clearStats()
00075 {
00076     inherited::clearStats();
00077     lateral_weights_pos_stats.clear();
00078     lateral_weights_neg_stats.clear();
00079 }
00080 
00081 void RBMLateralBinomialLayer::forget()
00082 {
00083     inherited::forget();
00084     //real bu;
00085     //for( int i=0; i<lateral_weights.length(); i++)
00086     //    for( int j=0; j<lateral_weights.width(); j++)
00087     //    {
00088     //        bu = random_gen->bounded_uniform(-1.0/size,1.0/size);
00089     //        lateral_weights(i,j) = bu;
00090     //        lateral_weights(j,i) = bu;
00091     //    }
00092     lateral_weights.clear();
00093     // Set diagonal to 0
00094     if( lateral_weights.length() != 0 )
00095     {
00096         real *d = lateral_weights.data();
00097         for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
00098             *d = 0;
00099     }
00100 
00101     for( int i=0; i<topographic_lateral_weights.length(); i++ )
00102         //topographic_lateral_weights[i].clear();
00103         topographic_lateral_weights[i].fill( topographic_lateral_weights_init_value );
00104 
00105     mean_field_output_weights.clear();
00106     for( int i=0; i<mean_field_output_weights.length(); i++ )
00107         mean_field_output_weights(i,i) = 1;
00108     for( int i=0; i<mean_field_output_bias.length(); i++ )
00109         mean_field_output_bias[i] = -0.5;
00110 
00111 }
00112 
00114 // generateSample //
00116 void RBMLateralBinomialLayer::generateSample()
00117 {
00118     PLASSERT_MSG(random_gen,
00119                  "random_gen should be initialized before generating samples");
00120 
00121     PLCHECK_MSG(expectation_is_up_to_date, "Expectation should be computed "
00122             "before calling generateSample()");
00123 
00124     for( int i=0 ; i<size ; i++ )
00125         sample[i] = random_gen->binomial_sample( expectation[i] );
00126 }
00127 
00129 // generateSamples //
00131 void RBMLateralBinomialLayer::generateSamples()
00132 {
00133     PLASSERT_MSG(random_gen,
00134                  "random_gen should be initialized before generating samples");
00135 
00136     PLCHECK_MSG(expectations_are_up_to_date, "Expectations should be computed "
00137             "before calling generateSamples()");
00138 
00139     PLASSERT( samples.width() == size && samples.length() == batch_size );
00140 
00141     for (int k = 0; k < batch_size; k++) {
00142         for (int i=0 ; i<size ; i++)
00143             samples(k, i) = random_gen->binomial_sample( expectations(k, i) );
00144     }
00145 }
00146 
00148 // computeExpectation //
00150 void RBMLateralBinomialLayer::computeExpectation()
00151 {
00152     if( expectation_is_up_to_date )
00153         return;
00154 
00155     if( use_parametric_mean_field )
00156     {
00157         if (use_fast_approximations)
00158             for( int i=0 ; i<size ; i++ )
00159                 mean_field_input[i] = fastsigmoid( activation[i] );
00160         else
00161             for( int i=0 ; i<size ; i++ )
00162                 mean_field_input[i] = sigmoid( activation[i] );
00163 
00164         product(pre_sigmoid_mean_field_output, mean_field_output_weights, mean_field_input);
00165         pre_sigmoid_mean_field_output += mean_field_output_bias;
00166 
00167         if (use_fast_approximations)
00168             for( int i=0 ; i<size ; i++ )
00169                 expectation[i] = fastsigmoid( pre_sigmoid_mean_field_output[i] );
00170         else
00171             for( int i=0 ; i<size ; i++ )
00172                 expectation[i] = sigmoid( pre_sigmoid_mean_field_output[i] );
00173 
00174         // Update mean-field predictor, using KL-divergence gradient:
00175         //   dKL/dp_i = -activation[i] - \sum_{j \neq i} p_j + V_i h
00176         // where - V_i is the ith row of mean_field_output_weights
00177         //       - h is sigmoid(activation)
00178 
00179         real mean_field_i;
00180         product(temp_mean_field_gradient, lateral_weights, expectation);
00181         temp_mean_field_gradient += activation;
00182         for( int i=0 ; i<size ; i++ )
00183         {
00184             mean_field_i = expectation[i];
00185             temp_mean_field_gradient[i] = (pre_sigmoid_mean_field_output[i]
00186                                            - temp_mean_field_gradient[i])
00187                 * mean_field_i * (1 - mean_field_i);
00188         }
00189 
00190         externalProductScaleAcc( mean_field_output_weights, temp_mean_field_gradient,
00191                                  mean_field_input, -learning_rate );
00192         multiplyScaledAdd( temp_mean_field_gradient, 1.0, -learning_rate, mean_field_output_bias);
00193     }
00194     else
00195     {
00196         if( temp_output.length() != n_lateral_connections_passes+1 )
00197         {
00198             temp_output.resize(n_lateral_connections_passes+1);
00199             for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ )
00200                 temp_output[i].resize(size);
00201         }
00202 
00203         current_temp_output = temp_output[0];
00204         temp_output.last() = expectation;
00205 
00206         if (use_fast_approximations)
00207             for( int i=0 ; i<size ; i++ )
00208                 current_temp_output[i] = fastsigmoid( activation[i] );
00209         else
00210             for( int i=0 ; i<size ; i++ )
00211                 current_temp_output[i] = sigmoid( activation[i] );
00212 
00213         for( int t=0; t<n_lateral_connections_passes; t++ )
00214         {
00215             previous_temp_output = current_temp_output;
00216             current_temp_output = temp_output[t+1];
00217             if( topographic_lateral_weights.length() == 0 )
00218                 product(dampening_expectation, lateral_weights, previous_temp_output);
00219             else
00220                 productTopoLateralWeights( dampening_expectation, previous_temp_output );
00221             dampening_expectation += activation;
00222             if (use_fast_approximations)
00223             {
00224                 if( fast_exact_is_equal( dampening_factor, 0) )
00225                 {
00226                     for( int i=0 ; i<size ; i++ )
00227                         current_temp_output[i] = fastsigmoid( dampening_expectation[i] );
00228                 }
00229                 else
00230                 {
00231                     for( int i=0 ; i<size ; i++ )
00232                         current_temp_output[i] =
00233                             (1-dampening_factor) * fastsigmoid( dampening_expectation[i] )
00234                             + dampening_factor * previous_temp_output[i];
00235                 }
00236             }
00237             else
00238             {
00239                 if( fast_exact_is_equal( dampening_factor, 0) )
00240                 {
00241                     for( int i=0 ; i<size ; i++ )
00242                         current_temp_output[i] = sigmoid( dampening_expectation[i] );
00243                 }
00244                 else
00245                 {
00246                     for( int i=0 ; i<size ; i++ )
00247                         current_temp_output[i] =
00248                             (1-dampening_factor) * sigmoid( dampening_expectation[i] )
00249                             + dampening_factor * previous_temp_output[i];
00250                 }
00251             }
00252             if( !fast_exact_is_equal(mean_field_precision_threshold, 0.) &&
00253                 dist(current_temp_output, previous_temp_output,2)/size < mean_field_precision_threshold )
00254             {
00255                 expectation << current_temp_output;
00256                 break;
00257             }
00258             //cout << sqrt(max(square(current_temp_output-previous_temp_output))) << " ";
00259             //cout << dist(current_temp_output, previous_temp_output,2)/current_temp_output.length() << " ";
00260         }
00261         //cout << endl;
00262         //expectation << current_temp_output;
00263     }
00264     expectation_is_up_to_date = true;
00265 }
00266 
00268 // computeExpectations //
00270 void RBMLateralBinomialLayer::computeExpectations()
00271 {
00272     if( expectations_are_up_to_date )
00273         return;
00274 
00275     PLASSERT( expectations.width() == size
00276               && expectations.length() == batch_size );
00277 
00278     if( use_parametric_mean_field )
00279     {
00280         PLERROR("RBMLateralBinomialLayer::computeExpectations(): use_parametric_mean_field=true "
00281             "not implemented yet.");
00282     }
00283     else
00284     {
00285         dampening_expectations.resize( batch_size, size );
00286 
00287         if( temp_outputs.length() != n_lateral_connections_passes+1 )
00288         {
00289             temp_outputs.resize(n_lateral_connections_passes+1);
00290             for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ )
00291                 temp_outputs[i].resize( batch_size, size);
00292         }
00293 
00294         current_temp_outputs = temp_outputs[0];
00295         temp_outputs.last() = expectations;
00296 
00297         if (use_fast_approximations)
00298             for (int k = 0; k < batch_size; k++)
00299                 for (int i = 0 ; i < size ; i++)
00300                     current_temp_outputs(k, i) = fastsigmoid(activations(k, i));
00301         else
00302             for (int k = 0; k < batch_size; k++)
00303                 for (int i = 0 ; i < size ; i++)
00304                     current_temp_outputs(k, i) = sigmoid(activations(k, i));
00305 
00306         for( int t=0; t<n_lateral_connections_passes; t++ )
00307         {
00308             previous_temp_outputs = current_temp_outputs;
00309             current_temp_outputs = temp_outputs[t+1];
00310             if( topographic_lateral_weights.length() == 0 )
00311                 productTranspose(dampening_expectations, previous_temp_outputs,
00312                                  lateral_weights);
00313             else
00314                 for( int b = 0; b<dampening_expectations.length(); b++)
00315                     productTopoLateralWeights( dampening_expectations(b),
00316                                                previous_temp_outputs(b) );
00317 
00318             dampening_expectations += activations;
00319             if (use_fast_approximations)
00320             {
00321                 if( fast_exact_is_equal( dampening_factor, 0) )
00322                 {
00323                     for(int k = 0; k < batch_size; k++)
00324                         for( int i=0 ; i<size ; i++ )
00325                             current_temp_outputs(k, i) =
00326                                 fastsigmoid( dampening_expectations(k, i) );
00327                 }
00328                 else
00329                 {
00330                     for(int k = 0; k < batch_size; k++)
00331                         for( int i=0 ; i<size ; i++ )
00332                             current_temp_outputs(k, i) = (1-dampening_factor)
00333                                 * fastsigmoid( dampening_expectations(k, i) )
00334                                 + dampening_factor * previous_temp_outputs(k, i);
00335                 }
00336             }
00337             else
00338             {
00339                 if( fast_exact_is_equal( dampening_factor, 0) )
00340                 {
00341                     for(int k = 0; k < batch_size; k++)
00342                         for( int i=0 ; i<size ; i++ )
00343                             current_temp_outputs(k, i) =
00344                                 sigmoid( dampening_expectations(k, i) );
00345                 }
00346                 else
00347                 {
00348                     for(int k = 0; k < batch_size; k++)
00349                         for( int i=0 ; i<size ; i++ )
00350                             current_temp_outputs(k, i) = (1-dampening_factor)
00351                                 * sigmoid( dampening_expectations(k, i) )
00352                                 + dampening_factor * previous_temp_outputs(k, i);
00353                 }
00354             }
00355         }
00356         //expectations << current_temp_outputs;
00357     }
00358     expectations_are_up_to_date = true;
00359 }
00360 
00362 // fprop //
00364 void RBMLateralBinomialLayer::fprop( const Vec& input, Vec& output ) const
00365 {
00366     PLASSERT( input.size() == input_size );
00367     output.resize( output_size );
00368 
00369     add(bias, input, bias_plus_input);
00370 
00371     if( use_parametric_mean_field )
00372     {
00373         if (use_fast_approximations)
00374             for( int i=0 ; i<size ; i++ )
00375                 mean_field_input[i] = fastsigmoid( bias_plus_input[i] );
00376         else
00377             for( int i=0 ; i<size ; i++ )
00378                 mean_field_input[i] = sigmoid( bias_plus_input[i] );
00379 
00380         product(pre_sigmoid_mean_field_output, mean_field_output_weights, mean_field_input);
00381         pre_sigmoid_mean_field_output += mean_field_output_bias;
00382 
00383         if (use_fast_approximations)
00384             for( int i=0 ; i<size ; i++ )
00385                 output[i] = fastsigmoid( pre_sigmoid_mean_field_output[i] );
00386         else
00387             for( int i=0 ; i<size ; i++ )
00388                 output[i] = sigmoid( pre_sigmoid_mean_field_output[i] );
00389     }
00390     else
00391     {
00392 
00393         if( temp_output.length() != n_lateral_connections_passes+1 )
00394         {
00395             temp_output.resize(n_lateral_connections_passes+1);
00396             for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ )
00397                 temp_output[i].resize(size);
00398         }
00399 
00400         temp_output.last() = output;
00401         current_temp_output = temp_output[0];
00402 
00403         if (use_fast_approximations)
00404             for( int i=0 ; i<size ; i++ )
00405                 current_temp_output[i] = fastsigmoid( bias_plus_input[i] );
00406         else
00407             for( int i=0 ; i<size ; i++ )
00408                 current_temp_output[i] = sigmoid( bias_plus_input[i] );
00409 
00410         for( int t=0; t<n_lateral_connections_passes; t++ )
00411         {
00412             previous_temp_output = current_temp_output;
00413             current_temp_output = temp_output[t+1];
00414             if( topographic_lateral_weights.length() == 0 )
00415                 product(dampening_expectation, lateral_weights, previous_temp_output);
00416             else
00417                 productTopoLateralWeights( dampening_expectation, previous_temp_output );
00418             dampening_expectation += bias_plus_input;
00419             if (use_fast_approximations)
00420             {
00421                 if( fast_exact_is_equal( dampening_factor, 0) )
00422                 {
00423                     for( int i=0 ; i<size ; i++ )
00424                         current_temp_output[i] = fastsigmoid( dampening_expectation[i] );
00425                 }
00426                 else
00427                 {
00428                     for( int i=0 ; i<size ; i++ )
00429                         current_temp_output[i] =
00430                             (1-dampening_factor) * fastsigmoid( dampening_expectation[i] )
00431                             + dampening_factor * previous_temp_output[i];
00432                 }
00433             }
00434             else
00435             {
00436                 if( fast_exact_is_equal( dampening_factor, 0) )
00437                 {
00438                     for( int i=0 ; i<size ; i++ )
00439                         current_temp_output[i] = sigmoid( dampening_expectation[i] );
00440                 }
00441                 else
00442                 {
00443                     for( int i=0 ; i<size ; i++ )
00444                         current_temp_output[i] =
00445                             (1-dampening_factor) * sigmoid( dampening_expectation[i] )
00446                             + dampening_factor * previous_temp_output[i];
00447                 }
00448             }
00449         }
00450     }
00451 }
00452 
00453 void RBMLateralBinomialLayer::fprop( const Mat& inputs, Mat& outputs )
00454 {
00455     int mbatch_size = inputs.length();
00456     PLASSERT( inputs.width() == size );
00457     outputs.resize( mbatch_size, size );
00458 
00459     dampening_expectations.resize( mbatch_size, size );
00460 
00461     if( use_parametric_mean_field )
00462     {
00463         PLERROR("RBMLateralBinomialLayer::fprop: use_parametric_mean_field = true "
00464             "not implemented yet for batch mode.");
00465     }
00466     else
00467     {
00468         if(bias_plus_inputs.length() != inputs.length() ||
00469            bias_plus_inputs.width() != inputs.width())
00470             bias_plus_inputs.resize(inputs.length(), inputs.width());
00471         bias_plus_inputs << inputs;
00472         bias_plus_inputs += bias;
00473 
00474         if( temp_outputs.length() != n_lateral_connections_passes+1 )
00475         {
00476             temp_outputs.resize(n_lateral_connections_passes+1);
00477             for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ )
00478                 temp_outputs[i].resize(mbatch_size,size);
00479         }
00480 
00481         temp_outputs.last() = outputs;
00482         current_temp_outputs = temp_outputs[0];
00483 
00484         if (use_fast_approximations)
00485             for( int k = 0; k < mbatch_size; k++ )
00486                 for( int i = 0; i < size; i++ )
00487                     current_temp_outputs(k,i) = fastsigmoid( bias_plus_inputs(k,i) );
00488         else
00489             for( int k = 0; k < mbatch_size; k++ )
00490                 for( int i = 0; i < size; i++ )
00491                     current_temp_outputs(k,i) = sigmoid( bias_plus_inputs(k,i) );
00492 
00493         for( int t=0; t<n_lateral_connections_passes; t++ )
00494         {
00495             previous_temp_outputs = current_temp_outputs;
00496             current_temp_outputs = temp_outputs[t+1];
00497             if( topographic_lateral_weights.length() == 0 )
00498                 productTranspose(dampening_expectations, previous_temp_outputs,
00499                                  lateral_weights);
00500             else
00501                 for( int b = 0; b<dampening_expectations.length(); b++)
00502                     productTopoLateralWeights( dampening_expectations(b),
00503                                                previous_temp_outputs(b) );
00504 
00505             dampening_expectations += bias_plus_inputs;
00506             if (use_fast_approximations)
00507             {
00508                 if( fast_exact_is_equal( dampening_factor, 0) )
00509                 {
00510                     for(int k = 0; k < batch_size; k++)
00511                         for( int i=0 ; i<size ; i++ )
00512                             current_temp_outputs(k, i) =
00513                                 fastsigmoid( dampening_expectations(k, i) );
00514                 }
00515                 else
00516                 {
00517                     for(int k = 0; k < batch_size; k++)
00518                         for( int i=0 ; i<size ; i++ )
00519                             current_temp_outputs(k, i) = (1-dampening_factor)
00520                                 * fastsigmoid( dampening_expectations(k, i) )
00521                                 + dampening_factor * previous_temp_outputs(k, i);
00522                 }
00523             }
00524             else
00525             {
00526                 if( fast_exact_is_equal( dampening_factor, 0) )
00527                 {
00528                     for(int k = 0; k < batch_size; k++)
00529                         for( int i=0 ; i<size ; i++ )
00530                             current_temp_outputs(k, i) =
00531                                 sigmoid( dampening_expectations(k, i) );
00532                 }
00533                 else
00534                 {
00535                     for(int k = 0; k < batch_size; k++)
00536                         for( int i=0 ; i<size ; i++ )
00537                             current_temp_outputs(k, i) = (1-dampening_factor)
00538                                 * sigmoid( dampening_expectations(k, i) )
00539                                 + dampening_factor * previous_temp_outputs(k, i);
00540                 }
00541             }
00542         }
00543     }
00544 }
00545 
00546 void RBMLateralBinomialLayer::fprop( const Vec& input, const Vec& rbm_bias,
00547                               Vec& output ) const
00548 {
00549     PLASSERT( input.size() == input_size );
00550     PLASSERT( rbm_bias.size() == input_size );
00551     output.resize( output_size );
00552 
00553     add(rbm_bias, input, bias_plus_input);
00554 
00555     if( use_parametric_mean_field )
00556     {
00557         PLERROR("RBMLateralBinomialLayer::fprop: use_parametric_mean_field = true "
00558             "not implemented yet for rbm_bias input.");
00559     }
00560     else
00561     {
00562 
00563         if( temp_output.length() != n_lateral_connections_passes+1 )
00564         {
00565             temp_output.resize(n_lateral_connections_passes+1);
00566             for( int i=0 ; i<n_lateral_connections_passes+1 ; i++ )
00567                 temp_output[i].resize(size);
00568         }
00569 
00570         temp_output.last() = output;
00571         current_temp_output = temp_output[0];
00572 
00573         if (use_fast_approximations)
00574             for( int i=0 ; i<size ; i++ )
00575                 current_temp_output[i] = fastsigmoid( bias_plus_input[i] );
00576         else
00577             for( int i=0 ; i<size ; i++ )
00578                 current_temp_output[i] = sigmoid( bias_plus_input[i] );
00579 
00580         for( int t=0; t<n_lateral_connections_passes; t++ )
00581         {
00582             previous_temp_output = current_temp_output;
00583             current_temp_output = temp_output[t+1];
00584             if( topographic_lateral_weights.length() == 0 )
00585                 product(dampening_expectation, lateral_weights, previous_temp_output);
00586             else
00587                 productTopoLateralWeights( dampening_expectation, previous_temp_output );
00588             dampening_expectation += bias_plus_input;
00589             if (use_fast_approximations)
00590             {
00591                 if( fast_exact_is_equal( dampening_factor, 0) )
00592                 {
00593                     for( int i=0 ; i<size ; i++ )
00594                         current_temp_output[i] = fastsigmoid( dampening_expectation[i] );
00595                 }
00596                 else
00597                 {
00598                     for( int i=0 ; i<size ; i++ )
00599                         current_temp_output[i] =
00600                             (1-dampening_factor) * fastsigmoid( dampening_expectation[i] )
00601                             + dampening_factor * previous_temp_output[i];
00602                 }
00603             }
00604             else
00605             {
00606                 if( fast_exact_is_equal( dampening_factor, 0) )
00607                 {
00608                     for( int i=0 ; i<size ; i++ )
00609                         current_temp_output[i] = sigmoid( dampening_expectation[i] );
00610                 }
00611                 else
00612                 {
00613                     for( int i=0 ; i<size ; i++ )
00614                         current_temp_output[i] =
00615                             (1-dampening_factor) * sigmoid( dampening_expectation[i] )
00616                             + dampening_factor * previous_temp_output[i];
00617                 }
00618             }
00619         }
00620     }
00621 }
00622 
00623 // HUGO: NO 0.5! Computes mat[i][j] += 0.5 * (v1[i] * v2[j] + v1[j] * v2[i])
00624 // Computes mat[i][j] += (v1[i] * v2[j] + v1[j] * v2[i])
00625 void RBMLateralBinomialLayer::externalSymetricProductAcc(const Mat& mat, const Vec& v1, const Vec& v2)
00626 {
00627 #ifdef BOUNDCHECK
00628     if (v1.length()!=mat.length() || mat.width()!=v2.length()
00629         || v1.length() != v2.length())
00630         PLERROR("externalSymetricProductAcc(Mat,Vec,Vec), incompatible "
00631                 "arguments sizes");
00632 #endif
00633 
00634     real* v_1=v1.data();
00635     real* v_2=v2.data();
00636     real* mp = mat.data();
00637     int l = mat.length();
00638     int w = mat.width();
00639 
00640     if(mat.isCompact())
00641     {
00642         real* pv11 = v_1;
00643         real* pv21 = v_2;
00644         for(int i=0; i<l; i++)
00645         {
00646             real* pv22 = v_2;
00647             real* pv12 = v_1;
00648             real val1 = *pv11++;
00649             real val2 = *pv21++;
00650             for(int j=0; j<w; j++)
00651                 //*mp++ += 0.5 * (val1 * *pv22++ + val2 * *pv12++) ;
00652                 *mp++ += (val1 * *pv22++ + val2 * *pv12++) ;
00653         }
00654     }
00655     else
00656     {
00657         cerr << "!";
00658         for (int i=0;i<l;i++)
00659         {
00660             real* mi = mat[i];
00661             real v1i = v_1[i];
00662             real v2i = v_2[i];
00663             for (int j=0;j<w;j++)
00664                 //mi[j] += 0.5 * ( v1i * v_2[j] + v2i * v_1[j]);
00665                 mi[j] += ( v1i * v_2[j] + v2i * v_1[j]);
00666         }
00667     }
00668 }
00669 
00670 void RBMLateralBinomialLayer::productTopoLateralWeights(const Vec& result,
00671                                                         const Vec& input ) const
00672 {
00673     // Could be made faster, in terms of memory access
00674     result.clear();
00675     int connected_neuron;
00676     int wi;
00677     real* current_weights;
00678     int neuron_v, neuron_h;
00679     int vmin, vmax, hmin, hmax;
00680     for( int i=0; i<topographic_lateral_weights.length(); i++ )
00681     {
00682         neuron_v = i/topographic_width;
00683         neuron_h = i%topographic_width;
00684         wi = 0;
00685         current_weights = topographic_lateral_weights[i].data();
00686 
00687         vmin = neuron_v < topographic_patch_vradius ?
00688             - neuron_v : - topographic_patch_vradius;
00689         vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ?
00690             topographic_length - neuron_v - 1: topographic_patch_vradius;
00691 
00692         hmin = neuron_h < topographic_patch_hradius ?
00693             - neuron_h : - topographic_patch_hradius;
00694         hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ?
00695             topographic_width - neuron_h - 1: topographic_patch_hradius;
00696 
00697         for( int j = -1 * topographic_patch_vradius;
00698              j <= topographic_patch_vradius ; j++ )
00699         {
00700             for( int k = -1 * topographic_patch_hradius;
00701                  k <= topographic_patch_hradius; k++ )
00702             {
00703                 connected_neuron = (i+j*topographic_width)+k;
00704                 if( connected_neuron != i )
00705                 {
00706                     if( j >= vmin && j <= vmax &&
00707                         k >= hmin && k <= hmax )
00708                         result[i] += input[connected_neuron]
00709                             * current_weights[wi];
00710                     wi++;
00711                 }
00712             }
00713         }
00714     }
00715 }
00716 
00717 void RBMLateralBinomialLayer::productTopoLateralWeightsGradients(
00718     const Vec& input,
00719     const Vec& input_gradient,
00720     const Vec& result_gradient,
00721     const TVec< Vec >& weights_gradient
00722     )
00723 {
00724     // Could be made faster, in terms of memory access
00725     int connected_neuron;
00726     int wi;
00727     real* current_weights;
00728     real* current_weights_gradient;
00729     int neuron_v, neuron_h;
00730     int vmin, vmax, hmin, hmax;
00731     real result_gradient_i;
00732     real input_i;
00733     for( int i=0; i<topographic_lateral_weights.length(); i++ )
00734     {
00735         neuron_v = i/topographic_width;
00736         neuron_h = i%topographic_width;
00737         wi = 0;
00738         current_weights = topographic_lateral_weights[i].data();
00739         current_weights_gradient = weights_gradient[i].data();
00740 
00741         vmin = neuron_v < topographic_patch_vradius ?
00742             - neuron_v : - topographic_patch_vradius;
00743         vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ?
00744             topographic_length - neuron_v - 1: topographic_patch_vradius;
00745 
00746         hmin = neuron_h < topographic_patch_hradius ?
00747             - neuron_h : - topographic_patch_hradius;
00748         hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ?
00749             topographic_width - neuron_h - 1: topographic_patch_hradius;
00750 
00751         result_gradient_i = result_gradient[i];
00752         input_i = input[i];
00753 
00754         for( int j = -1 * topographic_patch_vradius;
00755              j <= topographic_patch_vradius ; j++ )
00756         {
00757             for( int k = -1 * topographic_patch_hradius;
00758                  k <= topographic_patch_hradius; k++ )
00759             {
00760                 connected_neuron = (i+j*topographic_width)+k;
00761                 if( connected_neuron != i )
00762                 {
00763                     if( j >= vmin && j <= vmax &&
00764                         k >= hmin && k <= hmax )
00765                     {
00766                         input_gradient[connected_neuron] +=
00767                             result_gradient_i * current_weights[wi];
00768                         current_weights_gradient[wi] +=
00769                             //0.5 * ( result_gradient_i * input[connected_neuron] +
00770                             ( result_gradient_i * input[connected_neuron] +
00771                               input_i * result_gradient[connected_neuron] );
00772                     }
00773                     wi++;
00774                 }
00775             }
00776         }
00777     }
00778 }
00779 
00780 void RBMLateralBinomialLayer::updateTopoLateralWeightsCD(
00781     const Vec& pos_values,
00782     const Vec& neg_values  )
00783 {
00784     if( !do_not_learn_topographic_lateral_weights )
00785     {
00786 
00787         // Could be made faster, in terms of memory access
00788         int connected_neuron;
00789         int wi;
00790         int neuron_v, neuron_h;
00791         int vmin, vmax, hmin, hmax;
00792         real* current_weights;
00793         real pos_values_i;
00794         real neg_values_i;
00795         for( int i=0; i<topographic_lateral_weights.length(); i++ )
00796         {
00797             neuron_v = i/topographic_width;
00798             neuron_h = i%topographic_width;
00799             wi = 0;
00800 
00801             vmin = neuron_v < topographic_patch_vradius ?
00802                 - neuron_v : - topographic_patch_vradius;
00803             vmax = topographic_length - neuron_v - 1 < topographic_patch_vradius ?
00804                 topographic_length - neuron_v - 1: topographic_patch_vradius;
00805 
00806             hmin = neuron_h < topographic_patch_hradius ?
00807                 - neuron_h : - topographic_patch_hradius;
00808             hmax = topographic_width - neuron_h - 1 < topographic_patch_hradius ?
00809                 topographic_width - neuron_h - 1: topographic_patch_hradius;
00810 
00811             current_weights = topographic_lateral_weights[i].data();
00812             pos_values_i = pos_values[i];
00813             neg_values_i = neg_values[i];
00814 
00815             for( int j = - topographic_patch_vradius;
00816                  j <= topographic_patch_vradius ; j++ )
00817             {
00818                 for( int k = -topographic_patch_hradius;
00819                      k <= topographic_patch_hradius; k++ )
00820                 {
00821                     connected_neuron = (i+j*topographic_width)+k;
00822                     if( connected_neuron != i )
00823                     {
00824                         if( j >= vmin && j <= vmax &&
00825                             k >= hmin && k <= hmax )
00826                         {
00827                             current_weights[wi] +=
00828                                 //learning_rate * 0.5 * (
00829                                 learning_rate * (
00830                                     pos_values_i * pos_values[connected_neuron] -
00831                                     neg_values_i * neg_values[connected_neuron] );
00832                         }
00833                         wi++;
00834                     }
00835                 }
00836             }
00837         }
00838     }
00839 }
00840 
00842 // bpropUpdate //
00844 void RBMLateralBinomialLayer::bpropUpdate(const Vec& input, const Vec& output,
00845                                    Vec& input_gradient,
00846                                    const Vec& output_gradient,
00847                                    bool accumulate)
00848 {
00849     PLASSERT( input.size() == size );
00850     PLASSERT( output.size() == size );
00851     PLASSERT( output_gradient.size() == size );
00852 
00853     if( accumulate )
00854         PLASSERT_MSG( input_gradient.size() == size,
00855                       "Cannot resize input_gradient AND accumulate into it" );
00856     else
00857     {
00858         input_gradient.resize( size );
00859         input_gradient.clear();
00860     }
00861 
00862     //if( momentum != 0. )
00863     //    bias_inc.resize( size );
00864 
00865     if( use_parametric_mean_field )
00866     {
00867         real mean_field_i;
00868         for( int i=0 ; i<size ; i++ )
00869         {
00870             mean_field_i = output[i];
00871             temp_mean_field_gradient[i] = output_gradient[i] * mean_field_i * (1 - mean_field_i);
00872         }
00873 
00874         transposeProductAcc( input_gradient, mean_field_output_weights, temp_mean_field_gradient );
00875 
00876         externalProductScaleAcc( mean_field_output_weights, temp_mean_field_gradient,
00877                                  mean_field_input, -learning_rate );
00878         multiplyScaledAdd( temp_mean_field_gradient, 1.0, -learning_rate, mean_field_output_bias);
00879 
00880         real input_mean_field_i;
00881         for( int i=0 ; i<size ; i++ )
00882         {
00883             input_mean_field_i = mean_field_input[i];
00884             input_gradient[i] = input_gradient[i] * input_mean_field_i * (1 - input_mean_field_i);
00885         }
00886     }
00887     else
00888     {
00889         temp_input_gradient.clear();
00890         temp_mean_field_gradient << output_gradient;
00891         current_temp_output = output;
00892         lateral_weights_gradient.clear();
00893         for( int i=0; i<topographic_lateral_weights_gradient.length(); i++)
00894             topographic_lateral_weights_gradient[i].clear();
00895 
00896         real output_i;
00897         for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- )
00898         {
00899             for( int i=0 ; i<size ; i++ )
00900             {
00901                 output_i = current_temp_output[i];
00902 
00903                 // Contribution from the mean field approximation
00904                 temp_mean_field_gradient2[i] =  (1-dampening_factor)*
00905                     output_i * (1-output_i) * temp_mean_field_gradient[i];
00906 
00907                 // Contribution from the dampening
00908                 temp_mean_field_gradient[i] *= dampening_factor;
00909             }
00910 
00911             // Input gradient contribution
00912             temp_input_gradient += temp_mean_field_gradient2;
00913 
00914             // Lateral weights gradient contribution
00915             if( topographic_lateral_weights.length() == 0)
00916             {
00917                 externalSymetricProductAcc( lateral_weights_gradient,
00918                                             temp_mean_field_gradient2,
00919                                             temp_output[t] );
00920 
00921                 transposeProductAcc(temp_mean_field_gradient, lateral_weights,
00922                                     temp_mean_field_gradient2);
00923             }
00924             else
00925             {
00926                 productTopoLateralWeightsGradients(
00927                     temp_output[t],
00928                     temp_mean_field_gradient,
00929                     temp_mean_field_gradient2,
00930                     topographic_lateral_weights_gradient);
00931             }
00932 
00933             current_temp_output = temp_output[t];
00934         }
00935 
00936         for( int i=0 ; i<size ; i++ )
00937         {
00938             output_i = current_temp_output[i];
00939             temp_mean_field_gradient[i] *= output_i * (1-output_i);
00940         }
00941 
00942         temp_input_gradient += temp_mean_field_gradient;
00943 
00944         input_gradient += temp_input_gradient;
00945 
00946         // Update bias
00947         real in_grad_i;
00948         for( int i=0 ; i<size ; i++ )
00949         {
00950             in_grad_i = temp_input_gradient[i];
00951             if( momentum == 0. )
00952             {
00953                 // update the bias: bias -= learning_rate * input_gradient
00954                 bias[i] -= learning_rate * in_grad_i;
00955             }
00956             else
00957             {
00958                 // The update rule becomes:
00959                 // bias_inc = momentum * bias_inc - learning_rate * input_gradient
00960                 // bias += bias_inc
00961                 bias_inc[i] = momentum * bias_inc[i] - learning_rate * in_grad_i;
00962                 bias[i] += bias_inc[i];
00963             }
00964         }
00965 
00966         if( topographic_lateral_weights.length() == 0)
00967         {
00968             if( momentum == 0. )
00969             {
00970                 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate,
00971                                    lateral_weights);
00972             }
00973             else
00974             {
00975                 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate,
00976                                    lateral_weights_inc);
00977                 lateral_weights += lateral_weights_inc;
00978             }
00979         }
00980         else
00981         {
00982             if( !do_not_learn_topographic_lateral_weights )
00983             {
00984                 if( momentum == 0. )
00985                     for( int i=0; i<topographic_lateral_weights.length(); i++ )
00986                         multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0,
00987                                            -learning_rate,
00988                                            topographic_lateral_weights[i]);
00989 
00990                 else
00991                     PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
00992                             "topographic weights");
00993             }
00994         }
00995 
00996         // Set diagonal to 0
00997         if( lateral_weights.length() != 0 )
00998         {
00999             real *d = lateral_weights.data();
01000             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01001                 *d = 0;
01002         }
01003     }
01004 }
01005 
01006 void RBMLateralBinomialLayer::bpropUpdate(const Mat& inputs, const Mat& outputs,
01007                                    Mat& input_gradients,
01008                                    const Mat& output_gradients,
01009                                    bool accumulate)
01010 {
01011     PLASSERT( inputs.width() == size );
01012     PLASSERT( outputs.width() == size );
01013     PLASSERT( output_gradients.width() == size );
01014 
01015     int mbatch_size = inputs.length();
01016     PLASSERT( outputs.length() == mbatch_size );
01017     PLASSERT( output_gradients.length() == mbatch_size );
01018 
01019     if( accumulate )
01020     {
01021         PLASSERT_MSG( input_gradients.width() == size &&
01022                 input_gradients.length() == mbatch_size,
01023                 "Cannot resize input_gradients and accumulate into it" );
01024     }
01025     else
01026     {
01027         input_gradients.resize(mbatch_size, size);
01028         input_gradients.clear();
01029     }
01030 
01031     //if( momentum != 0. )
01032     //    bias_inc.resize( size );
01033 
01034     // TODO Can we do this more efficiently? (using BLAS)
01035 
01036     // We use the average gradient over the mini-batch.
01037     real avg_lr = learning_rate / inputs.length();
01038 
01039     if( use_parametric_mean_field )
01040     {
01041         PLERROR("RBMLateralBinomialLayer::bpropUpdate: use_parametric_mean_field=true "
01042             "not implemented yet for batch mode.");
01043     }
01044     else
01045     {
01046         lateral_weights_gradient.clear();
01047         real output_i;
01048         for (int j = 0; j < mbatch_size; j++)
01049         {
01050             temp_input_gradient.clear();
01051             temp_mean_field_gradient << output_gradients(j);
01052             current_temp_output = outputs(j);
01053 
01054             for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- )
01055             {
01056 
01057                 for( int i=0 ; i<size ; i++ )
01058                 {
01059                     output_i = current_temp_output[i];
01060 
01061                     // Contribution from the mean field approximation
01062                     temp_mean_field_gradient2[i] =  (1-dampening_factor)*
01063                         output_i * (1-output_i) * temp_mean_field_gradient[i];
01064 
01065                     // Contribution from the dampening
01066                     temp_mean_field_gradient[i] *= dampening_factor;
01067                 }
01068 
01069                 // Input gradient contribution
01070                 temp_input_gradient += temp_mean_field_gradient2;
01071 
01072                 // Lateral weights gradient contribution
01073                 if( topographic_lateral_weights.length() == 0)
01074                 {
01075 
01076                     externalSymetricProductAcc( lateral_weights_gradient,
01077                                                 temp_mean_field_gradient2,
01078                                                 temp_outputs[t](j) );
01079 
01080                     transposeProductAcc(temp_mean_field_gradient, lateral_weights,
01081                                         temp_mean_field_gradient2);
01082                 }
01083                 else
01084                 {
01085                     productTopoLateralWeightsGradients(
01086                         temp_outputs[t](j),
01087                         temp_mean_field_gradient,
01088                         temp_mean_field_gradient2,
01089                         topographic_lateral_weights_gradient);
01090                 }
01091 
01092                 current_temp_output = temp_outputs[t](j);
01093             }
01094 
01095             for( int i=0 ; i<size ; i++ )
01096             {
01097                 output_i = current_temp_output[i];
01098                 temp_mean_field_gradient[i] *= output_i * (1-output_i);
01099             }
01100 
01101             temp_input_gradient += temp_mean_field_gradient;
01102 
01103             input_gradients(j) += temp_input_gradient;
01104 
01105             // Update bias
01106             real in_grad_i;
01107             for( int i=0 ; i<size ; i++ )
01108             {
01109                 in_grad_i = temp_input_gradient[i];
01110                 if( momentum == 0. )
01111                 {
01112                     // update the bias: bias -= learning_rate * input_gradient
01113                     bias[i] -= avg_lr * in_grad_i;
01114                 }
01115                 else
01116                     PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
01117                             "momentum with mini-batches");
01118             }
01119         }
01120 
01121         if( topographic_lateral_weights.length() == 0)
01122         {
01123             if( momentum == 0. )
01124                 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate,
01125                                    lateral_weights);
01126             else
01127                 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
01128                         "momentum with mini-batches");
01129         }
01130         else
01131         {
01132             if( !do_not_learn_topographic_lateral_weights )
01133             {
01134                 if( momentum == 0. )
01135                     for( int i=0; i<topographic_lateral_weights.length(); i++ )
01136                         multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0,
01137                                            -learning_rate,
01138                                            topographic_lateral_weights[i]);
01139 
01140                 else
01141                     PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
01142                             "topographic weights");
01143             }
01144 
01145         }
01146 
01147         // Set diagonal to 0
01148         if( lateral_weights.length() != 0 )
01149         {
01150             real *d = lateral_weights.data();
01151             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01152                 *d = 0;
01153         }
01154     }
01155 }
01156 
01157 
01159 void RBMLateralBinomialLayer::bpropUpdate(const Vec& input, const Vec& rbm_bias,
01160                                    const Vec& output,
01161                                    Vec& input_gradient, Vec& rbm_bias_gradient,
01162                                    const Vec& output_gradient)
01163 {
01164     PLASSERT( input.size() == size );
01165     PLASSERT( rbm_bias.size() == size );
01166     PLASSERT( output.size() == size );
01167     PLASSERT( output_gradient.size() == size );
01168     input_gradient.resize( size );
01169     rbm_bias_gradient.resize( size );
01170 
01171     if( use_parametric_mean_field )
01172     {
01173         PLERROR("RBMLateralBinomialLayer::bpropUpdate: use_parametric_mean_field=true "
01174                 "not implemented yet for bias input.");
01175     }
01176     else
01177     {
01178         temp_input_gradient.clear();
01179         temp_mean_field_gradient << output_gradient;
01180         current_temp_output = output;
01181         lateral_weights_gradient.clear();
01182 
01183         real output_i;
01184         for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- )
01185         {
01186 
01187             for( int i=0 ; i<size ; i++ )
01188             {
01189                 output_i = current_temp_output[i];
01190 
01191                 // Contribution from the mean field approximation
01192                 temp_mean_field_gradient2[i] =  (1-dampening_factor)*
01193                     output_i * (1-output_i) * temp_mean_field_gradient[i];
01194 
01195                 // Contribution from the dampening
01196                 temp_mean_field_gradient[i] *= dampening_factor;
01197             }
01198 
01199             // Input gradient contribution
01200             temp_input_gradient += temp_mean_field_gradient2;
01201 
01202             // Lateral weights gradient contribution
01203             if( topographic_lateral_weights.length() == 0)
01204             {
01205 
01206                 externalSymetricProductAcc( lateral_weights_gradient,
01207                                             temp_mean_field_gradient2,
01208                                             temp_output[t] );
01209 
01210                 transposeProductAcc(temp_mean_field_gradient, lateral_weights,
01211                                     temp_mean_field_gradient2);
01212             }
01213             else
01214             {
01215                 productTopoLateralWeightsGradients(
01216                     temp_output[t],
01217                     temp_mean_field_gradient,
01218                     temp_mean_field_gradient2,
01219                     topographic_lateral_weights_gradient);
01220             }
01221 
01222             current_temp_output = temp_output[t];
01223         }
01224 
01225         for( int i=0 ; i<size ; i++ )
01226         {
01227             output_i = current_temp_output[i];
01228             temp_mean_field_gradient[i] *= output_i * (1-output_i);
01229         }
01230 
01231         temp_input_gradient += temp_mean_field_gradient;
01232 
01233         input_gradient << temp_input_gradient;
01234         rbm_bias_gradient << temp_input_gradient;
01235 
01236         if( topographic_lateral_weights.length() == 0)
01237         {
01238             if( momentum == 0. )
01239             {
01240                 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate,
01241                                    lateral_weights);
01242             }
01243             else
01244             {
01245                 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate,
01246                                    lateral_weights_inc);
01247                 lateral_weights += lateral_weights_inc;
01248             }
01249         }
01250         else
01251         {
01252             if( !do_not_learn_topographic_lateral_weights )
01253             {
01254                 if( momentum == 0. )
01255                     for( int i=0; i<topographic_lateral_weights.length(); i++ )
01256                         multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0,
01257                                            -learning_rate,
01258                                            topographic_lateral_weights[i]);
01259 
01260                 else
01261                     PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
01262                             "topographic weights");
01263             }
01264         }
01265 
01266         // Set diagonal to 0
01267         if( lateral_weights.length() != 0 )
01268         {
01269             real *d = lateral_weights.data();
01270             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01271                 *d = 0;
01272         }
01273     }
01274 }
01275 
01276 real RBMLateralBinomialLayer::fpropNLL(const Vec& target)
01277 {
01278     PLASSERT( target.size() == input_size );
01279     computeExpectation();
01280 
01281     real ret = 0;
01282     real target_i, expectation_i;
01283     for( int i=0 ; i<size ; i++ )
01284     {
01285         target_i = target[i];
01286         expectation_i = expectation[i];
01287         // TODO: implement more numerically stable version
01288         if(!fast_exact_is_equal(target_i,0.0))
01289             ret -= target_i*safeflog(expectation_i) ;
01290         if(!fast_exact_is_equal(target_i,1.0))
01291             ret -= (1-target_i)*safeflog(1-expectation_i);
01292     }
01293     return ret;
01294 }
01295 
01296 void RBMLateralBinomialLayer::fpropNLL(const Mat& targets, const Mat& costs_column)
01297 {
01298     computeExpectations();
01299 
01300     PLASSERT( targets.width() == input_size );
01301     PLASSERT( targets.length() == batch_size );
01302     PLASSERT( costs_column.width() == 1 );
01303     PLASSERT( costs_column.length() == batch_size );
01304 
01305     for (int k=0;k<batch_size;k++) // loop over minibatch
01306     {
01307         real nll = 0;
01308         real* expectation = expectations[k];
01309         real* target = targets[k];
01310         for( int i=0 ; i<size ; i++ ) // loop over outputs
01311         {
01312             // TODO: implement more numerically stable version
01313             if(!fast_exact_is_equal(target[i],0.0))
01314                 nll -= target[i]*safeflog(expectation[i]) ;
01315             if(!fast_exact_is_equal(target[i],1.0))
01316                 nll -= (1-target[i])*safeflog(1-expectation[i]);
01317         }
01318         costs_column(k,0) = nll;
01319     }
01320 }
01321 
01322 void RBMLateralBinomialLayer::bpropNLL(const Vec& target, real nll, Vec& bias_gradient)
01323 {
01324     computeExpectation();
01325 
01326     PLASSERT( target.size() == input_size );
01327     bias_gradient.resize( size );
01328     bias_gradient.clear();
01329 
01330     if( use_parametric_mean_field )
01331     {
01332         PLERROR("RBMLateralBinomialLayer::bpropNLL: use_parametric_mean_field=true "
01333                 "not implemented yet.");
01334     }
01335     else
01336     {
01337         // bias_gradient = expectation - target
01338         substract(expectation, target, temp_mean_field_gradient);
01339 
01340         current_temp_output = expectation;
01341         lateral_weights_gradient.clear();
01342 
01343         real output_i;
01344         for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- )
01345         {
01346             for( int i=0 ; i<size ; i++ )
01347             {
01348                 output_i = current_temp_output[i];
01349 
01350                 // Contribution from the mean field approximation
01351                 temp_mean_field_gradient2[i] =  (1-dampening_factor)*
01352                     output_i * (1-output_i) * temp_mean_field_gradient[i];
01353 
01354                 // Contribution from the dampening
01355                 temp_mean_field_gradient[i] *= dampening_factor;
01356             }
01357 
01358             // Input gradient contribution
01359             bias_gradient += temp_mean_field_gradient2;
01360 
01361             // Lateral weights gradient contribution
01362             if( topographic_lateral_weights.length() == 0)
01363             {
01364                 externalSymetricProductAcc( lateral_weights_gradient,
01365                                             temp_mean_field_gradient2,
01366                                             temp_output[t] );
01367 
01368                 transposeProductAcc(temp_mean_field_gradient, lateral_weights,
01369                                     temp_mean_field_gradient2);
01370             }
01371             else
01372             {
01373                 productTopoLateralWeightsGradients(
01374                     temp_output[t],
01375                     temp_mean_field_gradient,
01376                     temp_mean_field_gradient2,
01377                     topographic_lateral_weights_gradient);
01378             }
01379 
01380             current_temp_output = temp_output[t];
01381         }
01382 
01383         for( int i=0 ; i<size ; i++ )
01384         {
01385             output_i = current_temp_output[i];
01386             temp_mean_field_gradient[i] *= output_i * (1-output_i);
01387         }
01388 
01389         bias_gradient += temp_mean_field_gradient;
01390 
01391         if( topographic_lateral_weights.length() == 0)
01392         {
01393             // Update lateral connections
01394             if( momentum == 0. )
01395             {
01396                 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate,
01397                                    lateral_weights);
01398             }
01399             else
01400             {
01401                 multiplyScaledAdd( lateral_weights_gradient, momentum, -learning_rate,
01402                                    lateral_weights_inc);
01403                 lateral_weights += lateral_weights_inc;
01404             }
01405         }
01406         else
01407         {
01408             if( !do_not_learn_topographic_lateral_weights )
01409             {
01410                 if( momentum == 0. )
01411                     for( int i=0; i<topographic_lateral_weights.length(); i++ )
01412                         multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0,
01413                                            -learning_rate,
01414                                            topographic_lateral_weights[i]);
01415 
01416                 else
01417                     PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for "
01418                             "topographic weights");
01419             }
01420         }
01421         // Set diagonal to 0
01422         if( lateral_weights.length() != 0 )
01423         {
01424             real *d = lateral_weights.data();
01425             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01426                 *d = 0;
01427         }
01428     }
01429 }
01430 
01431 void RBMLateralBinomialLayer::bpropNLL(const Mat& targets, const Mat& costs_column,
01432                                 Mat& bias_gradients)
01433 {
01434     computeExpectations();
01435 
01436     PLASSERT( targets.width() == input_size );
01437     PLASSERT( targets.length() == batch_size );
01438     PLASSERT( costs_column.width() == 1 );
01439     PLASSERT( costs_column.length() == batch_size );
01440     bias_gradients.resize( batch_size, size );
01441     bias_gradients.clear();
01442 
01443 
01444     // TODO Can we do this more efficiently? (using BLAS)
01445 
01446     if( use_parametric_mean_field )
01447     {
01448         PLERROR("RBMLateralBinomialLayer::bpropNLL: use_parametric_mean_field=true "
01449                 "not implemented yet.");
01450     }
01451     else
01452     {
01453 
01454         // We use the average gradient over the mini-batch.
01455         lateral_weights_gradient.clear();
01456         real output_i;
01457         for (int j = 0; j < batch_size; j++)
01458         {
01459             // top_gradient = expectations(j) - targets(j)
01460             substract(expectations(j), targets(j), temp_mean_field_gradient);
01461             current_temp_output = expectations(j);
01462 
01463             for( int t=n_lateral_connections_passes-1 ; t>=0 ; t-- )
01464             {
01465                 for( int i=0 ; i<size ; i++ )
01466                 {
01467                     output_i = current_temp_output[i];
01468 
01469                     // Contribution from the mean field approximation
01470                     temp_mean_field_gradient2[i] =  (1-dampening_factor)*
01471                         output_i * (1-output_i) * temp_mean_field_gradient[i];
01472 
01473                     // Contribution from the dampening
01474                     temp_mean_field_gradient[i] *= dampening_factor;
01475                 }
01476 
01477                 // Input gradient contribution
01478                 bias_gradients(j) += temp_mean_field_gradient2;
01479 
01480                 // Lateral weights gradient contribution
01481                 if( topographic_lateral_weights.length() == 0)
01482                 {
01483 
01484                     externalSymetricProductAcc( lateral_weights_gradient,
01485                                                 temp_mean_field_gradient2,
01486                                                 temp_outputs[t](j) );
01487 
01488                     transposeProductAcc(temp_mean_field_gradient, lateral_weights,
01489                                         temp_mean_field_gradient2);
01490                 }
01491                 else
01492                 {
01493                     productTopoLateralWeightsGradients(
01494                         temp_outputs[t](j),
01495                         temp_mean_field_gradient,
01496                         temp_mean_field_gradient2,
01497                         topographic_lateral_weights_gradient);
01498                 }
01499                 current_temp_output = temp_outputs[t](j);
01500             }
01501 
01502             for( int i=0 ; i<size ; i++ )
01503             {
01504                 output_i = current_temp_output[i];
01505                 temp_mean_field_gradient[i] *= output_i * (1-output_i);
01506             }
01507 
01508             bias_gradients(j) += temp_mean_field_gradient;
01509         }
01510 
01511         // Update lateral connections
01512         if( topographic_lateral_weights.length() == 0 )
01513         {
01514             if( momentum == 0. )
01515                 multiplyScaledAdd( lateral_weights_gradient, 1.0, -learning_rate,
01516                                    lateral_weights);
01517             else
01518                 PLERROR("In RBMLateralBinomialLayer:bpropUpdate - Not implemented for "
01519                         "momentum with mini-batches");
01520         }
01521         else
01522         {
01523             if( !do_not_learn_topographic_lateral_weights )
01524             {
01525                 if( momentum == 0. )
01526                     for( int i=0; i<topographic_lateral_weights.length(); i++ )
01527                         multiplyScaledAdd( topographic_lateral_weights_gradient[i], 1.0,
01528                                            -learning_rate,
01529                                            topographic_lateral_weights[i]);
01530 
01531                 else
01532                     PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for "
01533                             "topographic weights");
01534             }
01535         }
01536 
01537         // Set diagonal to 0
01538         if( lateral_weights.length() != 0 )
01539         {
01540             real *d = lateral_weights.data();
01541             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01542                 *d = 0;
01543         }
01544     }
01545 }
01546 
01547 void RBMLateralBinomialLayer::accumulatePosStats( const Vec& pos_values )
01548 {
01549     inherited::accumulatePosStats( pos_values);
01550     externalProductAcc(lateral_weights_pos_stats, pos_values, pos_values);
01551 }
01552 
01553 void RBMLateralBinomialLayer::accumulatePosStats( const Mat& pos_values )
01554 {
01555     inherited::accumulatePosStats( pos_values);
01556     transposeProductAcc(lateral_weights_pos_stats, pos_values, pos_values);
01557 }
01558 
01559 void RBMLateralBinomialLayer::accumulateNegStats( const Vec& neg_values )
01560 {
01561     inherited::accumulateNegStats( neg_values);
01562     externalProductAcc(lateral_weights_neg_stats, neg_values, neg_values);
01563 }
01564 
01565 void RBMLateralBinomialLayer::accumulateNegStats( const Mat& neg_values )
01566 {
01567     inherited::accumulateNegStats( neg_values);
01568     transposeProductAcc(lateral_weights_neg_stats, neg_values, neg_values);
01569 }
01570 
01571 
01572 void RBMLateralBinomialLayer::update()
01573 {
01574     //real pos_factor = 0.5 * learning_rate / pos_count;
01575     //real neg_factor = - 0.5 * learning_rate / neg_count;
01576     real pos_factor = learning_rate / pos_count;
01577     real neg_factor = - learning_rate / neg_count;
01578 
01579     if( topographic_lateral_weights.length() != 0 )
01580         PLERROR("In RBMLateralBinomialLayer:update - Not implemented for "
01581                 "topographic weights");
01582 
01583     // Update lateral connections
01584     if( momentum == 0. )
01585     {
01586         multiplyScaledAdd( lateral_weights_pos_stats, neg_factor, pos_factor,
01587                            lateral_weights_neg_stats);
01588         lateral_weights += lateral_weights_neg_stats;
01589     }
01590     else
01591     {
01592         multiplyScaledAdd( lateral_weights_pos_stats, neg_factor, pos_factor,
01593                            lateral_weights_neg_stats);
01594         multiplyScaledAdd( lateral_weights_neg_stats, momentum, 1.0,
01595                            lateral_weights_inc);
01596         lateral_weights += lateral_weights_inc;
01597     }
01598 
01599     // Set diagonal to 0
01600     if( lateral_weights.length() != 0 )
01601     {
01602         real *d = lateral_weights.data();
01603         for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01604             *d = 0;
01605     }
01606 
01607     // Call to update() must be at the end, since update() calls clearStats()!
01608     inherited::update();
01609 }
01610 
01611 void RBMLateralBinomialLayer::update( const Vec& grad)
01612 {
01613     inherited::update( grad );
01614     PLWARNING("RBMLateralBinomialLayer::update( grad ): does not update the\n"
01615         "lateral connections.");
01616 }
01617 
01618 void RBMLateralBinomialLayer::update( const Vec& pos_values, const Vec& neg_values )
01619 {
01620     // Update lateral connections
01621     if( topographic_lateral_weights.length() == 0 )
01622     {
01623         if( momentum == 0. )
01624         {
01625             externalProductScaleAcc(lateral_weights, pos_values, pos_values,
01626                                     //0.5 * learning_rate);
01627                                     learning_rate);
01628             externalProductScaleAcc(lateral_weights, neg_values, neg_values,
01629                                     //- 0.5 * learning_rate);
01630                                     -learning_rate);
01631         }
01632         else
01633         {
01634             lateral_weights_inc *= momentum;
01635             externalProductScaleAcc(lateral_weights_inc, pos_values, pos_values,
01636                                     //0.5 * learning_rate);
01637                                     learning_rate);
01638             externalProductScaleAcc(lateral_weights_inc, neg_values, neg_values,
01639                                     //- 0.5 * learning_rate);
01640                                     - learning_rate);
01641             lateral_weights += lateral_weights_inc;
01642         }
01643 
01644         // Set diagonal to 0
01645         if( lateral_weights.length() != 0 )
01646         {
01647             real *d = lateral_weights.data();
01648             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01649                 *d = 0;
01650         }
01651     }
01652     else
01653     {
01654         if( momentum == 0. )
01655             updateTopoLateralWeightsCD(pos_values, neg_values);
01656         else
01657             PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for "
01658                     "topographic weights");
01659     }
01660 
01661     inherited::update( pos_values, neg_values );
01662 }
01663 
01664 void RBMLateralBinomialLayer::update( const Mat& pos_values, const Mat& neg_values )
01665 {
01666     int n = pos_values.length();
01667     PLASSERT( neg_values.length() == n );
01668 
01669     // We take the average gradient over the mini-batch.
01670     //real avg_lr = 0.5 * learning_rate / n;
01671     real avg_lr = learning_rate / n;
01672 
01673     // Update lateral connections
01674     if( topographic_lateral_weights.length() == 0 )
01675     {
01676         if( momentum == 0. )
01677         {
01678             transposeProductScaleAcc(lateral_weights, pos_values, pos_values,
01679                                      avg_lr, 1);
01680             transposeProductScaleAcc(lateral_weights, neg_values, neg_values,
01681                                      -avg_lr, 1);
01682         }
01683         else
01684         {
01685             lateral_weights_inc *= momentum;
01686             transposeProductScaleAcc(lateral_weights_inc, pos_values, pos_values,
01687                                      avg_lr, 1);
01688             transposeProductScaleAcc(lateral_weights_inc, neg_values, neg_values,
01689                                      -avg_lr, 1);
01690             lateral_weights += lateral_weights_inc;
01691         }
01692 
01693         // Set diagonal to 0
01694         if( lateral_weights.length() != 0 )
01695         {
01696             real *d = lateral_weights.data();
01697             for (int i=0; i<lateral_weights.length(); i++,d+=lateral_weights.mod()+1)
01698                 *d = 0;
01699         }
01700     }
01701     else
01702     {
01703         if( momentum == 0. )
01704         {
01705             for(int b=0; b<pos_values.length(); b++)
01706                 updateTopoLateralWeightsCD(pos_values(b), neg_values(b));
01707 
01708         }
01709         else
01710             PLERROR("In RBMLateralBinomialLayer:bpropNLL - Not implemented for "
01711                     "topographic weights");
01712     }
01713 
01714     inherited::update( pos_values, neg_values );
01715 }
01716 
01717 void RBMLateralBinomialLayer::updateCDandGibbs( const Mat& pos_values,
01718                                  const Mat& cd_neg_values,
01719                                  const Mat& gibbs_neg_values,
01720                                  real background_gibbs_update_ratio )
01721 {
01722     inherited::updateCDandGibbs( pos_values, cd_neg_values,
01723                                  gibbs_neg_values, background_gibbs_update_ratio );
01724     PLERROR("In RBMLateralBinomialLayer::updateCDandGibbs(): not implemented yet.");
01725 }
01726 
01727 void RBMLateralBinomialLayer::updateGibbs( const Mat& pos_values,
01728                                            const Mat& gibbs_neg_values)
01729 {
01730     inherited::updateGibbs( pos_values, gibbs_neg_values );
01731     PLERROR("In RBMLateralBinomialLayer::updateCDandGibbs(): not implemented yet.");
01732 }
01733 
01734 void RBMLateralBinomialLayer::declareOptions(OptionList& ol)
01735 {
01736     declareOption(ol, "n_lateral_connections_passes",
01737                   &RBMLateralBinomialLayer::n_lateral_connections_passes,
01738                   OptionBase::buildoption,
01739                   "Number of passes through the lateral connections.\n");
01740 
01741     declareOption(ol, "dampening_factor",
01742                   &RBMLateralBinomialLayer::dampening_factor,
01743                   OptionBase::buildoption,
01744                   "Dampening factor ( expectation_t = (1-df) * currrent mean field"
01745                   " + df * expectation_{t-1}).\n");
01746 
01747     declareOption(ol, "mean_field_precision_threshold",
01748                   &RBMLateralBinomialLayer::mean_field_precision_threshold,
01749                   OptionBase::buildoption,
01750                   "Mean-field precision threshold that, once reached, stops the mean-field\n"
01751                   "expectation approximation computation. Used only in computeExpectation().\n"
01752                   "Precision is computed as:\n"
01753                   "  dist(last_mean_field, current_mean_field) / size\n");
01754 
01755     declareOption(ol, "topographic_length",
01756                   &RBMLateralBinomialLayer::topographic_length,
01757                   OptionBase::buildoption,
01758                   "Length of the topographic map.\n");
01759 
01760     declareOption(ol, "topographic_width",
01761                   &RBMLateralBinomialLayer::topographic_width,
01762                   OptionBase::buildoption,
01763                   "Width of the topographic map.\n");
01764 
01765     declareOption(ol, "topographic_patch_vradius",
01766                   &RBMLateralBinomialLayer::topographic_patch_vradius,
01767                   OptionBase::buildoption,
01768                   "Vertical radius of the topographic local weight patches.\n");
01769 
01770     declareOption(ol, "topographic_patch_hradius",
01771                   &RBMLateralBinomialLayer::topographic_patch_hradius,
01772                   OptionBase::buildoption,
01773                   "Horizontal radius of the topographic local weight patches.\n");
01774 
01775     declareOption(ol, "topographic_lateral_weights_init_value",
01776                   &RBMLateralBinomialLayer::topographic_lateral_weights_init_value,
01777                   OptionBase::buildoption,
01778                   "Initial value for the topographic_lateral_weights.\n");
01779 
01780     declareOption(ol, "do_not_learn_topographic_lateral_weights",
01781                   &RBMLateralBinomialLayer::do_not_learn_topographic_lateral_weights,
01782                   OptionBase::buildoption,
01783                   "Indication that the topographic_lateral_weights should\n"
01784                   "be fixed at their initial value.\n");
01785 
01786     declareOption(ol, "lateral_weights",
01787                   &RBMLateralBinomialLayer::lateral_weights,
01788                   OptionBase::learntoption,
01789                   "Lateral connections.\n");
01790 
01791     declareOption(ol, "topographic_lateral_weights",
01792                   &RBMLateralBinomialLayer::topographic_lateral_weights,
01793                   OptionBase::learntoption,
01794                   "Local topographic lateral connections.\n");
01795 
01796     declareOption(ol, "use_parametric_mean_field",
01797                   &RBMLateralBinomialLayer::use_parametric_mean_field,
01798                   OptionBase::buildoption,
01799                   "Indication that a parametric predictor of the mean-field\n"
01800                   "approximation of the hidden layer conditional distribution.\n");
01801 
01802     declareOption(ol, "mean_field_output_weights",
01803                   &RBMLateralBinomialLayer::mean_field_output_weights,
01804                   OptionBase::learntoption,
01805                   "Output weights of the mean field predictor.\n");
01806 
01807     declareOption(ol, "mean_field_output_bias",
01808                   &RBMLateralBinomialLayer::mean_field_output_bias,
01809                   OptionBase::learntoption,
01810                   "Output bias of the mean field predictor.\n");
01811 
01812     // Now call the parent class' declareOptions
01813     inherited::declareOptions(ol);
01814 }
01815 
01816 void RBMLateralBinomialLayer::build_()
01817 {
01818     if( n_lateral_connections_passes == 0 &&
01819         !fast_exact_is_equal(dampening_factor, 0) )
01820         PLERROR("In RBMLateralBinomialLayer::build_(): when not using the lateral\n"
01821                 "connections, dampening_factor should be 0.");
01822 
01823     if( dampening_factor < 0 || dampening_factor > 1)
01824         PLERROR("In RBMLateralBinomialLayer::build_(): dampening_factor should be\n"
01825                 "in [0,1].");
01826 
01827     if( n_lateral_connections_passes < 0 )
01828         PLERROR("In RBMLateralBinomialLayer::build_(): n_lateral_connections_passes\n"
01829                 " should be >= 0.");
01830 
01831     if( use_parametric_mean_field && topographic_length > 0 && topographic_width > 0 )
01832         PLERROR("RBMLateralBinomialLayer::build_(): can't use parametric mean field "
01833             "and topographic lateral connections.");
01834 
01835     if( use_parametric_mean_field )
01836     {
01837         mean_field_output_weights.resize(size,size);
01838         mean_field_output_bias.resize(size);
01839         mean_field_input.resize(size);
01840         pre_sigmoid_mean_field_output.resize(size);
01841     }
01842 
01843     if( topographic_length <= 0 || topographic_width <= 0)
01844     {
01845         lateral_weights.resize(size,size);
01846 
01847         lateral_weights_gradient.resize(size,size);
01848         lateral_weights_pos_stats.resize(size,size);
01849         lateral_weights_neg_stats.resize(size,size);
01850         if( momentum != 0. )
01851         {
01852             bias_inc.resize( size );
01853             lateral_weights_inc.resize(size,size);
01854         }
01855     }
01856     else
01857     {
01858         if( size != topographic_length * topographic_width )
01859             PLERROR( "In RBMLateralBinomialLayer::build_(): size != "
01860                      "topographic_length * topographic_width.\n" );
01861 
01862         if( topographic_length-1 <= 2*topographic_patch_vradius )
01863             PLERROR( "In RBMLateralBinomialLayer::build_(): "
01864                      "topographic_patch_vradius is too large.\n" );
01865 
01866         if( topographic_width-1 <= 2*topographic_patch_hradius )
01867             PLERROR( "In RBMLateralBinomialLayer::build_(): "
01868                      "topographic_patch_hradius is too large.\n" );
01869 
01870         topographic_lateral_weights.resize(size);
01871         topographic_lateral_weights_gradient.resize(size);
01872         for( int i=0; i<size; i++ )
01873         {
01874             topographic_lateral_weights[i].resize(
01875                 ( 2 * topographic_patch_hradius + 1 ) *
01876                 ( 2 * topographic_patch_vradius + 1 ) - 1 );
01877             topographic_lateral_weights_gradient[i].resize(
01878                 ( 2 * topographic_patch_hradius + 1 ) *
01879                 ( 2 * topographic_patch_vradius + 1 ) - 1 );
01880         }
01881 
01882         // Should probably have separate lateral_weights_*_stats
01883     }
01884 
01885     // Resizing temporary variables
01886     dampening_expectation.resize(size);
01887     temp_input_gradient.resize(size);
01888     temp_mean_field_gradient.resize(size);
01889     temp_mean_field_gradient2.resize(size);
01890 }
01891 
01892 void RBMLateralBinomialLayer::build()
01893 {
01894     inherited::build();
01895     build_();
01896 }
01897 
01898 
01899 void RBMLateralBinomialLayer::makeDeepCopyFromShallowCopy(CopiesMap& copies)
01900 {
01901     inherited::makeDeepCopyFromShallowCopy(copies);
01902     deepCopyField(lateral_weights,copies);
01903     deepCopyField(topographic_lateral_weights,copies);
01904     deepCopyField(lateral_weights_pos_stats,copies);
01905     deepCopyField(lateral_weights_neg_stats,copies);
01906     deepCopyField(dampening_expectation,copies);
01907     deepCopyField(dampening_expectations,copies);
01908     deepCopyField(mean_field_input,copies);
01909     deepCopyField(pre_sigmoid_mean_field_output,copies);
01910     deepCopyField(temp_output,copies);
01911     deepCopyField(temp_outputs,copies);
01912     deepCopyField(current_temp_output,copies);
01913     deepCopyField(previous_temp_output,copies);
01914     deepCopyField(current_temp_outputs,copies);
01915     deepCopyField(previous_temp_outputs,copies);
01916     deepCopyField(bias_plus_input,copies);
01917     deepCopyField(bias_plus_inputs,copies);
01918     deepCopyField(temp_input_gradient,copies);
01919     deepCopyField(temp_mean_field_gradient,copies);
01920     deepCopyField(temp_mean_field_gradient2,copies);
01921     deepCopyField(lateral_weights_gradient,copies);
01922     deepCopyField(lateral_weights_inc,copies);
01923     deepCopyField(topographic_lateral_weights_gradient,copies);
01924     deepCopyField(mean_field_output_weights,copies);
01925     deepCopyField(mean_field_output_bias,copies);
01926 }
01927 
01928 real RBMLateralBinomialLayer::energy(const Vec& unit_values) const
01929 {
01930     if( topographic_lateral_weights.length() == 0 )
01931         product(dampening_expectation, lateral_weights, unit_values);
01932     else
01933         productTopoLateralWeights( dampening_expectation, unit_values );
01934     return -dot(unit_values, bias) - 0.5 * dot(unit_values, dampening_expectation);
01935 }
01936 
01937 real RBMLateralBinomialLayer::freeEnergyContribution(const Vec& unit_activations)
01938     const
01939 {
01940     PLERROR(
01941         "In RBMLateralBinomialLayer::freeEnergyContribution(): not implemented.");
01942     return -1;
01943 }
01944 
01945 int RBMLateralBinomialLayer::getConfigurationCount()
01946 {
01947     return size < 31 ? 1<<size : INFINITE_CONFIGURATIONS;
01948 }
01949 
01950 void RBMLateralBinomialLayer::getConfiguration(int conf_index, Vec& output)
01951 {
01952     PLASSERT( output.length() == size );
01953     PLASSERT( conf_index >= 0 && conf_index < getConfigurationCount() );
01954 
01955     for ( int i = 0; i < size; ++i ) {
01956         output[i] = conf_index & 1;
01957         conf_index >>= 1;
01958     }
01959 }
01960 
01961 } // end of namespace PLearn
01962 
01963 
01964 /*
01965   Local Variables:
01966   mode:c++
01967   c-basic-offset:4
01968   c-file-style:"stroustrup"
01969   c-file-offsets:((innamespace . 0)(inline-open . 0))
01970   indent-tabs-mode:nil
01971   fill-column:79
01972   End:
01973 */
01974 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines