PLearn 0.1
RBMTruncExpLayer.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // RBMTruncExpLayer.cc
00004 //
00005 // Copyright (C) 2006 Pascal Lamblin & Dan Popovici
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin & Dan Popovici
00036 
00041 #include "RBMTruncExpLayer.h"
00042 #include <plearn/math/TMat_maths.h>
00043 #include "RBMConnection.h"
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00048 PLEARN_IMPLEMENT_OBJECT(
00049     RBMTruncExpLayer,
00050     "RBM Layer where unit distribution is a truncated exponential in [0,1]",
00051     "");
00052 
00053 RBMTruncExpLayer::RBMTruncExpLayer( real the_learning_rate ) :
00054     inherited( the_learning_rate )
00055 {
00056 }
00057 
00058 RBMTruncExpLayer::RBMTruncExpLayer( int the_size, real the_learning_rate ) :
00059     inherited( the_learning_rate )
00060 {
00061     size = the_size;
00062     activation.resize( the_size );
00063     sample.resize( the_size );
00064     expectation.resize( the_size );
00065     bias.resize( the_size );
00066     bias_pos_stats.resize( the_size );
00067     bias_neg_stats.resize( the_size );
00068 }
00069 
00070 void RBMTruncExpLayer::generateSample()
00071 {
00072     PLASSERT_MSG(random_gen,
00073                  "random_gen should be initialized before generating samples");
00074 
00075     /* The cumulative is :
00076      * C(U) = P(u<U | x) = (1 - exp(U a)) / (1 - exp(a)) if 0 < U < 1,
00077      *        0 if U <= 0 and
00078      *        1 if 1 <= U
00079      *
00080      * And the inverse, if 0 <= s <=1:
00081      * C^{-1}(s) = log(1 - s*(1 - exp(a)) / a
00082      */
00083 
00084     for( int i=0 ; i<size ; i++ )
00085     {
00086         real s = random_gen->uniform_sample();
00087         real a_i = activation[i];
00088 
00089         // Polynomial approximation to avoid numerical instability if a ~ 0
00090         // C^{-1}(s) ~ s + (s - s^2)/2 * a + O(a^2)
00091         if( fabs( a_i ) <= 1e-5 )
00092             sample[i] = s + a_i*( s*(1 - s)/2 );
00093         else
00094             sample[i] = logadd( pl_log( 1-s ), pl_log(s) + a_i ) / a_i;
00095     }
00096 }
00097 
00098 void RBMTruncExpLayer::generateSamples()
00099 {
00100     PLASSERT_MSG(random_gen,
00101                  "random_gen should be initialized before generating samples");
00102 
00103     PLCHECK_MSG(expectations_are_up_to_date, "Expectations should be computed "
00104             "before calling generateSamples()");
00105 
00106     PLASSERT( samples.width() == size && samples.length() == batch_size );
00107 
00108     for (int k = 0; k < batch_size; k++)
00109         for (int i=0 ; i<size ; i++)
00110         {
00111             real s = random_gen->uniform_sample();
00112             real a_i = activations(k,i);
00113             if( fabs( a_i ) <= 1e-5 )
00114                 samples(k, i) = s + a_i*( s*(1 - s)/2 );
00115             else
00116                 samples(k, i) = logadd( pl_log( 1-s ), pl_log(s) + a_i ) / a_i;
00117         }
00118 
00119 }
00120 
00121 void RBMTruncExpLayer::computeExpectation()
00122 {
00123     if( expectation_is_up_to_date )
00124         return;
00125 
00126     /* Conditional expectation:
00127      * E[u|x] = 1/(1-exp(-a)) - 1/a
00128      */
00129 
00130     for( int i=0 ; i<size ; i++ )
00131     {
00132         real a_i = activation[i];
00133 
00134         // Polynomial approximation to avoid numerical instability
00135         // f(a) = 1/2 + a/12 - a^3/720 + O(a^5)
00136         if( fabs( a_i ) <= 0.01 )
00137             expectation[i] = 0.5 + a_i*(1./12. - a_i*a_i/720.);
00138         else
00139             expectation[i] = 1/(1-exp(-a_i)) - 1/a_i;
00140     }
00141 
00142     expectation_is_up_to_date = true;
00143 }
00144 
00145 
00146 void RBMTruncExpLayer::computeExpectations()
00147 {
00148     if( expectations_are_up_to_date )
00149         return;
00150 
00151     /* Conditional expectation:
00152      * E[u|x] = 1/(1-exp(-a)) - 1/a
00153      */
00154 
00155     PLASSERT( expectations.width() == size
00156               && expectations.length() == batch_size );
00157 
00158     for (int k = 0; k < batch_size; k++)
00159         for( int i=0 ; i<size ; i++ )
00160         {
00161             real a_i = activations(k,i);
00162 
00163             // Polynomial approximation to avoid numerical instability
00164             // f(a) = 1/2 + a/12 - a^3/720 + O(a^5)
00165             if( fabs( a_i ) <= 0.01 )
00166                 expectations(k,i) = 0.5 + a_i*(1./12. - a_i*a_i/720.);
00167             else
00168                 expectations(k,i) = 1/(1-exp(-a_i)) - 1/a_i;
00169         }
00170 
00171     expectations_are_up_to_date = true;
00172 }
00173 
00174 
00175 void RBMTruncExpLayer::fprop( const Vec& input, Vec& output ) const
00176 {
00177     PLASSERT( input.size() == input_size );
00178     output.resize( output_size );
00179 
00180     for( int i=0 ; i<size ; i++ )
00181     {
00182         real a_i = input[i] + bias[i];
00183 
00184         // Polynomial approximation to avoid numerical instability
00185         // f(a) = 1/(1-exp(-a) - 1/a
00186         // f(a) = 1/2 + a/12 - a^3/720 + O(a^5)
00187         if( fabs( a_i ) <= 0.01 )
00188             output[i] = 0.5 + a_i*(1./12. - a_i*a_i/720.);
00189         else
00190             output[i] = 1/(1-exp(-a_i)) - 1/a_i;
00191     }
00192 }
00193 
00194 void RBMTruncExpLayer::bpropUpdate(const Vec& input, const Vec& output,
00195                                    Vec& input_gradient,
00196                                    const Vec& output_gradient,
00197                                    bool accumulate)
00198 {
00199     PLASSERT( input.size() == size );
00200     PLASSERT( output.size() == size );
00201     PLASSERT( output_gradient.size() == size );
00202 
00203     if( accumulate )
00204     {
00205         PLASSERT_MSG( input_gradient.size() == size,
00206                       "Cannot resize input_gradient AND accumulate into it" );
00207     }
00208     else
00209     {
00210         input_gradient.resize( size );
00211         input_gradient.clear();
00212     }
00213 
00214     if( momentum != 0. )
00215         bias_inc.resize( size );
00216 
00217     // df/da = exp(a)/(1-exp(a))^2 - 1/a^2
00218 
00219     for( int i=0 ; i<size ; i++ )
00220     {
00221         real a_i = input[i] + bias[i];
00222         real in_grad_i;
00223 
00224         // Polynomial approximation to avoid numerical instability
00225         // df/da = -1/12 + a^2/240 + O(a^4)
00226         if( fabs( a_i ) <= 0.01 )
00227         {
00228             in_grad_i = output_gradient[i] * ( -1./12. + a_i * a_i / 240. );
00229         }
00230         else
00231         {
00232             real ea_i = exp( a_i );
00233             in_grad_i = output_gradient[i] * (
00234                 ea_i/( (1 - ea_i) * (1 - ea_i) ) + 1/(a_i * a_i) );
00235         }
00236 
00237         input_gradient[i] += in_grad_i;
00238 
00239         if( momentum == 0. )
00240         {
00241             // update the bias: bias -= learning_rate * input_gradient
00242             bias[i] -= learning_rate * in_grad_i;
00243         }
00244         else
00245         {
00246             // The update rule becomes:
00247             // bias_inc = momentum * bias_inc - learning_rate * input_gradient
00248             // bias += bias_inc
00249             bias_inc[i] = momentum * bias_inc[i] - learning_rate * in_grad_i;
00250             bias[i] += bias_inc[i];
00251         }
00252     }
00253 
00254     applyBiasDecay();
00255 }
00256 
00257 
00258 
00259 void RBMTruncExpLayer::declareOptions(OptionList& ol)
00260 {
00261 /*
00262     declareOption(ol, "size", &RBMTruncExpLayer::size,
00263                   OptionBase::buildoption,
00264                   "Number of units.");
00265 */
00266     // Now call the parent class' declareOptions
00267     inherited::declareOptions(ol);
00268 }
00269 
00270 void RBMTruncExpLayer::build_()
00271 {
00272 }
00273 
00274 void RBMTruncExpLayer::build()
00275 {
00276     inherited::build();
00277     build_();
00278 }
00279 
00280 
00281 void RBMTruncExpLayer::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00282 {
00283     inherited::makeDeepCopyFromShallowCopy(copies);
00284 }
00285 
00286 
00287 } // end of namespace PLearn
00288 
00289 
00290 /*
00291   Local Variables:
00292   mode:c++
00293   c-basic-offset:4
00294   c-file-style:"stroustrup"
00295   c-file-offsets:((innamespace . 0)(inline-open . 0))
00296   indent-tabs-mode:nil
00297   fill-column:79
00298   End:
00299 */
00300 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines