PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // RBMTruncExpLayer.cc 00004 // 00005 // Copyright (C) 2006 Pascal Lamblin & Dan Popovici 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin & Dan Popovici 00036 00041 #include "RBMTruncExpLayer.h" 00042 #include <plearn/math/TMat_maths.h> 00043 #include "RBMConnection.h" 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00048 PLEARN_IMPLEMENT_OBJECT( 00049 RBMTruncExpLayer, 00050 "RBM Layer where unit distribution is a truncated exponential in [0,1]", 00051 ""); 00052 00053 RBMTruncExpLayer::RBMTruncExpLayer( real the_learning_rate ) : 00054 inherited( the_learning_rate ) 00055 { 00056 } 00057 00058 RBMTruncExpLayer::RBMTruncExpLayer( int the_size, real the_learning_rate ) : 00059 inherited( the_learning_rate ) 00060 { 00061 size = the_size; 00062 activation.resize( the_size ); 00063 sample.resize( the_size ); 00064 expectation.resize( the_size ); 00065 bias.resize( the_size ); 00066 bias_pos_stats.resize( the_size ); 00067 bias_neg_stats.resize( the_size ); 00068 } 00069 00070 void RBMTruncExpLayer::generateSample() 00071 { 00072 PLASSERT_MSG(random_gen, 00073 "random_gen should be initialized before generating samples"); 00074 00075 /* The cumulative is : 00076 * C(U) = P(u<U | x) = (1 - exp(U a)) / (1 - exp(a)) if 0 < U < 1, 00077 * 0 if U <= 0 and 00078 * 1 if 1 <= U 00079 * 00080 * And the inverse, if 0 <= s <=1: 00081 * C^{-1}(s) = log(1 - s*(1 - exp(a)) / a 00082 */ 00083 00084 for( int i=0 ; i<size ; i++ ) 00085 { 00086 real s = random_gen->uniform_sample(); 00087 real a_i = activation[i]; 00088 00089 // Polynomial approximation to avoid numerical instability if a ~ 0 00090 // C^{-1}(s) ~ s + (s - s^2)/2 * a + O(a^2) 00091 if( fabs( a_i ) <= 1e-5 ) 00092 sample[i] = s + a_i*( s*(1 - s)/2 ); 00093 else 00094 sample[i] = logadd( pl_log( 1-s ), pl_log(s) + a_i ) / a_i; 00095 } 00096 } 00097 00098 void RBMTruncExpLayer::generateSamples() 00099 { 00100 PLASSERT_MSG(random_gen, 00101 "random_gen should be initialized before generating samples"); 00102 00103 PLCHECK_MSG(expectations_are_up_to_date, "Expectations should be computed " 00104 "before calling generateSamples()"); 00105 00106 PLASSERT( samples.width() == size && samples.length() == batch_size ); 00107 00108 for (int k = 0; k < batch_size; k++) 00109 for (int i=0 ; i<size ; i++) 00110 { 00111 real s = random_gen->uniform_sample(); 00112 real a_i = activations(k,i); 00113 if( fabs( a_i ) <= 1e-5 ) 00114 samples(k, i) = s + a_i*( s*(1 - s)/2 ); 00115 else 00116 samples(k, i) = logadd( pl_log( 1-s ), pl_log(s) + a_i ) / a_i; 00117 } 00118 00119 } 00120 00121 void RBMTruncExpLayer::computeExpectation() 00122 { 00123 if( expectation_is_up_to_date ) 00124 return; 00125 00126 /* Conditional expectation: 00127 * E[u|x] = 1/(1-exp(-a)) - 1/a 00128 */ 00129 00130 for( int i=0 ; i<size ; i++ ) 00131 { 00132 real a_i = activation[i]; 00133 00134 // Polynomial approximation to avoid numerical instability 00135 // f(a) = 1/2 + a/12 - a^3/720 + O(a^5) 00136 if( fabs( a_i ) <= 0.01 ) 00137 expectation[i] = 0.5 + a_i*(1./12. - a_i*a_i/720.); 00138 else 00139 expectation[i] = 1/(1-exp(-a_i)) - 1/a_i; 00140 } 00141 00142 expectation_is_up_to_date = true; 00143 } 00144 00145 00146 void RBMTruncExpLayer::computeExpectations() 00147 { 00148 if( expectations_are_up_to_date ) 00149 return; 00150 00151 /* Conditional expectation: 00152 * E[u|x] = 1/(1-exp(-a)) - 1/a 00153 */ 00154 00155 PLASSERT( expectations.width() == size 00156 && expectations.length() == batch_size ); 00157 00158 for (int k = 0; k < batch_size; k++) 00159 for( int i=0 ; i<size ; i++ ) 00160 { 00161 real a_i = activations(k,i); 00162 00163 // Polynomial approximation to avoid numerical instability 00164 // f(a) = 1/2 + a/12 - a^3/720 + O(a^5) 00165 if( fabs( a_i ) <= 0.01 ) 00166 expectations(k,i) = 0.5 + a_i*(1./12. - a_i*a_i/720.); 00167 else 00168 expectations(k,i) = 1/(1-exp(-a_i)) - 1/a_i; 00169 } 00170 00171 expectations_are_up_to_date = true; 00172 } 00173 00174 00175 void RBMTruncExpLayer::fprop( const Vec& input, Vec& output ) const 00176 { 00177 PLASSERT( input.size() == input_size ); 00178 output.resize( output_size ); 00179 00180 for( int i=0 ; i<size ; i++ ) 00181 { 00182 real a_i = input[i] + bias[i]; 00183 00184 // Polynomial approximation to avoid numerical instability 00185 // f(a) = 1/(1-exp(-a) - 1/a 00186 // f(a) = 1/2 + a/12 - a^3/720 + O(a^5) 00187 if( fabs( a_i ) <= 0.01 ) 00188 output[i] = 0.5 + a_i*(1./12. - a_i*a_i/720.); 00189 else 00190 output[i] = 1/(1-exp(-a_i)) - 1/a_i; 00191 } 00192 } 00193 00194 void RBMTruncExpLayer::bpropUpdate(const Vec& input, const Vec& output, 00195 Vec& input_gradient, 00196 const Vec& output_gradient, 00197 bool accumulate) 00198 { 00199 PLASSERT( input.size() == size ); 00200 PLASSERT( output.size() == size ); 00201 PLASSERT( output_gradient.size() == size ); 00202 00203 if( accumulate ) 00204 { 00205 PLASSERT_MSG( input_gradient.size() == size, 00206 "Cannot resize input_gradient AND accumulate into it" ); 00207 } 00208 else 00209 { 00210 input_gradient.resize( size ); 00211 input_gradient.clear(); 00212 } 00213 00214 if( momentum != 0. ) 00215 bias_inc.resize( size ); 00216 00217 // df/da = exp(a)/(1-exp(a))^2 - 1/a^2 00218 00219 for( int i=0 ; i<size ; i++ ) 00220 { 00221 real a_i = input[i] + bias[i]; 00222 real in_grad_i; 00223 00224 // Polynomial approximation to avoid numerical instability 00225 // df/da = -1/12 + a^2/240 + O(a^4) 00226 if( fabs( a_i ) <= 0.01 ) 00227 { 00228 in_grad_i = output_gradient[i] * ( -1./12. + a_i * a_i / 240. ); 00229 } 00230 else 00231 { 00232 real ea_i = exp( a_i ); 00233 in_grad_i = output_gradient[i] * ( 00234 ea_i/( (1 - ea_i) * (1 - ea_i) ) + 1/(a_i * a_i) ); 00235 } 00236 00237 input_gradient[i] += in_grad_i; 00238 00239 if( momentum == 0. ) 00240 { 00241 // update the bias: bias -= learning_rate * input_gradient 00242 bias[i] -= learning_rate * in_grad_i; 00243 } 00244 else 00245 { 00246 // The update rule becomes: 00247 // bias_inc = momentum * bias_inc - learning_rate * input_gradient 00248 // bias += bias_inc 00249 bias_inc[i] = momentum * bias_inc[i] - learning_rate * in_grad_i; 00250 bias[i] += bias_inc[i]; 00251 } 00252 } 00253 00254 applyBiasDecay(); 00255 } 00256 00257 00258 00259 void RBMTruncExpLayer::declareOptions(OptionList& ol) 00260 { 00261 /* 00262 declareOption(ol, "size", &RBMTruncExpLayer::size, 00263 OptionBase::buildoption, 00264 "Number of units."); 00265 */ 00266 // Now call the parent class' declareOptions 00267 inherited::declareOptions(ol); 00268 } 00269 00270 void RBMTruncExpLayer::build_() 00271 { 00272 } 00273 00274 void RBMTruncExpLayer::build() 00275 { 00276 inherited::build(); 00277 build_(); 00278 } 00279 00280 00281 void RBMTruncExpLayer::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00282 { 00283 inherited::makeDeepCopyFromShallowCopy(copies); 00284 } 00285 00286 00287 } // end of namespace PLearn 00288 00289 00290 /* 00291 Local Variables: 00292 mode:c++ 00293 c-basic-offset:4 00294 c-file-style:"stroustrup" 00295 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00296 indent-tabs-mode:nil 00297 fill-column:79 00298 End: 00299 */ 00300 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :