PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // TanhModule.cc 00004 // 00005 // Copyright (C) 2005 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: TanhModule.cc,v 1.2 2005/12/30 19:53:56 lamblinp Exp $ 00037 ******************************************************* */ 00038 00039 // Authors: Pascal Lamblin 00040 00044 #include "TanhModule.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 PLEARN_IMPLEMENT_OBJECT( 00050 TanhModule, 00051 "Propagates a (possibly scaled) Tanh function", 00052 "The output is ex_scale( in_scale * input ), for each coordinate.\n" 00053 "Usually in_scale = ex_scale = 1, but the values in_scale = 2/3 and\n" 00054 "ex_scale = 1.7159 are also used.\n" 00055 ); 00056 00057 TanhModule::TanhModule() : 00058 in_scale(1), 00059 ex_scale(1) 00060 { 00061 } 00062 00063 // Applies tanh to input, and propagates to output 00064 void TanhModule::fprop(const Vec& input, Vec& output) const 00065 { 00066 int in_size = input.size(); 00067 00068 // size check 00069 if( in_size != input_size ) 00070 { 00071 PLERROR("TanhModule::fprop: 'input.size()' should be equal\n" 00072 " to 'input_size' (%i != %i)\n", in_size, input_size); 00073 } 00074 00075 output.resize( output_size ); 00076 if (use_fast_approximations) 00077 for( int i=0 ; i<output_size ; i++ ) 00078 output[i] = ex_scale * fasttanh( in_scale * input[i] ); 00079 else 00080 for( int i=0 ; i<output_size ; i++ ) 00081 output[i] = ex_scale * tanh( in_scale * input[i] ); 00082 } 00083 00084 void TanhModule::fprop(const Mat& inputs, Mat& outputs) 00085 { 00086 int mbs=inputs.length(); 00087 outputs.resize(mbs,output_size); 00088 for (int i=0;i<mbs;i++) 00089 { 00090 Vec in_i = inputs(i); 00091 Vec out_i = outputs(i); 00092 fprop(in_i,out_i); 00093 } 00094 } 00095 00096 void TanhModule::bpropUpdate(const Mat& inputs, const Mat& outputs, 00097 Mat& input_gradients, const Mat& output_gradients, 00098 bool accumulate) 00099 { 00100 int mbs=inputs.length(); 00101 PLASSERT(mbs==outputs.length() && 00102 mbs==output_gradients.length()); 00103 input_gradients.resize(mbs,input_size); 00104 for (int i=0;i<mbs;i++) 00105 { 00106 Vec in_i = inputs(i); 00107 Vec out_i = outputs(i); 00108 Vec gin_i = input_gradients(i); 00109 Vec gout_i = output_gradients(i); 00110 bpropUpdate(in_i,out_i,gin_i,gout_i, 00111 accumulate); 00112 } 00113 } 00114 00115 // Nothing to update 00116 void TanhModule::bpropUpdate(const Vec& input, const Vec& output, 00117 const Vec& output_gradient) 00118 { 00119 int in_size = input.size(); 00120 int out_size = output.size(); 00121 int og_size = output_gradient.size(); 00122 00123 // size check 00124 if( in_size != input_size ) 00125 { 00126 PLERROR("TanhModule::bpropUpdate: 'input.size()' should be equal\n" 00127 " to 'input_size' (%i != %i)\n", in_size, input_size); 00128 } 00129 if( out_size != output_size ) 00130 { 00131 PLERROR("TanhModule::bpropUpdate: 'output.size()' should be equal\n" 00132 " to 'output_size' (%i != %i)\n", out_size, output_size); 00133 } 00134 if( og_size != output_size ) 00135 { 00136 PLERROR("TanhModule::bpropUpdate: 'output_gradient.size()' should\n" 00137 " be equal to 'output_size' (%i != %i)\n", 00138 og_size, output_size); 00139 } 00140 } 00141 00142 // Simply propagates output_gradient to input_gradient 00143 void TanhModule::bpropUpdate(const Vec& input, const Vec& output, 00144 Vec& input_gradient, const Vec& output_gradient, 00145 bool accumulate) 00146 { 00147 int in_size = input.size(); 00148 int out_size = output.size(); 00149 int og_size = output_gradient.size(); 00150 00151 // size check 00152 if( in_size != input_size ) 00153 { 00154 PLERROR("TanhModule::bpropUpdate: 'input.size()' should be equal\n" 00155 " to 'input_size' (%i != %i)\n", in_size, input_size); 00156 } 00157 if( out_size != output_size ) 00158 { 00159 PLERROR("TanhModule::bpropUpdate: 'output.size()' should be equal\n" 00160 " to 'output_size' (%i != %i)\n", out_size, output_size); 00161 } 00162 if( og_size != output_size ) 00163 { 00164 PLERROR("TanhModule::bpropUpdate: 'output_gradient.size()' should\n" 00165 " be equal to 'output_size' (%i != %i)\n", 00166 og_size, output_size); 00167 } 00168 00169 if( accumulate ) 00170 { 00171 PLASSERT_MSG( input_gradient.size() == input_size, 00172 "Cannot resize input_gradient AND accumulate into it" ); 00173 } 00174 else 00175 { 00176 input_gradient.resize( input_size ); 00177 input_gradient.clear(); 00178 } 00179 00180 for( int i=0 ; i<input_size ; i++ ) 00181 { 00182 real output_i = output[i]; 00183 input_gradient[i] += in_scale * 00184 (ex_scale - output_i*output_i/ex_scale)*output_gradient[i]; 00185 } 00186 00187 } 00188 00189 // Nothing to update 00190 void TanhModule::bbpropUpdate(const Vec& input, const Vec& output, 00191 const Vec& output_gradient, 00192 const Vec& output_diag_hessian) 00193 { 00194 int odh_size = output_diag_hessian.size(); 00195 if( odh_size != output_size ) 00196 { 00197 PLERROR("TanhModule::bbpropUpdate : 'output_diag_hessian.size()'\n" 00198 " should be equal to 'output_size' (%i != %i)\n", 00199 odh_size, output_size); 00200 } 00201 00202 bpropUpdate( input, output, output_gradient ); 00203 00204 } 00205 00206 // Propagates back output_gradient and output_diag_hessian 00207 void TanhModule::bbpropUpdate(const Vec& input, const Vec& output, 00208 Vec& input_gradient, 00209 const Vec& output_gradient, 00210 Vec& input_diag_hessian, 00211 const Vec& output_diag_hessian, 00212 bool accumulate) 00213 { 00214 int odh_size = output_diag_hessian.size(); 00215 00216 // size check 00217 // others size checks will be done in bpropUpdate() 00218 if( odh_size != output_size ) 00219 { 00220 PLERROR("TanhModule::bbpropUpdate : 'output_diag_hessian.size()'\n" 00221 " should be equal to 'output_size' (%i != %i)\n", 00222 odh_size, output_size); 00223 } 00224 00225 if( accumulate ) 00226 { 00227 PLASSERT_MSG( input_diag_hessian.size() == input_size, 00228 "Cannot resize input_diag_hessian AND accumulate into it" 00229 ); 00230 } 00231 else 00232 { 00233 input_diag_hessian.resize( input_size ); 00234 input_diag_hessian.clear(); 00235 } 00236 00237 for( int i=0 ; i<input_size ; i++ ) 00238 { 00239 real output_i = output[i]; 00240 real fprime_i = in_scale * (ex_scale-output_i*output_i / ex_scale); 00241 00242 if( estimate_simpler_diag_hessian ) 00243 input_diag_hessian[i] += 00244 fprime_i*fprime_i*output_diag_hessian[i]; 00245 else 00246 input_diag_hessian[i] += 00247 fprime_i*fprime_i*output_diag_hessian[i] 00248 - 2*in_scale/ex_scale*fprime_i*output_i*output_gradient[i]; 00249 } 00250 00251 bpropUpdate( input, output, input_gradient, output_gradient, accumulate ); 00252 } 00253 00254 00255 // Nothing to forget 00256 void TanhModule::forget() 00257 {} 00258 00259 00260 // ### Nothing to add here, simply calls build_ 00261 void TanhModule::build() 00262 { 00263 inherited::build(); 00264 build_(); 00265 } 00266 00267 void TanhModule::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00268 { 00269 inherited::makeDeepCopyFromShallowCopy(copies); 00270 } 00271 00272 void TanhModule::declareOptions(OptionList& ol) 00273 { 00274 declareOption(ol, "in_scale", &TanhModule::in_scale, 00275 OptionBase::buildoption, 00276 "Inner scale"); 00277 00278 declareOption(ol, "ex_scale", &TanhModule::ex_scale, 00279 OptionBase::buildoption, 00280 "External scale"); 00281 00282 inherited::declareOptions(ol); 00283 00284 redeclareOption(ol, "output_size", &TanhModule::output_size, 00285 OptionBase::learntoption, 00286 "output_size = input_size"); 00287 00288 } 00289 00290 void TanhModule::build_() 00291 { 00292 if( input_size < 0 ) 00293 PLERROR("TanhModule::build_: 'input_size' (%d) < 0", input_size); 00294 00295 output_size = input_size; 00296 } 00297 00298 00299 00300 00301 } // end of namespace PLearn 00302 00303 00304 /* 00305 Local Variables: 00306 mode:c++ 00307 c-basic-offset:4 00308 c-file-style:"stroustrup" 00309 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00310 indent-tabs-mode:nil 00311 fill-column:79 00312 End: 00313 */ 00314 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :