PLearn 0.1
NLLErrModule.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // NLLErrModule.cc
00004 //
00005 // Copyright (C) 2005 Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036    * $Id: NLLErrModule.cc,v 1.5 2005/12/30 19:53:56 lamblinp Exp $
00037    ******************************************************* */
00038 
00039 // Authors: Pascal Lamblin
00040 
00044 #include "NLLErrModule.h"
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 PLEARN_IMPLEMENT_OBJECT(
00050     NLLErrModule,
00051     "NLLError Module",
00052     "This class computes the Negative Log-Likelihood of the input, given the\n"
00053     "desired 'target'. Also propagates gradient and diagonal of Hessian\n"
00054     "backwards.\n"
00055     "If output_size = 2, the second output is the classification error.\n"
00056     );
00057 
00058 NLLErrModule::NLLErrModule():
00059     target_size(1)
00060     /* ### Initialize all fields to their default value */
00061 {
00062     output_size = 1;
00063 }
00064 
00065 
00066 // retrieve target from input vector
00067 int NLLErrModule::getTarget(const Vec& input) const
00068 {
00069     int t_size = input.size() - input_size;
00070     int target = -1;
00071 
00072     // size check
00073     if( t_size == 1 )
00074     {
00075         target = (int) input[ input_size ];
00076     }
00077     else if( t_size == input_size )
00078     {
00079         /*
00080         PLWARNING("NLLErrModule::getTarget: You're giving a target the same\n"
00081                   "size as the input, instead of an integer. Checking if\n"
00082                   "this is a one-hot vector from this integer.\n");
00083          */
00084 
00085         Vec the_target = input.subVec( input_size, t_size );
00086         // get position of '1'
00087         target = argmax( the_target );
00088 
00089 #ifdef BOUNDCHECK
00090         // check if vector matches with a one-hot one
00091         PLASSERT( is_equal( the_target[target], 1. ) ) ;
00092         for( int i=0 ; i<input_size ; i++ )
00093             PLASSERT( is_equal( the_target[i], 0. ) || i == target );
00094 #endif
00095     }
00096     else
00097     {
00098         PLERROR("NLLErrModule::getTarget: target.size() is %i,\n"
00099                 " but should be 1. 'target' should contain an integer.\n",
00100                 t_size);
00101     }
00102 
00103     if( target < 0 || target >= input_size )
00104         PLERROR("NLLErrModule::getTarget: target should be between 0 and"
00105                 "input_size (%i).\n", input_size);
00106 
00107     return target;
00108 }
00109 
00110 // output = error = log(softmax(input))[target]
00111 void NLLErrModule::fprop(const Vec& input, Vec& output) const
00112 {
00113     int target = getTarget( input );
00114     // size check is done in getTarget()
00115 
00116     Vec input_ = input.subVec( 0, input_size );
00117     output.resize( output_size );
00118 
00119     fp_sm = softmax( input_ );
00120     output[0] = - pl_log( fp_sm[target] );
00121 
00122 
00123     if( output_size > 1 )
00124         output[1] = ( argmax( input_ ) == target ) ? 0 : 1;
00125 }
00126 
00127 // Don't modify class
00128 void NLLErrModule::bpropUpdate(const Vec& input, const Vec& output,
00129                                const Vec& output_gradient)
00130 {
00131     int out_size = output.size();
00132     int og_size = output_gradient.size();
00133 
00134     // for size check
00135     getTarget( input );
00136 
00137     // size check
00138     if( out_size != output_size )
00139     {
00140         PLWARNING("NLLErrModule::bpropUpdate: output.size()' should be\n"
00141                   " equal to 'output_size' (%i != %i)\n",
00142                   out_size, output_size);
00143     }
00144     if( og_size != output_size )
00145     {
00146         PLWARNING("NLLErrModule::bpropUpdate: 'output_gradient.size()'\n"
00147                   " should be equal to 'output_size' (%i != %i)\n",
00148                   og_size, output_size);
00149     }
00150 }
00151 
00152 // We don't care about output_gradient, we consider we're the last variable.
00153 // So we compute the gradient of the error of this variable.
00154 void NLLErrModule::bpropUpdate(const Vec& input, const Vec& output,
00155                                Vec& input_gradient,
00156                                const Vec& output_gradient)
00157 {
00158     int out_size = output.size();
00159     int og_size = output_gradient.size();
00160     int target = getTarget( input );
00161     bool is_final_cost = false; // if yes, output_gradient is 1
00162 
00163     // size check
00164     if( out_size != output_size )
00165     {
00166         PLERROR("NLLErrModule::bpropUpdate: output.size()' should be\n"
00167                 " equal to 'output_size' (%i != %i)\n",
00168                 out_size, output_size);
00169     }
00170     if( og_size == 0 )
00171     {
00172         /*
00173         PLWARNING("NLLErrModule::bpropUpdate: you are not providing"
00174                   "output_gradient.\n"
00175                   "Assuming this is the final cost, and output_gradient=1.\n");
00176          */
00177         is_final_cost = true;
00178     }
00179     else if( og_size != output_size )
00180     {
00181         PLERROR("NLLErrModule::bpropUpdate: 'output_gradient.size()'\n"
00182                 " should be equal to 'output_size' (%i != %i)\n",
00183                 og_size, output_size);
00184     }
00185 
00186     // input_gradient[i] = output_gradient*( softmax(input)[i] ) if i!=target
00187     // input_gradient[target] = output_gradient*( softmax(input)[target] )
00188     input_gradient.resize( input_size );
00189     input_gradient << fp_sm;
00190 
00191     input_gradient[target] -= 1;
00192     if( !is_final_cost )
00193         input_gradient *= output_gradient[0];
00194 
00195 }
00196 
00197 // Does nothing (just checks and warns)
00198 void NLLErrModule::bbpropUpdate(const Vec& input, const Vec& output,
00199                                 const Vec& output_gradient,
00200                                 const Vec& output_diag_hessian)
00201 {
00202     int odh_size = output_diag_hessian.size();
00203     if( odh_size != output_size )
00204     {
00205         PLWARNING("NLLErrModule::bbpropUpdate:"
00206                   " 'output_diag_hessian.size()'\n"
00207                   " should be equal to 'output_size' (%i != %i)\n",
00208                   odh_size, output_size);
00209     }
00210 
00211     bpropUpdate( input, output, output_gradient );
00212 }
00213 
00214 // Propagates back output_gradient and output_diag_hessian
00215 void NLLErrModule::bbpropUpdate(const Vec& input, const Vec& output,
00216                                 Vec& input_gradient,
00217                                 const Vec& output_gradient,
00218                                 Vec& input_diag_hessian,
00219                                 const Vec& output_diag_hessian)
00220 {
00221     int odh_size = output_diag_hessian.size();
00222     int target = getTarget( input );
00223     bool is_final_cost = false; // if yes, output_diag_hessian is 0
00224 
00225     // size check
00226     // others size checks will be done in bpropUpdate()
00227     if( odh_size == 0 )
00228     {
00229         PLWARNING("NLLErrModule::bbpropUpdate: you are not providing"
00230                   " output_diag_hessian.\n"
00231                   "Assuming this is the final cost,"
00232                   " and output_diag_hessian=0.\n");
00233         is_final_cost = true;
00234     }
00235     else if( odh_size != output_size )
00236     {
00237         PLERROR("NLLErrModule::bbpropUpdate:"
00238                 " 'output_diag_hessian.size()'\n"
00239                 " should be equal to 'output_size' (%i != %i)\n",
00240                 odh_size, output_size);
00241     }
00242 
00243     bpropUpdate( input, output, input_gradient, output_gradient );
00244 
00245     Vec input_ = input.subVec( 0, input_size );
00246     input_diag_hessian.resize( input_size );
00247     Vec softmax_in = softmax( input_ );
00248 
00249     // computation of term dC/dy d²y/dx²,
00250     // skipped if estimate_simpler_diag_hessian, unless it is final cost
00251     if( estimate_simpler_diag_hessian && !is_final_cost )
00252     {
00253         input_diag_hessian.clear();
00254     }
00255     else
00256     {
00257         for( int i=0 ; i<input_size ; i++ )
00258         {
00259             real sm_i = softmax_in[i];
00260             input_diag_hessian[i] = sm_i*( 1-sm_i);
00261         }
00262 
00263         if( !is_final_cost )
00264             input_diag_hessian *= output_gradient[0];
00265 
00266     }
00267 
00268     // computation of term d²C/dy² (dy/dx)²,
00269     // skipped if it is final cost, because then d²C/dy² == d²C/dC² == 0
00270     if( !is_final_cost )
00271     {
00272         Vec fprime = softmax_in;
00273         fprime[target] -= 1;
00274         fprime *= fprime;
00275 
00276         input_diag_hessian += output_diag_hessian[0] * fprime;
00277     }
00278 
00279 }
00280 
00281 
00282 
00283 
00284 //
00285 void NLLErrModule::forget()
00286 {
00287 }
00288 
00289 
00290 // ### Nothing to add here, simply calls build_
00291 void NLLErrModule::build()
00292 {
00293     inherited::build();
00294     build_();
00295 }
00296 
00297 void NLLErrModule::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00298 {
00299     inherited::makeDeepCopyFromShallowCopy(copies);
00300 
00301 }
00302 
00303 void NLLErrModule::declareOptions(OptionList& ol)
00304 {
00305     inherited::declareOptions(ol);
00306 }
00307 
00308 void NLLErrModule::build_()
00309 {
00310     if( input_size < 0 )
00311     {
00312         PLWARNING("NLLErrModule::build_: 'input_size' is < 0.\n"
00313                   "You should set it to a positive integer.\n"
00314                   "Defaulting to '1' (like a sigmoid function ?)\n");
00315         input_size = 1;
00316     }
00317     if( output_size != 1 && output_size != 2 )
00318     {
00319         PLWARNING("NLLErrModule::build_: 'output_size' (%i) should be 1.\n"
00320                   "Setting 'output_size' to 1.\n", output_size);
00321         output_size = 1;
00322     }
00323 
00324     target_size = 1;
00325 
00326     fp_sm.resize(input_size);
00327 }
00328 
00329 
00330 
00331 
00332 } // end of namespace PLearn
00333 
00334 
00335 /*
00336   Local Variables:
00337   mode:c++
00338   c-basic-offset:4
00339   c-file-style:"stroustrup"
00340   c-file-offsets:((innamespace . 0)(inline-open . 0))
00341   indent-tabs-mode:nil
00342   fill-column:79
00343   End:
00344 */
00345 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines