PLearn 0.1
CrossEntropyCostModule.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // CrossEntropyCostModule.cc
00004 //
00005 // Copyright (C) 2007 Pascal Lamblin, Dumitru Erhan and Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin, Dumitru Erhan and Hugo Larochelle
00036 
00041 #include "CrossEntropyCostModule.h"
00042 
00043 namespace PLearn {
00044 using namespace std;
00045 
00046 PLEARN_IMPLEMENT_OBJECT(CrossEntropyCostModule,
00047     "Computes the CrossEntropy, given two activation vectors",
00048     "and back-propagates the gradient\n");
00049 
00050 CrossEntropyCostModule::CrossEntropyCostModule()
00051 {
00052     output_size = 1;
00053 }
00054 
00055 void CrossEntropyCostModule::declareOptions(OptionList& ol)
00056 {
00057     // Now call the parent class' declareOptions
00058     inherited::declareOptions(ol);
00059 
00060     redeclareOption(ol, "target_size", &CrossEntropyCostModule::target_size,
00061                      OptionBase::nosave,
00062                      "equals to input_size");
00063 
00064 }
00065 
00066 void CrossEntropyCostModule::build_()
00067 {
00068     target_size = input_size;
00069 }
00070 
00071 // ### Nothing to add here, simply calls build_
00072 void CrossEntropyCostModule::build()
00073 {
00074     inherited::build();
00075     build_();
00076 }
00077 
00078 
00079 void CrossEntropyCostModule::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00080 {
00081     inherited::makeDeepCopyFromShallowCopy(copies);
00082 }
00083 
00084 
00086 // fprop //
00088 void CrossEntropyCostModule::fprop(const Vec& input, const Vec& target, real& cost) const
00089 {
00090     PLASSERT( input.size() == input_size );
00091     PLASSERT( target.size() == target_size );
00092 
00093     cost = 0;
00094 
00095     real target_i, activation_i;
00096     for( int i=0 ; i < target_size ; i++ )
00097     {
00098         // nll = - target*log(sigmoid(act)) -(1-target)*log(1-sigmoid(act))
00099         // but it is numerically unstable, so use instead the following:
00100         //     = target*softplus(-act) +(1-target)*(act+softplus(-act))
00101         //     = act + softplus(-act) - target*act
00102         //     = softplus(act) - target*act
00103         target_i = target[i];
00104         activation_i = input[i];
00105         cost += softplus(activation_i) - target_i * activation_i;
00106     }
00107 }
00108 
00109 void CrossEntropyCostModule::fprop(const Vec& input, const Vec& target, Vec& cost) const
00110 {
00111     cost.resize( output_size );
00112     fprop( input, target, cost[0] );
00113 }
00114 
00115 void CrossEntropyCostModule::fprop(const Mat& inputs, const Mat& targets, Mat& costs) const
00116 {
00117     costs.resize( inputs.length(), output_size );
00118 
00119     for (int i = 0; i < inputs.length(); i++)
00120         fprop(inputs(i), targets(i), costs(i,0));
00121 
00122 }
00123 
00124 
00125 
00127 // bpropUpdate //
00129 void CrossEntropyCostModule::bpropUpdate(const Vec& input, const Vec& target,
00130                                          real cost, Vec& input_gradient,
00131                                          bool accumulate)
00132 {
00133     PLASSERT( input.size() == input_size );
00134     PLASSERT( target.size() == target_size );
00135 
00136     if (accumulate)
00137     {
00138         PLASSERT_MSG( input_gradient.size() == input_size,
00139                       "Cannot resize input_gradient AND accumulate into it" );
00140     }
00141     else
00142     {
00143         input_gradient.resize(input_size);
00144         input_gradient.clear();
00145     }
00146 
00147     for (int i=0; i < target_size; i++)
00148         input_gradient[i] += sigmoid(input[i]) - target[i];
00149 }
00150 
00151 void CrossEntropyCostModule::bpropUpdate(const Mat& inputs, const Mat& targets,
00152                                          const Vec& costs,
00153                                          Mat& input_gradients, bool accumulate)
00154 {
00155     PLASSERT( inputs.width() == input_size );
00156     PLASSERT( targets.width() == target_size );
00157 
00158     int batch_size = inputs.length();
00159 
00160     if (accumulate)
00161     {
00162         PLASSERT_MSG( input_gradients.width() == input_size &&
00163                       input_gradients.length() == batch_size,
00164                       "Cannot resize input_gradients AND accumulate into it" );
00165     }
00166     else
00167     {
00168         input_gradients.resize(batch_size, input_size);
00169         input_gradients.clear();
00170     }
00171 
00172     for (int i=0; i < batch_size; i++)
00173         for (int j=0; j < target_size; j++)
00174             input_gradients(i, j) += sigmoid(inputs(i, j)) - targets(i, j);
00175 }
00176 
00177 void CrossEntropyCostModule::bpropAccUpdate(const TVec<Mat*>& ports_value,
00178                                    const TVec<Mat*>& ports_gradient)
00179 {
00180     PLASSERT( ports_value.length() == nPorts() );
00181     PLASSERT( ports_gradient.length() == nPorts() );
00182 
00183     Mat* prediction = ports_value[0];
00184     Mat* target = ports_value[1];
00185 #ifndef NDEBUG
00186     Mat* cost = ports_value[2];
00187 #endif
00188     Mat* prediction_grad = ports_gradient[0];
00189     Mat* target_grad = ports_gradient[1];
00190     Mat* cost_grad = ports_gradient[2];
00191 
00192     // If we have cost_grad and we want prediction_grad
00193     if( prediction_grad && prediction_grad->isEmpty()
00194         && cost_grad && !cost_grad->isEmpty() )
00195     {
00196         PLASSERT( prediction );
00197         PLASSERT( target );
00198         PLASSERT( cost );
00199         PLASSERT( !target_grad );
00200 
00201         PLASSERT( prediction->width() == getPortSizes()(0,1) );
00202         PLASSERT( target->width() == getPortSizes()(1,1) );
00203         PLASSERT( cost->width() == getPortSizes()(2,1) );
00204         PLASSERT( prediction_grad->width() == getPortSizes()(0,1) );
00205         PLASSERT( cost_grad->width() == getPortSizes()(2,1) );
00206 
00207         int batch_size = prediction->length();
00208         PLASSERT( target->length() == batch_size );
00209         PLASSERT( cost->length() == batch_size );
00210         PLASSERT( cost_grad->length() == batch_size );
00211 
00212         prediction_grad->resize(batch_size, getPortSizes()(0,1));
00213 
00214         for( int i=0; i < batch_size; i++ )
00215             for ( int j=0; j < target->width(); j++ )
00216                 (*prediction_grad)(i, j) +=
00217                 (*cost_grad)(i,0)*(sigmoid((*prediction)(i,j)) - (*target)(i,j));
00218     }
00219 
00220     else if( !prediction_grad && !target_grad &&
00221                (!cost_grad || !cost_grad->isEmpty()) )
00222         // We do not care about the gradient w.r.t prediction and target, and
00223         // either we do not care about the gradient w.r.t. cost or there is a
00224         // gradient provided (that we will not use).
00225         // In such situations, there is nothing to do.
00226         return;
00227     else if( !cost_grad && prediction_grad && prediction_grad->isEmpty() )
00228         PLERROR("In CrossEntropyCostModule::bpropAccUpdate - cost gradient is NULL,\n"
00229                 "cannot compute prediction gradient. Maybe you should set\n"
00230                 "\"propagate_gradient = 0\" on the incoming connection.\n");
00231     else
00232         PLERROR("In OnlineLearningModule::bpropAccUpdate - Port configuration "
00233                 "not implemented for class '%s'", classname().c_str());
00234 }
00235 
00236 TVec<string> CrossEntropyCostModule::costNames()
00237 {
00238     if (name == "" || name == classname())
00239         return TVec<string>(1, "CrossEntropy");
00240     else
00241         return TVec<string>(1, name + ".CrossEntropy");
00242 }
00243 
00244 } // end of namespace PLearn
00245 
00246 
00247 /*
00248   Local Variables:
00249   mode:c++
00250   c-basic-offset:4
00251   c-file-style:"stroustrup"
00252   c-file-offsets:((innamespace . 0)(inline-open . 0))
00253   indent-tabs-mode:nil
00254   fill-column:79
00255   End:
00256 */
00257 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines