PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // CrossEntropyCostModule.cc 00004 // 00005 // Copyright (C) 2007 Pascal Lamblin, Dumitru Erhan and Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin, Dumitru Erhan and Hugo Larochelle 00036 00041 #include "CrossEntropyCostModule.h" 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 PLEARN_IMPLEMENT_OBJECT(CrossEntropyCostModule, 00047 "Computes the CrossEntropy, given two activation vectors", 00048 "and back-propagates the gradient\n"); 00049 00050 CrossEntropyCostModule::CrossEntropyCostModule() 00051 { 00052 output_size = 1; 00053 } 00054 00055 void CrossEntropyCostModule::declareOptions(OptionList& ol) 00056 { 00057 // Now call the parent class' declareOptions 00058 inherited::declareOptions(ol); 00059 00060 redeclareOption(ol, "target_size", &CrossEntropyCostModule::target_size, 00061 OptionBase::nosave, 00062 "equals to input_size"); 00063 00064 } 00065 00066 void CrossEntropyCostModule::build_() 00067 { 00068 target_size = input_size; 00069 } 00070 00071 // ### Nothing to add here, simply calls build_ 00072 void CrossEntropyCostModule::build() 00073 { 00074 inherited::build(); 00075 build_(); 00076 } 00077 00078 00079 void CrossEntropyCostModule::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00080 { 00081 inherited::makeDeepCopyFromShallowCopy(copies); 00082 } 00083 00084 00086 // fprop // 00088 void CrossEntropyCostModule::fprop(const Vec& input, const Vec& target, real& cost) const 00089 { 00090 PLASSERT( input.size() == input_size ); 00091 PLASSERT( target.size() == target_size ); 00092 00093 cost = 0; 00094 00095 real target_i, activation_i; 00096 for( int i=0 ; i < target_size ; i++ ) 00097 { 00098 // nll = - target*log(sigmoid(act)) -(1-target)*log(1-sigmoid(act)) 00099 // but it is numerically unstable, so use instead the following: 00100 // = target*softplus(-act) +(1-target)*(act+softplus(-act)) 00101 // = act + softplus(-act) - target*act 00102 // = softplus(act) - target*act 00103 target_i = target[i]; 00104 activation_i = input[i]; 00105 cost += softplus(activation_i) - target_i * activation_i; 00106 } 00107 } 00108 00109 void CrossEntropyCostModule::fprop(const Vec& input, const Vec& target, Vec& cost) const 00110 { 00111 cost.resize( output_size ); 00112 fprop( input, target, cost[0] ); 00113 } 00114 00115 void CrossEntropyCostModule::fprop(const Mat& inputs, const Mat& targets, Mat& costs) const 00116 { 00117 costs.resize( inputs.length(), output_size ); 00118 00119 for (int i = 0; i < inputs.length(); i++) 00120 fprop(inputs(i), targets(i), costs(i,0)); 00121 00122 } 00123 00124 00125 00127 // bpropUpdate // 00129 void CrossEntropyCostModule::bpropUpdate(const Vec& input, const Vec& target, 00130 real cost, Vec& input_gradient, 00131 bool accumulate) 00132 { 00133 PLASSERT( input.size() == input_size ); 00134 PLASSERT( target.size() == target_size ); 00135 00136 if (accumulate) 00137 { 00138 PLASSERT_MSG( input_gradient.size() == input_size, 00139 "Cannot resize input_gradient AND accumulate into it" ); 00140 } 00141 else 00142 { 00143 input_gradient.resize(input_size); 00144 input_gradient.clear(); 00145 } 00146 00147 for (int i=0; i < target_size; i++) 00148 input_gradient[i] += sigmoid(input[i]) - target[i]; 00149 } 00150 00151 void CrossEntropyCostModule::bpropUpdate(const Mat& inputs, const Mat& targets, 00152 const Vec& costs, 00153 Mat& input_gradients, bool accumulate) 00154 { 00155 PLASSERT( inputs.width() == input_size ); 00156 PLASSERT( targets.width() == target_size ); 00157 00158 int batch_size = inputs.length(); 00159 00160 if (accumulate) 00161 { 00162 PLASSERT_MSG( input_gradients.width() == input_size && 00163 input_gradients.length() == batch_size, 00164 "Cannot resize input_gradients AND accumulate into it" ); 00165 } 00166 else 00167 { 00168 input_gradients.resize(batch_size, input_size); 00169 input_gradients.clear(); 00170 } 00171 00172 for (int i=0; i < batch_size; i++) 00173 for (int j=0; j < target_size; j++) 00174 input_gradients(i, j) += sigmoid(inputs(i, j)) - targets(i, j); 00175 } 00176 00177 void CrossEntropyCostModule::bpropAccUpdate(const TVec<Mat*>& ports_value, 00178 const TVec<Mat*>& ports_gradient) 00179 { 00180 PLASSERT( ports_value.length() == nPorts() ); 00181 PLASSERT( ports_gradient.length() == nPorts() ); 00182 00183 Mat* prediction = ports_value[0]; 00184 Mat* target = ports_value[1]; 00185 #ifndef NDEBUG 00186 Mat* cost = ports_value[2]; 00187 #endif 00188 Mat* prediction_grad = ports_gradient[0]; 00189 Mat* target_grad = ports_gradient[1]; 00190 Mat* cost_grad = ports_gradient[2]; 00191 00192 // If we have cost_grad and we want prediction_grad 00193 if( prediction_grad && prediction_grad->isEmpty() 00194 && cost_grad && !cost_grad->isEmpty() ) 00195 { 00196 PLASSERT( prediction ); 00197 PLASSERT( target ); 00198 PLASSERT( cost ); 00199 PLASSERT( !target_grad ); 00200 00201 PLASSERT( prediction->width() == getPortSizes()(0,1) ); 00202 PLASSERT( target->width() == getPortSizes()(1,1) ); 00203 PLASSERT( cost->width() == getPortSizes()(2,1) ); 00204 PLASSERT( prediction_grad->width() == getPortSizes()(0,1) ); 00205 PLASSERT( cost_grad->width() == getPortSizes()(2,1) ); 00206 00207 int batch_size = prediction->length(); 00208 PLASSERT( target->length() == batch_size ); 00209 PLASSERT( cost->length() == batch_size ); 00210 PLASSERT( cost_grad->length() == batch_size ); 00211 00212 prediction_grad->resize(batch_size, getPortSizes()(0,1)); 00213 00214 for( int i=0; i < batch_size; i++ ) 00215 for ( int j=0; j < target->width(); j++ ) 00216 (*prediction_grad)(i, j) += 00217 (*cost_grad)(i,0)*(sigmoid((*prediction)(i,j)) - (*target)(i,j)); 00218 } 00219 00220 else if( !prediction_grad && !target_grad && 00221 (!cost_grad || !cost_grad->isEmpty()) ) 00222 // We do not care about the gradient w.r.t prediction and target, and 00223 // either we do not care about the gradient w.r.t. cost or there is a 00224 // gradient provided (that we will not use). 00225 // In such situations, there is nothing to do. 00226 return; 00227 else if( !cost_grad && prediction_grad && prediction_grad->isEmpty() ) 00228 PLERROR("In CrossEntropyCostModule::bpropAccUpdate - cost gradient is NULL,\n" 00229 "cannot compute prediction gradient. Maybe you should set\n" 00230 "\"propagate_gradient = 0\" on the incoming connection.\n"); 00231 else 00232 PLERROR("In OnlineLearningModule::bpropAccUpdate - Port configuration " 00233 "not implemented for class '%s'", classname().c_str()); 00234 } 00235 00236 TVec<string> CrossEntropyCostModule::costNames() 00237 { 00238 if (name == "" || name == classname()) 00239 return TVec<string>(1, "CrossEntropy"); 00240 else 00241 return TVec<string>(1, name + ".CrossEntropy"); 00242 } 00243 00244 } // end of namespace PLearn 00245 00246 00247 /* 00248 Local Variables: 00249 mode:c++ 00250 c-basic-offset:4 00251 c-file-style:"stroustrup" 00252 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00253 indent-tabs-mode:nil 00254 fill-column:79 00255 End: 00256 */ 00257 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :