PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2003 Pascal Vincent, Yoshua Bengio, 00006 // Olivier Delalleau and University of Montreal 00007 // 00008 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 00039 00040 /* ******************************************************* 00041 * $Id: AdaptGradientOptimizer.h 4774 2006-01-10 20:05:24Z tihocan $ 00042 * This file is part of the PLearn library. 00043 ******************************************************* */ 00044 00045 00048 #ifndef AdaptGradientOptimizer_INC 00049 #define AdaptGradientOptimizer_INC 00050 00051 #include "Optimizer.h" 00052 00053 namespace PLearn { 00054 using namespace std; 00055 00056 00064 class AdaptGradientOptimizer : public Optimizer 00065 { 00066 typedef Optimizer inherited; 00067 00068 public: 00069 00070 00071 int adapt_every; 00072 00073 real adapt_coeff1; 00074 real adapt_coeff2; 00075 00076 real decrease_constant; 00077 00080 real learning_rate; // current learning rate 00081 00087 int learning_rate_adaptation; 00088 00089 real max_learning_rate; 00090 real min_learning_rate; 00091 00092 int mini_batch; 00093 00094 real start_learning_rate; 00095 00096 private: 00097 00098 bool stochastic_hack; // true when we're computing a stochastic gradient 00099 Vec learning_rates; // used to store the individual learning rates 00100 Vec gradient; // used to store the gradient 00101 Vec tmp_storage; // used to store various stuff 00102 // used to store the previous weights evolution, it can be used to 00103 // see how many times a weight has increased / decreased consecutively 00104 Vec old_evol; 00105 Array<Mat> oldgradientlocations; // used for the stochastic hack 00106 Vec store_var_grad; // used to store the gradient variance 00107 Vec store_grad; // used to store the gradient 00108 Vec store_quad_grad; // used to store the gradient^2 00109 int count_updates; // used to count how many examples we went through 00110 00111 public: 00112 00113 AdaptGradientOptimizer(); 00114 00115 PLEARN_DECLARE_OBJECT(AdaptGradientOptimizer); 00116 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies) { inherited::makeDeepCopyFromShallowCopy(copies); } 00117 00118 virtual void build() 00119 { 00120 inherited::build(); 00121 build_(); 00122 } 00123 00124 private: 00125 00126 void build_(); 00127 00128 public: 00129 00130 virtual real optimize(); 00131 virtual bool optimizeN(VecStatsCollector& stats_coll); 00132 00133 private: 00134 00135 // Basic learning rate adaptation 00136 // If grad(i) > 0 : lr(i) = lr(i) + lr(i) * adapt_coeff1 00137 // else : lr(i) = lr(i) - lr(i) * adapt_coeff2 00138 void adaptLearningRateBasic( 00139 Vec old_params, 00140 Vec new_evol); 00141 00142 // ALAP1 formula learning rate adaptation 00143 // lr = lr + adapt_coeff1 * dot(grad(k-1), grad(k)) 00144 // NB: has not been tested 00145 void adaptLearningRateALAP1( 00146 Vec old_gradient, 00147 Vec new_gradient); 00148 00149 // Learning rate adaptation depending on the variance : 00150 // If var(i) is low, lr(i) = max_learning_rate 00151 // else lr(i) = min_learning_rate 00152 void adaptLearningRateVariance(); 00153 00154 protected: 00155 00156 static void declareOptions(OptionList& ol); 00157 00158 }; 00159 00160 } // end of namespace PLearn 00161 00162 #endif 00163 00164 00165 /* 00166 Local Variables: 00167 mode:c++ 00168 c-basic-offset:4 00169 c-file-style:"stroustrup" 00170 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00171 indent-tabs-mode:nil 00172 fill-column:79 00173 End: 00174 */ 00175 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :