PLearn 0.1
AutoScaledGradientOptimizer.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent and Yoshua Bengio
00006 // Copyright (C) 1999-2002, 2006 University of Montreal
00007 //
00008 
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************      
00039  * $Id: AutoScaledGradientOptimizer.h 8247 2007-11-12 20:22:12Z nouiz $
00040  * This file is part of the PLearn library.
00041  ******************************************************* */
00042 
00043 
00046 #ifndef AutoScaledGradientOptimizer_INC
00047 #define AutoScaledGradientOptimizer_INC
00048 
00049 #include <plearn/opt/Optimizer.h>
00050 
00051 namespace PLearn {
00052 using namespace std;
00053 
00054 
00055 class AutoScaledGradientOptimizer : public Optimizer
00056 {
00057     typedef Optimizer inherited;
00058       
00059 public:
00060 
00063     real learning_rate; // current learning rate
00064 
00065     // Options (also available through setOption)
00066     real start_learning_rate;
00067     real decrease_constant;
00068 
00069     // optionally the user can instead of using the decrease_constant
00070     // use a fixed schedule. This matrix has 2 columns: iteration_threshold and learning_rate_factor
00071     // As soon as the iteration number goes above the iteration_threshold, the corresponding learning_rate_factor
00072     // is applied (multiplied) to the start_learning_rate to obtain the learning_rate.
00073     Mat lr_schedule;
00074 
00075     int verbosity;
00076 
00077     // every how-many steps should the mean and scaling be reevaluated
00078     int evaluate_scaling_every;
00079     // how many steps should be used to re-evaluate the mean and scaling
00080     int evaluate_scaling_during; 
00081     // scaling will be 1/(mean_abs_grad + epsilon)
00082     real epsilon;
00083 
00084     AutoScaledGradientOptimizer();
00085 
00086     PLEARN_DECLARE_OBJECT(AutoScaledGradientOptimizer);
00087 
00088     virtual void setToOptimize(const VarArray& the_params, Var the_cost, VarArray the_other_costs = VarArray(0), TVec<VarArray> the_other_params = TVec<VarArray>(0), real the_other_weight = 1);
00089 
00090     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies)
00091     { inherited::makeDeepCopyFromShallowCopy(copies); }
00092 
00093     virtual void build()
00094     {
00095         inherited::build();
00096         build_();
00097     }
00098 
00099 protected:
00100     Vec scaling; // by how much to multiply the gradient before performing an update
00101     Vec meanabsgrad; // the mean absolute value of the gradient computed for 
00102     int nsteps_remaining_for_evaluation;
00103 
00104     // Vecs pointing to the value and graident of parameters (setup with the makeSharedValue and makeShared Gradient hack)
00105     Vec param_values;
00106     Vec param_gradients;
00107 
00108 private:
00109     void build_()
00110     {}
00111     
00112 public:
00113 
00114     // virtual void oldwrite(ostream& out) const;
00115     // virtual void oldread(istream& in);
00116     //virtual real optimize();
00117     virtual bool optimizeN(VecStatsCollector& stats_coll);
00118 
00119 protected:
00120 
00121     static void declareOptions(OptionList& ol);
00122 };
00123 
00124 DECLARE_OBJECT_PTR(AutoScaledGradientOptimizer);
00125 
00126 } // end of namespace PLearn
00127 
00128 #endif
00129 
00130 
00131 /*
00132   Local Variables:
00133   mode:c++
00134   c-basic-offset:4
00135   c-file-style:"stroustrup"
00136   c-file-offsets:((innamespace . 0)(inline-open . 0))
00137   indent-tabs-mode:nil
00138   fill-column:79
00139   End:
00140 */
00141 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines