PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // OnlineGramNaturalGradientOptimizer.h 00004 // 00005 // Copyright (C) 2007 Pierre-Antoine Manzagol 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pierre-Antoine Manzagol 00036 00040 #ifndef ONLINEGRAMNATURALGRADIENTOPTIMIZER_INC 00041 #define ONLINEGRAMNATURALGRADIENTOPTIMIZER_INC 00042 00043 #include <plearn/opt/Optimizer.h> 00044 00045 #include <ctime> 00046 00047 namespace PLearn { 00048 using namespace std; 00049 00059 class OnlineGramNaturalGradientOptimizer : public Optimizer 00060 { 00061 typedef Optimizer inherited; 00062 00063 public: 00064 //##### Public Build Options ############################################ 00065 00066 real learning_rate; 00067 real gamma; 00068 real reg; 00069 int opt_batch_size; 00070 int n_eigen; 00071 00072 00073 public: 00074 //##### Public Member Functions ######################################### 00075 00076 OnlineGramNaturalGradientOptimizer(); 00077 00078 void gramEigenNaturalGradient(); 00079 00080 //##### Optimizer Member Functions ####################################### 00081 00082 virtual bool optimizeN(VecStatsCollector& stats_coll); 00083 00084 //##### PLearn::Object Protocol ######################################### 00085 PLEARN_DECLARE_OBJECT(OnlineGramNaturalGradientOptimizer); 00086 00087 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00088 00089 virtual void build() 00090 { 00091 inherited::build(); 00092 build_(); 00093 } 00094 00095 protected: 00096 //##### Protected Options ############################################### 00097 00098 // ### Declare protected option fields (such as learned parameters) here 00099 00100 00101 protected: 00102 //##### Protected Member Functions ###################################### 00103 00105 // (PLEASE IMPLEMENT IN .cc) 00106 static void declareOptions(OptionList& ol); 00107 00108 private: 00109 //##### Private Member Functions ######################################## 00110 00112 // (PLEASE IMPLEMENT IN .cc) 00113 void build_(); 00114 00115 private: 00116 //##### Private Data Members ############################################ 00117 00118 int n_optimizeN_calls; 00119 00120 // The batch index 00121 int bi; 00122 int n_eigen_cur; 00123 int n_eigen_old; 00124 00125 // Holds the gradients and their mean 00126 Mat gradients; 00127 Vec mu; 00128 00129 real total_variance, variance_percentage; 00130 00131 // the gram matrix - G = UDU' or in our case U'DU 00132 Mat gram; 00133 Mat U; 00134 Vec D; 00135 00136 // The covariance matrix is C = VDV' (with the eigen vectors in V's columns) 00137 // or in our case cov_eigen_vectors' diag(cov_eigen_values) cov_eigen_vectors 00138 Mat cov_eigen_vec; 00139 Mat old_cov_eigen_vec; 00140 Mat swap; 00141 Vec cov_eigen_val; 00142 00143 Mat cov_norm_eigen_vec; 00144 00145 // 00146 Vec dot_prod; 00147 Vec scaled_dot_prod; 00148 00149 // The natural gradient, ie C^{-1} mu 00150 Vec naturalg; 00151 00152 clock_t t0, t1, t2, t3; 00153 00154 }; 00155 00156 DECLARE_OBJECT_PTR(OnlineGramNaturalGradientOptimizer); 00157 00158 00159 } // end of namespace PLearn 00160 00161 #endif 00162 00163 00164 /* 00165 Local Variables: 00166 mode:c++ 00167 c-basic-offset:4 00168 c-file-style:"stroustrup" 00169 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00170 indent-tabs-mode:nil 00171 fill-column:79 00172 End: 00173 */ 00174 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :