PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // SubsamplingDBN.h 00004 // 00005 // Copyright (C) 2006 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin 00036 00040 #ifndef SubsamplingDBN_INC 00041 #define SubsamplingDBN_INC 00042 00043 #include <plearn_learners/generic/PLearner.h> 00044 #include <plearn_learners/online/OnlineLearningModule.h> 00045 #include <plearn_learners/online/CostModule.h> 00046 #include <plearn_learners/online/NLLCostModule.h> 00047 #include <plearn_learners/online/RBMClassificationModule.h> 00048 #include <plearn_learners/online/RBMLayer.h> 00049 #include <plearn_learners/online/RBMMixedLayer.h> 00050 #include <plearn_learners/online/RBMConnection.h> 00051 #include <plearn/misc/PTimer.h> 00052 #include <plearn/sys/Profiler.h> 00053 00054 namespace PLearn { 00055 00061 class SubsamplingDBN : public PLearner 00062 { 00063 typedef PLearner inherited; 00064 00065 public: 00066 //##### Public Build Options ############################################ 00067 00069 real cd_learning_rate; 00070 00072 real grad_learning_rate; 00073 00074 int batch_size; 00075 00077 real grad_decrease_ct; 00078 00079 /* NOT IMPLEMENTED YET 00081 real grad_weight_decay; 00082 */ 00083 00089 int n_classes; 00090 00098 TVec<int> training_schedule; 00099 00103 bool use_classification_cost; 00104 00112 bool reconstruct_layerwise; 00113 00115 TVec< PP<RBMLayer> > layers; 00116 00118 TVec< PP<RBMConnection> > connections; 00119 00126 PP<OnlineLearningModule> final_module; 00127 00132 PP<CostModule> final_cost; 00133 00137 TVec< PP<CostModule> > partial_costs; 00138 00141 bool independent_biases; 00142 00145 TVec< PP<OnlineLearningModule> > subsampling_modules; 00146 00147 //##### Public Learnt Options ########################################### 00149 PP<RBMClassificationModule> classification_module; 00150 00152 int n_layers; 00153 00155 TVec<string> cost_names; 00156 00158 bool online; 00159 00160 // Coefficient between 0 and 1. If non-zero, run a background 00161 // Gibbs chain and use the visible-hidden statistics to 00162 // contribute in the negative phase update (in proportion 00163 // background_gibbs_update_ratio wrt the contrastive divergence 00164 // negative phase statistics). If = 1, then do not perform any 00165 // contrastive divergence negative phase (use only the Gibbs chain 00166 // statistics). 00167 real background_gibbs_update_ratio; 00168 00171 bool top_layer_joint_cd; 00172 00176 int gibbs_chain_reinit_freq; 00177 00182 TVec< PP<RBMLayer> > reduced_layers; 00183 00184 //##### Not Options ##################################################### 00185 00187 PP<PTimer> timer; 00188 00191 PP<NLLCostModule> classification_cost; 00192 00195 PP<RBMMixedLayer> joint_layer; 00196 00197 00198 public: 00199 //##### Public Member Functions ######################################### 00200 00202 SubsamplingDBN(); 00203 00204 00205 //##### PLearner Member Functions ####################################### 00206 00209 virtual int outputsize() const; 00210 00214 // (PLEASE IMPLEMENT IN .cc) 00215 virtual void forget(); 00216 00220 // (PLEASE IMPLEMENT IN .cc) 00221 virtual void train(); 00222 00224 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00225 VMat testoutputs=0, VMat testcosts=0) const; 00226 00228 // (PLEASE IMPLEMENT IN .cc) 00229 virtual void computeOutput(const Vec& input, Vec& output) const; 00230 00232 // (PLEASE IMPLEMENT IN .cc) 00233 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00234 const Vec& target, Vec& costs) const; 00235 00238 // (PLEASE IMPLEMENT IN .cc) 00239 virtual TVec<std::string> getTestCostNames() const; 00240 00243 // (PLEASE IMPLEMENT IN .cc) 00244 virtual TVec<std::string> getTrainCostNames() const; 00245 00246 00247 void onlineStep( const Vec& input, const Vec& target, Vec& train_costs ); 00248 00249 void onlineStep( const Mat& inputs, const Mat& targets, Mat& train_costs ); 00250 00251 void greedyStep( const Vec& input, const Vec& target, int index ); 00252 00254 void greedyStep(const Mat& inputs, const Mat& targets, int index, Mat& train_costs_m); 00255 00256 void jointGreedyStep( const Vec& input, const Vec& target ); 00257 00258 void fineTuningStep( const Vec& input, const Vec& target, 00259 Vec& train_costs ); 00260 00262 void fineTuningStep( const Mat& inputs, const Mat& targets, 00263 Mat& train_costs ); 00264 00267 void contrastiveDivergenceStep( const PP<RBMLayer>& down_layer, 00268 const PP<RBMConnection>& connection, 00269 const PP<RBMLayer>& up_layer, 00270 int layer_index, 00271 bool nofprop=false); 00272 00273 00274 // *** SUBCLASS WRITING: *** 00275 // While in general not necessary, in case of particular needs 00276 // (efficiency concerns for ex) you may also want to overload 00277 // some of the following methods: 00278 // virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00279 // Vec& output, Vec& costs) const; 00280 // virtual void computeCostsOnly(const Vec& input, const Vec& target, 00281 // Vec& costs) const; 00282 // virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00283 // VMat testoutputs=0, VMat testcosts=0) const; 00284 // virtual int nTestCosts() const; 00285 // virtual int nTrainCosts() const; 00286 // virtual void resetInternalState(); 00287 // virtual bool isStatefulLearner() const; 00288 00289 00290 //##### PLearn::Object Protocol ######################################### 00291 00292 // Declares other standard object methods. 00293 // ### If your class is not instantiatable (it has pure virtual methods) 00294 // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS 00295 PLEARN_DECLARE_OBJECT(SubsamplingDBN); 00296 00297 // Simply calls inherited::build() then build_() 00298 virtual void build(); 00299 00301 // (PLEASE IMPLEMENT IN .cc) 00302 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00303 00304 protected: 00305 00306 int minibatch_size; 00307 00308 //##### Not Options ##################################################### 00309 00310 // whether to re-initialize Gibbs chain next time around 00311 bool initialize_gibbs_chain; 00312 00315 mutable TVec<Vec> activation_gradients; 00316 mutable TVec<Mat> activations_gradients; 00317 00320 mutable TVec<Vec> expectation_gradients; 00321 mutable TVec<Mat> expectations_gradients; 00322 00323 mutable TVec<Vec> subsampling_gradients; 00324 00325 mutable Vec final_cost_input; 00326 mutable Mat final_cost_inputs; 00327 00328 mutable Vec final_cost_value; 00329 mutable Mat final_cost_values; 00330 00331 mutable Vec final_cost_output; 00332 00333 mutable Vec class_output; 00334 00335 mutable Vec class_gradient; 00336 00338 mutable Vec final_cost_gradient; 00339 mutable Mat final_cost_gradients; 00340 00342 mutable Vec save_layer_activation; 00343 00344 Mat save_layer_activations; 00345 00347 mutable Vec save_layer_expectation; 00348 00349 Mat save_layer_expectations; 00350 00352 bool final_module_has_learning_rate; 00353 00355 bool final_cost_has_learning_rate; 00356 00358 mutable Vec pos_down_val; 00359 mutable Vec pos_up_val; 00360 mutable Mat pos_down_vals; 00361 mutable Mat pos_up_vals; 00362 mutable Mat cd_neg_down_vals; 00363 mutable Mat cd_neg_up_vals; 00364 00366 mutable TVec<Mat> gibbs_down_state; 00367 00369 Vec optimized_costs; 00370 00372 mutable Vec reconstruction_costs; 00373 00375 int nll_cost_index; 00376 00378 int class_cost_index; 00379 00381 int final_cost_index; 00382 00384 TVec<int> partial_costs_indices; 00385 00387 int reconstruction_cost_index; 00388 00390 int training_cpu_time_cost_index; 00391 00393 int cumulative_training_time_cost_index; 00394 00396 int cumulative_testing_time_cost_index; 00397 00399 real cumulative_training_time; 00400 00402 mutable real cumulative_testing_time; 00403 00405 TVec<int> cumulative_schedule; 00406 00407 mutable Vec layer_input; 00408 mutable Mat layer_inputs; 00409 00410 protected: 00411 //##### Protected Member Functions ###################################### 00412 00414 static void declareOptions(OptionList& ol); 00415 00416 private: 00417 //##### Private Member Functions ######################################## 00418 00420 void build_(); 00421 00422 void build_layers_and_connections(); 00423 00424 void build_costs(); 00425 00426 void build_classification_cost(); 00427 00428 void build_final_cost(); 00429 00430 void setLearningRate( real the_learning_rate ); 00431 00432 private: 00433 //##### Private Data Members ############################################ 00434 00435 // The rest of the private stuff goes here 00436 }; 00437 00438 // Declares a few other classes and functions related to this class 00439 DECLARE_OBJECT_PTR(SubsamplingDBN); 00440 00441 } // end of namespace PLearn 00442 00443 #endif 00444 00445 00446 /* 00447 Local Variables: 00448 mode:c++ 00449 c-basic-offset:4 00450 c-file-style:"stroustrup" 00451 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00452 indent-tabs-mode:nil 00453 fill-column:79 00454 End: 00455 */ 00456 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :