PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // DeepBeliefNet.h 00004 // 00005 // Copyright (C) 2006 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin 00036 00039 #ifndef DeepBeliefNet_INC 00040 #define DeepBeliefNet_INC 00041 00042 #include <plearn_learners/generic/PLearner.h> 00043 #include "OnlineLearningModule.h" 00044 #include "CostModule.h" 00045 #include "NLLCostModule.h" 00046 #include "RBMClassificationModule.h" 00047 #include "RBMLayer.h" 00048 #include "RBMMixedLayer.h" 00049 #include "RBMConnection.h" 00050 #include <plearn/misc/PTimer.h> 00051 #include <plearn/sys/Profiler.h> 00052 00053 namespace PLearn { 00054 using namespace std; 00055 00061 class DeepBeliefNet : public PLearner 00062 { 00063 typedef PLearner inherited; 00064 00065 public: 00066 //##### Public Build Options ############################################ 00067 00069 real cd_learning_rate; 00070 00073 real cd_decrease_ct; 00074 00077 real up_down_learning_rate; 00078 00081 real up_down_decrease_ct; 00082 00084 real grad_learning_rate; 00085 00087 real grad_decrease_ct; 00088 00089 /* NOT IMPLEMENTED YET 00091 real grad_weight_decay; 00092 */ 00093 00095 int batch_size; 00096 00102 int n_classes; 00103 00111 TVec<int> training_schedule; 00112 00117 int up_down_nstages; 00118 00122 bool use_classification_cost; 00123 00131 bool reconstruct_layerwise; 00132 00134 TVec< PP<RBMLayer> > layers; 00135 00138 int i_output_layer; 00139 00142 string learnerExpdir; 00143 00145 bool save_learner_before_fine_tuning; 00146 00148 TVec< PP<RBMConnection> > connections; 00149 00151 TVec< PP<RBMMultinomialLayer> > greedy_target_layers; 00152 00154 TVec< PP<RBMMatrixConnection> > greedy_target_connections; 00155 00162 PP<OnlineLearningModule> final_module; 00163 00168 PP<CostModule> final_cost; 00169 00173 TVec< PP<CostModule> > partial_costs; 00174 00177 bool use_sample_for_up_layer; 00178 00185 string use_corrupted_posDownVal; 00186 00188 string noise_type; 00189 00192 real fraction_of_masked_inputs; 00193 00196 bool mask_with_pepper_salt; 00197 00200 real prob_salt_noise; 00201 00202 00203 //##### Public Learnt Options ########################################### 00205 PP<RBMClassificationModule> classification_module; 00206 00208 int n_layers; 00209 00211 TVec<string> cost_names; 00212 00214 bool online; 00215 00216 // Coefficient between 0 and 1. If non-zero, run a background 00217 // Gibbs chain and use the visible-hidden statistics to 00218 // contribute in the negative phase update (in proportion 00219 // background_gibbs_update_ratio wrt the contrastive divergence 00220 // negative phase statistics). If = 1, then do not perform any 00221 // contrastive divergence negative phase (use only the Gibbs chain 00222 // statistics). 00223 real background_gibbs_update_ratio; 00224 00227 bool top_layer_joint_cd; 00228 00232 int gibbs_chain_reinit_freq; 00233 00237 real mean_field_contrastive_divergence_ratio; 00238 00241 int train_stats_window; 00242 00243 //##### Not Options ##################################################### 00244 00246 PP<PTimer> timer; 00247 00250 PP<NLLCostModule> classification_cost; 00251 00254 PP<RBMMixedLayer> joint_layer; 00255 00256 public: 00257 //##### Public Member Functions ######################################### 00258 00260 DeepBeliefNet(); 00261 00262 00263 //##### PLearner Member Functions ####################################### 00264 00267 virtual int outputsize() const; 00268 00272 virtual void forget(); 00273 00277 virtual void train(); 00278 00280 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00281 VMat testoutputs=0, VMat testcosts=0) const; 00282 00284 virtual void computeOutput(const Vec& input, Vec& output) const; 00285 00286 virtual void computeOutputsAndCosts(const Mat& inputs, const Mat& targets, 00287 Mat& outputs, Mat& costs) const; 00288 00290 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00291 const Vec& target, Vec& costs) const; 00292 virtual void computeClassifAndFinalCostsFromOutputs( 00293 const Mat& inputs, const Mat& outputs, 00294 const Mat& targets, Mat& costs) const; 00295 00298 // (PLEASE IMPLEMENT IN .cc) 00299 virtual TVec<std::string> getTestCostNames() const; 00300 00303 // (PLEASE IMPLEMENT IN .cc) 00304 virtual TVec<std::string> getTrainCostNames() const; 00305 00306 void onlineStep(const Vec& input, const Vec& target, Vec& train_costs); 00307 void onlineStep(const Mat& inputs, const Mat& targets, Mat& train_costs); 00308 00309 void greedyStep(const Vec& input, const Vec& target, int index); 00310 void greedyStep(const Mat& inputs, const Mat& targets, int index, 00311 Mat& train_costs_m); 00312 00313 00314 void jointGreedyStep(const Vec& input, const Vec& target); 00315 void jointGreedyStep(const Mat& inputs, const Mat& targets); 00316 00317 void upDownStep(const Vec& input, const Vec& target, Vec& train_costs); 00318 void upDownStep(const Mat& inputs, const Mat& targets, Mat& train_costs); 00319 00320 void fineTuningStep(const Vec& input, const Vec& target, Vec& train_costs); 00321 void fineTuningStep(const Mat& inputs, const Mat& targets, 00322 Mat& train_costs); 00323 00326 void contrastiveDivergenceStep( const PP<RBMLayer>& down_layer, 00327 const PP<RBMConnection>& connection, 00328 const PP<RBMLayer>& up_layer, 00329 int layer_index, 00330 bool nofprop=false); 00331 00332 00333 // *** SUBCLASS WRITING: *** 00334 // While in general not necessary, in case of particular needs 00335 // (efficiency concerns for ex) you may also want to overload 00336 // some of the following methods: 00337 // virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00338 // Vec& output, Vec& costs) const; 00339 // virtual void computeCostsOnly(const Vec& input, const Vec& target, 00340 // Vec& costs) const; 00341 // virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00342 // VMat testoutputs=0, VMat testcosts=0) const; 00343 // virtual int nTestCosts() const; 00344 // virtual int nTrainCosts() const; 00345 // virtual void resetInternalState(); 00346 // virtual bool isStatefulLearner() const; 00347 00348 00349 //##### PLearn::Object Protocol ######################################### 00350 00351 // Declares other standard object methods. 00352 PLEARN_DECLARE_OBJECT(DeepBeliefNet); 00353 00354 // Simply calls inherited::build() then build_() 00355 virtual void build(); 00356 00358 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00359 00360 protected: 00362 int minibatch_size; 00363 00364 //##### Not Options ##################################################### 00365 00366 // whether to re-initialize Gibbs chain next time around 00367 bool initialize_gibbs_chain; 00368 00371 mutable TVec<Vec> activation_gradients; 00372 mutable TVec<Mat> activations_gradients; 00373 00376 mutable TVec<Vec> expectation_gradients; 00377 mutable TVec<Mat> expectations_gradients; 00378 00380 mutable TVec< TVec<Vec> > greedy_target_expectations; 00381 mutable TVec< TVec<Vec> > greedy_target_activations; 00382 mutable TVec< TVec<Vec> > greedy_target_expectation_gradients; 00383 mutable TVec< TVec<Vec> > greedy_target_activation_gradients; 00384 mutable TVec< Vec > greedy_target_probability_gradients; 00385 mutable TVec< PP<RBMLayer> > greedy_joint_layers; 00386 mutable TVec< PP<RBMConnection> > greedy_joint_connections; 00387 00388 mutable Vec final_cost_input; 00389 mutable Mat final_cost_inputs; 00390 00391 mutable Vec final_cost_value; 00392 mutable Mat final_cost_values; 00393 00394 mutable Vec final_cost_output; 00395 00396 mutable Vec class_output; 00397 00398 mutable Vec class_gradient; 00399 00401 mutable Vec final_cost_gradient; 00402 mutable Mat final_cost_gradients; 00403 00405 mutable Vec save_layer_activation; 00406 Mat save_layer_activations; 00407 00409 mutable Vec save_layer_expectation; 00410 Mat save_layer_expectations; 00411 00413 mutable Vec pos_down_val; 00414 mutable Vec corrupted_pos_down_val; 00415 mutable Vec pos_up_val; 00416 mutable Mat pos_down_vals; 00417 mutable Mat pos_up_vals; 00418 mutable Mat cd_neg_down_vals; 00419 mutable Mat cd_neg_up_vals; 00420 mutable Mat mf_cd_neg_down_vals; 00421 mutable Mat mf_cd_neg_up_vals; 00422 mutable Vec mf_cd_neg_down_val; 00423 mutable Vec mf_cd_neg_up_val; 00424 00426 mutable TVec<Mat> gibbs_down_state; 00427 00429 Vec optimized_costs; 00430 00432 mutable Vec target_one_hot; 00433 00435 mutable Vec reconstruction_costs; 00436 00438 int nll_cost_index; 00439 00441 int class_cost_index; 00442 00444 int final_cost_index; 00445 00447 TVec<int> partial_costs_indices; 00448 00450 int reconstruction_cost_index; 00451 00453 int greedy_target_layer_nlls_index; 00454 00456 int training_cpu_time_cost_index; 00457 00459 int cumulative_training_time_cost_index; 00460 00462 int cumulative_testing_time_cost_index; 00463 00465 real cumulative_training_time; 00466 00468 mutable real cumulative_testing_time; 00469 00471 TVec<int> cumulative_schedule; 00472 00474 int up_down_stage; 00475 00476 mutable Vec layer_input; 00477 mutable Mat layer_inputs; 00478 00481 TVec< PP<RBMConnection> > generative_connections; 00482 00483 mutable TVec<Vec> up_sample; 00484 mutable TVec<Vec> down_sample; 00485 00487 TVec< TVec<int> > expectation_indices; 00488 00489 protected: 00490 //##### Protected Member Functions ###################################### 00491 00493 static void declareOptions(OptionList& ol); 00494 00496 static void declareMethods(RemoteMethodMap& rmm); 00497 00498 private: 00499 //##### Private Member Functions ######################################## 00500 00502 void build_(); 00503 00504 void build_layers_and_connections(); 00505 00506 void build_costs(); 00507 00508 void build_classification_cost(); 00509 00510 void build_final_cost(); 00511 00512 void corrupt_input(const Vec& input, Vec& corrupted_input, int layer); 00513 00514 void setLearningRate( real the_learning_rate ); 00515 00516 TVec<Vec> fantasizeKTime(const int KTime, const Vec& srcImg, const Vec& sample, 00517 bool alwaysFromSrcImg); 00518 TVec<Vec> fantasizeKTimeOnMultiSrcImg(const int KTime, const Mat& srcImg, const Vec& sample, 00519 bool alwaysFromSrcImg); 00520 00521 private: 00522 //##### Private Data Members ############################################ 00523 00524 // The rest of the private stuff goes here 00525 }; 00526 00527 // Declares a few other classes and functions related to this class 00528 DECLARE_OBJECT_PTR(DeepBeliefNet); 00529 00530 } // end of namespace PLearn 00531 00532 #endif 00533 00534 00535 /* 00536 Local Variables: 00537 mode:c++ 00538 c-basic-offset:4 00539 c-file-style:"stroustrup" 00540 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00541 indent-tabs-mode:nil 00542 fill-column:79 00543 End: 00544 */ 00545 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :