PLearn 0.1
DeepBeliefNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DeepBeliefNet.h
00004 //
00005 // Copyright (C) 2006 Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin
00036 
00039 #ifndef DeepBeliefNet_INC
00040 #define DeepBeliefNet_INC
00041 
00042 #include <plearn_learners/generic/PLearner.h>
00043 #include "OnlineLearningModule.h"
00044 #include "CostModule.h"
00045 #include "NLLCostModule.h"
00046 #include "RBMClassificationModule.h"
00047 #include "RBMLayer.h"
00048 #include "RBMMixedLayer.h"
00049 #include "RBMConnection.h"
00050 #include <plearn/misc/PTimer.h>
00051 #include <plearn/sys/Profiler.h>
00052 
00053 namespace PLearn {
00054 using namespace std;
00055 
00061 class DeepBeliefNet : public PLearner
00062 {
00063     typedef PLearner inherited;
00064 
00065 public:
00066     //#####  Public Build Options  ############################################
00067 
00069     real cd_learning_rate;
00070 
00073     real cd_decrease_ct;
00074 
00077     real up_down_learning_rate;
00078 
00081     real up_down_decrease_ct;
00082 
00084     real grad_learning_rate;
00085 
00087     real grad_decrease_ct;
00088 
00089     /* NOT IMPLEMENTED YET
00091     real grad_weight_decay;
00092     */
00093 
00095     int batch_size;
00096 
00102     int n_classes;
00103 
00111     TVec<int> training_schedule;
00112 
00117     int up_down_nstages;
00118 
00122     bool use_classification_cost;
00123 
00131     bool reconstruct_layerwise;
00132 
00134     TVec< PP<RBMLayer> > layers;
00135 
00138     int i_output_layer;
00139 
00142     string learnerExpdir;
00143  
00145     bool save_learner_before_fine_tuning;
00146   
00148     TVec< PP<RBMConnection> > connections;
00149 
00151     TVec< PP<RBMMultinomialLayer> > greedy_target_layers;
00152 
00154     TVec< PP<RBMMatrixConnection> > greedy_target_connections;
00155 
00162     PP<OnlineLearningModule> final_module;
00163 
00168     PP<CostModule> final_cost;
00169 
00173     TVec< PP<CostModule> > partial_costs;
00174 
00177     bool use_sample_for_up_layer;
00178 
00185     string use_corrupted_posDownVal;
00186     
00188     string noise_type;
00189     
00192     real fraction_of_masked_inputs;
00193     
00196     bool mask_with_pepper_salt;
00197     
00200     real prob_salt_noise;
00201 
00202 
00203     //#####  Public Learnt Options  ###########################################
00205     PP<RBMClassificationModule> classification_module;
00206 
00208     int n_layers;
00209 
00211     TVec<string> cost_names;
00212 
00214     bool online;
00215 
00216     // Coefficient between 0 and 1. If non-zero, run a background
00217     // Gibbs chain and use the visible-hidden statistics to
00218     // contribute in the negative phase update (in proportion
00219     // background_gibbs_update_ratio wrt the contrastive divergence
00220     // negative phase statistics). If = 1, then do not perform any
00221     // contrastive divergence negative phase (use only the Gibbs chain
00222     // statistics).
00223     real background_gibbs_update_ratio;
00224 
00227     bool top_layer_joint_cd;
00228 
00232     int gibbs_chain_reinit_freq;
00233 
00237     real mean_field_contrastive_divergence_ratio;
00238 
00241     int train_stats_window;
00242 
00243     //#####  Not Options  #####################################################
00244 
00246     PP<PTimer> timer;
00247 
00250     PP<NLLCostModule> classification_cost;
00251 
00254     PP<RBMMixedLayer> joint_layer;
00255 
00256 public:
00257     //#####  Public Member Functions  #########################################
00258 
00260     DeepBeliefNet();
00261 
00262 
00263     //#####  PLearner Member Functions  #######################################
00264 
00267     virtual int outputsize() const;
00268 
00272     virtual void forget();
00273 
00277     virtual void train();
00278 
00280     virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00281                       VMat testoutputs=0, VMat testcosts=0) const;
00282 
00284     virtual void computeOutput(const Vec& input, Vec& output) const;
00285 
00286     virtual void computeOutputsAndCosts(const Mat& inputs, const Mat& targets,
00287                                         Mat& outputs, Mat& costs) const;
00288 
00290     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00291                                          const Vec& target, Vec& costs) const;
00292     virtual void computeClassifAndFinalCostsFromOutputs(
00293             const Mat& inputs, const Mat& outputs,
00294             const Mat& targets, Mat& costs) const;
00295 
00298     // (PLEASE IMPLEMENT IN .cc)
00299     virtual TVec<std::string> getTestCostNames() const;
00300 
00303     // (PLEASE IMPLEMENT IN .cc)
00304     virtual TVec<std::string> getTrainCostNames() const;
00305 
00306     void onlineStep(const Vec& input, const Vec& target, Vec& train_costs);
00307     void onlineStep(const Mat& inputs, const Mat& targets, Mat& train_costs);
00308 
00309     void greedyStep(const Vec& input, const Vec& target, int index);
00310     void greedyStep(const Mat& inputs, const Mat& targets, int index,
00311                     Mat& train_costs_m);
00312 
00313 
00314     void jointGreedyStep(const Vec& input, const Vec& target);
00315     void jointGreedyStep(const Mat& inputs, const Mat& targets);
00316 
00317     void upDownStep(const Vec& input, const Vec& target, Vec& train_costs);
00318     void upDownStep(const Mat& inputs, const Mat& targets, Mat& train_costs);
00319 
00320     void fineTuningStep(const Vec& input, const Vec& target, Vec& train_costs);
00321     void fineTuningStep(const Mat& inputs, const Mat& targets,
00322                         Mat& train_costs);
00323 
00326     void contrastiveDivergenceStep( const PP<RBMLayer>& down_layer,
00327                                     const PP<RBMConnection>& connection,
00328                                     const PP<RBMLayer>& up_layer,
00329                                     int layer_index,
00330                                     bool nofprop=false);
00331 
00332 
00333     // *** SUBCLASS WRITING: ***
00334     // While in general not necessary, in case of particular needs
00335     // (efficiency concerns for ex) you may also want to overload
00336     // some of the following methods:
00337     // virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00338     //                                    Vec& output, Vec& costs) const;
00339     // virtual void computeCostsOnly(const Vec& input, const Vec& target,
00340     //                               Vec& costs) const;
00341     // virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00342     //                   VMat testoutputs=0, VMat testcosts=0) const;
00343     // virtual int nTestCosts() const;
00344     // virtual int nTrainCosts() const;
00345     // virtual void resetInternalState();
00346     // virtual bool isStatefulLearner() const;
00347 
00348 
00349     //#####  PLearn::Object Protocol  #########################################
00350 
00351     // Declares other standard object methods.
00352     PLEARN_DECLARE_OBJECT(DeepBeliefNet);
00353 
00354     // Simply calls inherited::build() then build_()
00355     virtual void build();
00356 
00358     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00359 
00360 protected:
00362     int minibatch_size;
00363 
00364     //#####  Not Options  #####################################################
00365 
00366     // whether to re-initialize Gibbs chain next time around
00367     bool initialize_gibbs_chain;
00368 
00371     mutable TVec<Vec> activation_gradients;
00372     mutable TVec<Mat> activations_gradients; 
00373 
00376     mutable TVec<Vec> expectation_gradients;
00377     mutable TVec<Mat> expectations_gradients; 
00378     
00380     mutable TVec< TVec<Vec> > greedy_target_expectations;
00381     mutable TVec< TVec<Vec> > greedy_target_activations;
00382     mutable TVec< TVec<Vec> > greedy_target_expectation_gradients;
00383     mutable TVec< TVec<Vec> > greedy_target_activation_gradients;
00384     mutable TVec< Vec > greedy_target_probability_gradients;
00385     mutable TVec< PP<RBMLayer> > greedy_joint_layers;
00386     mutable TVec< PP<RBMConnection> > greedy_joint_connections;
00387 
00388     mutable Vec final_cost_input;
00389     mutable Mat final_cost_inputs; 
00390 
00391     mutable Vec final_cost_value;
00392     mutable Mat final_cost_values; 
00393 
00394     mutable Vec final_cost_output;
00395 
00396     mutable Vec class_output;
00397 
00398     mutable Vec class_gradient;
00399 
00401     mutable Vec final_cost_gradient;
00402     mutable Mat final_cost_gradients; 
00403 
00405     mutable Vec save_layer_activation;
00406     Mat save_layer_activations; 
00407 
00409     mutable Vec save_layer_expectation;
00410     Mat save_layer_expectations; 
00411 
00413     mutable Vec pos_down_val;
00414     mutable Vec corrupted_pos_down_val;
00415     mutable Vec pos_up_val;
00416     mutable Mat pos_down_vals;
00417     mutable Mat pos_up_vals;
00418     mutable Mat cd_neg_down_vals;
00419     mutable Mat cd_neg_up_vals;
00420     mutable Mat mf_cd_neg_down_vals;
00421     mutable Mat mf_cd_neg_up_vals;
00422     mutable Vec mf_cd_neg_down_val;
00423     mutable Vec mf_cd_neg_up_val;
00424 
00426     mutable TVec<Mat> gibbs_down_state;
00427 
00429     Vec optimized_costs;
00430 
00432     mutable Vec target_one_hot;
00433 
00435     mutable Vec reconstruction_costs;
00436 
00438     int nll_cost_index;
00439 
00441     int class_cost_index;
00442 
00444     int final_cost_index;
00445 
00447     TVec<int> partial_costs_indices;
00448 
00450     int reconstruction_cost_index;
00451 
00453     int greedy_target_layer_nlls_index;
00454 
00456     int training_cpu_time_cost_index;
00457 
00459     int cumulative_training_time_cost_index;
00460 
00462     int cumulative_testing_time_cost_index;
00463 
00465     real cumulative_training_time;
00466 
00468     mutable real cumulative_testing_time;
00469 
00471     TVec<int> cumulative_schedule;
00472 
00474     int up_down_stage;
00475 
00476     mutable Vec layer_input;
00477     mutable Mat layer_inputs;
00478 
00481     TVec< PP<RBMConnection> > generative_connections;
00482 
00483     mutable TVec<Vec> up_sample;
00484     mutable TVec<Vec> down_sample;
00485 
00487     TVec< TVec<int> > expectation_indices;
00488 
00489 protected:
00490     //#####  Protected Member Functions  ######################################
00491 
00493     static void declareOptions(OptionList& ol);
00494    
00496     static void declareMethods(RemoteMethodMap& rmm);
00497 
00498 private:
00499     //#####  Private Member Functions  ########################################
00500 
00502     void build_();
00503 
00504     void build_layers_and_connections();
00505 
00506     void build_costs();
00507 
00508     void build_classification_cost();
00509 
00510     void build_final_cost();
00511 
00512     void corrupt_input(const Vec& input, Vec& corrupted_input, int layer);
00513 
00514     void setLearningRate( real the_learning_rate );
00515 
00516     TVec<Vec> fantasizeKTime(const int KTime, const Vec& srcImg, const Vec& sample,
00517                          bool alwaysFromSrcImg);
00518     TVec<Vec> fantasizeKTimeOnMultiSrcImg(const int KTime, const Mat& srcImg, const Vec& sample,
00519                          bool alwaysFromSrcImg);
00520 
00521 private:
00522     //#####  Private Data Members  ############################################
00523 
00524     // The rest of the private stuff goes here
00525 };
00526 
00527 // Declares a few other classes and functions related to this class
00528 DECLARE_OBJECT_PTR(DeepBeliefNet);
00529 
00530 } // end of namespace PLearn
00531 
00532 #endif
00533 
00534 
00535 /*
00536   Local Variables:
00537   mode:c++
00538   c-basic-offset:4
00539   c-file-style:"stroustrup"
00540   c-file-offsets:((innamespace . 0)(inline-open . 0))
00541   indent-tabs-mode:nil
00542   fill-column:79
00543   End:
00544 */
00545 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines