PLearn 0.1
SubsamplingDBN.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // SubsamplingDBN.h
00004 //
00005 // Copyright (C) 2006 Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin
00036 
00040 #ifndef SubsamplingDBN_INC
00041 #define SubsamplingDBN_INC
00042 
00043 #include <plearn_learners/generic/PLearner.h>
00044 #include <plearn_learners/online/OnlineLearningModule.h>
00045 #include <plearn_learners/online/CostModule.h>
00046 #include <plearn_learners/online/NLLCostModule.h>
00047 #include <plearn_learners/online/RBMClassificationModule.h>
00048 #include <plearn_learners/online/RBMLayer.h>
00049 #include <plearn_learners/online/RBMMixedLayer.h>
00050 #include <plearn_learners/online/RBMConnection.h>
00051 #include <plearn/misc/PTimer.h>
00052 #include <plearn/sys/Profiler.h>
00053 
00054 namespace PLearn {
00055 
00061 class SubsamplingDBN : public PLearner
00062 {
00063     typedef PLearner inherited;
00064 
00065 public:
00066     //#####  Public Build Options  ############################################
00067 
00069     real cd_learning_rate;
00070 
00072     real grad_learning_rate;
00073 
00074     int batch_size;
00075 
00077     real grad_decrease_ct;
00078 
00079     /* NOT IMPLEMENTED YET
00081     real grad_weight_decay;
00082     */
00083 
00089     int n_classes;
00090 
00098     TVec<int> training_schedule;
00099 
00103     bool use_classification_cost;
00104 
00112     bool reconstruct_layerwise;
00113 
00115     TVec< PP<RBMLayer> > layers;
00116 
00118     TVec< PP<RBMConnection> > connections;
00119 
00126     PP<OnlineLearningModule> final_module;
00127 
00132     PP<CostModule> final_cost;
00133 
00137     TVec< PP<CostModule> > partial_costs;
00138 
00141     bool independent_biases;
00142 
00145     TVec< PP<OnlineLearningModule> > subsampling_modules;
00146 
00147     //#####  Public Learnt Options  ###########################################
00149     PP<RBMClassificationModule> classification_module;
00150 
00152     int n_layers;
00153 
00155     TVec<string> cost_names;
00156 
00158     bool online;
00159 
00160     // Coefficient between 0 and 1. If non-zero, run a background
00161     // Gibbs chain and use the visible-hidden statistics to
00162     // contribute in the negative phase update (in proportion
00163     // background_gibbs_update_ratio wrt the contrastive divergence
00164     // negative phase statistics). If = 1, then do not perform any
00165     // contrastive divergence negative phase (use only the Gibbs chain
00166     // statistics).
00167     real background_gibbs_update_ratio;
00168 
00171     bool top_layer_joint_cd;
00172 
00176     int gibbs_chain_reinit_freq;
00177 
00182     TVec< PP<RBMLayer> > reduced_layers;
00183 
00184     //#####  Not Options  #####################################################
00185 
00187     PP<PTimer> timer;
00188 
00191     PP<NLLCostModule> classification_cost;
00192 
00195     PP<RBMMixedLayer> joint_layer;
00196 
00197 
00198 public:
00199     //#####  Public Member Functions  #########################################
00200 
00202     SubsamplingDBN();
00203 
00204 
00205     //#####  PLearner Member Functions  #######################################
00206 
00209     virtual int outputsize() const;
00210 
00214     // (PLEASE IMPLEMENT IN .cc)
00215     virtual void forget();
00216 
00220     // (PLEASE IMPLEMENT IN .cc)
00221     virtual void train();
00222 
00224     virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00225                       VMat testoutputs=0, VMat testcosts=0) const;
00226 
00228     // (PLEASE IMPLEMENT IN .cc)
00229     virtual void computeOutput(const Vec& input, Vec& output) const;
00230 
00232     // (PLEASE IMPLEMENT IN .cc)
00233     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00234                                          const Vec& target, Vec& costs) const;
00235 
00238     // (PLEASE IMPLEMENT IN .cc)
00239     virtual TVec<std::string> getTestCostNames() const;
00240 
00243     // (PLEASE IMPLEMENT IN .cc)
00244     virtual TVec<std::string> getTrainCostNames() const;
00245 
00246 
00247     void onlineStep( const Vec& input, const Vec& target, Vec& train_costs );
00248 
00249     void onlineStep( const Mat& inputs, const Mat& targets, Mat& train_costs );
00250 
00251     void greedyStep( const Vec& input, const Vec& target, int index );
00252 
00254     void greedyStep(const Mat& inputs, const Mat& targets, int index, Mat& train_costs_m);
00255 
00256     void jointGreedyStep( const Vec& input, const Vec& target );
00257 
00258     void fineTuningStep( const Vec& input, const Vec& target,
00259                          Vec& train_costs );
00260 
00262     void fineTuningStep( const Mat& inputs, const Mat& targets,
00263                          Mat& train_costs );
00264 
00267     void contrastiveDivergenceStep( const PP<RBMLayer>& down_layer,
00268                                     const PP<RBMConnection>& connection,
00269                                     const PP<RBMLayer>& up_layer,
00270                                     int layer_index,
00271                                     bool nofprop=false);
00272 
00273 
00274     // *** SUBCLASS WRITING: ***
00275     // While in general not necessary, in case of particular needs
00276     // (efficiency concerns for ex) you may also want to overload
00277     // some of the following methods:
00278     // virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00279     //                                    Vec& output, Vec& costs) const;
00280     // virtual void computeCostsOnly(const Vec& input, const Vec& target,
00281     //                               Vec& costs) const;
00282     // virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00283     //                   VMat testoutputs=0, VMat testcosts=0) const;
00284     // virtual int nTestCosts() const;
00285     // virtual int nTrainCosts() const;
00286     // virtual void resetInternalState();
00287     // virtual bool isStatefulLearner() const;
00288 
00289 
00290     //#####  PLearn::Object Protocol  #########################################
00291 
00292     // Declares other standard object methods.
00293     // ### If your class is not instantiatable (it has pure virtual methods)
00294     // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS
00295     PLEARN_DECLARE_OBJECT(SubsamplingDBN);
00296 
00297     // Simply calls inherited::build() then build_()
00298     virtual void build();
00299 
00301     // (PLEASE IMPLEMENT IN .cc)
00302     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00303 
00304 protected:
00305 
00306     int minibatch_size;
00307 
00308     //#####  Not Options  #####################################################
00309 
00310     // whether to re-initialize Gibbs chain next time around
00311     bool initialize_gibbs_chain;
00312 
00315     mutable TVec<Vec> activation_gradients;
00316     mutable TVec<Mat> activations_gradients; 
00317 
00320     mutable TVec<Vec> expectation_gradients;
00321     mutable TVec<Mat> expectations_gradients; 
00322 
00323     mutable TVec<Vec> subsampling_gradients;
00324 
00325     mutable Vec final_cost_input;
00326     mutable Mat final_cost_inputs; 
00327 
00328     mutable Vec final_cost_value;
00329     mutable Mat final_cost_values; 
00330 
00331     mutable Vec final_cost_output;
00332 
00333     mutable Vec class_output;
00334 
00335     mutable Vec class_gradient;
00336 
00338     mutable Vec final_cost_gradient;
00339     mutable Mat final_cost_gradients; 
00340 
00342     mutable Vec save_layer_activation;
00343 
00344     Mat save_layer_activations; 
00345 
00347     mutable Vec save_layer_expectation;
00348 
00349     Mat save_layer_expectations; 
00350 
00352     bool final_module_has_learning_rate;
00353 
00355     bool final_cost_has_learning_rate;
00356 
00358     mutable Vec pos_down_val;
00359     mutable Vec pos_up_val;
00360     mutable Mat pos_down_vals;
00361     mutable Mat pos_up_vals;
00362     mutable Mat cd_neg_down_vals;
00363     mutable Mat cd_neg_up_vals;
00364 
00366     mutable TVec<Mat> gibbs_down_state;
00367 
00369     Vec optimized_costs;
00370 
00372     mutable Vec reconstruction_costs;
00373 
00375     int nll_cost_index;
00376 
00378     int class_cost_index;
00379 
00381     int final_cost_index;
00382 
00384     TVec<int> partial_costs_indices;
00385 
00387     int reconstruction_cost_index;
00388 
00390     int training_cpu_time_cost_index;
00391 
00393     int cumulative_training_time_cost_index;
00394 
00396     int cumulative_testing_time_cost_index;
00397 
00399     real cumulative_training_time;
00400 
00402     mutable real cumulative_testing_time;
00403 
00405     TVec<int> cumulative_schedule;
00406 
00407     mutable Vec layer_input;
00408     mutable Mat layer_inputs;
00409 
00410 protected:
00411     //#####  Protected Member Functions  ######################################
00412 
00414     static void declareOptions(OptionList& ol);
00415 
00416 private:
00417     //#####  Private Member Functions  ########################################
00418 
00420     void build_();
00421 
00422     void build_layers_and_connections();
00423 
00424     void build_costs();
00425 
00426     void build_classification_cost();
00427 
00428     void build_final_cost();
00429 
00430     void setLearningRate( real the_learning_rate );
00431 
00432 private:
00433     //#####  Private Data Members  ############################################
00434 
00435     // The rest of the private stuff goes here
00436 };
00437 
00438 // Declares a few other classes and functions related to this class
00439 DECLARE_OBJECT_PTR(SubsamplingDBN);
00440 
00441 } // end of namespace PLearn
00442 
00443 #endif
00444 
00445 
00446 /*
00447   Local Variables:
00448   mode:c++
00449   c-basic-offset:4
00450   c-file-style:"stroustrup"
00451   c-file-offsets:((innamespace . 0)(inline-open . 0))
00452   indent-tabs-mode:nil
00453   fill-column:79
00454   End:
00455 */
00456 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines