PLearn 0.1
PartSupervisedDBN.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PartSupervisedDBN.h
00004 //
00005 // Copyright (C) 2006 Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin
00036 
00040 #ifndef PartSupervisedDBN_INC
00041 #define PartSupervisedDBN_INC
00042 
00043 #include <plearn_learners/distributions/PDistribution.h>
00044 
00045 namespace PLearn {
00046 
00047 class RBMLayer;
00048 class RBMMixedLayer;
00049 class RBMMultinomialLayer;
00050 class RBMParameters;
00051 class RBMLLParameters;
00052 class RBMJointLLParameters;
00053 class OnlineLearningModule;
00054 
00062 class PartSupervisedDBN : public PDistribution
00063 {
00064     typedef PDistribution inherited;
00065 
00066 public:
00067     //#####  Public Build Options  ############################################
00068 
00070     real learning_rate;
00071 
00073     Vec supervised_learning_rates;
00074 
00076     real fine_tuning_learning_rate;
00077 
00079     real initial_momentum;
00080 
00082     real final_momentum;
00083 
00086     int momentum_switch_time;
00087 
00089     real weight_decay;
00090 
00096     string initialization_method;
00097 
00100     int n_layers;
00101 
00104     TVec< PP<RBMLayer> > layers;
00105 
00107     PP<RBMLayer> last_layer;
00108 
00110     PP<RBMMultinomialLayer> target_layer;
00111 
00113     PP<RBMMixedLayer> joint_layer;
00114 
00117     TVec< PP<RBMLLParameters> > params;
00118 
00120     PP<RBMLLParameters> target_params;
00121 
00124     PP<RBMJointLLParameters> joint_params;
00125 
00128     TVec< PP<OnlineLearningModule> > regressors;
00129 
00134     int parallelization_minibatch_size;
00135 
00138     bool sum_parallel_contributions;
00139 
00142     TVec<int> training_schedule;
00143 
00150     string fine_tuning_method;
00151 
00152 //    bool use_sample_rather_than_expectation_in_positive_phase_statistics;
00153 
00165     TVec<int> use_sample_or_expectation;
00166 
00167 public:
00168     //#####  Public Member Functions  #########################################
00169 
00171     // ### Make sure the implementation in the .cc
00172     // ### initializes all fields to reasonable default values.
00173     PartSupervisedDBN();
00174 
00175 
00176     //#####  PDistribution Member Functions  ##################################
00177 
00179     virtual real density(const Vec& y) const;
00180 
00182     virtual real log_density(const Vec& y) const;
00183 
00185     virtual real survival_fn(const Vec& y) const;
00186 
00188     virtual real cdf(const Vec& y) const;
00189 
00191     virtual void expectation(Vec& mu) const;
00192 
00194     virtual void variance(Mat& cov) const;
00195 
00198     virtual void generate(Vec& y) const;
00199 
00200     //### Override this method if you need it (and if your distribution can
00201     //### handle it. Default version calls PLERROR.
00206     // virtual void generatePredictorGivenPredicted(Vec& x, const Vec& y);
00207 
00209     //### See help in PDistribution.h.
00210     virtual bool setPredictorPredictedSizes(int the_predictor_size,
00211                                             int the_predicted_size,
00212                                             bool call_parent = true);
00213 
00215     //### See help in PDistribution.h.
00216     virtual void setPredictor(const Vec& predictor, bool call_parent = true)
00217                               const;
00218 
00219     // ### These methods may be overridden for efficiency purpose:
00220     /*
00221     //### Default version calls setPredictorPredictedSises(0,-1) and generate
00226     virtual void generateJoint(Vec& xy);
00227 
00228     //### Default version calls generateJoint and discards y
00233     virtual void generatePredictor(Vec& x);
00234 
00235     //### Default version calls generateJoint and discards x
00240     virtual void generatePredicted(Vec& y);
00241     */
00242 
00243 
00244     //#####  PLearner Member Functions  #######################################
00245 
00246     // ### Default version of inputsize returns learner->inputsize()
00247     // ### If this is not appropriate, you should uncomment this and define
00248     // ### it properly in the .cc
00249     // virtual int inputsize() const;
00250 
00258     virtual void forget();
00259 
00263     virtual void train();
00264 
00268     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00269                                          const Vec& target, Vec& costs) const;
00270 
00271     virtual TVec<string> getTestCostNames() const;
00272     virtual TVec<string> getTrainCostNames() const;
00273 
00275 #if USING_MPI
00276 
00277 
00278 
00279 
00280 
00281 
00282 
00283     virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00284                       VMat testoutputs=0, VMat testcosts=0) const;
00285 #endif
00286 
00287 
00288     //#####  PLearn::Object Protocol  #########################################
00289 
00290     // Declares other standard object methods.
00291     // ### If your class is not instantiatable (it has pure virtual methods)
00292     // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS
00293     PLEARN_DECLARE_OBJECT(PartSupervisedDBN);
00294 
00295     // Simply calls inherited::build() then build_()
00296     virtual void build();
00297 
00299     // (PLEASE IMPLEMENT IN .cc)
00300     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00301 
00302 protected:
00303     //#####  Protected Options  ###############################################
00304 
00305     // ### Declare protected option fields (such as learned parameters) here
00306     // ...
00307 
00309     mutable TVec< Vec > activation_gradients;
00310 
00312     mutable TVec< Vec > expectation_gradients;
00313 
00315     mutable Vec output_gradient;
00316 
00317 
00318 protected:
00319     //#####  Protected Member Functions  ######################################
00320 
00321     virtual void contrastiveDivergenceStep(
00322         const PP<RBMLayer>& down_layer,
00323         const PP<RBMParameters>& parameters,
00324         const PP<RBMLayer>& up_layer );
00325 
00326     virtual real supervisedContrastiveDivergenceStep(
00327         const PP<RBMLayer>& down_layer,
00328         const PP<RBMParameters>& parameters,
00329         const PP<RBMLayer>& up_layer,
00330         const Vec& target,
00331         int index );
00332 
00333     virtual real greedyStep( const Vec& predictor, int params_index );
00334     virtual real jointGreedyStep( const Vec& input );
00335     virtual void fineTuneByGradientDescent( const Vec& input,
00336                                             const Vec& train_costs );
00337 
00339     static void declareOptions(OptionList& ol);
00340 
00341 private:
00342     //#####  Private Member Functions  ########################################
00343 
00345     void build_();
00346 
00348     void build_layers();
00349 
00351     void build_params();
00352 
00354     void build_regressors();
00355 
00356 #if USING_MPI
00357     void shareParamsMPI();
00358 #endif
00359 
00360 private:
00361     //#####  Private Data Members  ############################################
00362 
00363     // The rest of the private stuff goes here
00364 
00365 #if USING_MPI
00366 
00367     Vec global_params;
00370     Vec previous_global_params;
00371 #endif
00372 };
00373 
00374 // Declares a few other classes and functions related to this class
00375 DECLARE_OBJECT_PTR(PartSupervisedDBN);
00376 
00377 } // end of namespace PLearn
00378 
00379 #endif
00380 
00381 
00382 /*
00383   Local Variables:
00384   mode:c++
00385   c-basic-offset:4
00386   c-file-style:"stroustrup"
00387   c-file-offsets:((innamespace . 0)(inline-open . 0))
00388   indent-tabs-mode:nil
00389   fill-column:79
00390   End:
00391 */
00392 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines