PLearn 0.1
DiscriminativeDeepBeliefNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DiscriminativeDeepBeliefNet.h
00004 //
00005 // Copyright (C) 2007 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Hugo Larochelle
00036 
00040 #ifndef DiscriminativeDeepBeliefNet_INC
00041 #define DiscriminativeDeepBeliefNet_INC
00042 
00043 #include <plearn/vmat/ClassSubsetVMatrix.h>
00044 #include <plearn_learners/generic/PLearner.h>
00045 #include <plearn_learners/online/GradNNetLayerModule.h>
00046 #include <plearn_learners/online/OnlineLearningModule.h>
00047 #include <plearn_learners/online/CostModule.h>
00048 #include <plearn_learners/online/ModuleStackModule.h>
00049 #include <plearn_learners/online/NLLCostModule.h>
00050 #include <plearn_learners/online/ClassErrorCostModule.h>
00051 #include <plearn_learners/online/CombiningCostsModule.h>
00052 #include <plearn_learners/online/RBMClassificationModule.h>
00053 #include <plearn_learners/online/RBMLayer.h>
00054 #include <plearn_learners/online/RBMMixedLayer.h>
00055 #include <plearn_learners/online/RBMConnection.h>
00056 #include <plearn_learners/online/SoftmaxModule.h>
00057 #include <plearn/misc/PTimer.h>
00058 
00059 namespace PLearn {
00060 
00064 class DiscriminativeDeepBeliefNet : public PLearner
00065 {
00066     typedef PLearner inherited;
00067 
00068 public:
00069     //#####  Public Build Options  ############################################
00070 
00072     real cd_learning_rate;
00073     
00075     real cd_decrease_ct;
00076 
00078     real fine_tuning_learning_rate;
00079 
00082     real fine_tuning_decrease_ct;
00083 
00086     TVec<int> training_schedule;
00087 
00089     TVec< PP<RBMLayer> > layers;
00090 
00092     TVec< PP<RBMConnection> > connections;
00093 
00095     TVec< PP<RBMLayer> > unsupervised_layers;
00096 
00098     TVec< PP<RBMConnection> > unsupervised_connections;
00099 
00102     int k_neighbors;
00103 
00105     int n_classes;
00106 
00108     real discriminative_criteria_weight;
00109 
00111     real output_weights_l1_penalty_factor;
00112 
00114     real output_weights_l2_penalty_factor;
00115 
00119     bool compare_joint_in_discriminative_criteria;
00120 
00123     bool do_not_use_generative_criteria;
00124 
00125 //    //! Indication that the discriminative and generative criteria should cancel
00126 //    //! their normalization terms. This is for the compare_joint_in_discriminative_criteria
00127 //    //! option, and this option ignores the value of discriminative_criteria_weight.
00128 //    bool cancel_normalization_terms;
00129 
00130     //#####  Public Learnt Options  ###########################################
00131 
00133     int n_layers;
00134 
00135 public:
00136     //#####  Public Member Functions  #########################################
00137 
00139     DiscriminativeDeepBeliefNet();
00140 
00141     //#####  PLearner Member Functions  #######################################
00142 
00145     virtual int outputsize() const;
00146 
00150     virtual void forget();
00151 
00155     virtual void train();
00156 
00158     virtual void computeOutput(const Vec& input, Vec& output) const;
00159 
00161     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00162                                          const Vec& target, Vec& costs) const;
00163 
00166     virtual TVec<std::string> getTestCostNames() const;
00167 
00170     virtual TVec<std::string> getTrainCostNames() const;
00171 
00179     virtual void setTrainingSet(VMat training_set, bool call_forget=true);
00180 
00181     void updateNearestNeighbors();
00182 
00183     void greedyStep( const Vec& input, const Vec& target, int index, 
00184                      Vec train_costs, int stage, Vec dissimilar_example);
00185 
00186     void fineTuningStep( const Vec& input, const Vec& target,
00187                          Vec& train_costs);
00188 
00189     void computeRepresentation( const Vec& input, 
00190                                 Vec& representation, int layer) const;
00191 
00192     //#####  PLearn::Object Protocol  #########################################
00193 
00194     // Declares other standard object methods.
00195     // ### If your class is not instantiatable (it has pure virtual methods)
00196     // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS
00197     PLEARN_DECLARE_OBJECT(DiscriminativeDeepBeliefNet);
00198 
00199     // Simply calls inherited::build() then build_()
00200     virtual void build();
00201 
00203     // (PLEASE IMPLEMENT IN .cc)
00204     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00205 
00206 protected:
00207     //#####  Not Options  #####################################################
00208 
00210     bool nearest_neighbors_are_up_to_date;
00211 
00214     mutable TVec<Vec> activations;
00215 
00218     mutable TVec<Vec> expectations;
00219 
00223     mutable TVec<Vec> activation_gradients;
00224 
00228     mutable TVec<Vec> expectation_gradients;
00229 
00231     TVec< PP<RBMLayer> > greedy_layers;
00232 
00234     TVec< PP<RBMConnection> > greedy_connections;
00235 
00237     Vec dissimilar_example_representation;
00238 
00240     mutable Vec input_representation;
00241 
00243     Vec pos_down_val;
00245     Vec pos_up_val;
00247     Vec neg_down_val;
00249     Vec neg_up_val;
00250 
00252     Vec disc_pos_down_val1;
00254     Vec disc_pos_up_val1;
00256     Vec disc_pos_down_val2;
00258     Vec disc_pos_up_val2;
00260     Vec disc_neg_down_val;
00262     Vec disc_neg_up_val;
00263 
00265     mutable Vec final_cost_input;
00267     mutable Vec final_cost_value;
00269     mutable Vec final_cost_gradient;
00270 
00272     TVec< PP<ClassSubsetVMatrix> > other_class_datasets;
00273 
00275     TMat<int> nearest_neighbors_indices;
00276 
00278     TVec<int> greedy_stages;
00279 
00282     int currently_trained_layer;
00283 
00285     PP<OnlineLearningModule> final_module;
00286 
00288     PP<CostModule> final_cost;
00289 
00290 protected:
00291     //#####  Protected Member Functions  ######################################
00292 
00294     static void declareOptions(OptionList& ol);
00295 
00296 private:
00297     //#####  Private Member Functions  ########################################
00298 
00300     void build_();
00301 
00302     void build_layers_and_connections();
00303 
00304     void build_output_layer_and_cost();
00305 
00306     void setLearningRate( real the_learning_rate );
00307 
00308 private:
00309     //#####  Private Data Members  ############################################
00310 
00311     // The rest of the private stuff goes here    
00312 };
00313 
00314 // Declares a few other classes and functions related to this class
00315 DECLARE_OBJECT_PTR(DiscriminativeDeepBeliefNet);
00316 
00317 } // end of namespace PLearn
00318 
00319 #endif
00320 
00321 
00322 /*
00323   Local Variables:
00324   mode:c++
00325   c-basic-offset:4
00326   c-file-style:"stroustrup"
00327   c-file-offsets:((innamespace . 0)(inline-open . 0))
00328   indent-tabs-mode:nil
00329   fill-column:79
00330   End:
00331 */
00332 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines