PLearn 0.1
PseudolikelihoodRBM.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PseudolikelihoodRBM.h
00004 //
00005 // Copyright (C) 2008 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Hugo Larochelle
00036 
00039 #ifndef PseudolikelihoodRBM_INC
00040 #define PseudolikelihoodRBM_INC
00041 
00042 #include <plearn_learners/generic/PLearner.h>
00043 #include <plearn_learners/online/OnlineLearningModule.h>
00044 #include <plearn_learners/online/CostModule.h>
00045 #include <plearn_learners/online/CrossEntropyCostModule.h>
00046 #include <plearn_learners/online/NLLCostModule.h>
00047 #include <plearn_learners/online/RBMClassificationModule.h>
00048 #include <plearn_learners/online/RBMMatrixTransposeConnection.h>
00049 #include <plearn_learners/online/RBMMultitaskClassificationModule.h>
00050 #include <plearn_learners/online/RBMLayer.h>
00051 #include <plearn_learners/online/RBMMixedLayer.h>
00052 #include <plearn_learners/online/RBMConnection.h>
00053 #include <plearn/misc/PTimer.h>
00054 #include <plearn/sys/Profiler.h>
00055 
00056 namespace PLearn {
00057 using namespace std;
00058 
00062 class PseudolikelihoodRBM : public PLearner
00063 {
00064     typedef PLearner inherited;
00065 
00066 public:
00067     //#####  Public Build Options  ############################################
00068 
00070     real learning_rate;
00071 
00073     real decrease_ct;
00074 
00076     real cd_learning_rate;
00077 
00079     real cd_decrease_ct;
00080 
00082     int cd_n_gibbs;
00083 
00086     real persistent_cd_weight;
00087 
00090     int n_gibbs_chains;
00091 
00094     bool use_mean_field_cd;
00095 
00097     real denoising_learning_rate;
00098 
00100     real denoising_decrease_ct;
00101 
00103     real fraction_of_masked_inputs;
00104 
00106     bool only_reconstruct_masked_inputs;
00107 
00109     int n_classes;
00110     
00113     bool input_is_sparse;
00114 
00116     int factorized_connection_rank;
00117 
00119     int n_selected_inputs_pseudolikelihood;
00120 
00122     int n_selected_inputs_cd;
00123 
00126     //int select_among_k_most_frequent;
00127 
00133     bool compute_input_space_nll;
00134 
00136     bool compute_Z_exactly;
00137 
00140     bool use_ais_to_compute_Z;
00141 
00143     int n_ais_chains;
00144 
00145     // Schedule information for the betas in AIS. 
00148     Vec ais_beta_begin;
00151     Vec ais_beta_end;
00153     TVec<int> ais_beta_n_steps;
00154 
00155     // Each row gives
00161     Mat ais_beta_schedule;
00162 
00166     int pseudolikelihood_context_size;
00167 
00177     string pseudolikelihood_context_type;
00178 
00180     int k_most_correlated;
00181 
00183     real generative_learning_weight;
00184 
00186     real sparsity_bias_decay;
00187 
00191     real semi_sup_learning_weight;
00192 
00194     PP<RBMLayer> input_layer;
00195 
00197     PP<RBMLayer> hidden_layer;
00198 
00200     PP<RBMMatrixConnection> connection;
00201 
00203     //real target_weights_L1_penalty_factor;
00204     //
00206     //real target_weights_L2_penalty_factor;
00207 
00208     //#####  Public Learnt Options  ###########################################
00210     TVec<string> cost_names;
00211 
00212     PP<RBMMatrixTransposeConnection> transpose_connection;
00213 
00215     PP<RBMLayer> target_layer;
00216 
00218     PP<RBMMatrixConnection> target_connection;
00219 
00221     Mat U;
00222     
00225     Mat V;
00226 
00227 public:
00228     //#####  Public Member Functions  #########################################
00229 
00231     PseudolikelihoodRBM();
00232 
00233     //#####  PLearner Member Functions  #######################################
00234 
00237     virtual int outputsize() const;
00238 
00242     // (PLEASE IMPLEMENT IN .cc)
00243     virtual void forget();
00244 
00248     // (PLEASE IMPLEMENT IN .cc)
00249     virtual void train();
00250 
00259     virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 
00260                       VMat testoutputs=0, VMat testcosts=0) const;
00261 
00263     // (PLEASE IMPLEMENT IN .cc)
00264     virtual void computeOutput(const Vec& input, Vec& output) const;
00265 
00267     // (PLEASE IMPLEMENT IN .cc)
00268     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00269                                          const Vec& target, Vec& costs) const;
00270 
00273     // (PLEASE IMPLEMENT IN .cc)
00274     virtual TVec<std::string> getTestCostNames() const;
00275 
00278     // (PLEASE IMPLEMENT IN .cc)
00279     virtual TVec<std::string> getTrainCostNames() const;
00280 
00281 
00282     // *** SUBCLASS WRITING: ***
00283     // While in general not necessary, in case of particular needs
00284     // (efficiency concerns for ex) you may also want to overload
00285     // some of the following methods:
00286     // virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00287     //                                    Vec& output, Vec& costs) const;
00288     // virtual void computeCostsOnly(const Vec& input, const Vec& target,
00289     //                               Vec& costs) const;
00290     // virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00291     //                   VMat testoutputs=0, VMat testcosts=0) const;
00292     // virtual int nTestCosts() const;
00293     // virtual int nTrainCosts() const;
00294     // virtual void resetInternalState();
00295     // virtual bool isStatefulLearner() const;
00296 
00297 
00298     //#####  PLearn::Object Protocol  #########################################
00299 
00300     // Declares other standard object methods.
00301     // ### If your class is not instantiatable (it has pure virtual methods)
00302     // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS
00303     PLEARN_DECLARE_OBJECT(PseudolikelihoodRBM);
00304 
00305     // Simply calls inherited::build() then build_()
00306     virtual void build();
00307 
00309     // (PLEASE IMPLEMENT IN .cc)
00310     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00311 
00312 protected:
00313 
00314     //#####  Not Options  #####################################################
00315 
00317     mutable Vec target_one_hot;
00318 
00320     mutable Vec input_gradient;
00321     mutable Vec class_output;
00322     mutable Vec class_gradient;
00323     mutable Vec hidden_activation_pos_i;
00324     mutable Vec hidden_activation_neg_i;
00325     mutable Vec hidden_activation_gradient;
00326     mutable Vec hidden_activation_pos_i_gradient;
00327     mutable Vec hidden_activation_neg_i_gradient;
00328     mutable Mat connection_gradient;
00329     mutable TVec<int> context_indices;
00330     mutable TMat<int> context_indices_per_i;
00331     mutable Mat correlations_per_i;
00332     mutable TVec< TVec< int > > context_most_correlated;
00333     mutable Mat hidden_activations_context;
00334     mutable Vec hidden_activations_context_k_gradient;
00335     mutable Vec nums;
00336     mutable Vec nums_act;
00337     mutable Vec context_probs;
00338     mutable Vec gnums_act;
00339     mutable Vec conf;
00340     mutable Vec pos_input;
00341     mutable Vec pos_target;
00342     mutable Vec pos_hidden;
00343     mutable Vec neg_input;
00344     mutable Vec neg_target;
00345     mutable Vec neg_hidden;
00346     mutable Vec reconstruction_activation_gradient;
00347     mutable Vec hidden_layer_expectation_gradient;
00348     mutable Vec hidden_layer_activation_gradient;
00349     mutable Vec masked_autoencoder_input;
00350     mutable TVec<int> autoencoder_input_indices;
00351     mutable TVec<Vec> pers_cd_hidden;
00352 
00354     Vec Vx;
00355     Mat U_gradient;
00356     Vec Vx_gradient;
00357     Mat V_gradients;
00358     TVec<bool> input_is_active;
00359     TVec<int> input_indices;
00360     TVec<bool> input_is_selected;
00361     Vec hidden_act_non_selected;
00362     Vec pos_input_sparse;
00363 
00365     int nll_cost_index;
00366 
00368     int log_Z_cost_index;
00370     int log_Z_ais_cost_index;
00373     int log_Z_interval_lower_cost_index;
00376     int log_Z_interval_upper_cost_index;
00377 
00379     int class_cost_index;
00380 
00382     int training_cpu_time_cost_index;
00383     int cumulative_training_time_cost_index;
00384     //real cumulative_testing_time_cost_index;
00385 
00387     real cumulative_training_time;
00388     //real cumulative_testing_time;
00389     
00391     mutable real log_Z;
00393     mutable real log_Z_ais;
00394 
00396     mutable real log_Z_down;
00398     mutable real log_Z_up;
00399 
00401     mutable bool Z_is_up_to_date;
00402 
00404     mutable bool Z_ais_is_up_to_date;
00405 
00408     mutable TVec<bool> persistent_gibbs_chain_is_started;
00409 
00410 protected:
00411     //#####  Protected Member Functions  ######################################
00412 
00414     static void declareOptions(OptionList& ol);
00415 
00416 private:
00417     //#####  Private Member Functions  ########################################
00418 
00420     void build_();
00421 
00422     void build_layers_and_connections();
00423 
00424     void build_costs();
00425 
00426     void setLearningRate( real the_learning_rate );
00427 
00428     void compute_Z() const;
00429 
00430 private:
00431     //#####  Private Data Members  ############################################
00432 
00433     // The rest of the private stuff goes here
00434 };
00435 
00436 // Declares a few other classes and functions related to this class
00437 DECLARE_OBJECT_PTR(PseudolikelihoodRBM);
00438 
00439 } // end of namespace PLearn
00440 
00441 #endif
00442 
00443 
00444 /*
00445   Local Variables:
00446   mode:c++
00447   c-basic-offset:4
00448   c-file-style:"stroustrup"
00449   c-file-offsets:((innamespace . 0)(inline-open . 0))
00450   indent-tabs-mode:nil
00451   fill-column:79
00452   End:
00453 */
00454 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines