PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PseudolikelihoodRBM.h 00004 // 00005 // Copyright (C) 2008 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Hugo Larochelle 00036 00039 #ifndef PseudolikelihoodRBM_INC 00040 #define PseudolikelihoodRBM_INC 00041 00042 #include <plearn_learners/generic/PLearner.h> 00043 #include <plearn_learners/online/OnlineLearningModule.h> 00044 #include <plearn_learners/online/CostModule.h> 00045 #include <plearn_learners/online/CrossEntropyCostModule.h> 00046 #include <plearn_learners/online/NLLCostModule.h> 00047 #include <plearn_learners/online/RBMClassificationModule.h> 00048 #include <plearn_learners/online/RBMMatrixTransposeConnection.h> 00049 #include <plearn_learners/online/RBMMultitaskClassificationModule.h> 00050 #include <plearn_learners/online/RBMLayer.h> 00051 #include <plearn_learners/online/RBMMixedLayer.h> 00052 #include <plearn_learners/online/RBMConnection.h> 00053 #include <plearn/misc/PTimer.h> 00054 #include <plearn/sys/Profiler.h> 00055 00056 namespace PLearn { 00057 using namespace std; 00058 00062 class PseudolikelihoodRBM : public PLearner 00063 { 00064 typedef PLearner inherited; 00065 00066 public: 00067 //##### Public Build Options ############################################ 00068 00070 real learning_rate; 00071 00073 real decrease_ct; 00074 00076 real cd_learning_rate; 00077 00079 real cd_decrease_ct; 00080 00082 int cd_n_gibbs; 00083 00086 real persistent_cd_weight; 00087 00090 int n_gibbs_chains; 00091 00094 bool use_mean_field_cd; 00095 00097 real denoising_learning_rate; 00098 00100 real denoising_decrease_ct; 00101 00103 real fraction_of_masked_inputs; 00104 00106 bool only_reconstruct_masked_inputs; 00107 00109 int n_classes; 00110 00113 bool input_is_sparse; 00114 00116 int factorized_connection_rank; 00117 00119 int n_selected_inputs_pseudolikelihood; 00120 00122 int n_selected_inputs_cd; 00123 00126 //int select_among_k_most_frequent; 00127 00133 bool compute_input_space_nll; 00134 00136 bool compute_Z_exactly; 00137 00140 bool use_ais_to_compute_Z; 00141 00143 int n_ais_chains; 00144 00145 // Schedule information for the betas in AIS. 00148 Vec ais_beta_begin; 00151 Vec ais_beta_end; 00153 TVec<int> ais_beta_n_steps; 00154 00155 // Each row gives 00161 Mat ais_beta_schedule; 00162 00166 int pseudolikelihood_context_size; 00167 00177 string pseudolikelihood_context_type; 00178 00180 int k_most_correlated; 00181 00183 real generative_learning_weight; 00184 00186 real sparsity_bias_decay; 00187 00191 real semi_sup_learning_weight; 00192 00194 PP<RBMLayer> input_layer; 00195 00197 PP<RBMLayer> hidden_layer; 00198 00200 PP<RBMMatrixConnection> connection; 00201 00203 //real target_weights_L1_penalty_factor; 00204 // 00206 //real target_weights_L2_penalty_factor; 00207 00208 //##### Public Learnt Options ########################################### 00210 TVec<string> cost_names; 00211 00212 PP<RBMMatrixTransposeConnection> transpose_connection; 00213 00215 PP<RBMLayer> target_layer; 00216 00218 PP<RBMMatrixConnection> target_connection; 00219 00221 Mat U; 00222 00225 Mat V; 00226 00227 public: 00228 //##### Public Member Functions ######################################### 00229 00231 PseudolikelihoodRBM(); 00232 00233 //##### PLearner Member Functions ####################################### 00234 00237 virtual int outputsize() const; 00238 00242 // (PLEASE IMPLEMENT IN .cc) 00243 virtual void forget(); 00244 00248 // (PLEASE IMPLEMENT IN .cc) 00249 virtual void train(); 00250 00259 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00260 VMat testoutputs=0, VMat testcosts=0) const; 00261 00263 // (PLEASE IMPLEMENT IN .cc) 00264 virtual void computeOutput(const Vec& input, Vec& output) const; 00265 00267 // (PLEASE IMPLEMENT IN .cc) 00268 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00269 const Vec& target, Vec& costs) const; 00270 00273 // (PLEASE IMPLEMENT IN .cc) 00274 virtual TVec<std::string> getTestCostNames() const; 00275 00278 // (PLEASE IMPLEMENT IN .cc) 00279 virtual TVec<std::string> getTrainCostNames() const; 00280 00281 00282 // *** SUBCLASS WRITING: *** 00283 // While in general not necessary, in case of particular needs 00284 // (efficiency concerns for ex) you may also want to overload 00285 // some of the following methods: 00286 // virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00287 // Vec& output, Vec& costs) const; 00288 // virtual void computeCostsOnly(const Vec& input, const Vec& target, 00289 // Vec& costs) const; 00290 // virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00291 // VMat testoutputs=0, VMat testcosts=0) const; 00292 // virtual int nTestCosts() const; 00293 // virtual int nTrainCosts() const; 00294 // virtual void resetInternalState(); 00295 // virtual bool isStatefulLearner() const; 00296 00297 00298 //##### PLearn::Object Protocol ######################################### 00299 00300 // Declares other standard object methods. 00301 // ### If your class is not instantiatable (it has pure virtual methods) 00302 // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS 00303 PLEARN_DECLARE_OBJECT(PseudolikelihoodRBM); 00304 00305 // Simply calls inherited::build() then build_() 00306 virtual void build(); 00307 00309 // (PLEASE IMPLEMENT IN .cc) 00310 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00311 00312 protected: 00313 00314 //##### Not Options ##################################################### 00315 00317 mutable Vec target_one_hot; 00318 00320 mutable Vec input_gradient; 00321 mutable Vec class_output; 00322 mutable Vec class_gradient; 00323 mutable Vec hidden_activation_pos_i; 00324 mutable Vec hidden_activation_neg_i; 00325 mutable Vec hidden_activation_gradient; 00326 mutable Vec hidden_activation_pos_i_gradient; 00327 mutable Vec hidden_activation_neg_i_gradient; 00328 mutable Mat connection_gradient; 00329 mutable TVec<int> context_indices; 00330 mutable TMat<int> context_indices_per_i; 00331 mutable Mat correlations_per_i; 00332 mutable TVec< TVec< int > > context_most_correlated; 00333 mutable Mat hidden_activations_context; 00334 mutable Vec hidden_activations_context_k_gradient; 00335 mutable Vec nums; 00336 mutable Vec nums_act; 00337 mutable Vec context_probs; 00338 mutable Vec gnums_act; 00339 mutable Vec conf; 00340 mutable Vec pos_input; 00341 mutable Vec pos_target; 00342 mutable Vec pos_hidden; 00343 mutable Vec neg_input; 00344 mutable Vec neg_target; 00345 mutable Vec neg_hidden; 00346 mutable Vec reconstruction_activation_gradient; 00347 mutable Vec hidden_layer_expectation_gradient; 00348 mutable Vec hidden_layer_activation_gradient; 00349 mutable Vec masked_autoencoder_input; 00350 mutable TVec<int> autoencoder_input_indices; 00351 mutable TVec<Vec> pers_cd_hidden; 00352 00354 Vec Vx; 00355 Mat U_gradient; 00356 Vec Vx_gradient; 00357 Mat V_gradients; 00358 TVec<bool> input_is_active; 00359 TVec<int> input_indices; 00360 TVec<bool> input_is_selected; 00361 Vec hidden_act_non_selected; 00362 Vec pos_input_sparse; 00363 00365 int nll_cost_index; 00366 00368 int log_Z_cost_index; 00370 int log_Z_ais_cost_index; 00373 int log_Z_interval_lower_cost_index; 00376 int log_Z_interval_upper_cost_index; 00377 00379 int class_cost_index; 00380 00382 int training_cpu_time_cost_index; 00383 int cumulative_training_time_cost_index; 00384 //real cumulative_testing_time_cost_index; 00385 00387 real cumulative_training_time; 00388 //real cumulative_testing_time; 00389 00391 mutable real log_Z; 00393 mutable real log_Z_ais; 00394 00396 mutable real log_Z_down; 00398 mutable real log_Z_up; 00399 00401 mutable bool Z_is_up_to_date; 00402 00404 mutable bool Z_ais_is_up_to_date; 00405 00408 mutable TVec<bool> persistent_gibbs_chain_is_started; 00409 00410 protected: 00411 //##### Protected Member Functions ###################################### 00412 00414 static void declareOptions(OptionList& ol); 00415 00416 private: 00417 //##### Private Member Functions ######################################## 00418 00420 void build_(); 00421 00422 void build_layers_and_connections(); 00423 00424 void build_costs(); 00425 00426 void setLearningRate( real the_learning_rate ); 00427 00428 void compute_Z() const; 00429 00430 private: 00431 //##### Private Data Members ############################################ 00432 00433 // The rest of the private stuff goes here 00434 }; 00435 00436 // Declares a few other classes and functions related to this class 00437 DECLARE_OBJECT_PTR(PseudolikelihoodRBM); 00438 00439 } // end of namespace PLearn 00440 00441 #endif 00442 00443 00444 /* 00445 Local Variables: 00446 mode:c++ 00447 c-basic-offset:4 00448 c-file-style:"stroustrup" 00449 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00450 indent-tabs-mode:nil 00451 fill-column:79 00452 End: 00453 */ 00454 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :