PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // StackedAutoassociatorsNet.h 00004 // 00005 // Copyright (C) 2007 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Hugo Larochelle 00036 00040 #ifndef StackedAutoassociatorsNet_INC 00041 #define StackedAutoassociatorsNet_INC 00042 00043 #include <plearn_learners/generic/PLearner.h> 00044 #include <plearn_learners/online/OnlineLearningModule.h> 00045 #include <plearn_learners/online/CostModule.h> 00046 #include <plearn_learners/online/NLLCostModule.h> 00047 #include <plearn_learners/online/RBMClassificationModule.h> 00048 #include <plearn_learners/online/RBMLayer.h> 00049 #include <plearn_learners/online/RBMMixedLayer.h> 00050 #include <plearn_learners/online/RBMConnection.h> 00051 #include <plearn/misc/PTimer.h> 00052 00053 namespace PLearn { 00054 00060 class StackedAutoassociatorsNet : public PLearner 00061 { 00062 typedef PLearner inherited; 00063 00064 public: 00065 //##### Public Build Options ############################################ 00066 00069 real greedy_learning_rate; 00070 00075 real greedy_decrease_ct; 00076 00078 real fine_tuning_learning_rate; 00079 00082 real fine_tuning_decrease_ct; 00083 00086 real l1_neuron_decay; 00087 00091 real l1_neuron_decay_center; 00092 00094 int batch_size; 00095 00098 TVec<int> training_schedule; 00099 00101 bool online; 00102 00104 TVec< PP<RBMLayer> > layers; 00105 00110 TVec< PP<RBMLayer> > reconstruction_layers; 00111 00113 TVec< PP<RBMConnection> > connections; 00114 00116 TVec< PP<RBMConnection> > reconstruction_connections; 00117 00121 TVec< PP<RBMConnection> > correlation_connections; 00122 00126 mutable TVec< PP<RBMConnection> > direct_connections; 00127 00131 PP<OnlineLearningModule> final_module; 00132 00137 PP<CostModule> final_cost; 00138 00142 TVec< PP<CostModule> > partial_costs; 00143 00146 Vec partial_costs_weights; 00147 00153 TVec< PP<RBMConnection> > greedy_target_connections; 00154 00157 bool compute_all_test_costs; 00158 00161 bool reconstruct_hidden; 00162 00164 string noise_type; 00165 00168 string missing_data_method; 00169 00172 real corrupted_data_weight; 00173 00176 real data_weight; 00177 00180 real fraction_of_masked_inputs; 00181 00184 real probability_of_masked_inputs; 00185 00188 real probability_of_masked_target; 00189 00191 bool mask_with_mean; 00192 00195 bool mask_with_pepper_salt; 00196 00203 real pep_salt_zero_centered; 00204 00208 bool renoising; 00209 00214 int noisy; 00215 00217 real prob_salt_noise; 00218 00220 real gaussian_std; 00221 00224 real binary_sampling_noise_parameter; 00225 00227 int unsupervised_nstages; 00228 00231 real unsupervised_fine_tuning_learning_rate; 00232 00235 real unsupervised_fine_tuning_decrease_ct; 00236 00239 int nb_corrupted_layer; 00240 00243 bool mask_input_layer_only; 00244 00247 bool mask_input_layer_only_in_unsupervised_fine_tuning; 00248 00251 int train_stats_window; 00252 00255 string learnerExpdir; 00256 00258 bool save_learner_before_fine_tuning; 00259 00262 bool keep_online_representations; 00263 00264 //##### Public Learnt Options ########################################### 00265 00267 int n_layers; 00268 00270 int unsupervised_stage; 00271 00272 public: 00273 //##### Public Member Functions ######################################### 00274 00276 StackedAutoassociatorsNet(); 00277 00278 //##### PLearner Member Functions ####################################### 00279 00282 virtual int outputsize() const; 00283 00287 virtual void forget(); 00288 00292 virtual void train(); 00293 00295 virtual void computeOutput(const Vec& input, Vec& output) const; 00296 00298 virtual void computeOutputs(const Mat& input, Mat& output) const; 00299 00301 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00302 const Vec& target, Vec& costs) const; 00303 00304 virtual void computeOutputsAndCosts(const Mat& input, const Mat& target, 00305 Mat& output, Mat& costs) const; 00306 00309 virtual TVec<std::string> getTestCostNames() const; 00310 00313 virtual TVec<std::string> getTrainCostNames() const; 00314 00316 inline TVec< Vec > getTrainRepresentations() const 00317 { 00318 return train_representations; 00319 } 00320 00321 inline void remote_setCurrentlyTrainedLayer(int new_currently_trained_layer) 00322 { 00323 currently_trained_layer = new_currently_trained_layer; 00324 } 00325 00326 00327 void greedyStep(const Vec& input, const Vec& target, int index, 00328 Vec train_costs, Vec& representation); 00329 void greedyStep(const Mat& inputs, const Mat& targets, int index, 00330 Mat& train_costs); 00331 00332 void unsupervisedFineTuningStep(const Vec& input, const Vec& target, 00333 Vec& train_costs); 00334 void unsupervisedFineTuningStep(const Mat& inputs, const Mat& targets, 00335 Mat& train_costs); 00336 00337 void fineTuningStep(const Vec& input, const Vec& target, 00338 Vec& train_costs); 00339 void fineTuningStep(const Mat& inputs, const Mat& targets, 00340 Mat& train_costs); 00341 00342 void onlineStep(const Vec& input, const Vec& target, 00343 Vec& train_costs); 00344 void onlineStep(const Mat& inputs, const Mat& targets, 00345 Mat& train_costs); 00346 00347 00348 //##### PLearn::Object Protocol ######################################### 00349 00350 // Declares other standard object methods. 00351 // ### If your class is not instantiatable (it has pure virtual methods) 00352 // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS 00353 PLEARN_DECLARE_OBJECT(StackedAutoassociatorsNet); 00354 00355 // Simply calls inherited::build() then build_() 00356 virtual void build(); 00357 00359 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00360 00361 protected: 00362 00364 int minibatch_size; 00365 00366 //##### Not Options ##################################################### 00367 00370 mutable TVec<Vec> activations; 00371 mutable TVec<Mat> activations_m; 00372 00375 mutable TVec<Vec> expectations; 00376 mutable TVec<Mat> expectations_m; 00379 mutable TVec< Vec > doubled_expectations; 00380 00384 mutable TVec<Vec> activation_gradients; 00385 mutable TVec<Mat> activation_gradients_m; 00386 00390 mutable TVec<Vec> expectation_gradients; 00391 mutable TVec<Mat> expectation_gradients_m; 00393 mutable TVec< Vec > doubled_expectation_gradients; 00394 00396 mutable Vec reconstruction_activations; 00397 mutable Mat reconstruction_activations_m; 00398 00400 mutable Vec reconstruction_activation_gradients; 00401 mutable Mat reconstruction_activation_gradients_m; 00402 00404 mutable Vec reconstruction_expectation_gradients; 00405 mutable Mat reconstruction_expectation_gradients_m; 00406 00408 TVec< Vec > fine_tuning_reconstruction_activations; 00409 00411 TVec< Vec > fine_tuning_reconstruction_expectations; 00412 00414 TVec< Vec > fine_tuning_reconstruction_activation_gradients; 00415 00417 TVec< Vec > fine_tuning_reconstruction_expectation_gradients; 00418 00420 mutable Vec reconstruction_activation_gradients_from_hid_rec; 00421 00423 mutable Vec reconstruction_expectation_gradients_from_hid_rec; 00424 00426 mutable Vec hidden_reconstruction_activations; 00427 00429 mutable Vec hidden_reconstruction_activation_gradients; 00430 00432 mutable TVec<Vec> correlation_activations; 00433 mutable TVec<Mat> correlation_activations_m; 00434 00436 mutable TVec<Vec> correlation_expectations; 00437 mutable TVec<Mat> correlation_expectations_m; 00438 00440 mutable TVec<Vec> correlation_activation_gradients; 00441 mutable TVec<Mat> correlation_activation_gradients_m; 00442 00444 mutable TVec<Vec> correlation_expectation_gradients; 00445 mutable TVec<Mat> correlation_expectation_gradients_m; 00446 00448 mutable TVec< PP<RBMLayer> > correlation_layers; 00449 00451 mutable Vec direct_activations; 00452 00454 mutable Vec direct_and_reconstruction_activations; 00455 00458 mutable Vec direct_and_reconstruction_activation_gradients; 00459 00461 mutable TVec<int> partial_costs_positions; 00462 00464 mutable Vec partial_cost_value; 00465 mutable Mat partial_cost_values; 00466 mutable Vec partial_cost_values_0; 00467 00469 mutable Vec final_cost_input; 00470 mutable Mat final_cost_inputs; 00471 00473 mutable Vec final_cost_value; 00474 mutable Mat final_cost_values; 00475 mutable Vec final_cost_values_0; 00476 00478 mutable Vec final_cost_gradient; 00479 mutable Mat final_cost_gradients; 00480 00482 TVec< Vec > corrupted_autoassociator_expectations; 00483 00490 TVec< Vec > second_corrupted_autoassociator_expectations; 00491 00498 mutable Vec reconstruction_weights; 00499 00501 mutable TVec< Vec > train_representations; 00502 00504 TVec< Vec > binary_masks; 00505 00507 Vec tmp_mask; 00508 00510 TVec< TVec<int> > autoassociator_expectation_indices; 00511 00513 TVec<Vec> expectation_means; 00514 00516 Vec target_vec; 00517 Vec target_vec_gradient; 00519 TVec< Vec > targets_vec; 00520 TVec< Vec > targets_vec_gradient; 00521 00523 TVec<int> greedy_stages; 00524 00527 int currently_trained_layer; 00528 00529 protected: 00530 //##### Protected Member Functions ###################################### 00531 00533 static void declareOptions(OptionList& ol); 00534 00536 static void declareMethods(RemoteMethodMap& rmm); 00537 00538 private: 00539 //##### Private Member Functions ######################################## 00540 00542 void build_(); 00543 00544 void build_layers_and_connections(); 00545 00546 void build_classification_cost(); 00547 00548 void build_costs(); 00549 00550 void setLearningRate( real the_learning_rate ); 00551 00552 void corrupt_input(const Vec& input, Vec& corrupted_input, int layer); 00553 00556 void double_input(const Vec& input, Vec& doubled_input, bool double_grad=false) const; 00557 00560 void divide_input(const Vec& input, Vec& divided_input) const ; 00561 00575 TVec<Vec> fantasizeKTime(const int KTime, const Vec& srcImg, const Vec& sample, 00576 const Vec& maskNoiseFractOrProb, bool alwaysFromSrcImg); 00577 00579 TVec<Vec> fantasizeKTimeOnMultiSrcImg(const int KTime, const Mat& srcImg, const Vec& sample, 00580 const Vec& maskNoiseFractOrProb, bool alwaysFromSrcImg); 00581 00582 void corrupt_input(const Vec& input, Vec& corrupted_input, int layer, Vec& binary_mask); 00583 00585 mutable Vec tmp_output; 00586 mutable Mat tmp_output_mat; 00587 00588 private: 00589 //##### Private Data Members ############################################ 00590 00591 // The rest of the private stuff goes here 00592 }; 00593 00594 // Declares a few other classes and functions related to this class 00595 DECLARE_OBJECT_PTR(StackedAutoassociatorsNet); 00596 00597 } // end of namespace PLearn 00598 00599 #endif 00600 00601 00602 /* 00603 Local Variables: 00604 mode:c++ 00605 c-basic-offset:4 00606 c-file-style:"stroustrup" 00607 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00608 indent-tabs-mode:nil 00609 fill-column:79 00610 End: 00611 */ 00612 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :