PLearn 0.1
StackedAutoassociatorsNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // StackedAutoassociatorsNet.h
00004 //
00005 // Copyright (C) 2007 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Hugo Larochelle
00036 
00040 #ifndef StackedAutoassociatorsNet_INC
00041 #define StackedAutoassociatorsNet_INC
00042 
00043 #include <plearn_learners/generic/PLearner.h>
00044 #include <plearn_learners/online/OnlineLearningModule.h>
00045 #include <plearn_learners/online/CostModule.h>
00046 #include <plearn_learners/online/NLLCostModule.h>
00047 #include <plearn_learners/online/RBMClassificationModule.h>
00048 #include <plearn_learners/online/RBMLayer.h>
00049 #include <plearn_learners/online/RBMMixedLayer.h>
00050 #include <plearn_learners/online/RBMConnection.h>
00051 #include <plearn/misc/PTimer.h>
00052 
00053 namespace PLearn {
00054 
00060 class StackedAutoassociatorsNet : public PLearner
00061 {
00062     typedef PLearner inherited;
00063 
00064 public:
00065     //#####  Public Build Options  ############################################
00066 
00069     real greedy_learning_rate;
00070 
00075     real greedy_decrease_ct;
00076 
00078     real fine_tuning_learning_rate;
00079 
00082     real fine_tuning_decrease_ct;
00083 
00086     real l1_neuron_decay;
00087 
00091     real l1_neuron_decay_center;
00092 
00094     int batch_size;
00095 
00098     TVec<int> training_schedule;
00099 
00101     bool online;
00102 
00104     TVec< PP<RBMLayer> > layers;
00105 
00110     TVec< PP<RBMLayer> > reconstruction_layers;
00111 
00113     TVec< PP<RBMConnection> > connections;
00114 
00116     TVec< PP<RBMConnection> > reconstruction_connections;
00117 
00121     TVec< PP<RBMConnection> > correlation_connections;
00122 
00126     mutable TVec< PP<RBMConnection> > direct_connections;
00127 
00131     PP<OnlineLearningModule> final_module;
00132 
00137     PP<CostModule> final_cost;
00138 
00142     TVec< PP<CostModule> > partial_costs;
00143 
00146     Vec partial_costs_weights;
00147 
00153     TVec< PP<RBMConnection> > greedy_target_connections;
00154 
00157     bool compute_all_test_costs;
00158 
00161     bool reconstruct_hidden;
00162 
00164     string noise_type;
00165 
00168     string missing_data_method;
00169 
00172     real corrupted_data_weight;
00173 
00176     real data_weight;
00177 
00180     real fraction_of_masked_inputs;
00181 
00184     real probability_of_masked_inputs;
00185 
00188     real probability_of_masked_target;
00189 
00191     bool mask_with_mean;
00192 
00195     bool mask_with_pepper_salt;
00196 
00203     real pep_salt_zero_centered;
00204 
00208     bool renoising;
00209 
00214     int noisy;   
00215 
00217     real prob_salt_noise;
00218 
00220     real gaussian_std;
00221 
00224     real binary_sampling_noise_parameter;
00225 
00227     int unsupervised_nstages;
00228 
00231     real unsupervised_fine_tuning_learning_rate;
00232 
00235     real unsupervised_fine_tuning_decrease_ct;
00236 
00239     int nb_corrupted_layer;
00240 
00243     bool mask_input_layer_only;
00244 
00247     bool mask_input_layer_only_in_unsupervised_fine_tuning;
00248 
00251     int train_stats_window;
00252 
00255     string learnerExpdir;
00256 
00258     bool save_learner_before_fine_tuning;
00259     
00262     bool keep_online_representations;
00263 
00264     //#####  Public Learnt Options  ###########################################
00265 
00267     int n_layers;
00268 
00270     int unsupervised_stage;
00271 
00272 public:
00273     //#####  Public Member Functions  #########################################
00274 
00276     StackedAutoassociatorsNet();
00277 
00278     //#####  PLearner Member Functions  #######################################
00279 
00282     virtual int outputsize() const;
00283 
00287     virtual void forget();
00288 
00292     virtual void train();
00293 
00295     virtual void computeOutput(const Vec& input, Vec& output) const;
00296 
00298     virtual void computeOutputs(const Mat& input, Mat& output) const;
00299 
00301     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00302                                          const Vec& target, Vec& costs) const;
00303 
00304     virtual void computeOutputsAndCosts(const Mat& input, const Mat& target,
00305                                         Mat& output, Mat& costs) const;
00306 
00309     virtual TVec<std::string> getTestCostNames() const;
00310 
00313     virtual TVec<std::string> getTrainCostNames() const;
00314 
00316     inline TVec< Vec > getTrainRepresentations() const
00317     {
00318         return train_representations;
00319     }
00320 
00321     inline void remote_setCurrentlyTrainedLayer(int new_currently_trained_layer)
00322     {
00323         currently_trained_layer = new_currently_trained_layer;
00324     }
00325         
00326 
00327     void greedyStep(const Vec& input, const Vec& target, int index,
00328                     Vec train_costs, Vec& representation);
00329     void greedyStep(const Mat& inputs, const Mat& targets, int index,
00330                     Mat& train_costs);
00331 
00332     void unsupervisedFineTuningStep(const Vec& input, const Vec& target,
00333                                     Vec& train_costs);
00334     void unsupervisedFineTuningStep(const Mat& inputs, const Mat& targets,
00335                                     Mat& train_costs);
00336 
00337     void fineTuningStep(const Vec& input, const Vec& target,
00338                         Vec& train_costs);
00339     void fineTuningStep(const Mat& inputs, const Mat& targets,
00340                         Mat& train_costs);
00341 
00342     void onlineStep(const Vec& input, const Vec& target,
00343                     Vec& train_costs);
00344     void onlineStep(const Mat& inputs, const Mat& targets,
00345                     Mat& train_costs);
00346 
00347 
00348     //#####  PLearn::Object Protocol  #########################################
00349 
00350     // Declares other standard object methods.
00351     // ### If your class is not instantiatable (it has pure virtual methods)
00352     // ### you should replace this by PLEARN_DECLARE_ABSTRACT_OBJECT_METHODS
00353     PLEARN_DECLARE_OBJECT(StackedAutoassociatorsNet);
00354 
00355     // Simply calls inherited::build() then build_()
00356     virtual void build();
00357 
00359     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00360 
00361 protected:
00362 
00364     int minibatch_size;
00365 
00366     //#####  Not Options  #####################################################
00367 
00370     mutable TVec<Vec> activations;
00371     mutable TVec<Mat> activations_m;
00372 
00375     mutable TVec<Vec> expectations;
00376     mutable TVec<Mat> expectations_m;
00379     mutable TVec< Vec > doubled_expectations;
00380 
00384     mutable TVec<Vec> activation_gradients;
00385     mutable TVec<Mat> activation_gradients_m;
00386 
00390     mutable TVec<Vec> expectation_gradients;
00391     mutable TVec<Mat> expectation_gradients_m;
00393     mutable TVec< Vec > doubled_expectation_gradients;
00394 
00396     mutable Vec reconstruction_activations;
00397     mutable Mat reconstruction_activations_m;
00398 
00400     mutable Vec reconstruction_activation_gradients;
00401     mutable Mat reconstruction_activation_gradients_m;
00402 
00404     mutable Vec reconstruction_expectation_gradients;
00405     mutable Mat reconstruction_expectation_gradients_m;
00406 
00408     TVec< Vec > fine_tuning_reconstruction_activations;
00409 
00411     TVec< Vec > fine_tuning_reconstruction_expectations;
00412 
00414     TVec< Vec > fine_tuning_reconstruction_activation_gradients;
00415 
00417     TVec< Vec > fine_tuning_reconstruction_expectation_gradients;
00418 
00420     mutable Vec reconstruction_activation_gradients_from_hid_rec;
00421 
00423     mutable Vec reconstruction_expectation_gradients_from_hid_rec;
00424 
00426     mutable Vec hidden_reconstruction_activations;
00427 
00429     mutable Vec hidden_reconstruction_activation_gradients;
00430 
00432     mutable TVec<Vec> correlation_activations;
00433     mutable TVec<Mat> correlation_activations_m;
00434 
00436     mutable TVec<Vec> correlation_expectations;
00437     mutable TVec<Mat> correlation_expectations_m;
00438 
00440     mutable TVec<Vec> correlation_activation_gradients;
00441     mutable TVec<Mat> correlation_activation_gradients_m;
00442 
00444     mutable TVec<Vec> correlation_expectation_gradients;
00445     mutable TVec<Mat> correlation_expectation_gradients_m;
00446 
00448     mutable TVec< PP<RBMLayer> > correlation_layers;
00449 
00451     mutable Vec direct_activations;
00452 
00454     mutable Vec direct_and_reconstruction_activations;
00455 
00458     mutable Vec direct_and_reconstruction_activation_gradients;
00459 
00461     mutable TVec<int> partial_costs_positions;
00462 
00464     mutable Vec partial_cost_value;
00465     mutable Mat partial_cost_values;
00466     mutable Vec partial_cost_values_0;
00467 
00469     mutable Vec final_cost_input;
00470     mutable Mat final_cost_inputs;
00471 
00473     mutable Vec final_cost_value;
00474     mutable Mat final_cost_values;
00475     mutable Vec final_cost_values_0;
00476 
00478     mutable Vec final_cost_gradient;
00479     mutable Mat final_cost_gradients;
00480 
00482     TVec< Vec > corrupted_autoassociator_expectations;
00483 
00490     TVec< Vec > second_corrupted_autoassociator_expectations;
00491 
00498     mutable Vec reconstruction_weights;
00499 
00501     mutable TVec< Vec > train_representations;
00502 
00504     TVec< Vec > binary_masks;
00505 
00507     Vec tmp_mask;
00508 
00510     TVec< TVec<int> > autoassociator_expectation_indices;
00511 
00513     TVec<Vec> expectation_means;
00514 
00516     Vec target_vec;
00517     Vec target_vec_gradient;
00519     TVec< Vec > targets_vec;
00520     TVec< Vec > targets_vec_gradient;
00521 
00523     TVec<int> greedy_stages;
00524 
00527     int currently_trained_layer;
00528 
00529 protected:
00530     //#####  Protected Member Functions  ######################################
00531 
00533     static void declareOptions(OptionList& ol);
00534 
00536     static void declareMethods(RemoteMethodMap& rmm);
00537 
00538 private:
00539     //#####  Private Member Functions  ########################################
00540 
00542     void build_();
00543 
00544     void build_layers_and_connections();
00545 
00546     void build_classification_cost();
00547 
00548     void build_costs();
00549 
00550     void setLearningRate( real the_learning_rate );
00551 
00552     void corrupt_input(const Vec& input, Vec& corrupted_input, int layer);
00553 
00556     void double_input(const Vec& input, Vec& doubled_input, bool double_grad=false) const;
00557 
00560     void divide_input(const Vec& input, Vec& divided_input) const ;
00561 
00575     TVec<Vec> fantasizeKTime(const int KTime, const Vec& srcImg, const Vec& sample,
00576                         const Vec& maskNoiseFractOrProb, bool alwaysFromSrcImg);
00577 
00579     TVec<Vec> fantasizeKTimeOnMultiSrcImg(const int KTime, const Mat& srcImg, const Vec& sample,
00580                         const Vec& maskNoiseFractOrProb, bool alwaysFromSrcImg);
00581 
00582     void corrupt_input(const Vec& input, Vec& corrupted_input, int layer, Vec& binary_mask);
00583 
00585     mutable Vec tmp_output;
00586     mutable Mat tmp_output_mat;
00587 
00588 private:
00589     //#####  Private Data Members  ############################################
00590 
00591     // The rest of the private stuff goes here
00592 };
00593 
00594 // Declares a few other classes and functions related to this class
00595 DECLARE_OBJECT_PTR(StackedAutoassociatorsNet);
00596 
00597 } // end of namespace PLearn
00598 
00599 #endif
00600 
00601 
00602 /*
00603   Local Variables:
00604   mode:c++
00605   c-basic-offset:4
00606   c-file-style:"stroustrup"
00607   c-file-offsets:((innamespace . 0)(inline-open . 0))
00608   indent-tabs-mode:nil
00609   fill-column:79
00610   End:
00611 */
00612 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines