PLearn 0.1
DenoisingRecurrentNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DenoisingRecurrentNet.h
00004 //
00005 // Copyright (C) 2006 Stanislas Lauly
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Stanislas Lauly
00036 
00041 #ifndef DenoisingRecurrentNet_INC
00042 #define DenoisingRecurrentNet_INC
00043 
00044 #include <plearn_learners/generic/PLearner.h>
00045 #include <plearn_learners/online/OnlineLearningModule.h>
00046 #include <plearn_learners/online/RBMClassificationModule.h>
00047 #include <plearn_learners/online/RBMLayer.h>
00048 #include <plearn_learners/online/RBMMixedLayer.h>
00049 #include <plearn_learners/online/RBMConnection.h>
00050 #include <plearn_learners/online/RBMMatrixConnection.h>
00051 #include <plearn/vmat/AutoVMatrix.h>
00052 #include <plearn_learners/online/RBMMatrixTransposeConnection.h>
00053 
00054 #include <plearn_learners/online/GradNNetLayerModule.h>
00055 
00056 namespace PLearn {
00057 
00061 class DenoisingRecurrentNet : public PLearner
00062 {
00063     typedef PLearner inherited;
00064 
00065 public:
00066     //#####  Public Build Options  ############################################
00067 
00069     //real rbm_learning_rate;
00070 
00072     //int rbm_nstages;
00073     //int nCost;
00074 
00076     Vec target_layers_weights;
00077     
00081     bool use_target_layers_masks;
00082 
00084     real end_of_sequence_symbol;
00085 
00087     PP<RBMLayer> input_layer;
00088 
00090     TVec< PP<RBMLayer> > target_layers;
00091 
00093     PP<RBMLayer> hidden_layer;
00094 
00096     PP<RBMLayer> hidden_layer2;
00097 
00099     PP<RBMConnection> dynamic_connections;
00100 
00102     PP<RBMConnection> dynamic_reconstruction_connections;
00103 
00105     PP<RBMConnection> hidden_connections;
00106 
00108     PP<RBMConnection> input_connections;
00109 
00111     TVec< PP<RBMConnection> > target_connections;
00112 
00113     //#####  Public Learnt Options  ###########################################
00114 
00117     TVec<int> target_layers_n_of_target_elements;
00118 
00120     TVec<int> input_symbol_sizes;
00121     
00123     TVec< TVec<int> > target_symbol_sizes;
00124 
00127     string encoding;
00128 
00129     bool noise;
00130 
00132     real L1_penalty_factor;
00133 
00135     real L2_penalty_factor;
00136     
00138     int input_window_size;
00139 
00140     // Phase greedy (unsupervised)
00141     bool tied_input_reconstruction_weights;
00142     double input_noise_prob;
00143     double input_reconstruction_lr;
00144     double hidden_noise_prob;
00145     double hidden_reconstruction_lr;
00146     bool tied_hidden_reconstruction_weights;
00147 
00148     // Phase noisy recurrent (supervised): uses input_noise_prob
00149     // this phase *also* uses dynamic_gradient_scale_factor;
00150     double noisy_recurrent_lr;    
00151     double dynamic_gradient_scale_factor;
00152     
00153     // Phase recurrent no noise (supervised fine tuning)
00154     double recurrent_lr;
00155 
00156     // When training with trainUnconditionalPredictor, this is simply used to store the avg encoded frame
00157     Vec mean_encoded_vec;
00158 
00159     // learnt bias for input reconstruction
00160     Vec input_reconstruction_bias;
00161 
00162     // learnt bias for hidden reconstruction
00163     Vec hidden_reconstruction_bias;
00164 
00165     Vec hidden_reconstruction_bias2;
00166     
00167     double prediction_cost_weight;
00168     double input_reconstruction_cost_weight;
00169     double hidden_reconstruction_cost_weight;
00170 
00171     double nb_stage_reconstruction;
00172     double nb_stage_target;
00173 
00174     //#####  Not Options  #####################################################
00175 
00176 
00177 public:
00178     //#####  Public static Functions  #########################################
00179         
00180     // Finding sequence end indexes
00181     static void locateSequenceBoundaries(VMat dataset, TVec<int>& boundaries, real end_of_sequence_symbol);
00182 
00183     // encodings
00184 
00187     void encodeSequence(Mat sequence, Mat& encoded_seq) const;
00188 
00189     static void encode_onehot_diffNote_duration(Mat sequence, Mat& encoded_sequence,
00190                                                   bool use_silence, int duration_nbits=20);
00191 
00192     static void encode_onehot_note_octav_duration(Mat sequence, Mat& encoded_sequence, int prepend_zero_rows,
00193                                                   bool use_silence, int octav_nbits, int duration_nbits=20);
00194     
00195     static void encode_onehot_timeframe(Mat sequence, Mat& encoded_sequence, 
00196                                         int prepend_zero_rows, bool use_silence=false); 
00197 
00198 
00199     static int duration_to_number_of_timeframes(int duration);
00200     static int getDurationBit(int duration);
00201 
00202 
00203     // input noise injection
00204     void inject_zero_forcing_noise(Mat sequence, double noise_prob) const;
00205 
00206     // noise injection
00207     void inject_zero_forcing_noise(Vec sequence, double noise_prob) const;
00208 
00209     inline static Vec getInputWindow(Mat sequence, int startpos, int winsize)
00210     { return sequence.subMatRows(startpos, winsize).toVec(); }
00211           
00212     // 
00213     inline static void getNoteAndOctave(int midi_number, int& note, int& octave)
00214     {
00215         note = midi_number%12;
00216         octave = midi_number/12;
00217     }
00218     
00219 
00220 
00221 public:
00222     //#####  Public Member Functions  #########################################
00223 
00225     DenoisingRecurrentNet();
00226 
00227 
00228     //#####  PLearner Member Functions  #######################################
00229 
00232     virtual int outputsize() const;
00233 
00234     void setTrainingSet(VMat training_set, bool call_forget=true);
00235 
00239     virtual void forget();
00240 
00244     virtual void train();
00245 
00248     void setLearningRate( real the_learning_rate );
00249 
00251     virtual void computeOutput(const Vec& input, Vec& output) const;
00252 
00254     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00255                                          const Vec& target, Vec& costs) const;
00256 
00259     virtual TVec<std::string> getTestCostNames() const;
00260 
00262     int nSequences() const
00263     { return trainset_boundaries.length(); }
00264 
00266     void getSequence(int i, Mat& seq) const;
00267 
00269     void generate(int t, int n);
00270 
00272     void generateArtificial();
00273 
00274 //    //! Generate a part of the data in a folder
00275 //    void gen();
00276 
00279     virtual TVec<std::string> getTrainCostNames() const;
00280 
00282     void partition(TVec<double> part, TVec<double> periode, TVec<double> vel ) const;
00283     
00285     void clamp_units(const Vec layer_vector, PP<RBMLayer> layer,
00286                      TVec<int> symbol_sizes) const;
00287 
00290     void clamp_units(const Vec layer_vector, PP<RBMLayer> layer,
00291                      TVec<int> symbol_sizes, const Vec original_mask,
00292                      Vec &formated_mask) const;
00293     
00297     void recurrentUpdate(real input_reconstruction_weight,
00298                          real hidden_reconstruction_cost_weight,
00299                          real temporal_gradient_contribution,
00300                          real prediction_cost_weight,
00301                          real inputAndDynamicPart,
00302                          Vec train_costs,
00303                          Vec train_n_items);
00304 
00305     virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00306                       VMat testoutputs=0, VMat testcosts=0) const;
00307 
00308     
00309 
00310 
00311     // *** SUBCLASS WRITING: ***
00312     // While in general not necessary, in case of particular needs
00313     // (efficiency concerns for ex) you may also want to overload
00314     // some of the following methods:
00315     // virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00316     //                                    Vec& output, Vec& costs) const;
00317     // virtual void computeCostsOnly(const Vec& input, const Vec& target,
00318     //                               Vec& costs) const;
00319     // virtual int nTestCosts() const;
00320     // virtual int nTrainCosts() const;
00321     // virtual void resetInternalState();
00322     // virtual bool isStatefulLearner() const;
00323 
00324 
00325     //#####  PLearn::Object Protocol  #########################################
00326 
00327     // Declares other standard object methods.
00328     PLEARN_DECLARE_OBJECT(DenoisingRecurrentNet);
00329 
00330     // Simply calls inherited::build() then build_()
00331     virtual void build();
00332 
00334     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00335 
00336 protected:
00337     //#####  Not Options  #####################################################
00338     mutable double current_learning_rate;
00339 
00341     PP<AutoVMatrix> data;
00342    
00343     mutable TVec< Mat > acc_target_connections_gr;
00344 
00345     mutable Mat acc_input_connections_gr;
00346 
00347     mutable Mat acc_dynamic_connections_gr;
00348 
00349     mutable Mat acc_reconstruction_dynamic_connections_gr;
00350 
00352     mutable Vec acc_target_bias_gr;
00353 
00355     mutable Vec acc_hidden_bias_gr;
00356 
00358     mutable Vec acc_recons_bias_gr;
00359 
00361     mutable Vec bias_gradient;
00362     
00364     mutable Vec visi_bias_gradient;
00365 
00367     mutable Vec hidden_gradient;
00368     
00370     mutable Vec hidden_temporal_gradient;
00371 
00373     // mutable TVec< Vec > hidden_list;
00374     mutable Mat hidden_list;
00375     // mutable TVec< Vec > hidden_act_no_bias_list;
00376     mutable Mat hidden_act_no_bias_list;
00377 
00379     // mutable TVec< Vec > hidden2_list;
00380     mutable Mat hidden2_list;
00381     // mutable TVec< Vec > hidden2_act_no_bias_list;
00382     mutable Mat hidden2_act_no_bias_list;
00383 
00385     // mutable TVec< TVec< Vec > > target_prediction_list;
00386     mutable TVec<Mat> target_prediction_list;
00387     // mutable TVec< TVec< Vec > > target_prediction_act_no_bias_list;
00388     mutable TVec<Mat> target_prediction_act_no_bias_list;
00389 
00391     mutable TVec< Vec > input_list;
00392 
00394     // mutable TVec< TVec< Vec > > targets_list;
00395     mutable TVec<Mat> targets_list;
00396 
00398     mutable Mat nll_list;
00399 
00401     // mutable TVec< TVec< Vec > > masks_list;
00402     mutable TVec< Mat > masks_list;
00403 
00405     mutable Vec dynamic_act_no_bias_contribution;
00406 
00407     TVec<int> trainset_boundaries;
00408     mutable TVec<int> testset_boundaries;
00409 
00410     mutable Mat seq; // contains the current train or test sequence
00411     mutable Mat encoded_seq; // contains encoded version of current train or test sequence (possibly corrupted by noise)
00412     mutable Mat clean_encoded_seq; // copy of clean sequence contains encoded version of current train or test sequence
00413 
00414     //mutable Vec input_reconstruction_activation; // temporary Vec to hold input reconstruction activation (before softmax)
00415     mutable Vec input_reconstruction_prob;       // temporary Vec to hold input reconstruction prob (after applying softmax)
00416     mutable Vec hidden_reconstruction_prob;
00417 
00418 protected:
00419     //#####  Protected Member Functions  ######################################
00420 
00422     static void declareOptions(OptionList& ol);
00423 
00424 private:
00425     //#####  Private Member Functions  ########################################
00426 
00428     void build_();
00429 
00430 
00431     void applyMultipleSoftmaxToInputWindow(Vec input_reconstruction_activation, Vec input_reconstruction_prob);
00432 
00433     // note: the following functions are declared const because they have
00434     // to be called by test (which is const). Similarly, the members they 
00435     // manipulate are all declared mutable.
00436     void recurrentFprop(Vec train_costs, Vec train_n_items, bool useDynamicConnections=true) const;
00437 
00439     void encodeSequenceAndPopulateLists(Mat seq, bool doNoise) const;
00440 
00442     void encodeAndCreateSupervisedSequence(Mat seq) const;
00443 
00444     void encodeAndCreateSupervisedSequence2(Mat seq) const;
00445 
00447     void splitRawMaskedSupervisedSequence(Mat seq, bool doNoise) const;
00448 
00449     void encode_artificialData(Mat seq) const; 
00450 
00451     void resize_lists(int l) const;
00452 
00453     void trainUnconditionalPredictor();
00454     void unconditionalFprop(Vec train_costs, Vec train_n_items) const;
00455 
00456     Mat getTargetConnectionsWeightMatrix(int tar);
00457 
00458     Mat getInputConnectionsWeightMatrix();
00459 
00460     Mat getDynamicConnectionsWeightMatrix();
00461 
00462     Mat getDynamicReconstructionConnectionsWeightMatrix();
00463 
00464     void updateTargetLayer( Vec& grad, 
00465                             Vec& bias , 
00466                             real& lr );
00467     
00468     void bpropUpdateConnection(const Vec& input, 
00469                                const Vec& output,
00470                                Vec& input_gradient,
00471                                const Vec& output_gradient,
00472                                Mat& weights,
00473                                Mat& acc_weights_gr,
00474                                int& down_size,
00475                                int& up_size,
00476                                real& lr,
00477                                bool accumulate,
00478                                bool using_penalty_factor);
00479 
00480     void bpropUpdateHiddenLayer(const Vec& input, 
00481                                 const Vec& output,
00482                                 Vec& input_gradient,
00483                                 const Vec& output_gradient,
00484                                 Vec& bias,
00485                                 real& lr);
00486 
00487 
00488     void applyWeightPenalty(Mat& weights, Mat& acc_weights_gr, int& down_size, int& up_size, real& lr);
00489 
00494     
00495     double fpropUpdateInputReconstructionFromHidden(Vec hidden, Mat& reconstruction_weights, Mat& acc_weights_gr, Vec& input_reconstruction_bias, Vec& input_reconstruction_prob, 
00496                                                 Vec clean_input, Vec hidden_gradient, double input_reconstruction_cost_weight, double lr);
00497 
00498     
00501     double fpropInputReconstructionFromHidden(Vec hidden, Mat reconstruction_weights, Vec& input_reconstruction_bias, Vec& input_reconstruction_prob, 
00502                                               Vec clean_input);
00503 
00506     void updateInputReconstructionFromHidden(Vec hidden, Mat& reconstruction_weights, Mat& acc_weights_gr, Vec& input_reconstruction_bias, Vec input_reconstruction_prob, 
00507                                              Vec clean_input, Vec hidden_gradient, double input_reconstruction_cost_weight, double lr);
00508 
00509     double fpropHiddenReconstructionFromLastHidden2(Vec theInput, Vec hidden, Mat reconstruction_weights, Mat& acc_weights_gr, Vec& reconstruction_bias, Vec& reconstruction_bias2, Vec hidden_reconstruction_activation_grad, Vec& reconstruction_prob, 
00510                                                                           Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr);
00511 
00512     double fpropHiddenReconstructionFromLastHidden(Vec theInput, Vec hidden, Mat reconstruction_weights, Mat& acc_weights_gr, Vec& reconstruction_bias, Vec& reconstruction_bias2, Vec hidden_reconstruction_activation_grad, Vec& reconstruction_prob, 
00513                                                                           Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr);
00514     
00515     double fpropHiddenSymmetricDynamicMatrix(Vec hidden, Mat reconstruction_weights, Vec& reconstruction_prob, 
00516                                                                           Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr);
00517 private:
00518     //#####  Private Data Members  ############################################
00519 
00520     // The rest of the private stuff goes here
00521 };
00522 
00523 // Declares a few other classes and functions related to this class
00524 DECLARE_OBJECT_PTR(DenoisingRecurrentNet);
00525 
00526 } // end of namespace PLearn
00527 
00528 #endif
00529 
00530 
00531 /*
00532   Local Variables:
00533   mode:c++
00534   c-basic-offset:4
00535   c-file-style:"stroustrup"
00536   c-file-offsets:((innamespace . 0)(inline-open . 0))
00537   indent-tabs-mode:nil
00538   fill-column:79
00539   End:
00540 */
00541 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines