PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // DenoisingRecurrentNet.h 00004 // 00005 // Copyright (C) 2006 Stanislas Lauly 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Stanislas Lauly 00036 00041 #ifndef DenoisingRecurrentNet_INC 00042 #define DenoisingRecurrentNet_INC 00043 00044 #include <plearn_learners/generic/PLearner.h> 00045 #include <plearn_learners/online/OnlineLearningModule.h> 00046 #include <plearn_learners/online/RBMClassificationModule.h> 00047 #include <plearn_learners/online/RBMLayer.h> 00048 #include <plearn_learners/online/RBMMixedLayer.h> 00049 #include <plearn_learners/online/RBMConnection.h> 00050 #include <plearn_learners/online/RBMMatrixConnection.h> 00051 #include <plearn/vmat/AutoVMatrix.h> 00052 #include <plearn_learners/online/RBMMatrixTransposeConnection.h> 00053 00054 #include <plearn_learners/online/GradNNetLayerModule.h> 00055 00056 namespace PLearn { 00057 00061 class DenoisingRecurrentNet : public PLearner 00062 { 00063 typedef PLearner inherited; 00064 00065 public: 00066 //##### Public Build Options ############################################ 00067 00069 //real rbm_learning_rate; 00070 00072 //int rbm_nstages; 00073 //int nCost; 00074 00076 Vec target_layers_weights; 00077 00081 bool use_target_layers_masks; 00082 00084 real end_of_sequence_symbol; 00085 00087 PP<RBMLayer> input_layer; 00088 00090 TVec< PP<RBMLayer> > target_layers; 00091 00093 PP<RBMLayer> hidden_layer; 00094 00096 PP<RBMLayer> hidden_layer2; 00097 00099 PP<RBMConnection> dynamic_connections; 00100 00102 PP<RBMConnection> dynamic_reconstruction_connections; 00103 00105 PP<RBMConnection> hidden_connections; 00106 00108 PP<RBMConnection> input_connections; 00109 00111 TVec< PP<RBMConnection> > target_connections; 00112 00113 //##### Public Learnt Options ########################################### 00114 00117 TVec<int> target_layers_n_of_target_elements; 00118 00120 TVec<int> input_symbol_sizes; 00121 00123 TVec< TVec<int> > target_symbol_sizes; 00124 00127 string encoding; 00128 00129 bool noise; 00130 00132 real L1_penalty_factor; 00133 00135 real L2_penalty_factor; 00136 00138 int input_window_size; 00139 00140 // Phase greedy (unsupervised) 00141 bool tied_input_reconstruction_weights; 00142 double input_noise_prob; 00143 double input_reconstruction_lr; 00144 double hidden_noise_prob; 00145 double hidden_reconstruction_lr; 00146 bool tied_hidden_reconstruction_weights; 00147 00148 // Phase noisy recurrent (supervised): uses input_noise_prob 00149 // this phase *also* uses dynamic_gradient_scale_factor; 00150 double noisy_recurrent_lr; 00151 double dynamic_gradient_scale_factor; 00152 00153 // Phase recurrent no noise (supervised fine tuning) 00154 double recurrent_lr; 00155 00156 // When training with trainUnconditionalPredictor, this is simply used to store the avg encoded frame 00157 Vec mean_encoded_vec; 00158 00159 // learnt bias for input reconstruction 00160 Vec input_reconstruction_bias; 00161 00162 // learnt bias for hidden reconstruction 00163 Vec hidden_reconstruction_bias; 00164 00165 Vec hidden_reconstruction_bias2; 00166 00167 double prediction_cost_weight; 00168 double input_reconstruction_cost_weight; 00169 double hidden_reconstruction_cost_weight; 00170 00171 double nb_stage_reconstruction; 00172 double nb_stage_target; 00173 00174 //##### Not Options ##################################################### 00175 00176 00177 public: 00178 //##### Public static Functions ######################################### 00179 00180 // Finding sequence end indexes 00181 static void locateSequenceBoundaries(VMat dataset, TVec<int>& boundaries, real end_of_sequence_symbol); 00182 00183 // encodings 00184 00187 void encodeSequence(Mat sequence, Mat& encoded_seq) const; 00188 00189 static void encode_onehot_diffNote_duration(Mat sequence, Mat& encoded_sequence, 00190 bool use_silence, int duration_nbits=20); 00191 00192 static void encode_onehot_note_octav_duration(Mat sequence, Mat& encoded_sequence, int prepend_zero_rows, 00193 bool use_silence, int octav_nbits, int duration_nbits=20); 00194 00195 static void encode_onehot_timeframe(Mat sequence, Mat& encoded_sequence, 00196 int prepend_zero_rows, bool use_silence=false); 00197 00198 00199 static int duration_to_number_of_timeframes(int duration); 00200 static int getDurationBit(int duration); 00201 00202 00203 // input noise injection 00204 void inject_zero_forcing_noise(Mat sequence, double noise_prob) const; 00205 00206 // noise injection 00207 void inject_zero_forcing_noise(Vec sequence, double noise_prob) const; 00208 00209 inline static Vec getInputWindow(Mat sequence, int startpos, int winsize) 00210 { return sequence.subMatRows(startpos, winsize).toVec(); } 00211 00212 // 00213 inline static void getNoteAndOctave(int midi_number, int& note, int& octave) 00214 { 00215 note = midi_number%12; 00216 octave = midi_number/12; 00217 } 00218 00219 00220 00221 public: 00222 //##### Public Member Functions ######################################### 00223 00225 DenoisingRecurrentNet(); 00226 00227 00228 //##### PLearner Member Functions ####################################### 00229 00232 virtual int outputsize() const; 00233 00234 void setTrainingSet(VMat training_set, bool call_forget=true); 00235 00239 virtual void forget(); 00240 00244 virtual void train(); 00245 00248 void setLearningRate( real the_learning_rate ); 00249 00251 virtual void computeOutput(const Vec& input, Vec& output) const; 00252 00254 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00255 const Vec& target, Vec& costs) const; 00256 00259 virtual TVec<std::string> getTestCostNames() const; 00260 00262 int nSequences() const 00263 { return trainset_boundaries.length(); } 00264 00266 void getSequence(int i, Mat& seq) const; 00267 00269 void generate(int t, int n); 00270 00272 void generateArtificial(); 00273 00274 // //! Generate a part of the data in a folder 00275 // void gen(); 00276 00279 virtual TVec<std::string> getTrainCostNames() const; 00280 00282 void partition(TVec<double> part, TVec<double> periode, TVec<double> vel ) const; 00283 00285 void clamp_units(const Vec layer_vector, PP<RBMLayer> layer, 00286 TVec<int> symbol_sizes) const; 00287 00290 void clamp_units(const Vec layer_vector, PP<RBMLayer> layer, 00291 TVec<int> symbol_sizes, const Vec original_mask, 00292 Vec &formated_mask) const; 00293 00297 void recurrentUpdate(real input_reconstruction_weight, 00298 real hidden_reconstruction_cost_weight, 00299 real temporal_gradient_contribution, 00300 real prediction_cost_weight, 00301 real inputAndDynamicPart, 00302 Vec train_costs, 00303 Vec train_n_items); 00304 00305 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00306 VMat testoutputs=0, VMat testcosts=0) const; 00307 00308 00309 00310 00311 // *** SUBCLASS WRITING: *** 00312 // While in general not necessary, in case of particular needs 00313 // (efficiency concerns for ex) you may also want to overload 00314 // some of the following methods: 00315 // virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00316 // Vec& output, Vec& costs) const; 00317 // virtual void computeCostsOnly(const Vec& input, const Vec& target, 00318 // Vec& costs) const; 00319 // virtual int nTestCosts() const; 00320 // virtual int nTrainCosts() const; 00321 // virtual void resetInternalState(); 00322 // virtual bool isStatefulLearner() const; 00323 00324 00325 //##### PLearn::Object Protocol ######################################### 00326 00327 // Declares other standard object methods. 00328 PLEARN_DECLARE_OBJECT(DenoisingRecurrentNet); 00329 00330 // Simply calls inherited::build() then build_() 00331 virtual void build(); 00332 00334 virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies); 00335 00336 protected: 00337 //##### Not Options ##################################################### 00338 mutable double current_learning_rate; 00339 00341 PP<AutoVMatrix> data; 00342 00343 mutable TVec< Mat > acc_target_connections_gr; 00344 00345 mutable Mat acc_input_connections_gr; 00346 00347 mutable Mat acc_dynamic_connections_gr; 00348 00349 mutable Mat acc_reconstruction_dynamic_connections_gr; 00350 00352 mutable Vec acc_target_bias_gr; 00353 00355 mutable Vec acc_hidden_bias_gr; 00356 00358 mutable Vec acc_recons_bias_gr; 00359 00361 mutable Vec bias_gradient; 00362 00364 mutable Vec visi_bias_gradient; 00365 00367 mutable Vec hidden_gradient; 00368 00370 mutable Vec hidden_temporal_gradient; 00371 00373 // mutable TVec< Vec > hidden_list; 00374 mutable Mat hidden_list; 00375 // mutable TVec< Vec > hidden_act_no_bias_list; 00376 mutable Mat hidden_act_no_bias_list; 00377 00379 // mutable TVec< Vec > hidden2_list; 00380 mutable Mat hidden2_list; 00381 // mutable TVec< Vec > hidden2_act_no_bias_list; 00382 mutable Mat hidden2_act_no_bias_list; 00383 00385 // mutable TVec< TVec< Vec > > target_prediction_list; 00386 mutable TVec<Mat> target_prediction_list; 00387 // mutable TVec< TVec< Vec > > target_prediction_act_no_bias_list; 00388 mutable TVec<Mat> target_prediction_act_no_bias_list; 00389 00391 mutable TVec< Vec > input_list; 00392 00394 // mutable TVec< TVec< Vec > > targets_list; 00395 mutable TVec<Mat> targets_list; 00396 00398 mutable Mat nll_list; 00399 00401 // mutable TVec< TVec< Vec > > masks_list; 00402 mutable TVec< Mat > masks_list; 00403 00405 mutable Vec dynamic_act_no_bias_contribution; 00406 00407 TVec<int> trainset_boundaries; 00408 mutable TVec<int> testset_boundaries; 00409 00410 mutable Mat seq; // contains the current train or test sequence 00411 mutable Mat encoded_seq; // contains encoded version of current train or test sequence (possibly corrupted by noise) 00412 mutable Mat clean_encoded_seq; // copy of clean sequence contains encoded version of current train or test sequence 00413 00414 //mutable Vec input_reconstruction_activation; // temporary Vec to hold input reconstruction activation (before softmax) 00415 mutable Vec input_reconstruction_prob; // temporary Vec to hold input reconstruction prob (after applying softmax) 00416 mutable Vec hidden_reconstruction_prob; 00417 00418 protected: 00419 //##### Protected Member Functions ###################################### 00420 00422 static void declareOptions(OptionList& ol); 00423 00424 private: 00425 //##### Private Member Functions ######################################## 00426 00428 void build_(); 00429 00430 00431 void applyMultipleSoftmaxToInputWindow(Vec input_reconstruction_activation, Vec input_reconstruction_prob); 00432 00433 // note: the following functions are declared const because they have 00434 // to be called by test (which is const). Similarly, the members they 00435 // manipulate are all declared mutable. 00436 void recurrentFprop(Vec train_costs, Vec train_n_items, bool useDynamicConnections=true) const; 00437 00439 void encodeSequenceAndPopulateLists(Mat seq, bool doNoise) const; 00440 00442 void encodeAndCreateSupervisedSequence(Mat seq) const; 00443 00444 void encodeAndCreateSupervisedSequence2(Mat seq) const; 00445 00447 void splitRawMaskedSupervisedSequence(Mat seq, bool doNoise) const; 00448 00449 void encode_artificialData(Mat seq) const; 00450 00451 void resize_lists(int l) const; 00452 00453 void trainUnconditionalPredictor(); 00454 void unconditionalFprop(Vec train_costs, Vec train_n_items) const; 00455 00456 Mat getTargetConnectionsWeightMatrix(int tar); 00457 00458 Mat getInputConnectionsWeightMatrix(); 00459 00460 Mat getDynamicConnectionsWeightMatrix(); 00461 00462 Mat getDynamicReconstructionConnectionsWeightMatrix(); 00463 00464 void updateTargetLayer( Vec& grad, 00465 Vec& bias , 00466 real& lr ); 00467 00468 void bpropUpdateConnection(const Vec& input, 00469 const Vec& output, 00470 Vec& input_gradient, 00471 const Vec& output_gradient, 00472 Mat& weights, 00473 Mat& acc_weights_gr, 00474 int& down_size, 00475 int& up_size, 00476 real& lr, 00477 bool accumulate, 00478 bool using_penalty_factor); 00479 00480 void bpropUpdateHiddenLayer(const Vec& input, 00481 const Vec& output, 00482 Vec& input_gradient, 00483 const Vec& output_gradient, 00484 Vec& bias, 00485 real& lr); 00486 00487 00488 void applyWeightPenalty(Mat& weights, Mat& acc_weights_gr, int& down_size, int& up_size, real& lr); 00489 00494 00495 double fpropUpdateInputReconstructionFromHidden(Vec hidden, Mat& reconstruction_weights, Mat& acc_weights_gr, Vec& input_reconstruction_bias, Vec& input_reconstruction_prob, 00496 Vec clean_input, Vec hidden_gradient, double input_reconstruction_cost_weight, double lr); 00497 00498 00501 double fpropInputReconstructionFromHidden(Vec hidden, Mat reconstruction_weights, Vec& input_reconstruction_bias, Vec& input_reconstruction_prob, 00502 Vec clean_input); 00503 00506 void updateInputReconstructionFromHidden(Vec hidden, Mat& reconstruction_weights, Mat& acc_weights_gr, Vec& input_reconstruction_bias, Vec input_reconstruction_prob, 00507 Vec clean_input, Vec hidden_gradient, double input_reconstruction_cost_weight, double lr); 00508 00509 double fpropHiddenReconstructionFromLastHidden2(Vec theInput, Vec hidden, Mat reconstruction_weights, Mat& acc_weights_gr, Vec& reconstruction_bias, Vec& reconstruction_bias2, Vec hidden_reconstruction_activation_grad, Vec& reconstruction_prob, 00510 Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr); 00511 00512 double fpropHiddenReconstructionFromLastHidden(Vec theInput, Vec hidden, Mat reconstruction_weights, Mat& acc_weights_gr, Vec& reconstruction_bias, Vec& reconstruction_bias2, Vec hidden_reconstruction_activation_grad, Vec& reconstruction_prob, 00513 Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr); 00514 00515 double fpropHiddenSymmetricDynamicMatrix(Vec hidden, Mat reconstruction_weights, Vec& reconstruction_prob, 00516 Vec clean_input, Vec hidden_gradient, double hidden_reconstruction_cost_weight, double lr); 00517 private: 00518 //##### Private Data Members ############################################ 00519 00520 // The rest of the private stuff goes here 00521 }; 00522 00523 // Declares a few other classes and functions related to this class 00524 DECLARE_OBJECT_PTR(DenoisingRecurrentNet); 00525 00526 } // end of namespace PLearn 00527 00528 #endif 00529 00530 00531 /* 00532 Local Variables: 00533 mode:c++ 00534 c-basic-offset:4 00535 c-file-style:"stroustrup" 00536 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00537 indent-tabs-mode:nil 00538 fill-column:79 00539 End: 00540 */ 00541 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :