PLearn 0.1
NeuralProbabilisticLanguageModel.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // NeuralProbabilisticLanguageModel.h
00004 // Copyright (c) 1998-2002 Pascal Vincent
00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal
00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00038 #ifndef NeuralProbabilisticLanguageModel_INC
00039 #define NeuralProbabilisticLanguageModel_INC
00040 
00041 #include "PLearner.h"
00042 #include <plearn/math/PRandom.h>
00043 #include <plearn/feat/FeatureSet.h>
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00058 class NeuralProbabilisticLanguageModel: public PLearner
00059 {
00060 
00061 private:
00062 
00063     typedef PLearner inherited;
00064     
00066     mutable Vec target_values;
00068     mutable Vec output_comp;
00070     mutable Vec row;
00073     mutable Vec last_layer;
00075     mutable Vec gradient_last_layer;
00077     mutable TVec< TVec<int> > feats;
00078 
00082     mutable Vec gradient, neg_energies, densities;
00083     mutable string str;
00084     mutable real * pval1, * pval2, * pval3, * pval4, * pval5;
00085     mutable real val, val2, grad;
00086     mutable int offset;
00087     mutable int ni,nj,nk,id,nfeats,ifeats;
00088     mutable int* f;
00089 
00090 protected:
00091 
00093     int total_output_size;
00095     int total_updates;
00097     int n_feat_sets;
00101     int total_feats_per_token;
00103     mutable int reind_target;
00105     mutable Vec feat_input;
00107     Vec gradient_feat_input;
00109     Vec nnet_input;
00111     Vec gradient_nnet_input;
00113     Vec hiddenv;
00115     Vec gradient_hiddenv;
00117     Vec gradient_act_hiddenv;
00119     Vec hidden2v;
00121     Vec gradient_hidden2v;
00123     Vec gradient_act_hidden2v;
00125     Vec gradient_outputv;
00127     Vec gradient_act_outputv;
00129     PP<PRandom> rgen;
00131     Vec feats_since_last_update;
00133     Vec target_values_since_last_update;
00135     mutable VMat val_string_reference_set;
00137     mutable VMat target_values_reference_set;
00139     Vec importance_sampling_ratios;
00141     Vec sample;
00143     Vec generated_samples;
00144 
00145 public: 
00147     Mat w1;
00149     Mat gradient_w1;
00151     Vec b1;
00153     Vec gradient_b1;
00155     Mat w2;
00157     Mat gradient_w2;
00159     Vec b2;
00161     Vec gradient_b2;
00163     Mat wout;
00165     Mat gradient_wout;
00167     Vec bout;
00169     Vec gradient_bout;
00171     Mat direct_wout;
00173     Mat gradient_direct_wout;
00175     Vec direct_bout;
00177     Vec gradient_direct_bout;
00180     Mat wout_dist_rep;
00183     Mat gradient_wout_dist_rep;
00186     Vec bout_dist_rep;
00189     Vec gradient_bout_dist_rep;
00190 
00191 public:
00192 
00193     // Build options:
00194 
00196     int nhidden;
00198     int nhidden2; 
00200     real weight_decay; 
00202     real bias_decay; 
00205     real layer1_weight_decay; 
00208     real layer1_bias_decay;   
00211     real layer2_weight_decay; 
00214     real layer2_bias_decay;   
00217     real output_layer_weight_decay; 
00220     real output_layer_bias_decay;
00223     real direct_in_to_out_weight_decay;
00226     real output_layer_dist_rep_weight_decay; 
00229     real output_layer_dist_rep_bias_decay;
00232     real margin; 
00235     bool fixed_output_weights;
00238     bool direct_in_to_out;
00241     string penalty_type; 
00243     string output_transfer_func; 
00246     string hidden_transfer_func; 
00248     TVec<string> cost_funcs;  
00250     real start_learning_rate;
00252     real decrease_constant;
00255     int batch_size; 
00258     bool stochastic_gradient_descent_speedup;
00260     string initialization_method;
00263     int dist_rep_dim;
00266     bool possible_targets_vary;
00268     TVec<PP<FeatureSet> > feat_sets;
00275     PP<PDistribution> proposal_distribution;
00278     bool train_proposal_distribution;
00280     int sampling_block_size;
00282     int minimum_effective_sample_size;
00283 
00284 private:
00285     void build_();
00286 
00291     void compute_softmax(const Vec& x, const Vec& y) const;
00292 
00294     real nll(const Vec& outputv, int target) const;
00295     
00297     real classification_loss(const Vec& outputv, int target) const;
00298     
00305     int my_argmax(const Vec& vec, int default_compare=0) const;
00306 
00307 public:
00308 
00309     NeuralProbabilisticLanguageModel();
00310     virtual ~NeuralProbabilisticLanguageModel();
00311     PLEARN_DECLARE_OBJECT(NeuralProbabilisticLanguageModel);
00312 
00313     virtual void build();
00314     virtual void forget(); // simply calls initializeParams()
00315 
00316     virtual int outputsize() const;
00317     virtual TVec<string> getTrainCostNames() const;
00318     virtual TVec<string> getTestCostNames() const;
00319 
00320     virtual void train();
00321 
00322     virtual void computeOutput(const Vec& input, Vec& output) const;
00323 
00324     virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00325                                        Vec& output, Vec& costs) const;
00326 
00327     virtual void computeCostsFromOutputs(const Vec& input, 
00328                                          const Vec& output, 
00329                                          const Vec& target, 
00330                                          Vec& costs) const;
00331 
00332     virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies);
00333 
00334 protected:
00335     static void declareOptions(OptionList& ol);
00336 
00338     void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00339 
00341     void fpropOutput(const Vec& inputv, Vec& outputv) const;
00342 
00345     void fpropBeforeOutputWeights(const Vec& inputv) const;
00346 
00348     void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00349 
00355     void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1);
00356 
00358     void update();
00359 
00361     void update_affine_transform(Vec input, Mat weights, Vec bias,
00362                                  Mat gweights, Vec gbias,
00363                                  bool input_is_sparse, bool output_is_sparse,
00364                                  Vec output_indices);
00365     
00368     void clearProppathGradient();
00369 
00375     virtual void initializeParams(bool set_seed = true);
00376 
00379     void add_transfer_func(const Vec& input, 
00380                           string transfer_func = "default") const;
00381 
00390     void gradient_transfer_func(Vec& output, Vec& gradient_input, 
00391                                 Vec& gradient_output,                   
00392                                 string transfer_func = "default",
00393                                 int nll_softmax_speed_up_target=-1);
00394 
00399     void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 
00400                               bool input_is_sparse, bool output_is_sparse,
00401                               Vec output_indices = Vec(0)) const;
00402 
00407     void gradient_affine_transform(Vec input, Mat weights, Vec bias, 
00408                                    Vec ginput, Mat gweights, Vec gbias, Vec goutput, 
00409                                    bool input_is_sparse, bool output_is_sparse,
00410                                    real learning_rate,
00411                                    real weight_decay, real bias_decay,
00412                                    Vec output_indices = Vec(0));
00413 
00416     void gradient_penalty(Vec input, Mat weights, Vec bias, 
00417                           Mat gweights, Vec gbias,  
00418                           bool input_is_sparse, bool output_is_sparse,
00419                           real learning_rate,
00420                           real weight_decay, real bias_decay,
00421                           Vec output_indices = Vec(0));
00422     
00425     void importance_sampling_gradient_update(Vec& inputv, Vec& targetv, 
00426                                              real learning_rate, int n_samples, 
00427                                              real train_sample_weight=1);
00428 
00431     void getNegativeEnergyValues(Vec samples, Vec neg_energies);
00432 
00435     void fillWeights(const Mat& weights);
00436 
00438     void verify_gradient(Vec& input, Vec target, real step);
00439 
00441     void verify_gradient_affine_transform(
00442         Vec global_input, Vec& global_output, Vec& global_targetv, 
00443         Vec& global_costs, real sampleweight,
00444         Vec input, Mat weights, Vec bias,
00445         Mat est_gweights, Vec est_gbias, 
00446         bool input_is_sparse, bool output_is_sparse,
00447         real step,
00448         Vec output_indices = Vec(0)) const;
00449     
00450     void output_gradient_verification(Vec grad, Vec est_grad);
00451 
00453     void batchComputeOutputAndConfidence(VMat inputs, real probability,
00454                                          VMat outputs_and_confidence) const;
00456     virtual void use(VMat testset, VMat outputs) const;
00458     virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 
00459                       VMat testoutputs=0, VMat testcosts=0) const;
00461     virtual VMat processDataSet(VMat dataset) const;
00462         
00463 };
00464 
00465 DECLARE_OBJECT_PTR(NeuralProbabilisticLanguageModel);
00466 
00467 } // end of namespace PLearn
00468 
00469 #endif
00470 
00471 
00472 /*
00473   Local Variables:
00474   mode:c++
00475   c-basic-offset:4
00476   c-file-style:"stroustrup"
00477   c-file-offsets:((innamespace . 0)(inline-open . 0))
00478   indent-tabs-mode:nil
00479   fill-column:79
00480   End:
00481 */
00482 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines