PLearn 0.1
FeatureSetNNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // FeatureSetNNet.h
00004 // Copyright (c) 1998-2002 Pascal Vincent
00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal
00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 #ifndef FeatureSetNNet_INC
00038 #define FeatureSetNNet_INC
00039 
00040 #include "PLearner.h"
00041 #include <plearn/math/PRandom.h>
00042 #include <plearn/feat/FeatureSet.h>
00043 
00044 namespace PLearn {
00045 using namespace std;
00046 
00055 class FeatureSetNNet: public PLearner
00056 {
00057 
00058 private:
00059 
00060     typedef PLearner inherited;
00061     
00063     mutable Vec target_values;
00065     mutable Vec output_comp;
00067     mutable Vec row;
00070     mutable Vec last_layer;
00072     mutable Vec gradient_last_layer;
00074     mutable TVec< TVec<int> > feats;
00075 
00079     mutable Vec gradient;
00080     mutable string str;
00081     mutable real * pval1, * pval2, * pval3, * pval4, * pval5;
00082     mutable real val, val2, grad;
00083     mutable int offset;
00084     mutable int ni,nj,nk,id,nfeats,ifeats;
00085     mutable int* f;
00086 
00087 protected:
00088 
00090     int total_output_size;
00092     int total_updates;
00094     int n_feat_sets;
00098     int total_feats_per_token;
00100     mutable int reind_target;
00102     mutable Vec feat_input;
00104     Vec gradient_feat_input;
00106     mutable Vec nnet_input;
00108     Vec gradient_nnet_input;
00110     Vec hiddenv;
00112     Vec gradient_hiddenv;
00114     Vec gradient_act_hiddenv;
00116     Vec hidden2v;
00118     Vec gradient_hidden2v;
00120     Vec gradient_act_hidden2v;
00122     Vec gradient_outputv;
00124     Vec gradient_act_outputv;
00126     PP<PRandom> rgen;
00128     Vec feats_since_last_update;
00130     Vec target_values_since_last_update;
00132     mutable VMat val_string_reference_set;
00134     mutable VMat target_values_reference_set;
00135 
00136 public: 
00138     Mat w1;
00140     Mat gradient_w1;
00142     Vec b1;
00144     Vec gradient_b1;
00146     Mat w2;
00148     Mat gradient_w2;
00150     Vec b2;
00152     Vec gradient_b2;
00154     Mat wout;
00156     Mat gradient_wout;
00158     Vec bout;
00160     Vec gradient_bout;
00162     Mat direct_wout;
00164     Mat gradient_direct_wout;
00166     Vec direct_bout;
00168     Vec gradient_direct_bout;
00171     Mat wout_dist_rep;
00174     Mat gradient_wout_dist_rep;
00177     Vec bout_dist_rep;
00180     Vec gradient_bout_dist_rep;
00183     Vec output_empirical_distribution;
00184 
00185 public:
00186 
00187     // Build options:
00188 
00190     int nhidden;
00192     int nhidden2; 
00194     real weight_decay; 
00196     real bias_decay; 
00199     real layer1_weight_decay; 
00202     real layer1_bias_decay;   
00205     real layer2_weight_decay; 
00208     real layer2_bias_decay;   
00211     real output_layer_weight_decay; 
00214     real output_layer_bias_decay;
00217     real direct_in_to_out_weight_decay;
00220     real output_layer_dist_rep_weight_decay; 
00223     real output_layer_dist_rep_bias_decay;
00226     real margin; 
00229     bool fixed_output_weights;
00232     bool direct_in_to_out;
00235     string penalty_type; 
00237     string output_transfer_func; 
00240     string hidden_transfer_func; 
00242     TVec<string> cost_funcs;  
00244     real start_learning_rate;
00246     real decrease_constant;
00249     int batch_size; 
00252     bool stochastic_gradient_descent_speedup;
00254     string initialization_method;
00257     int dist_rep_dim;
00260     bool possible_targets_vary;
00262     TVec<PP<FeatureSet> > feat_sets;
00263     //  //! Indication that the input IDs should be used as the feature ID.
00264     //  //! The ID/string mapping provided by the input VMatrix Dictionary
00265     //  //! objects is hence used.
00266     //  //! This implies that all VMatrices (even those at test time) that
00267     //  //! provide the input vectors should use the same Dictionary objects.
00268     //  bool use_input_as_feature;
00269     //  //! Indication that an estimate of the gradient using
00270     //  //! importance sampling should be used.
00271     //  //! See (Bengio and Sénécal, 2003) for more details.
00272     //  bool use_importance_sampling_gradient;
00273 
00274 private:
00275     void build_();
00276 
00281     void compute_softmax(const Vec& x, const Vec& y) const;
00282 
00284     real nll(const Vec& outputv, int target) const;
00285     
00287     real classification_loss(const Vec& outputv, int target) const;
00288     
00295     int my_argmax(const Vec& vec, int default_compare=0) const;
00296 
00297 public:
00298 
00299     FeatureSetNNet();
00300     virtual ~FeatureSetNNet();
00301     PLEARN_DECLARE_OBJECT(FeatureSetNNet);
00302 
00303     virtual void build();
00304     virtual void forget(); // simply calls initializeParams()
00305 
00306     virtual int outputsize() const;
00307     virtual TVec<string> getTrainCostNames() const;
00308     virtual TVec<string> getTestCostNames() const;
00309 
00310     virtual void train();
00311 
00312     virtual void computeOutput(const Vec& input, Vec& output) const;
00313 
00314     virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00315                                        Vec& output, Vec& costs) const;
00316 
00317     virtual void computeCostsFromOutputs(const Vec& input, 
00318                                          const Vec& output, 
00319                                          const Vec& target, 
00320                                          Vec& costs) const;
00321 
00322     virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies);
00323 
00324 protected:
00325     static void declareOptions(OptionList& ol);
00326 
00328     void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00329 
00331     void fpropOutput(const Vec& inputv, Vec& outputv) const;
00332 
00334     void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00335 
00341     void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1);
00342 
00344     void update();
00345 
00347     void update_affine_transform(Vec input, Mat weights, Vec bias,
00348                                  Mat gweights, Vec gbias,
00349                                  bool input_is_sparse, bool output_is_sparse,
00350                                  Vec output_indices);
00351     
00354     void clearProppathGradient();
00355 
00361     virtual void initializeParams(bool set_seed = true);
00362 
00365     void add_transfer_func(const Vec& input, 
00366                           string transfer_func = "default") const;
00367 
00376     void gradient_transfer_func(Vec& output, Vec& gradient_input, 
00377                                 Vec& gradient_output,                   
00378                                 string transfer_func = "default",
00379                                 int nll_softmax_speed_up_target=-1);
00380 
00385     void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 
00386                               bool input_is_sparse, bool output_is_sparse,
00387                               Vec output_indices = Vec(0)) const;
00388 
00393     void gradient_affine_transform(Vec input, Mat weights, Vec bias, 
00394                                    Vec ginput, Mat gweights, Vec gbias, Vec goutput, 
00395                                    bool input_is_sparse, bool output_is_sparse,
00396                                    real learning_rate,
00397                                    real weight_decay, real bias_decay,
00398                                    Vec output_indices = Vec(0));
00399 
00402     void gradient_penalty(Vec input, Mat weights, Vec bias, 
00403                           Mat gweights, Vec gbias,  
00404                           bool input_is_sparse, bool output_is_sparse,
00405                           real learning_rate,
00406                           real weight_decay, real bias_decay,
00407                           Vec output_indices = Vec(0));
00408     
00411     void fillWeights(const Mat& weights);
00412 
00414     void verify_gradient(Vec& input, Vec target, real step);
00415 
00417     void verify_gradient_affine_transform(
00418         Vec global_input, Vec& global_output, Vec& global_targetv, 
00419         Vec& global_costs, real sampleweight,
00420         Vec input, Mat weights, Vec bias,
00421         Mat est_gweights, Vec est_gbias, 
00422         bool input_is_sparse, bool output_is_sparse,
00423         real step,
00424         Vec output_indices = Vec(0)) const;
00425     
00426     void output_gradient_verification(Vec grad, Vec est_grad);
00427 
00429     void batchComputeOutputAndConfidence(VMat inputs, real probability,
00430                                          VMat outputs_and_confidence) const;
00432     virtual void use(VMat testset, VMat outputs) const;
00434     virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 
00435                       VMat testoutputs=0, VMat testcosts=0) const;
00437     virtual VMat processDataSet(VMat dataset) const;
00438         
00439 };
00440 
00441 DECLARE_OBJECT_PTR(FeatureSetNNet);
00442 
00443 } // end of namespace PLearn
00444 
00445 #endif
00446 
00447 
00448 /*
00449   Local Variables:
00450   mode:c++
00451   c-basic-offset:4
00452   c-file-style:"stroustrup"
00453   c-file-offsets:((innamespace . 0)(inline-open . 0))
00454   indent-tabs-mode:nil
00455   fill-column:79
00456   End:
00457 */
00458 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines