PLearn 0.1
FeatureSetSequentialCRF.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // FeatureSetSequentialCRF.h
00004 // Copyright (c) 1998-2002 Pascal Vincent
00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal
00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 #ifndef FeatureSetSequentialCRF_INC
00038 #define FeatureSetSequentialCRF_INC
00039 
00040 #include "PLearner.h"
00041 #include <plearn/math/PRandom.h>
00042 #include <plearn/feat/FeatureSet.h>
00043 
00044 namespace PLearn {
00045 using namespace std;
00046 
00055 class FeatureSetSequentialCRF: public PLearner
00056 {
00057 
00058 private:
00059 
00060     typedef PLearner inherited;
00061     
00063     mutable Vec target_values;
00065     mutable Vec output_comp;
00067     mutable Vec row;
00070     mutable Vec last_layer;
00072     mutable Vec gradient_last_layer;
00074     mutable TVec< TVec<int> > feats;
00075 
00079     mutable Vec gradient;
00080     mutable string str;
00081     mutable real * pval1, * pval2, * pval3, * pval4, * pval5;
00082     mutable real val, val2, grad;
00083     mutable int offset;
00084     mutable int ni,nj,nk,id,nfeats,ifeats;
00085     mutable int* f;
00086 
00087 protected:
00088 
00090     int total_output_size;
00092     int total_updates;
00094     int n_feat_sets;
00098     int total_feats_per_token;
00100     mutable int reind_target;
00102     mutable Vec feat_input;
00104     Vec gradient_feat_input;
00106     Vec nnet_input;
00108     Vec gradient_nnet_input;
00110     Vec hiddenv;
00112     Vec gradient_hiddenv;
00114     Vec gradient_act_hiddenv;
00116     Vec hidden2v;
00118     Vec gradient_hidden2v;
00120     Vec gradient_act_hidden2v;
00122     Vec gradient_outputv;
00124     Vec gradient_act_outputv;
00126     PP<PRandom> rgen;
00128     Vec feats_since_last_update;
00130     Vec target_values_since_last_update;
00132     mutable VMat val_string_reference_set;
00134     mutable VMat target_values_reference_set;
00135 
00136 public: 
00138     Mat w1;
00140     Mat gradient_w1;
00142     Vec b1;
00144     Vec gradient_b1;
00146     Mat w2;
00148     Mat gradient_w2;
00150     Vec b2;
00152     Vec gradient_b2;
00154     Mat wout;
00156     Mat gradient_wout;
00158     Vec bout;
00160     Vec gradient_bout;
00162     Mat direct_wout;
00164     Mat gradient_direct_wout;
00166     Vec direct_bout;
00168     Vec gradient_direct_bout;
00171     Mat wout_dist_rep;
00174     Mat gradient_wout_dist_rep;
00177     Vec bout_dist_rep;
00180     Vec gradient_bout_dist_rep;
00181 
00182 public:
00183 
00184     // Build options:
00185 
00187     int nhidden;
00189     int nhidden2; 
00191     real weight_decay; 
00193     real bias_decay; 
00196     real layer1_weight_decay; 
00199     real layer1_bias_decay;   
00202     real layer2_weight_decay; 
00205     real layer2_bias_decay;   
00208     real output_layer_weight_decay; 
00211     real output_layer_bias_decay;
00214     real direct_in_to_out_weight_decay;
00217     real output_layer_dist_rep_weight_decay; 
00220     real output_layer_dist_rep_bias_decay;
00223     real margin; 
00226     bool fixed_output_weights;
00229     bool direct_in_to_out;
00232     string penalty_type; 
00234     string output_transfer_func; 
00237     string hidden_transfer_func; 
00239     TVec<string> cost_funcs;  
00241     real start_learning_rate;
00243     real decrease_constant;
00246     int batch_size; 
00249     bool stochastic_gradient_descent_speedup;
00251     string initialization_method;
00254     int dist_rep_dim;
00257     bool possible_targets_vary;
00259     TVec<PP<FeatureSet> > feat_sets;
00265     bool use_input_as_feature;
00266 
00267 private:
00268     void build_();
00269 
00274     void compute_softmax(const Vec& x, const Vec& y) const;
00275 
00277     real nll(const Vec& outputv, int target) const;
00278     
00280     real classification_loss(const Vec& outputv, int target) const;
00281     
00288     int my_argmax(const Vec& vec, int default_compare=0) const;
00289 
00290 public:
00291 
00292     FeatureSetSequentialCRF();
00293     virtual ~FeatureSetSequentialCRF();
00294     PLEARN_DECLARE_OBJECT(FeatureSetSequentialCRF);
00295 
00296     virtual void build();
00297     virtual void forget(); // simply calls initializeParams()
00298 
00299     virtual int outputsize() const;
00300     virtual TVec<string> getTrainCostNames() const;
00301     virtual TVec<string> getTestCostNames() const;
00302 
00303     virtual void train();
00304 
00305     virtual void computeOutput(const Vec& input, Vec& output) const;
00306 
00307     virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00308                                        Vec& output, Vec& costs) const;
00309 
00310     virtual void computeCostsFromOutputs(const Vec& input, 
00311                                          const Vec& output, 
00312                                          const Vec& target, 
00313                                          Vec& costs) const;
00314 
00315     virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies);
00316 
00317 protected:
00318     static void declareOptions(OptionList& ol);
00319 
00321     void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00322 
00324     void fpropOutput(const Vec& inputv, Vec& outputv) const;
00325 
00327     void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const;
00328 
00334     void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1);
00335 
00337     void update();
00338 
00340     void update_affine_transform(Vec input, Mat weights, Vec bias,
00341                                  Mat gweights, Vec gbias,
00342                                  bool input_is_sparse, bool output_is_sparse,
00343                                  Vec output_indices);
00344     
00347     void clearProppathGradient();
00348 
00354     virtual void initializeParams(bool set_seed = true);
00355 
00358     void add_transfer_func(const Vec& input, 
00359                           string transfer_func = "default") const;
00360 
00369     void gradient_transfer_func(Vec& output, Vec& gradient_input, 
00370                                 Vec& gradient_output,                   
00371                                 string transfer_func = "default",
00372                                 int nll_softmax_speed_up_target=-1);
00373 
00378     void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 
00379                               bool input_is_sparse, bool output_is_sparse,
00380                               Vec output_indices = Vec(0)) const;
00381 
00386     void gradient_affine_transform(Vec input, Mat weights, Vec bias, 
00387                                    Vec ginput, Mat gweights, Vec gbias, Vec goutput, 
00388                                    bool input_is_sparse, bool output_is_sparse,
00389                                    real learning_rate,
00390                                    real weight_decay, real bias_decay,
00391                                    Vec output_indices = Vec(0));
00392 
00395     void gradient_penalty(Vec input, Mat weights, Vec bias, 
00396                           Mat gweights, Vec gbias,  
00397                           bool input_is_sparse, bool output_is_sparse,
00398                           real learning_rate,
00399                           real weight_decay, real bias_decay,
00400                           Vec output_indices = Vec(0));
00401     
00404     void fillWeights(const Mat& weights);
00405 
00407     void verify_gradient(Vec& input, Vec target, real step);
00408 
00410     void verify_gradient_affine_transform(
00411         Vec global_input, Vec& global_output, Vec& global_targetv, 
00412         Vec& global_costs, real sampleweight,
00413         Vec input, Mat weights, Vec bias,
00414         Mat est_gweights, Vec est_gbias, 
00415         bool input_is_sparse, bool output_is_sparse,
00416         real step,
00417         Vec output_indices = Vec(0)) const;
00418     
00419     void output_gradient_verification(Vec grad, Vec est_grad);
00420 
00422     void batchComputeOutputAndConfidence(VMat inputs, real probability,
00423                                          VMat outputs_and_confidence) const;
00425     virtual void use(VMat testset, VMat outputs) const;
00427     virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 
00428                       VMat testoutputs=0, VMat testcosts=0) const;
00430     virtual VMat processDataSet(VMat dataset) const;
00431         
00432 };
00433 
00434 DECLARE_OBJECT_PTR(FeatureSetSequentialCRF);
00435 
00436 } // end of namespace PLearn
00437 
00438 #endif
00439 
00440 
00441 /*
00442   Local Variables:
00443   mode:c++
00444   c-basic-offset:4
00445   c-file-style:"stroustrup"
00446   c-file-offsets:((innamespace . 0)(inline-open . 0))
00447   indent-tabs-mode:nil
00448   fill-column:79
00449   End:
00450 */
00451 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines