PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // FeatureSetSequentialCRF.h 00004 // Copyright (c) 1998-2002 Pascal Vincent 00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal 00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 #ifndef FeatureSetSequentialCRF_INC 00038 #define FeatureSetSequentialCRF_INC 00039 00040 #include "PLearner.h" 00041 #include <plearn/math/PRandom.h> 00042 #include <plearn/feat/FeatureSet.h> 00043 00044 namespace PLearn { 00045 using namespace std; 00046 00055 class FeatureSetSequentialCRF: public PLearner 00056 { 00057 00058 private: 00059 00060 typedef PLearner inherited; 00061 00063 mutable Vec target_values; 00065 mutable Vec output_comp; 00067 mutable Vec row; 00070 mutable Vec last_layer; 00072 mutable Vec gradient_last_layer; 00074 mutable TVec< TVec<int> > feats; 00075 00079 mutable Vec gradient; 00080 mutable string str; 00081 mutable real * pval1, * pval2, * pval3, * pval4, * pval5; 00082 mutable real val, val2, grad; 00083 mutable int offset; 00084 mutable int ni,nj,nk,id,nfeats,ifeats; 00085 mutable int* f; 00086 00087 protected: 00088 00090 int total_output_size; 00092 int total_updates; 00094 int n_feat_sets; 00098 int total_feats_per_token; 00100 mutable int reind_target; 00102 mutable Vec feat_input; 00104 Vec gradient_feat_input; 00106 Vec nnet_input; 00108 Vec gradient_nnet_input; 00110 Vec hiddenv; 00112 Vec gradient_hiddenv; 00114 Vec gradient_act_hiddenv; 00116 Vec hidden2v; 00118 Vec gradient_hidden2v; 00120 Vec gradient_act_hidden2v; 00122 Vec gradient_outputv; 00124 Vec gradient_act_outputv; 00126 PP<PRandom> rgen; 00128 Vec feats_since_last_update; 00130 Vec target_values_since_last_update; 00132 mutable VMat val_string_reference_set; 00134 mutable VMat target_values_reference_set; 00135 00136 public: 00138 Mat w1; 00140 Mat gradient_w1; 00142 Vec b1; 00144 Vec gradient_b1; 00146 Mat w2; 00148 Mat gradient_w2; 00150 Vec b2; 00152 Vec gradient_b2; 00154 Mat wout; 00156 Mat gradient_wout; 00158 Vec bout; 00160 Vec gradient_bout; 00162 Mat direct_wout; 00164 Mat gradient_direct_wout; 00166 Vec direct_bout; 00168 Vec gradient_direct_bout; 00171 Mat wout_dist_rep; 00174 Mat gradient_wout_dist_rep; 00177 Vec bout_dist_rep; 00180 Vec gradient_bout_dist_rep; 00181 00182 public: 00183 00184 // Build options: 00185 00187 int nhidden; 00189 int nhidden2; 00191 real weight_decay; 00193 real bias_decay; 00196 real layer1_weight_decay; 00199 real layer1_bias_decay; 00202 real layer2_weight_decay; 00205 real layer2_bias_decay; 00208 real output_layer_weight_decay; 00211 real output_layer_bias_decay; 00214 real direct_in_to_out_weight_decay; 00217 real output_layer_dist_rep_weight_decay; 00220 real output_layer_dist_rep_bias_decay; 00223 real margin; 00226 bool fixed_output_weights; 00229 bool direct_in_to_out; 00232 string penalty_type; 00234 string output_transfer_func; 00237 string hidden_transfer_func; 00239 TVec<string> cost_funcs; 00241 real start_learning_rate; 00243 real decrease_constant; 00246 int batch_size; 00249 bool stochastic_gradient_descent_speedup; 00251 string initialization_method; 00254 int dist_rep_dim; 00257 bool possible_targets_vary; 00259 TVec<PP<FeatureSet> > feat_sets; 00265 bool use_input_as_feature; 00266 00267 private: 00268 void build_(); 00269 00274 void compute_softmax(const Vec& x, const Vec& y) const; 00275 00277 real nll(const Vec& outputv, int target) const; 00278 00280 real classification_loss(const Vec& outputv, int target) const; 00281 00288 int my_argmax(const Vec& vec, int default_compare=0) const; 00289 00290 public: 00291 00292 FeatureSetSequentialCRF(); 00293 virtual ~FeatureSetSequentialCRF(); 00294 PLEARN_DECLARE_OBJECT(FeatureSetSequentialCRF); 00295 00296 virtual void build(); 00297 virtual void forget(); // simply calls initializeParams() 00298 00299 virtual int outputsize() const; 00300 virtual TVec<string> getTrainCostNames() const; 00301 virtual TVec<string> getTestCostNames() const; 00302 00303 virtual void train(); 00304 00305 virtual void computeOutput(const Vec& input, Vec& output) const; 00306 00307 virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00308 Vec& output, Vec& costs) const; 00309 00310 virtual void computeCostsFromOutputs(const Vec& input, 00311 const Vec& output, 00312 const Vec& target, 00313 Vec& costs) const; 00314 00315 virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies); 00316 00317 protected: 00318 static void declareOptions(OptionList& ol); 00319 00321 void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00322 00324 void fpropOutput(const Vec& inputv, Vec& outputv) const; 00325 00327 void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00328 00334 void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1); 00335 00337 void update(); 00338 00340 void update_affine_transform(Vec input, Mat weights, Vec bias, 00341 Mat gweights, Vec gbias, 00342 bool input_is_sparse, bool output_is_sparse, 00343 Vec output_indices); 00344 00347 void clearProppathGradient(); 00348 00354 virtual void initializeParams(bool set_seed = true); 00355 00358 void add_transfer_func(const Vec& input, 00359 string transfer_func = "default") const; 00360 00369 void gradient_transfer_func(Vec& output, Vec& gradient_input, 00370 Vec& gradient_output, 00371 string transfer_func = "default", 00372 int nll_softmax_speed_up_target=-1); 00373 00378 void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 00379 bool input_is_sparse, bool output_is_sparse, 00380 Vec output_indices = Vec(0)) const; 00381 00386 void gradient_affine_transform(Vec input, Mat weights, Vec bias, 00387 Vec ginput, Mat gweights, Vec gbias, Vec goutput, 00388 bool input_is_sparse, bool output_is_sparse, 00389 real learning_rate, 00390 real weight_decay, real bias_decay, 00391 Vec output_indices = Vec(0)); 00392 00395 void gradient_penalty(Vec input, Mat weights, Vec bias, 00396 Mat gweights, Vec gbias, 00397 bool input_is_sparse, bool output_is_sparse, 00398 real learning_rate, 00399 real weight_decay, real bias_decay, 00400 Vec output_indices = Vec(0)); 00401 00404 void fillWeights(const Mat& weights); 00405 00407 void verify_gradient(Vec& input, Vec target, real step); 00408 00410 void verify_gradient_affine_transform( 00411 Vec global_input, Vec& global_output, Vec& global_targetv, 00412 Vec& global_costs, real sampleweight, 00413 Vec input, Mat weights, Vec bias, 00414 Mat est_gweights, Vec est_gbias, 00415 bool input_is_sparse, bool output_is_sparse, 00416 real step, 00417 Vec output_indices = Vec(0)) const; 00418 00419 void output_gradient_verification(Vec grad, Vec est_grad); 00420 00422 void batchComputeOutputAndConfidence(VMat inputs, real probability, 00423 VMat outputs_and_confidence) const; 00425 virtual void use(VMat testset, VMat outputs) const; 00427 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00428 VMat testoutputs=0, VMat testcosts=0) const; 00430 virtual VMat processDataSet(VMat dataset) const; 00431 00432 }; 00433 00434 DECLARE_OBJECT_PTR(FeatureSetSequentialCRF); 00435 00436 } // end of namespace PLearn 00437 00438 #endif 00439 00440 00441 /* 00442 Local Variables: 00443 mode:c++ 00444 c-basic-offset:4 00445 c-file-style:"stroustrup" 00446 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00447 indent-tabs-mode:nil 00448 fill-column:79 00449 End: 00450 */ 00451 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :