PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // FeatureSetNNet.h 00004 // Copyright (c) 1998-2002 Pascal Vincent 00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal 00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 #ifndef FeatureSetNNet_INC 00038 #define FeatureSetNNet_INC 00039 00040 #include "PLearner.h" 00041 #include <plearn/math/PRandom.h> 00042 #include <plearn/feat/FeatureSet.h> 00043 00044 namespace PLearn { 00045 using namespace std; 00046 00055 class FeatureSetNNet: public PLearner 00056 { 00057 00058 private: 00059 00060 typedef PLearner inherited; 00061 00063 mutable Vec target_values; 00065 mutable Vec output_comp; 00067 mutable Vec row; 00070 mutable Vec last_layer; 00072 mutable Vec gradient_last_layer; 00074 mutable TVec< TVec<int> > feats; 00075 00079 mutable Vec gradient; 00080 mutable string str; 00081 mutable real * pval1, * pval2, * pval3, * pval4, * pval5; 00082 mutable real val, val2, grad; 00083 mutable int offset; 00084 mutable int ni,nj,nk,id,nfeats,ifeats; 00085 mutable int* f; 00086 00087 protected: 00088 00090 int total_output_size; 00092 int total_updates; 00094 int n_feat_sets; 00098 int total_feats_per_token; 00100 mutable int reind_target; 00102 mutable Vec feat_input; 00104 Vec gradient_feat_input; 00106 mutable Vec nnet_input; 00108 Vec gradient_nnet_input; 00110 Vec hiddenv; 00112 Vec gradient_hiddenv; 00114 Vec gradient_act_hiddenv; 00116 Vec hidden2v; 00118 Vec gradient_hidden2v; 00120 Vec gradient_act_hidden2v; 00122 Vec gradient_outputv; 00124 Vec gradient_act_outputv; 00126 PP<PRandom> rgen; 00128 Vec feats_since_last_update; 00130 Vec target_values_since_last_update; 00132 mutable VMat val_string_reference_set; 00134 mutable VMat target_values_reference_set; 00135 00136 public: 00138 Mat w1; 00140 Mat gradient_w1; 00142 Vec b1; 00144 Vec gradient_b1; 00146 Mat w2; 00148 Mat gradient_w2; 00150 Vec b2; 00152 Vec gradient_b2; 00154 Mat wout; 00156 Mat gradient_wout; 00158 Vec bout; 00160 Vec gradient_bout; 00162 Mat direct_wout; 00164 Mat gradient_direct_wout; 00166 Vec direct_bout; 00168 Vec gradient_direct_bout; 00171 Mat wout_dist_rep; 00174 Mat gradient_wout_dist_rep; 00177 Vec bout_dist_rep; 00180 Vec gradient_bout_dist_rep; 00183 Vec output_empirical_distribution; 00184 00185 public: 00186 00187 // Build options: 00188 00190 int nhidden; 00192 int nhidden2; 00194 real weight_decay; 00196 real bias_decay; 00199 real layer1_weight_decay; 00202 real layer1_bias_decay; 00205 real layer2_weight_decay; 00208 real layer2_bias_decay; 00211 real output_layer_weight_decay; 00214 real output_layer_bias_decay; 00217 real direct_in_to_out_weight_decay; 00220 real output_layer_dist_rep_weight_decay; 00223 real output_layer_dist_rep_bias_decay; 00226 real margin; 00229 bool fixed_output_weights; 00232 bool direct_in_to_out; 00235 string penalty_type; 00237 string output_transfer_func; 00240 string hidden_transfer_func; 00242 TVec<string> cost_funcs; 00244 real start_learning_rate; 00246 real decrease_constant; 00249 int batch_size; 00252 bool stochastic_gradient_descent_speedup; 00254 string initialization_method; 00257 int dist_rep_dim; 00260 bool possible_targets_vary; 00262 TVec<PP<FeatureSet> > feat_sets; 00263 // //! Indication that the input IDs should be used as the feature ID. 00264 // //! The ID/string mapping provided by the input VMatrix Dictionary 00265 // //! objects is hence used. 00266 // //! This implies that all VMatrices (even those at test time) that 00267 // //! provide the input vectors should use the same Dictionary objects. 00268 // bool use_input_as_feature; 00269 // //! Indication that an estimate of the gradient using 00270 // //! importance sampling should be used. 00271 // //! See (Bengio and Sénécal, 2003) for more details. 00272 // bool use_importance_sampling_gradient; 00273 00274 private: 00275 void build_(); 00276 00281 void compute_softmax(const Vec& x, const Vec& y) const; 00282 00284 real nll(const Vec& outputv, int target) const; 00285 00287 real classification_loss(const Vec& outputv, int target) const; 00288 00295 int my_argmax(const Vec& vec, int default_compare=0) const; 00296 00297 public: 00298 00299 FeatureSetNNet(); 00300 virtual ~FeatureSetNNet(); 00301 PLEARN_DECLARE_OBJECT(FeatureSetNNet); 00302 00303 virtual void build(); 00304 virtual void forget(); // simply calls initializeParams() 00305 00306 virtual int outputsize() const; 00307 virtual TVec<string> getTrainCostNames() const; 00308 virtual TVec<string> getTestCostNames() const; 00309 00310 virtual void train(); 00311 00312 virtual void computeOutput(const Vec& input, Vec& output) const; 00313 00314 virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00315 Vec& output, Vec& costs) const; 00316 00317 virtual void computeCostsFromOutputs(const Vec& input, 00318 const Vec& output, 00319 const Vec& target, 00320 Vec& costs) const; 00321 00322 virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies); 00323 00324 protected: 00325 static void declareOptions(OptionList& ol); 00326 00328 void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00329 00331 void fpropOutput(const Vec& inputv, Vec& outputv) const; 00332 00334 void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00335 00341 void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1); 00342 00344 void update(); 00345 00347 void update_affine_transform(Vec input, Mat weights, Vec bias, 00348 Mat gweights, Vec gbias, 00349 bool input_is_sparse, bool output_is_sparse, 00350 Vec output_indices); 00351 00354 void clearProppathGradient(); 00355 00361 virtual void initializeParams(bool set_seed = true); 00362 00365 void add_transfer_func(const Vec& input, 00366 string transfer_func = "default") const; 00367 00376 void gradient_transfer_func(Vec& output, Vec& gradient_input, 00377 Vec& gradient_output, 00378 string transfer_func = "default", 00379 int nll_softmax_speed_up_target=-1); 00380 00385 void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 00386 bool input_is_sparse, bool output_is_sparse, 00387 Vec output_indices = Vec(0)) const; 00388 00393 void gradient_affine_transform(Vec input, Mat weights, Vec bias, 00394 Vec ginput, Mat gweights, Vec gbias, Vec goutput, 00395 bool input_is_sparse, bool output_is_sparse, 00396 real learning_rate, 00397 real weight_decay, real bias_decay, 00398 Vec output_indices = Vec(0)); 00399 00402 void gradient_penalty(Vec input, Mat weights, Vec bias, 00403 Mat gweights, Vec gbias, 00404 bool input_is_sparse, bool output_is_sparse, 00405 real learning_rate, 00406 real weight_decay, real bias_decay, 00407 Vec output_indices = Vec(0)); 00408 00411 void fillWeights(const Mat& weights); 00412 00414 void verify_gradient(Vec& input, Vec target, real step); 00415 00417 void verify_gradient_affine_transform( 00418 Vec global_input, Vec& global_output, Vec& global_targetv, 00419 Vec& global_costs, real sampleweight, 00420 Vec input, Mat weights, Vec bias, 00421 Mat est_gweights, Vec est_gbias, 00422 bool input_is_sparse, bool output_is_sparse, 00423 real step, 00424 Vec output_indices = Vec(0)) const; 00425 00426 void output_gradient_verification(Vec grad, Vec est_grad); 00427 00429 void batchComputeOutputAndConfidence(VMat inputs, real probability, 00430 VMat outputs_and_confidence) const; 00432 virtual void use(VMat testset, VMat outputs) const; 00434 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00435 VMat testoutputs=0, VMat testcosts=0) const; 00437 virtual VMat processDataSet(VMat dataset) const; 00438 00439 }; 00440 00441 DECLARE_OBJECT_PTR(FeatureSetNNet); 00442 00443 } // end of namespace PLearn 00444 00445 #endif 00446 00447 00448 /* 00449 Local Variables: 00450 mode:c++ 00451 c-basic-offset:4 00452 c-file-style:"stroustrup" 00453 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00454 indent-tabs-mode:nil 00455 fill-column:79 00456 End: 00457 */ 00458 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :