PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // NeuralProbabilisticLanguageModel.h 00004 // Copyright (c) 1998-2002 Pascal Vincent 00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal 00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00038 #ifndef NeuralProbabilisticLanguageModel_INC 00039 #define NeuralProbabilisticLanguageModel_INC 00040 00041 #include "PLearner.h" 00042 #include <plearn/math/PRandom.h> 00043 #include <plearn/feat/FeatureSet.h> 00044 00045 namespace PLearn { 00046 using namespace std; 00047 00058 class NeuralProbabilisticLanguageModel: public PLearner 00059 { 00060 00061 private: 00062 00063 typedef PLearner inherited; 00064 00066 mutable Vec target_values; 00068 mutable Vec output_comp; 00070 mutable Vec row; 00073 mutable Vec last_layer; 00075 mutable Vec gradient_last_layer; 00077 mutable TVec< TVec<int> > feats; 00078 00082 mutable Vec gradient, neg_energies, densities; 00083 mutable string str; 00084 mutable real * pval1, * pval2, * pval3, * pval4, * pval5; 00085 mutable real val, val2, grad; 00086 mutable int offset; 00087 mutable int ni,nj,nk,id,nfeats,ifeats; 00088 mutable int* f; 00089 00090 protected: 00091 00093 int total_output_size; 00095 int total_updates; 00097 int n_feat_sets; 00101 int total_feats_per_token; 00103 mutable int reind_target; 00105 mutable Vec feat_input; 00107 Vec gradient_feat_input; 00109 Vec nnet_input; 00111 Vec gradient_nnet_input; 00113 Vec hiddenv; 00115 Vec gradient_hiddenv; 00117 Vec gradient_act_hiddenv; 00119 Vec hidden2v; 00121 Vec gradient_hidden2v; 00123 Vec gradient_act_hidden2v; 00125 Vec gradient_outputv; 00127 Vec gradient_act_outputv; 00129 PP<PRandom> rgen; 00131 Vec feats_since_last_update; 00133 Vec target_values_since_last_update; 00135 mutable VMat val_string_reference_set; 00137 mutable VMat target_values_reference_set; 00139 Vec importance_sampling_ratios; 00141 Vec sample; 00143 Vec generated_samples; 00144 00145 public: 00147 Mat w1; 00149 Mat gradient_w1; 00151 Vec b1; 00153 Vec gradient_b1; 00155 Mat w2; 00157 Mat gradient_w2; 00159 Vec b2; 00161 Vec gradient_b2; 00163 Mat wout; 00165 Mat gradient_wout; 00167 Vec bout; 00169 Vec gradient_bout; 00171 Mat direct_wout; 00173 Mat gradient_direct_wout; 00175 Vec direct_bout; 00177 Vec gradient_direct_bout; 00180 Mat wout_dist_rep; 00183 Mat gradient_wout_dist_rep; 00186 Vec bout_dist_rep; 00189 Vec gradient_bout_dist_rep; 00190 00191 public: 00192 00193 // Build options: 00194 00196 int nhidden; 00198 int nhidden2; 00200 real weight_decay; 00202 real bias_decay; 00205 real layer1_weight_decay; 00208 real layer1_bias_decay; 00211 real layer2_weight_decay; 00214 real layer2_bias_decay; 00217 real output_layer_weight_decay; 00220 real output_layer_bias_decay; 00223 real direct_in_to_out_weight_decay; 00226 real output_layer_dist_rep_weight_decay; 00229 real output_layer_dist_rep_bias_decay; 00232 real margin; 00235 bool fixed_output_weights; 00238 bool direct_in_to_out; 00241 string penalty_type; 00243 string output_transfer_func; 00246 string hidden_transfer_func; 00248 TVec<string> cost_funcs; 00250 real start_learning_rate; 00252 real decrease_constant; 00255 int batch_size; 00258 bool stochastic_gradient_descent_speedup; 00260 string initialization_method; 00263 int dist_rep_dim; 00266 bool possible_targets_vary; 00268 TVec<PP<FeatureSet> > feat_sets; 00275 PP<PDistribution> proposal_distribution; 00278 bool train_proposal_distribution; 00280 int sampling_block_size; 00282 int minimum_effective_sample_size; 00283 00284 private: 00285 void build_(); 00286 00291 void compute_softmax(const Vec& x, const Vec& y) const; 00292 00294 real nll(const Vec& outputv, int target) const; 00295 00297 real classification_loss(const Vec& outputv, int target) const; 00298 00305 int my_argmax(const Vec& vec, int default_compare=0) const; 00306 00307 public: 00308 00309 NeuralProbabilisticLanguageModel(); 00310 virtual ~NeuralProbabilisticLanguageModel(); 00311 PLEARN_DECLARE_OBJECT(NeuralProbabilisticLanguageModel); 00312 00313 virtual void build(); 00314 virtual void forget(); // simply calls initializeParams() 00315 00316 virtual int outputsize() const; 00317 virtual TVec<string> getTrainCostNames() const; 00318 virtual TVec<string> getTestCostNames() const; 00319 00320 virtual void train(); 00321 00322 virtual void computeOutput(const Vec& input, Vec& output) const; 00323 00324 virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00325 Vec& output, Vec& costs) const; 00326 00327 virtual void computeCostsFromOutputs(const Vec& input, 00328 const Vec& output, 00329 const Vec& target, 00330 Vec& costs) const; 00331 00332 virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies); 00333 00334 protected: 00335 static void declareOptions(OptionList& ol); 00336 00338 void fprop(const Vec& inputv, Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00339 00341 void fpropOutput(const Vec& inputv, Vec& outputv) const; 00342 00345 void fpropBeforeOutputWeights(const Vec& inputv) const; 00346 00348 void fpropCostsFromOutput(const Vec& inputv, const Vec& outputv, const Vec& targetv, Vec& costsv, real sampleweight=1) const; 00349 00355 void bprop(Vec& inputv, Vec& outputv, Vec& targetv, Vec& costsv, real learning_rate, real sampleweight=1); 00356 00358 void update(); 00359 00361 void update_affine_transform(Vec input, Mat weights, Vec bias, 00362 Mat gweights, Vec gbias, 00363 bool input_is_sparse, bool output_is_sparse, 00364 Vec output_indices); 00365 00368 void clearProppathGradient(); 00369 00375 virtual void initializeParams(bool set_seed = true); 00376 00379 void add_transfer_func(const Vec& input, 00380 string transfer_func = "default") const; 00381 00390 void gradient_transfer_func(Vec& output, Vec& gradient_input, 00391 Vec& gradient_output, 00392 string transfer_func = "default", 00393 int nll_softmax_speed_up_target=-1); 00394 00399 void add_affine_transform(Vec input, Mat weights, Vec bias, Vec output, 00400 bool input_is_sparse, bool output_is_sparse, 00401 Vec output_indices = Vec(0)) const; 00402 00407 void gradient_affine_transform(Vec input, Mat weights, Vec bias, 00408 Vec ginput, Mat gweights, Vec gbias, Vec goutput, 00409 bool input_is_sparse, bool output_is_sparse, 00410 real learning_rate, 00411 real weight_decay, real bias_decay, 00412 Vec output_indices = Vec(0)); 00413 00416 void gradient_penalty(Vec input, Mat weights, Vec bias, 00417 Mat gweights, Vec gbias, 00418 bool input_is_sparse, bool output_is_sparse, 00419 real learning_rate, 00420 real weight_decay, real bias_decay, 00421 Vec output_indices = Vec(0)); 00422 00425 void importance_sampling_gradient_update(Vec& inputv, Vec& targetv, 00426 real learning_rate, int n_samples, 00427 real train_sample_weight=1); 00428 00431 void getNegativeEnergyValues(Vec samples, Vec neg_energies); 00432 00435 void fillWeights(const Mat& weights); 00436 00438 void verify_gradient(Vec& input, Vec target, real step); 00439 00441 void verify_gradient_affine_transform( 00442 Vec global_input, Vec& global_output, Vec& global_targetv, 00443 Vec& global_costs, real sampleweight, 00444 Vec input, Mat weights, Vec bias, 00445 Mat est_gweights, Vec est_gbias, 00446 bool input_is_sparse, bool output_is_sparse, 00447 real step, 00448 Vec output_indices = Vec(0)) const; 00449 00450 void output_gradient_verification(Vec grad, Vec est_grad); 00451 00453 void batchComputeOutputAndConfidence(VMat inputs, real probability, 00454 VMat outputs_and_confidence) const; 00456 virtual void use(VMat testset, VMat outputs) const; 00458 virtual void test(VMat testset, PP<VecStatsCollector> test_stats, 00459 VMat testoutputs=0, VMat testcosts=0) const; 00461 virtual VMat processDataSet(VMat dataset) const; 00462 00463 }; 00464 00465 DECLARE_OBJECT_PTR(NeuralProbabilisticLanguageModel); 00466 00467 } // end of namespace PLearn 00468 00469 #endif 00470 00471 00472 /* 00473 Local Variables: 00474 mode:c++ 00475 c-basic-offset:4 00476 c-file-style:"stroustrup" 00477 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00478 indent-tabs-mode:nil 00479 fill-column:79 00480 End: 00481 */ 00482 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :