PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // DistRepNNet.h 00004 // Copyright (c) 1998-2002 Pascal Vincent 00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal 00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: DistRepNNet.h 3994 2005-08-25 13:35:03Z chapados $ 00039 ******************************************************* */ 00040 00041 00042 #ifndef DistRepNNet_INC 00043 #define DistRepNNet_INC 00044 00045 #include "PLearner.h" 00046 #include <plearn/opt/Optimizer.h> 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00051 class DistRepNNet: public PLearner 00052 { 00053 00054 private: 00055 00056 typedef PLearner inherited; 00057 00059 mutable Vec target_values; 00061 mutable Vec output_comp; 00063 mutable Vec row; 00065 mutable Vec tf; 00066 mutable TVec<string> options; 00067 00068 protected: 00069 00071 Var output; 00073 VarArray costs; 00075 VarArray partial_update_vars; 00077 VarArray penalties; 00080 Var training_cost; 00082 VarArray training_cost_extra_tasks; 00084 Var test_costs; 00086 VarArray invars; 00088 TVec< VarArray > invars_extra_tasks; 00090 VarArray params; 00093 //VarArray activated_weights; 00095 TVec<int> input_to_dict_index; 00097 int target_dict_index; 00099 real winputsparse_weight_decay; 00100 real winputsparse_bias_decay; 00101 00102 public: 00105 Vec paramsvalues; // values of all parameters 00107 Var input; 00109 Var dp_input; 00111 Var target; 00113 Var sampleweight; 00115 Var w1; 00117 VarArray winputsparse; 00119 //Var w1target; 00121 Var w1theta; 00123 VarArray winputdistrep; 00125 Var woutdistrep; 00127 Var w2; 00129 Var wout; 00131 Var direct_wout; 00133 //Var wouttarget; 00135 Var wouttheta; 00137 Var outbias; 00139 Var dp_all_targets; 00141 Var token_features; 00143 Var dist_rep; 00145 //VarArray dist_reps; 00147 TVec< PP<Dictionary> > dictionaries; 00149 //TVec<int> seen_target; 00151 //oTVec<int> unseen_target; 00153 mutable Func f; 00155 mutable Func test_costf; 00157 mutable Func token_to_dist_rep; 00159 mutable Func paramf; 00160 00161 public: 00162 00163 // Build options: 00164 00166 TVec<VMat> extra_tasks; 00168 int nhidden; 00170 int nhidden2; 00172 TVec<int> nhidden_extra_tasks; 00174 TVec<int> nhidden2_extra_tasks; 00176 int nhidden_theta_predictor; 00178 int nhidden_dist_rep_predictor; 00180 real weight_decay; 00182 real bias_decay; 00184 real input_dist_rep_predictor_bias_decay; 00186 real output_dist_rep_predictor_bias_decay; 00188 real input_dist_rep_predictor_weight_decay; 00190 real output_dist_rep_predictor_weight_decay; 00192 real layer1_weight_decay; 00194 real layer1_bias_decay; 00196 real layer1_theta_predictor_weight_decay; 00198 real layer1_theta_predictor_bias_decay; 00200 real layer2_weight_decay; 00202 real layer2_bias_decay; 00204 real output_layer_weight_decay; 00206 real output_layer_bias_decay; 00208 real output_layer_theta_predictor_weight_decay; 00210 real output_layer_theta_predictor_bias_decay; 00212 real direct_in_to_out_weight_decay; 00214 real direct_in_to_out_bias_decay; 00216 real margin; 00218 bool fixed_output_weights; 00220 bool direct_in_to_out; 00222 string penalty_type; 00224 string output_transfer_func; 00226 string hidden_transfer_func; // tanh, sigmoid, softplus, softmax, etc... (default: "tanh" means no transfer function) 00228 bool do_not_change_params; 00230 TVec<string> cost_funcs; 00232 PP<Optimizer> optimizer; 00234 TVec< PP<Optimizer> > optimizer_extra_tasks; 00237 int batch_size; 00239 string initialization_method; 00241 TVec<int> dist_rep_dim; 00243 int ntokens; 00246 TVec<int> ntokens_extra_tasks; 00248 int nfeatures_per_token; 00251 TVec<int> nfeatures_for_each_token; 00253 PP<Dictionary> target_dictionary; 00255 Mat target_dist_rep; 00257 bool use_dist_reps; 00259 bool use_output_weights_bases; 00262 //bool possible_targets_varies; 00265 bool use_extra_tasks_only_on_first_epoch; 00268 bool initialize_sparse_params_to_zero; 00270 //string nnet_architecture; 00271 00272 private: 00273 void build_(); 00274 00275 public: 00276 00277 DistRepNNet(); 00278 virtual ~DistRepNNet(); 00279 PLEARN_DECLARE_OBJECT(DistRepNNet); 00280 00281 virtual void build(); 00282 virtual void forget(); // simply calls initializeParams() 00283 00284 virtual int outputsize() const; 00285 virtual TVec<string> getTrainCostNames() const; 00286 virtual TVec<string> getTestCostNames() const; 00287 00288 virtual void train(); 00289 00290 virtual void computeOutput(const Vec& input, Vec& output) const; 00291 00292 virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00293 Vec& output, Vec& costs) const; 00294 00295 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00296 const Vec& target, Vec& costs) const; 00297 00298 virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies); 00299 00304 void getTokenDistRep(TVec<string>& token_features, Vec& dist_rep); 00305 00307 //virtual Mat getW1() {return w1->matValue;} 00308 //virtual Mat getW2() {return w2->matValue;} 00309 //virtual Mat getWout() {return wout->matValue;} 00310 00311 protected: 00312 static void declareOptions(OptionList& ol); 00313 00315 Var buildSparseAffineTransform(VarArray weights, Var input, TVec<int> input_to_dict_index, int begin); 00316 00318 Var buildSparseAffineTransformWeightPenalty(VarArray weights, Var input, TVec<int> input_to_dict_index, int begin, real weight_decay, real bias_decay=0, string penalty_type="L2_square"); 00319 00321 void buildVarGraph(int task_index); 00322 00328 virtual void initializeParams(bool set_seed = true, int task_index = -1); 00329 00332 Var add_transfer_func(const Var& input, string transfer_func = "default", VarArray mus=0, Var sigma=0); 00333 00335 void buildOutputFromInput(int task_index); 00336 00338 void buildCosts(const Var& output, const Var& target, int task_index = -1); 00339 00341 void buildFuncs(VarArray& invars); 00342 00346 void fillWeights(const Var& weights, bool clear_first_row, int use_this_to_scale=-1); 00347 00349 virtual void buildPenalties(int this_ntokens); 00350 00351 }; 00352 00353 DECLARE_OBJECT_PTR(DistRepNNet); 00354 00355 } // end of namespace PLearn 00356 00357 #endif 00358 00359 00360 /* 00361 Local Variables: 00362 mode:c++ 00363 c-basic-offset:4 00364 c-file-style:"stroustrup" 00365 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00366 indent-tabs-mode:nil 00367 fill-column:79 00368 End: 00369 */ 00370 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :