PLearn 0.1
DistRepNNet.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DistRepNNet.h
00004 // Copyright (c) 1998-2002 Pascal Vincent
00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal
00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 /* *******************************************************      
00038  * $Id: DistRepNNet.h 3994 2005-08-25 13:35:03Z chapados $
00039  ******************************************************* */
00040 
00041 
00042 #ifndef DistRepNNet_INC
00043 #define DistRepNNet_INC
00044 
00045 #include "PLearner.h"
00046 #include <plearn/opt/Optimizer.h>
00047 
00048 namespace PLearn {
00049 using namespace std;
00050 
00051 class DistRepNNet: public PLearner
00052 {
00053 
00054 private:
00055 
00056     typedef PLearner inherited;
00057     
00059     mutable Vec target_values;
00061     mutable Vec output_comp;
00063     mutable Vec row;
00065     mutable Vec tf;
00066     mutable TVec<string> options;
00067 
00068 protected:
00069 
00071     Var output;
00073     VarArray costs;
00075     VarArray partial_update_vars;
00077     VarArray penalties;
00080     Var training_cost;
00082     VarArray training_cost_extra_tasks;
00084     Var test_costs; 
00086     VarArray invars;
00088     TVec< VarArray > invars_extra_tasks;
00090     VarArray params; 
00093     //VarArray activated_weights;
00095     TVec<int> input_to_dict_index;
00097     int target_dict_index;
00099     real winputsparse_weight_decay;
00100     real winputsparse_bias_decay;
00101 
00102 public: 
00105     Vec paramsvalues; // values of all parameters
00107     Var input;  
00109     Var dp_input;  
00111     Var target; 
00113     Var sampleweight; 
00115     Var w1;
00117     VarArray winputsparse;
00119     //Var w1target;
00121     Var w1theta;  
00123     VarArray winputdistrep;
00125     Var woutdistrep;
00127     Var w2; 
00129     Var wout;
00131     Var direct_wout;
00133     //Var wouttarget; 
00135     Var wouttheta; 
00137     Var outbias;
00139     Var dp_all_targets;
00141     Var token_features;
00143     Var dist_rep;
00145     //VarArray dist_reps;    
00147     TVec< PP<Dictionary> > dictionaries;
00149     //TVec<int> seen_target;
00151     //oTVec<int> unseen_target;
00153     mutable Func f;
00155     mutable Func test_costf; 
00157     mutable Func token_to_dist_rep;
00159     mutable Func paramf;
00160 
00161 public:
00162 
00163     // Build options:
00164 
00166     TVec<VMat> extra_tasks;
00168     int nhidden;
00170     int nhidden2; 
00172     TVec<int> nhidden_extra_tasks;
00174     TVec<int> nhidden2_extra_tasks; 
00176     int nhidden_theta_predictor; 
00178     int nhidden_dist_rep_predictor; 
00180     real weight_decay; 
00182     real bias_decay; 
00184     real input_dist_rep_predictor_bias_decay;
00186     real output_dist_rep_predictor_bias_decay;
00188     real input_dist_rep_predictor_weight_decay;
00190     real output_dist_rep_predictor_weight_decay;
00192     real layer1_weight_decay; 
00194     real layer1_bias_decay;   
00196     real layer1_theta_predictor_weight_decay; 
00198     real layer1_theta_predictor_bias_decay;   
00200     real layer2_weight_decay; 
00202     real layer2_bias_decay;   
00204     real output_layer_weight_decay; 
00206     real output_layer_bias_decay;
00208     real output_layer_theta_predictor_weight_decay; 
00210     real output_layer_theta_predictor_bias_decay;
00212     real direct_in_to_out_weight_decay;
00214     real direct_in_to_out_bias_decay;
00216     real margin; 
00218     bool fixed_output_weights;
00220     bool direct_in_to_out;
00222     string penalty_type; 
00224     string output_transfer_func; 
00226     string hidden_transfer_func; // tanh, sigmoid, softplus, softmax, etc...  (default: "tanh" means no transfer function)
00228     bool do_not_change_params;
00230     TVec<string> cost_funcs;  
00232     PP<Optimizer> optimizer; 
00234     TVec< PP<Optimizer> > optimizer_extra_tasks;
00237     int batch_size; 
00239     string initialization_method;
00241     TVec<int> dist_rep_dim;
00243     int ntokens;
00246     TVec<int> ntokens_extra_tasks;
00248     int nfeatures_per_token;
00251     TVec<int> nfeatures_for_each_token;
00253     PP<Dictionary> target_dictionary;
00255     Mat target_dist_rep;
00257     bool use_dist_reps;
00259     bool use_output_weights_bases;
00262     //bool possible_targets_varies;
00265     bool use_extra_tasks_only_on_first_epoch;
00268     bool initialize_sparse_params_to_zero;
00270     //string nnet_architecture;
00271 
00272 private:
00273     void build_();
00274 
00275 public:
00276 
00277     DistRepNNet();
00278     virtual ~DistRepNNet();
00279     PLEARN_DECLARE_OBJECT(DistRepNNet);
00280 
00281     virtual void build();
00282     virtual void forget(); // simply calls initializeParams()
00283 
00284     virtual int outputsize() const;
00285     virtual TVec<string> getTrainCostNames() const;
00286     virtual TVec<string> getTestCostNames() const;
00287 
00288     virtual void train();
00289 
00290     virtual void computeOutput(const Vec& input, Vec& output) const;
00291 
00292     virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00293                                        Vec& output, Vec& costs) const;
00294 
00295     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 
00296                                          const Vec& target, Vec& costs) const;
00297 
00298     virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies);
00299 
00304     void getTokenDistRep(TVec<string>& token_features, Vec& dist_rep);
00305 
00307     //virtual Mat getW1() {return w1->matValue;}
00308     //virtual Mat getW2() {return w2->matValue;}
00309     //virtual Mat getWout() {return wout->matValue;}
00310 
00311 protected:
00312     static void declareOptions(OptionList& ol);
00313 
00315     Var buildSparseAffineTransform(VarArray weights, Var input, TVec<int> input_to_dict_index, int begin);
00316 
00318     Var buildSparseAffineTransformWeightPenalty(VarArray weights, Var input, TVec<int> input_to_dict_index, int begin, real weight_decay, real bias_decay=0, string penalty_type="L2_square");
00319 
00321     void buildVarGraph(int task_index);
00322 
00328     virtual void initializeParams(bool set_seed = true, int task_index = -1);
00329 
00332     Var add_transfer_func(const Var& input, string transfer_func = "default", VarArray mus=0, Var sigma=0);
00333 
00335     void buildOutputFromInput(int task_index);
00336 
00338     void buildCosts(const Var& output, const Var& target, int task_index = -1);
00339 
00341     void buildFuncs(VarArray& invars);
00342 
00346     void fillWeights(const Var& weights, bool clear_first_row, int use_this_to_scale=-1);
00347 
00349     virtual void buildPenalties(int this_ntokens);
00350 
00351 };
00352 
00353 DECLARE_OBJECT_PTR(DistRepNNet);
00354 
00355 } // end of namespace PLearn
00356 
00357 #endif
00358 
00359 
00360 /*
00361   Local Variables:
00362   mode:c++
00363   c-basic-offset:4
00364   c-file-style:"stroustrup"
00365   c-file-offsets:((innamespace . 0)(inline-open . 0))
00366   indent-tabs-mode:nil
00367   fill-column:79
00368   End:
00369 */
00370 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines