PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // NNet.h 00004 // Copyright (c) 1998-2002 Pascal Vincent 00005 // Copyright (C) 1999-2002 Yoshua Bengio and University of Montreal 00006 // Copyright (c) 2002 Jean-Sebastien Senecal, Xavier Saint-Mleux, Rejean Ducharme 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: NNet.h 9176 2008-06-26 17:49:35Z tihocan $ 00039 ******************************************************* */ 00040 00041 00042 #ifndef NNet_INC 00043 #define NNet_INC 00044 00045 #include "PLearner.h" 00046 #include <plearn/opt/Optimizer.h> 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00051 class NNet: public PLearner 00052 { 00053 00054 private: 00055 00056 typedef PLearner inherited; 00057 00058 protected: 00059 00060 Var rbf_centers; // n_classes (or n_classes-1) x rbf_layer_size = mu_i of RBF gaussians 00061 Var rbf_sigmas; // n_classes (or n_classes-1) entries (sigma's of the RBFs) 00062 Var junk_prob; // scalar background (junk) probability, if first_class_is_junk 00063 Var alpha_adaboost; 00064 Var output; 00065 Var predicted_input; 00066 VarArray costs; // all costs of interest 00067 VarArray penalties; 00068 Var training_cost; // weighted scalar costs[0] including penalties 00069 Var test_costs; // hconcat(costs) 00070 VarArray invars; 00071 VarArray params; // all arameter input vars 00072 00074 Var bag_inputs; 00075 00077 Mat store_bag_inputs; 00078 00080 Var bag_size; 00081 00083 Vec store_bag_size; 00084 00086 int n_training_bags; 00087 00088 // to put back later -- blip Vec paramsvalues; // values of all parameters 00089 00090 public: // to set these values instead of getting them by training 00091 Vec paramsvalues; // values of all parameters 00092 Var input; // Var(inputsize()) 00093 Var target; // Var(targetsize()-weightsize()) 00094 Var sampleweight; // Var(1) if train_set->hasWeights() 00095 00096 Var w1; // bias and weights of first hidden layer 00097 Var w2; // bias and weights of second hidden layer 00098 Var wout; // bias and weights of output layer 00099 Var outbias; // bias used only if fixed_output_weights 00100 Var wdirect; // bias and weights for direct in-to-out connection 00101 Var wrec; // input reconstruction weights (optional), from hidden layer to predicted input 00102 00105 VarArray v1, v2; 00106 00107 // first hidden layer 00108 Var hidden_layer; 00109 00110 public: 00111 00112 mutable Func input_to_output; // input -> output 00113 mutable Func test_costf; // input & target -> output & test_costs 00114 mutable Func output_and_target_to_cost; // output & target -> cost 00115 00116 public: 00117 00118 // Build options inherited from learner: 00119 // inputsize, outputszie, targetsize, experiment_name, save_at_every_epoch 00120 00121 00122 //##### Public Build Options ############################################ 00123 00125 int nhidden; 00126 00128 int nhidden2; 00129 00145 int noutputs; 00146 00147 bool operate_on_bags; 00148 int max_bag_size; 00149 00150 real weight_decay; // default: 0 00151 real bias_decay; // default: 0 00152 real layer1_weight_decay; // default: MISSING_VALUE 00153 real layer1_bias_decay; // default: MISSING_VALUE 00154 real layer2_weight_decay; // default: MISSING_VALUE 00155 real layer2_bias_decay; // default: MISSING_VALUE 00156 real output_layer_weight_decay; // default: MISSING_VALUE 00157 real output_layer_bias_decay; // default: MISSING_VALUE 00158 real direct_in_to_out_weight_decay; // default: MISSING_VALUE 00159 real classification_regularizer; // default: 0 00160 real margin; // default: 1, used with margin_perceptron_cost 00161 bool fixed_output_weights; 00162 00163 int rbf_layer_size; // number of representation units when adding an rbf layer in output 00164 bool first_class_is_junk; 00165 00166 string penalty_type; // default: "L2_square" 00167 bool L1_penalty; // default: false - deprecated, set "penalty_type" to "L1" 00168 00169 real input_reconstruction_penalty; // default = 0 00170 bool direct_in_to_out; // should we include direct input to output connecitons? default: false 00171 string output_transfer_func; // tanh, sigmoid, softplus, softmax, etc... (default: "" means no transfer function) 00172 string hidden_transfer_func; // tanh, sigmoid, softplus, softmax, etc... (default: "tanh" means no transfer function) 00173 real interval_minval, interval_maxval; // if output_transfer_func = interval(minval,maxval), these are the interval bounds 00174 00175 bool do_not_change_params; 00176 00177 Var first_hidden_layer; 00178 bool first_hidden_layer_is_output; 00179 bool transpose_first_hidden_layer; 00180 int n_non_params_in_first_hidden_layer; 00181 00183 TVec<string> cost_funcs; 00184 00185 // Build options related to the optimization: 00186 PP<Optimizer> optimizer; // the optimizer to use (no default) 00187 00188 int batch_size; // how many samples to use to estimate gradient before an update 00189 // 0 means the whole training set (default: 1) 00190 00191 string initialization_method; 00192 int ratio_rank; 00193 00194 00195 private: 00196 void build_(); 00197 00198 public: 00199 00200 NNet(); 00201 PLEARN_DECLARE_OBJECT(NNet); 00202 00203 virtual void build(); 00204 virtual void forget(); // simply calls initializeParams() 00205 00212 virtual void setTrainingSet(VMat training_set, bool call_forget=true); 00213 00214 virtual int outputsize() const; 00215 virtual TVec<string> getTrainCostNames() const; 00216 virtual TVec<string> getTestCostNames() const; 00217 00218 virtual void train(); 00219 00220 virtual void computeOutput(const Vec& input, Vec& output) const; 00221 00222 virtual void computeOutputAndCosts(const Vec& input, const Vec& target, 00223 Vec& output, Vec& costs) const; 00224 00225 virtual void computeCostsFromOutputs(const Vec& input, const Vec& output, 00226 const Vec& target, Vec& costs) const; 00227 00228 virtual void makeDeepCopyFromShallowCopy(CopiesMap &copies); 00229 00231 virtual Mat getW1() {return w1->matValue;} 00232 virtual Mat getW2() {return w2->matValue;} 00233 virtual Mat getWdirect() {return wdirect->matValue;} 00234 virtual Mat getWout() {return wout->matValue;} 00235 00236 protected: 00237 static void declareOptions(OptionList& ol); 00238 00242 virtual void initializeParams(bool set_seed = true); 00243 00247 Var hiddenLayer(const Var& input, const Var& weights, string transfer_func = "default", 00248 VarArray* ratio_quad_weights = NULL); 00249 00254 void buildOutputFromInput(const Var& the_input, Var& hidden_layer, Var& before_transfer_func); 00255 00260 void buildBagOutputFromBagInputs( 00261 const Var& input, Var& before_transfer_func, 00262 const Var& bag_inputs, const Var& bag_size, Var& bag_output); 00263 00265 void buildTargetAndWeight(); 00266 00268 void buildCosts(const Var& output, const Var& target, const Var& hidden_layer, const Var& before_transfer_func); 00269 00272 virtual Var getCost(const string& costname, const Var& output, 00273 const Var& target, const Var& before_transfer_func); 00274 00276 void buildFuncs(const Var& the_input, const Var& the_output, const Var& the_target, const Var& the_sampleweight, const Var& the_bag_size); 00277 00279 void applyTransferFunc(const Var& before_transfer_func, Var& output); 00280 00284 void fillWeights(const Var& weights, bool clear_first_row); 00285 00287 virtual void buildPenalties(const Var& hidden_layer); 00288 00289 }; 00290 00291 DECLARE_OBJECT_PTR(NNet); 00292 00293 } // end of namespace PLearn 00294 00295 #endif 00296 00297 00298 /* 00299 Local Variables: 00300 mode:c++ 00301 c-basic-offset:4 00302 c-file-style:"stroustrup" 00303 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00304 indent-tabs-mode:nil 00305 fill-column:79 00306 End: 00307 */ 00308 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :