PLearn 0.1
NnlmOnlineLearner.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // NnlmOnlineLearner.h
00004 //
00005 // Copyright (C) 2006 Pierre-Antoine Manzagol
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pierre-Antoine Manzagol
00036 
00041 #ifndef NnlmOnlineLearner_INC
00042 #define NnlmOnlineLearner_INC
00043 
00044 #include <plearn_learners/generic/PLearner.h>
00045 
00046 namespace PLearn {
00047 
00048 class OnlineLearningModule;
00049 class NnlmOutputLayer;
00050 class NGramDistribution;
00051 
00061 class NnlmOnlineLearner : public PLearner
00062 {
00063     typedef PLearner inherited;
00064 
00065 public:
00066     //#####  Public Build Options  ############################################
00067 
00069     string str_input_model;    // 'wrl' (default) or 'gnnl' (a gradnnetlayer)
00070     string str_output_model;    // 'gaussian' (default) or 'softmax' (a gradnnetlayer + softmax)
00071 
00072 
00074 
00076     int word_representation_size;
00077 
00079     int semantic_layer_size;
00080 
00082     real wrl_slr;
00083     real wrl_dc;
00084     real wrl_wd_l1;
00085     real wrl_wd_l2;
00086     real sl_slr;
00087     real sl_dc;
00088     real sl_wd_l1;
00089     real sl_wd_l2;
00090 
00092 
00094     string str_gaussian_model_train_cost;
00095     string str_gaussian_model_learning;
00096     real gaussian_model_sigma2_min;
00097     real gaussian_model_dl_slr;
00098     real gaussian_model_dl_dc;
00099 
00102     // What to do if multiple sources suggest the same word? Do we have less candidates? Or compensate?
00103     int shared_candidates_size;    // frequent words on which we don't want to make mistakes
00104     int ngram_candidates_size;     // from a bigram
00105     int self_candidates_size;      // from the model itself. Should be used after training has gone a while
00106                                    // to keep ourselves on track (some words getting too high a score?)
00107 
00110     VMat ngram_train_set;
00111 
00113 
00114     real sm_slr;
00115     real sm_dc;
00116     real sm_wd_l1;
00117     real sm_wd_l2;
00118 
00119 
00120     //#####  Public Learnt Options  ############################################
00121 
00125     TVec< PP<OnlineLearningModule> > modules;
00126     TVec< PP<OnlineLearningModule> > output_modules;
00127 
00128     //#####  Public NOT Options  ##############################################
00129 
00131     int vocabulary_size;
00132     int context_size;       // the train_set's input size -1 (because target is last input)
00133 
00135 
00136     // TODO THIS COULD BE A LEARNT OPTION
00139     PP<NGramDistribution> theNGram;
00140 
00141     // TODO THIS COULD BE A LEARNT OPTION
00143     TVec<int> shared_candidates;    // frequent (ie paying) words
00144     TVec< TVec<int> > candidates;   // context specific candidates
00145 
00146 
00147 public:
00148     //#####  Public Member Functions  #########################################
00149 
00151     NnlmOnlineLearner();
00152 
00153 
00154     //#####  PLearner Member Functions  #######################################
00155 
00157     void buildLayers();
00158 
00160     void buildCandidates();
00161     void reevaluateGaussianParameters() const;
00162     //void evaluateGaussianCounts() const;
00163 
00165     void myGetExample(const VMat& example_set, int& sample, Vec& input, Vec& target, real& weight) const;
00166 
00169     virtual int outputsize() const;
00170 
00174     virtual void forget();
00175 
00179     virtual void train();
00180 
00181     void test(VMat testset, PP<VecStatsCollector> test_stats, VMat testoutputs, VMat testcosts) const;
00182 
00184     virtual void computeOutput(const Vec& input, Vec& output) const;
00185 
00187     virtual void computeCostsFromOutputs(const Vec& input, const Vec& output,
00188                                          const Vec& target, Vec& costs) const;
00189 
00190     virtual void computeTrainCostsFromOutputs(const Vec& input, const Vec& output,
00191                                            const Vec& target, Vec& costs) const;
00192 
00195     virtual TVec<std::string> getTestCostNames() const;
00196 
00199     virtual TVec<std::string> getTrainCostNames() const;
00200 
00201 
00202     // *** SUBCLASS WRITING: ***
00203     // While in general not necessary, in case of particular needs
00204     // (efficiency concerns for ex) you may also want to overload
00205     // some of the following methods:
00206     // virtual void computeOutputAndCosts(const Vec& input, const Vec& target,
00207     //                                    Vec& output, Vec& costs) const;
00208     // virtual void computeCostsOnly(const Vec& input, const Vec& target,
00209     //                               Vec& costs) const;
00210     // virtual void test(VMat testset, PP<VecStatsCollector> test_stats,
00211     //                   VMat testoutputs=0, VMat testcosts=0) const;
00212     // virtual int nTestCosts() const;
00213     // virtual int nTrainCosts() const;
00214     // virtual void resetInternalState();
00215     // virtual bool isStatefulLearner() const;
00216 
00217 
00218     //#####  PLearn::Object Protocol  #########################################
00219 
00220     // Declares other standard object methods.
00221     PLEARN_DECLARE_OBJECT(NnlmOnlineLearner);
00222 
00223     // Simply calls inherited::build() then build_()
00224     virtual void build();
00225 
00227     virtual void makeDeepCopyFromShallowCopy(CopiesMap& copies);
00228 
00229 protected:
00230     //#####  Protected Options  ###############################################
00231 
00232     // ### Declare protected option fields (such as learned parameters) here
00233 
00234 
00235     //####  Not Options  ######################################################
00236 
00238     TVec<Vec> values;
00240     TVec<Vec> gradients;
00241 
00244     TVec<Vec> output_values;
00245     TVec<Vec> output_gradients;
00246 
00247 
00248 protected:
00249     //#####  Protected Member Functions  ######################################
00250 
00252     static void declareOptions(OptionList& ol);
00253 
00254 private:
00255     //#####  Private Member Functions  ########################################
00256 
00258     void build_();
00259 
00260 private:
00261     //#####  Private Data Members  ############################################
00262 
00264     int nmodules;
00265     int output_nmodules;
00266 
00268     int model_type;
00269 
00271     int gaussian_model_cost;
00272     int gaussian_model_learning;
00273 
00274     enum{MODEL_TYPE_GAUSSIAN=0, MODEL_TYPE_SOFTMAX=1};
00275     enum{GAUSSIAN_COST_DISCR=0, GAUSSIAN_COST_APPROX_DISCR=1, GAUSSIAN_COST_NON_DISCR=2};
00276     enum{GAUSSIAN_LEARNING_DISCR=0, GAUSSIAN_LEARNING_EMPIRICAL=1};
00277 };
00278 
00279 // Declares a few other classes and functions related to this class
00280 DECLARE_OBJECT_PTR(NnlmOnlineLearner);
00281 
00282 } // end of namespace PLearn
00283 
00284 #endif
00285 
00286 
00287 /*
00288   Local Variables:
00289   mode:c++
00290   c-basic-offset:4
00291   c-file-style:"stroustrup"
00292   c-file-offsets:((innamespace . 0)(inline-open . 0))
00293   indent-tabs-mode:nil
00294   fill-column:79
00295   End:
00296 */
00297 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines