PLearn 0.1
RBMTrainer.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // RBMTrainer.cc
00004 //
00005 // Copyright (C) 2007 Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Pascal Lamblin
00036 
00040 #include "RBMTrainer.h"
00041 
00042 #include <plearn_learners/online/RBMBinomialLayer.h>
00043 #include <plearn_learners/online/RBMGaussianLayer.h>
00044 #include <plearn_learners/online/RBMMatrixConnection.h>
00045 #include <plearn/vmat/AutoVMatrix.h>
00046 #include <plearn/vmat/SubVMatrix.h>
00047 #include <plearn/io/openFile.h>
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 PLEARN_IMPLEMENT_OBJECT(
00053     RBMTrainer,
00054     "Trains an RBM",
00055     "Glop"
00056     );
00057 
00058 RBMTrainer::RBMTrainer():
00059     n_visible(-1),
00060     n_hidden(-1),
00061     visible_type("binomial"),
00062     update_with_h0_sample(false),
00063     sample_v1_in_chain(true),
00064     compute_log_likelihood(true),
00065     n_stages(1),
00066     learning_rate(0.01),
00067     seed(1827),
00068     n_train(-1),
00069     n_valid(-1),
00070     n_test(-1),
00071     batch_size(1),
00072     print_debug(false),
00073     use_fast_approximations(false),
00074     n_ports(0),
00075     n_state_ports(0),
00076     nll_index(-1),
00077     visible_index(-1),
00078     rec_err_index(-1)
00079 {
00080 }
00081 
00082 // ### Nothing to add here, simply calls build_
00083 void RBMTrainer::build()
00084 {
00085     inherited::build();
00086     build_();
00087 }
00088 
00089 void RBMTrainer::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00090 {
00091     inherited::makeDeepCopyFromShallowCopy(copies);
00092 
00093     // deepCopyField(trainvec, copies);
00094 }
00095 
00096 void RBMTrainer::declareOptions(OptionList& ol)
00097 {
00098     declareOption(ol, "n_visible", &RBMTrainer::n_visible,
00099                   OptionBase::buildoption,
00100                   "n_visible");
00101 
00102     declareOption(ol, "n_hidden", &RBMTrainer::n_hidden,
00103                   OptionBase::buildoption,
00104                   "n_hidden");
00105 
00106     declareOption(ol, "visible_type", &RBMTrainer::visible_type,
00107                   OptionBase::buildoption,
00108                   "visible_type");
00109 
00110     declareOption(ol, "update_with_h0_sample",
00111                   &RBMTrainer::update_with_h0_sample,
00112                   OptionBase::buildoption,
00113                   "update_with_h0_sample");
00114 
00115     declareOption(ol, "sample_v1_in_chain", &RBMTrainer::sample_v1_in_chain,
00116                   OptionBase::buildoption,
00117                   "sample_v1_in_chain");
00118 
00119     declareOption(ol, "compute_log_likelihood",
00120                   &RBMTrainer::compute_log_likelihood,
00121                   OptionBase::buildoption,
00122                   "compute_log_likelihood");
00123 
00124     declareOption(ol, "n_stages", &RBMTrainer::n_stages,
00125                   OptionBase::buildoption,
00126                   "n_stages");
00127 
00128     declareOption(ol, "learning_rate", &RBMTrainer::learning_rate,
00129                   OptionBase::buildoption,
00130                   "learning_rate");
00131 
00132     declareOption(ol, "seed", &RBMTrainer::seed,
00133                   OptionBase::buildoption,
00134                   "seed");
00135 
00136     declareOption(ol, "n_train", &RBMTrainer::n_train,
00137                   OptionBase::buildoption,
00138                   "n_train");
00139 
00140     declareOption(ol, "n_valid", &RBMTrainer::n_valid,
00141                   OptionBase::buildoption,
00142                   "n_valid");
00143 
00144     declareOption(ol, "n_test", &RBMTrainer::n_test,
00145                   OptionBase::buildoption,
00146                   "n_test");
00147 
00148     declareOption(ol, "batch_size", &RBMTrainer::batch_size,
00149                   OptionBase::buildoption,
00150                   "batch_size");
00151 
00152     declareOption(ol, "data_filename", &RBMTrainer::data_filename,
00153                   OptionBase::buildoption,
00154                   "data_filename");
00155 
00156     declareOption(ol, "save_path", &RBMTrainer::save_path,
00157                   OptionBase::buildoption,
00158                   "save_path");
00159 
00160     declareOption(ol, "save_name", &RBMTrainer::save_name,
00161                   OptionBase::buildoption,
00162                   "save_name");
00163 
00164     declareOption(ol, "print_debug", &RBMTrainer::print_debug,
00165                   OptionBase::buildoption,
00166                   "print_debug");
00167 
00168     declareOption(ol, "use_fast_approximations",
00169                   &RBMTrainer::use_fast_approximations,
00170                   OptionBase::buildoption,
00171                   "use_fast_approximations");
00172 
00173     declareOption(ol, "data", &RBMTrainer::data,
00174                   OptionBase::learntoption,
00175                   "data");
00176 
00177     declareOption(ol, "train_input", &RBMTrainer::train_input,
00178                   OptionBase::learntoption,
00179                   "train_input");
00180 
00181     declareOption(ol, "valid_input", &RBMTrainer::valid_input,
00182                   OptionBase::learntoption,
00183                   "valid_input");
00184 
00185     declareOption(ol, "test_input", &RBMTrainer::test_input,
00186                   OptionBase::learntoption,
00187                   "test_input");
00188 
00189     declareOption(ol, "rbm", &RBMTrainer::rbm,
00190                   OptionBase::learntoption,
00191                   "rbm");
00192 
00193     declareOption(ol, "visible", &RBMTrainer::visible,
00194                   OptionBase::learntoption,
00195                   "visible");
00196 
00197     declareOption(ol, "hidden", &RBMTrainer::hidden,
00198                   OptionBase::learntoption,
00199                   "hidden");
00200 
00201     declareOption(ol, "connection", &RBMTrainer::connection,
00202                   OptionBase::learntoption,
00203                   "connection");
00204 
00205     /*
00206     declareOption(ol, "", &RBMTrainer::,
00207                   OptionBase::learntoption,
00208                   "");
00209     */
00210 
00211     // Now call the parent class' declareOptions
00212     inherited::declareOptions(ol);
00213 }
00214 
00215 void RBMTrainer::declareMethods(RemoteMethodMap& rmm)
00216 {
00217     // Make sure that inherited methods are declared
00218     rmm.inherited(inherited::_getRemoteMethodMap_());
00219 
00220     declareMethod(rmm, "NLL", &RBMTrainer::NLL,
00221                   (BodyDoc("Computes NLL"),
00222                    ArgDoc ("examples", "The examples"),
00223                    RetDoc ("The NLL")
00224                   ));
00225 
00226     declareMethod(rmm, "recError", &RBMTrainer::recError,
00227                   (BodyDoc("Computes reconstruction error"),
00228                    ArgDoc ("examples", "The examples"),
00229                    RetDoc ("The reconstruction error")
00230                   ));
00231 
00232     declareMethod(rmm, "CD1", &RBMTrainer::CD1,
00233                   (BodyDoc("Performs one step of CD"),
00234                    ArgDoc ("examples", "The examples")));
00235 }
00236 
00237 void RBMTrainer::build_()
00238 {
00239     visible_type = lowerstring(visible_type);
00240     PLCHECK( visible_type == "binomial" || visible_type == "gaussian" );
00241 
00242     // visible
00243     if (visible_type == "binomial")
00244         visible = new RBMBinomialLayer();
00245     else if (visible_type == "gaussian")
00246         visible = new RBMGaussianLayer();
00247     else
00248         PLERROR("Unknown visible_type (%s).", visible_type.c_str());
00249 
00250     visible->size = n_visible;
00251     visible->setLearningRate(learning_rate);
00252     visible->use_fast_approximations = use_fast_approximations;
00253     visible->build();
00254 
00255     // hidden
00256     hidden = new RBMBinomialLayer();
00257     hidden->size = n_hidden;
00258     hidden->setLearningRate(learning_rate);
00259     hidden->use_fast_approximations = use_fast_approximations;
00260     hidden->build();
00261 
00262     // connection
00263     connection = new RBMMatrixConnection();
00264     connection->down_size = n_visible;
00265     connection->up_size = n_hidden;
00266     connection->setLearningRate(learning_rate);
00267     connection->use_fast_approximations = use_fast_approximations;
00268     connection->build();
00269 
00270     // RBM
00271     rbm = new RBMModule();
00272     rbm->visible_layer = visible;
00273     rbm->hidden_layer = hidden;
00274     rbm->connection = connection;
00275     rbm->reconstruction_connection = connection;
00276     rbm->compute_log_likelihood = compute_log_likelihood;
00277     rbm->random_gen = new PRandom(seed);
00278     rbm->cd_learning_rate = learning_rate;
00279     rbm->use_fast_approximations = use_fast_approximations;
00280     rbm->build();
00281 
00282     // data
00283     if (!data_filename.isEmpty())
00284     {
00285         PP<AutoVMatrix> data_ = new AutoVMatrix();
00286         data_->filename = data_filename;
00287         data_->defineSizes(n_visible, 1);
00288         data_->build();
00289         data = get_pointer(data_);
00290 
00291         // train_input
00292         train_input = new SubVMatrix(data,      // source
00293                                      0,         // istart
00294                                      0,         // jstart
00295                                      n_train,   // length
00296                                      n_visible, // width
00297                                      true       // call_build
00298                                      );
00299 
00300         // valid_input
00301         valid_input = new SubVMatrix(data,
00302                                      n_train,
00303                                      0,
00304                                      n_valid,
00305                                      n_visible,
00306                                      true
00307                                      );
00308 
00309         // valid_input
00310         test_input = new SubVMatrix(data,
00311                                     n_train + n_valid,
00312                                     0,
00313                                     n_test,
00314                                     n_visible,
00315                                     true
00316                                     );
00317     }
00318 
00319     // ports
00320     ports = rbm->getPorts();
00321     n_ports = ports.length();
00322     for (int i=0; i<n_ports; i++)
00323     {
00324         if (ports[i].find(".state", 0) != string::npos)
00325         {
00326             state_ports.append(ports[i]);
00327             state_ports_indices.append(i);
00328         }
00329     }
00330 
00331     n_state_ports = state_ports.length();
00332     nll_index = rbm->getPortIndex("neg_log_likelihood");
00333     visible_index = rbm->getPortIndex("visible");
00334     rec_err_index = rbm->getPortIndex("reconstruction_error.state");
00335 
00336     // nll_values
00337     nll_values.resize(n_ports);
00338     for (int i=0; i<n_state_ports; i++)
00339         nll_values[state_ports_indices[i]] = new Mat();
00340 
00341     // rec_err_values
00342     rec_err_values.resize(n_ports);
00343     for (int i=0; i<n_state_ports; i++)
00344         rec_err_values[state_ports_indices[i]] = new Mat();
00345 }
00346 
00347 Mat RBMTrainer::NLL(const Mat& examples)
00348 {
00349     Mat nll;
00350 
00351     for (int i=0; i<n_state_ports; i++)
00352         nll_values[state_ports_indices[i]]->resize(0,0);
00353 
00354     nll_values[nll_index] = &nll;
00355     nll_values[visible_index] = const_cast<Mat *>(&examples);
00356 
00357     rbm->fprop(nll_values);
00358 
00359     if (print_debug)
00360     {
00361         pout << "In NLL:" << endl;
00362         for (int i=0; i<n_state_ports; i++)
00363         {
00364             int portnum = state_ports_indices[i];
00365             string portname = rbm->getPortName(portnum);
00366             pout << portname << ": " << nll_values[portnum]->size() << endl;
00367         }
00368     }
00369 
00370     PLASSERT(nll.length() == examples.length() && nll.width() == 1);
00371     return nll;
00372 }
00373 
00374 Mat RBMTrainer::recError(const Mat& examples)
00375 {
00376     Mat rec_err;
00377     for (int i=0; i<n_state_ports; i++)
00378         rec_err_values[state_ports_indices[i]]->resize(0,0);
00379 
00380     rec_err_values[rec_err_index] = &rec_err;
00381     rec_err_values[visible_index] = const_cast<Mat *>(&examples);
00382 
00383     rbm->fprop(rec_err_values);
00384 
00385     if (print_debug)
00386     {
00387         pout << "In recError:" << endl;
00388         for (int i=0; i<n_state_ports; i++)
00389         {
00390             int portnum = state_ports_indices[i];
00391             string portname = rbm->getPortName(portnum);
00392             pout << portname << ": " << rec_err_values[portnum]->size() << endl;
00393         }
00394     }
00395 
00396     PLASSERT(rec_err.length() == examples.length()
00397              && rec_err.width() == 1);
00398     return rec_err;
00399 }
00400 
00401 void RBMTrainer::CD1(const Mat& examples)
00402 {
00403     int n_examples = examples.length();
00404     if (print_debug)
00405     {
00406         pout << "v0 = " << endl << examples << endl;
00407     }
00408 
00409     // Positive phase
00410     connection->setAsDownInputs(examples);
00411     hidden->getAllActivations(connection, 0, true);
00412     hidden->computeExpectations();
00413     hidden->generateSamples();
00414 
00415     h0_a = hidden->activations;
00416     h0_s.resize(n_examples, n_hidden);
00417     h0_s << hidden->samples.copy();
00418 
00419     if (update_with_h0_sample)
00420     {
00421         h0 = h0_s;
00422     }
00423     else
00424     {
00425         h0_e.resize(n_examples, n_hidden);
00426         h0_e << hidden->getExpectations();
00427         h0 = h0_e;
00428     }
00429 
00430     if (print_debug)
00431     {
00432         pout << "h0_a = " << endl << h0_a << endl;
00433         pout << "h0_e = " << endl << hidden->getExpectations() << endl;
00434         pout << "h0_s = " << endl << h0_s << endl;
00435     }
00436 
00437     // Downward pass
00438     connection->setAsUpInputs(h0_s);
00439     visible->getAllActivations(connection, 0, true);
00440     visible->computeExpectations();
00441     visible->generateSamples();
00442 
00443     v1_a = visible->activations;
00444     v1_e = visible->getExpectations();
00445     v1_s = visible->samples;
00446 
00447     // Negative phase
00448     if (sample_v1_in_chain)
00449         v1 = v1_s;
00450     else
00451         v1 = v1_e;
00452 
00453     if (print_debug)
00454     {
00455         pout << "v1_a = " << endl << v1_a << endl;
00456         pout << "v1_e = " << endl << v1_e << endl;
00457         pout << "v1_s = " << endl << v1_s << endl;
00458     }
00459 
00460     connection->setAsDownInputs(v1);
00461     hidden->getAllActivations(connection, 0, true);
00462     hidden->computeExpectations();
00463 
00464     Mat h1 = hidden->getExpectations();
00465     if (print_debug)
00466         pout << "h1 = " << endl << h1 << endl;
00467 
00468     rbm->CDUpdate(examples, h0, v1, h1);
00469 }
00470 
00471 void RBMTrainer::run()
00472 {
00473     Mat results(n_stages+1, 6);
00474     for (int stage=0; stage<n_stages; stage++)
00475     {
00476         pout << "stage: " << stage << endl;
00477         results(stage, 0) = mean(NLL(train_input));
00478         results(stage, 3) = mean(recError(train_input));
00479 
00480         if (n_valid > 0)
00481         {
00482             results(stage, 1) = mean(NLL(valid_input));
00483             results(stage, 4) = mean(recError(valid_input));
00484         }
00485 
00486         if (n_test > 0)
00487         {
00488             results(stage, 2) = mean(NLL(test_input));
00489             results(stage, 5) = mean(recError(test_input));
00490         }
00491 
00492         pout << "NLL:      " << results(stage).subVec(0,3) << endl;
00493         pout << "RecError: " << results(stage).subVec(3,3) << endl;
00494 
00495         for (int i=0; i<n_train/batch_size; i++)
00496             CD1(train_input.subMatRows(i*batch_size, batch_size));
00497     }
00498     pout << "stage: " << n_stages << endl;
00499     results(n_stages, 0) = mean(NLL(train_input));
00500     results(n_stages, 3) = mean(recError(train_input));
00501     if (n_valid > 0)
00502     {
00503         results(n_stages, 1) = mean(NLL(valid_input));
00504         results(n_stages, 4) = mean(recError(valid_input));
00505     }
00506     if (n_test > 0)
00507     {
00508         results(n_stages, 2) = mean(NLL(test_input));
00509         results(n_stages, 5) = mean(recError(test_input));
00510     }
00511     pout << "NLL:      " << results(n_stages).subVec(0,3) << endl;
00512     pout << "RecError: " << results(n_stages).subVec(3,3) << endl;
00513     pout << "results = " << endl << results << endl;
00514 
00515     if (!save_path.isEmpty())
00516     {
00517         PPath filename;
00518         if (save_name.isEmpty())
00519             filename = save_path / "results.amat";
00520         else
00521             filename = save_path / save_name;
00522 
00523         PStream file = openFile(filename, PStream::raw_ascii, "w");
00524         file << results << endl;
00525     }
00526 }
00527 
00528 
00529 } // end of namespace PLearn
00530 
00531 
00532 /*
00533   Local Variables:
00534   mode:c++
00535   c-basic-offset:4
00536   c-file-style:"stroustrup"
00537   c-file-offsets:((innamespace . 0)(inline-open . 0))
00538   indent-tabs-mode:nil
00539   fill-column:79
00540   End:
00541 */
00542 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines