PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // RBMTrainer.cc 00004 // 00005 // Copyright (C) 2007 Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Pascal Lamblin 00036 00040 #include "RBMTrainer.h" 00041 00042 #include <plearn_learners/online/RBMBinomialLayer.h> 00043 #include <plearn_learners/online/RBMGaussianLayer.h> 00044 #include <plearn_learners/online/RBMMatrixConnection.h> 00045 #include <plearn/vmat/AutoVMatrix.h> 00046 #include <plearn/vmat/SubVMatrix.h> 00047 #include <plearn/io/openFile.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 PLEARN_IMPLEMENT_OBJECT( 00053 RBMTrainer, 00054 "Trains an RBM", 00055 "Glop" 00056 ); 00057 00058 RBMTrainer::RBMTrainer(): 00059 n_visible(-1), 00060 n_hidden(-1), 00061 visible_type("binomial"), 00062 update_with_h0_sample(false), 00063 sample_v1_in_chain(true), 00064 compute_log_likelihood(true), 00065 n_stages(1), 00066 learning_rate(0.01), 00067 seed(1827), 00068 n_train(-1), 00069 n_valid(-1), 00070 n_test(-1), 00071 batch_size(1), 00072 print_debug(false), 00073 use_fast_approximations(false), 00074 n_ports(0), 00075 n_state_ports(0), 00076 nll_index(-1), 00077 visible_index(-1), 00078 rec_err_index(-1) 00079 { 00080 } 00081 00082 // ### Nothing to add here, simply calls build_ 00083 void RBMTrainer::build() 00084 { 00085 inherited::build(); 00086 build_(); 00087 } 00088 00089 void RBMTrainer::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00090 { 00091 inherited::makeDeepCopyFromShallowCopy(copies); 00092 00093 // deepCopyField(trainvec, copies); 00094 } 00095 00096 void RBMTrainer::declareOptions(OptionList& ol) 00097 { 00098 declareOption(ol, "n_visible", &RBMTrainer::n_visible, 00099 OptionBase::buildoption, 00100 "n_visible"); 00101 00102 declareOption(ol, "n_hidden", &RBMTrainer::n_hidden, 00103 OptionBase::buildoption, 00104 "n_hidden"); 00105 00106 declareOption(ol, "visible_type", &RBMTrainer::visible_type, 00107 OptionBase::buildoption, 00108 "visible_type"); 00109 00110 declareOption(ol, "update_with_h0_sample", 00111 &RBMTrainer::update_with_h0_sample, 00112 OptionBase::buildoption, 00113 "update_with_h0_sample"); 00114 00115 declareOption(ol, "sample_v1_in_chain", &RBMTrainer::sample_v1_in_chain, 00116 OptionBase::buildoption, 00117 "sample_v1_in_chain"); 00118 00119 declareOption(ol, "compute_log_likelihood", 00120 &RBMTrainer::compute_log_likelihood, 00121 OptionBase::buildoption, 00122 "compute_log_likelihood"); 00123 00124 declareOption(ol, "n_stages", &RBMTrainer::n_stages, 00125 OptionBase::buildoption, 00126 "n_stages"); 00127 00128 declareOption(ol, "learning_rate", &RBMTrainer::learning_rate, 00129 OptionBase::buildoption, 00130 "learning_rate"); 00131 00132 declareOption(ol, "seed", &RBMTrainer::seed, 00133 OptionBase::buildoption, 00134 "seed"); 00135 00136 declareOption(ol, "n_train", &RBMTrainer::n_train, 00137 OptionBase::buildoption, 00138 "n_train"); 00139 00140 declareOption(ol, "n_valid", &RBMTrainer::n_valid, 00141 OptionBase::buildoption, 00142 "n_valid"); 00143 00144 declareOption(ol, "n_test", &RBMTrainer::n_test, 00145 OptionBase::buildoption, 00146 "n_test"); 00147 00148 declareOption(ol, "batch_size", &RBMTrainer::batch_size, 00149 OptionBase::buildoption, 00150 "batch_size"); 00151 00152 declareOption(ol, "data_filename", &RBMTrainer::data_filename, 00153 OptionBase::buildoption, 00154 "data_filename"); 00155 00156 declareOption(ol, "save_path", &RBMTrainer::save_path, 00157 OptionBase::buildoption, 00158 "save_path"); 00159 00160 declareOption(ol, "save_name", &RBMTrainer::save_name, 00161 OptionBase::buildoption, 00162 "save_name"); 00163 00164 declareOption(ol, "print_debug", &RBMTrainer::print_debug, 00165 OptionBase::buildoption, 00166 "print_debug"); 00167 00168 declareOption(ol, "use_fast_approximations", 00169 &RBMTrainer::use_fast_approximations, 00170 OptionBase::buildoption, 00171 "use_fast_approximations"); 00172 00173 declareOption(ol, "data", &RBMTrainer::data, 00174 OptionBase::learntoption, 00175 "data"); 00176 00177 declareOption(ol, "train_input", &RBMTrainer::train_input, 00178 OptionBase::learntoption, 00179 "train_input"); 00180 00181 declareOption(ol, "valid_input", &RBMTrainer::valid_input, 00182 OptionBase::learntoption, 00183 "valid_input"); 00184 00185 declareOption(ol, "test_input", &RBMTrainer::test_input, 00186 OptionBase::learntoption, 00187 "test_input"); 00188 00189 declareOption(ol, "rbm", &RBMTrainer::rbm, 00190 OptionBase::learntoption, 00191 "rbm"); 00192 00193 declareOption(ol, "visible", &RBMTrainer::visible, 00194 OptionBase::learntoption, 00195 "visible"); 00196 00197 declareOption(ol, "hidden", &RBMTrainer::hidden, 00198 OptionBase::learntoption, 00199 "hidden"); 00200 00201 declareOption(ol, "connection", &RBMTrainer::connection, 00202 OptionBase::learntoption, 00203 "connection"); 00204 00205 /* 00206 declareOption(ol, "", &RBMTrainer::, 00207 OptionBase::learntoption, 00208 ""); 00209 */ 00210 00211 // Now call the parent class' declareOptions 00212 inherited::declareOptions(ol); 00213 } 00214 00215 void RBMTrainer::declareMethods(RemoteMethodMap& rmm) 00216 { 00217 // Make sure that inherited methods are declared 00218 rmm.inherited(inherited::_getRemoteMethodMap_()); 00219 00220 declareMethod(rmm, "NLL", &RBMTrainer::NLL, 00221 (BodyDoc("Computes NLL"), 00222 ArgDoc ("examples", "The examples"), 00223 RetDoc ("The NLL") 00224 )); 00225 00226 declareMethod(rmm, "recError", &RBMTrainer::recError, 00227 (BodyDoc("Computes reconstruction error"), 00228 ArgDoc ("examples", "The examples"), 00229 RetDoc ("The reconstruction error") 00230 )); 00231 00232 declareMethod(rmm, "CD1", &RBMTrainer::CD1, 00233 (BodyDoc("Performs one step of CD"), 00234 ArgDoc ("examples", "The examples"))); 00235 } 00236 00237 void RBMTrainer::build_() 00238 { 00239 visible_type = lowerstring(visible_type); 00240 PLCHECK( visible_type == "binomial" || visible_type == "gaussian" ); 00241 00242 // visible 00243 if (visible_type == "binomial") 00244 visible = new RBMBinomialLayer(); 00245 else if (visible_type == "gaussian") 00246 visible = new RBMGaussianLayer(); 00247 else 00248 PLERROR("Unknown visible_type (%s).", visible_type.c_str()); 00249 00250 visible->size = n_visible; 00251 visible->setLearningRate(learning_rate); 00252 visible->use_fast_approximations = use_fast_approximations; 00253 visible->build(); 00254 00255 // hidden 00256 hidden = new RBMBinomialLayer(); 00257 hidden->size = n_hidden; 00258 hidden->setLearningRate(learning_rate); 00259 hidden->use_fast_approximations = use_fast_approximations; 00260 hidden->build(); 00261 00262 // connection 00263 connection = new RBMMatrixConnection(); 00264 connection->down_size = n_visible; 00265 connection->up_size = n_hidden; 00266 connection->setLearningRate(learning_rate); 00267 connection->use_fast_approximations = use_fast_approximations; 00268 connection->build(); 00269 00270 // RBM 00271 rbm = new RBMModule(); 00272 rbm->visible_layer = visible; 00273 rbm->hidden_layer = hidden; 00274 rbm->connection = connection; 00275 rbm->reconstruction_connection = connection; 00276 rbm->compute_log_likelihood = compute_log_likelihood; 00277 rbm->random_gen = new PRandom(seed); 00278 rbm->cd_learning_rate = learning_rate; 00279 rbm->use_fast_approximations = use_fast_approximations; 00280 rbm->build(); 00281 00282 // data 00283 if (!data_filename.isEmpty()) 00284 { 00285 PP<AutoVMatrix> data_ = new AutoVMatrix(); 00286 data_->filename = data_filename; 00287 data_->defineSizes(n_visible, 1); 00288 data_->build(); 00289 data = get_pointer(data_); 00290 00291 // train_input 00292 train_input = new SubVMatrix(data, // source 00293 0, // istart 00294 0, // jstart 00295 n_train, // length 00296 n_visible, // width 00297 true // call_build 00298 ); 00299 00300 // valid_input 00301 valid_input = new SubVMatrix(data, 00302 n_train, 00303 0, 00304 n_valid, 00305 n_visible, 00306 true 00307 ); 00308 00309 // valid_input 00310 test_input = new SubVMatrix(data, 00311 n_train + n_valid, 00312 0, 00313 n_test, 00314 n_visible, 00315 true 00316 ); 00317 } 00318 00319 // ports 00320 ports = rbm->getPorts(); 00321 n_ports = ports.length(); 00322 for (int i=0; i<n_ports; i++) 00323 { 00324 if (ports[i].find(".state", 0) != string::npos) 00325 { 00326 state_ports.append(ports[i]); 00327 state_ports_indices.append(i); 00328 } 00329 } 00330 00331 n_state_ports = state_ports.length(); 00332 nll_index = rbm->getPortIndex("neg_log_likelihood"); 00333 visible_index = rbm->getPortIndex("visible"); 00334 rec_err_index = rbm->getPortIndex("reconstruction_error.state"); 00335 00336 // nll_values 00337 nll_values.resize(n_ports); 00338 for (int i=0; i<n_state_ports; i++) 00339 nll_values[state_ports_indices[i]] = new Mat(); 00340 00341 // rec_err_values 00342 rec_err_values.resize(n_ports); 00343 for (int i=0; i<n_state_ports; i++) 00344 rec_err_values[state_ports_indices[i]] = new Mat(); 00345 } 00346 00347 Mat RBMTrainer::NLL(const Mat& examples) 00348 { 00349 Mat nll; 00350 00351 for (int i=0; i<n_state_ports; i++) 00352 nll_values[state_ports_indices[i]]->resize(0,0); 00353 00354 nll_values[nll_index] = &nll; 00355 nll_values[visible_index] = const_cast<Mat *>(&examples); 00356 00357 rbm->fprop(nll_values); 00358 00359 if (print_debug) 00360 { 00361 pout << "In NLL:" << endl; 00362 for (int i=0; i<n_state_ports; i++) 00363 { 00364 int portnum = state_ports_indices[i]; 00365 string portname = rbm->getPortName(portnum); 00366 pout << portname << ": " << nll_values[portnum]->size() << endl; 00367 } 00368 } 00369 00370 PLASSERT(nll.length() == examples.length() && nll.width() == 1); 00371 return nll; 00372 } 00373 00374 Mat RBMTrainer::recError(const Mat& examples) 00375 { 00376 Mat rec_err; 00377 for (int i=0; i<n_state_ports; i++) 00378 rec_err_values[state_ports_indices[i]]->resize(0,0); 00379 00380 rec_err_values[rec_err_index] = &rec_err; 00381 rec_err_values[visible_index] = const_cast<Mat *>(&examples); 00382 00383 rbm->fprop(rec_err_values); 00384 00385 if (print_debug) 00386 { 00387 pout << "In recError:" << endl; 00388 for (int i=0; i<n_state_ports; i++) 00389 { 00390 int portnum = state_ports_indices[i]; 00391 string portname = rbm->getPortName(portnum); 00392 pout << portname << ": " << rec_err_values[portnum]->size() << endl; 00393 } 00394 } 00395 00396 PLASSERT(rec_err.length() == examples.length() 00397 && rec_err.width() == 1); 00398 return rec_err; 00399 } 00400 00401 void RBMTrainer::CD1(const Mat& examples) 00402 { 00403 int n_examples = examples.length(); 00404 if (print_debug) 00405 { 00406 pout << "v0 = " << endl << examples << endl; 00407 } 00408 00409 // Positive phase 00410 connection->setAsDownInputs(examples); 00411 hidden->getAllActivations(connection, 0, true); 00412 hidden->computeExpectations(); 00413 hidden->generateSamples(); 00414 00415 h0_a = hidden->activations; 00416 h0_s.resize(n_examples, n_hidden); 00417 h0_s << hidden->samples.copy(); 00418 00419 if (update_with_h0_sample) 00420 { 00421 h0 = h0_s; 00422 } 00423 else 00424 { 00425 h0_e.resize(n_examples, n_hidden); 00426 h0_e << hidden->getExpectations(); 00427 h0 = h0_e; 00428 } 00429 00430 if (print_debug) 00431 { 00432 pout << "h0_a = " << endl << h0_a << endl; 00433 pout << "h0_e = " << endl << hidden->getExpectations() << endl; 00434 pout << "h0_s = " << endl << h0_s << endl; 00435 } 00436 00437 // Downward pass 00438 connection->setAsUpInputs(h0_s); 00439 visible->getAllActivations(connection, 0, true); 00440 visible->computeExpectations(); 00441 visible->generateSamples(); 00442 00443 v1_a = visible->activations; 00444 v1_e = visible->getExpectations(); 00445 v1_s = visible->samples; 00446 00447 // Negative phase 00448 if (sample_v1_in_chain) 00449 v1 = v1_s; 00450 else 00451 v1 = v1_e; 00452 00453 if (print_debug) 00454 { 00455 pout << "v1_a = " << endl << v1_a << endl; 00456 pout << "v1_e = " << endl << v1_e << endl; 00457 pout << "v1_s = " << endl << v1_s << endl; 00458 } 00459 00460 connection->setAsDownInputs(v1); 00461 hidden->getAllActivations(connection, 0, true); 00462 hidden->computeExpectations(); 00463 00464 Mat h1 = hidden->getExpectations(); 00465 if (print_debug) 00466 pout << "h1 = " << endl << h1 << endl; 00467 00468 rbm->CDUpdate(examples, h0, v1, h1); 00469 } 00470 00471 void RBMTrainer::run() 00472 { 00473 Mat results(n_stages+1, 6); 00474 for (int stage=0; stage<n_stages; stage++) 00475 { 00476 pout << "stage: " << stage << endl; 00477 results(stage, 0) = mean(NLL(train_input)); 00478 results(stage, 3) = mean(recError(train_input)); 00479 00480 if (n_valid > 0) 00481 { 00482 results(stage, 1) = mean(NLL(valid_input)); 00483 results(stage, 4) = mean(recError(valid_input)); 00484 } 00485 00486 if (n_test > 0) 00487 { 00488 results(stage, 2) = mean(NLL(test_input)); 00489 results(stage, 5) = mean(recError(test_input)); 00490 } 00491 00492 pout << "NLL: " << results(stage).subVec(0,3) << endl; 00493 pout << "RecError: " << results(stage).subVec(3,3) << endl; 00494 00495 for (int i=0; i<n_train/batch_size; i++) 00496 CD1(train_input.subMatRows(i*batch_size, batch_size)); 00497 } 00498 pout << "stage: " << n_stages << endl; 00499 results(n_stages, 0) = mean(NLL(train_input)); 00500 results(n_stages, 3) = mean(recError(train_input)); 00501 if (n_valid > 0) 00502 { 00503 results(n_stages, 1) = mean(NLL(valid_input)); 00504 results(n_stages, 4) = mean(recError(valid_input)); 00505 } 00506 if (n_test > 0) 00507 { 00508 results(n_stages, 2) = mean(NLL(test_input)); 00509 results(n_stages, 5) = mean(recError(test_input)); 00510 } 00511 pout << "NLL: " << results(n_stages).subVec(0,3) << endl; 00512 pout << "RecError: " << results(n_stages).subVec(3,3) << endl; 00513 pout << "results = " << endl << results << endl; 00514 00515 if (!save_path.isEmpty()) 00516 { 00517 PPath filename; 00518 if (save_name.isEmpty()) 00519 filename = save_path / "results.amat"; 00520 else 00521 filename = save_path / save_name; 00522 00523 PStream file = openFile(filename, PStream::raw_ascii, "w"); 00524 file << results << endl; 00525 } 00526 } 00527 00528 00529 } // end of namespace PLearn 00530 00531 00532 /* 00533 Local Variables: 00534 mode:c++ 00535 c-basic-offset:4 00536 c-file-style:"stroustrup" 00537 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00538 indent-tabs-mode:nil 00539 fill-column:79 00540 End: 00541 */ 00542 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :