PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // ModuleTester.cc 00004 // 00005 // Copyright (C) 2007 Olivier Delalleau 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Olivier Delalleau 00036 00040 #include "ModuleTester.h" 00041 #include <plearn/vmat/VMatrix.h> 00042 00043 #define PL_LOG_MODULE_NAME "ModuleTester" 00044 #include <plearn/io/pl_log.h> 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 PLEARN_IMPLEMENT_OBJECT( 00050 ModuleTester, 00051 "Test an OnlineLearningModule to ensure it is properly implemented.", 00052 "" 00053 ); 00054 00055 ModuleTester::ModuleTester(): 00056 seeds(TVec<int32_t>(1, int32_t(1827))), 00057 default_length(10), 00058 default_width(5), 00059 max_in(1), 00060 max_out_grad(MISSING_VALUE), 00061 min_in(0), 00062 min_out_grad(MISSING_VALUE), 00063 step(1e-6), 00064 absolute_tolerance_threshold(1), 00065 absolute_tolerance(1e-5), 00066 relative_tolerance(1e-5) 00067 {} 00068 00069 void ModuleTester::build() 00070 { 00071 inherited::build(); 00072 build_(); 00073 } 00074 00075 void ModuleTester::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00076 { 00077 inherited::makeDeepCopyFromShallowCopy(copies); 00078 00079 // ### Call deepCopyField on all "pointer-like" fields 00080 // ### that you wish to be deepCopied rather than 00081 // ### shallow-copied. 00082 // ### ex: 00083 // deepCopyField(trainvec, copies); 00084 00085 // ### Remove this line when you have fully implemented this method. 00086 PLERROR("ModuleTester::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00087 } 00088 00089 void ModuleTester::declareOptions(OptionList& ol) 00090 { 00091 declareOption(ol, "module", &ModuleTester::module, 00092 OptionBase::buildoption, 00093 "The module to be tested."); 00094 00095 declareOption(ol, "configurations", &ModuleTester::configurations, 00096 OptionBase::buildoption, 00097 "List of port configurations to test. Each element is a map from a\n" 00098 "string to a list of corresponding ports. This string can be one of:\n" 00099 " - 'in_grad': input ports for which a gradient must be computed\n" 00100 " - 'in_nograd': input ports for which no gradient is computed\n" 00101 " - 'out_grad': output ports for which a gradient must be provided\n" 00102 " - 'out_nograd': output ports for which no gradient is provided"); 00103 00104 declareOption(ol, "min_in", &ModuleTester::min_in, 00105 OptionBase::buildoption, 00106 "Minimum value used when uniformly sampling input data."); 00107 00108 declareOption(ol, "max_in", &ModuleTester::max_in, 00109 OptionBase::buildoption, 00110 "Maximum value used when uniformly sampling input data."); 00111 00112 declareOption(ol, "min_out_grad", &ModuleTester::min_out_grad, 00113 OptionBase::buildoption, 00114 "Minimum value used when uniformly sampling output gradient data.\n" 00115 "If missing, then 'min_in' is used."); 00116 00117 declareOption(ol, "max_out_grad", &ModuleTester::max_out_grad, 00118 OptionBase::buildoption, 00119 "Maximum value used when uniformly sampling output gradient data.\n" 00120 "If missing, then 'max_in' is used."); 00121 00122 declareOption(ol, "sampling_data", &ModuleTester::sampling_data, 00123 OptionBase::buildoption, 00124 "A map from port names to specific data to use when sampling (either\n" 00125 "input data or output gradient data, depending on whether the port\n" 00126 "is an input or output) for this port. This mean the port data is\n" 00127 "actually not sampled, but filled with the provided VMatrix (which\n" 00128 "might be a VMatrixFromDistribution if sampling is needed)."); 00129 00130 declareOption(ol, "seeds", &ModuleTester::seeds, 00131 OptionBase::buildoption, 00132 "Seeds used for random number generation. You can try different seeds " 00133 "if you want to test more situations."); 00134 00135 declareOption(ol, "default_length", &ModuleTester::default_length, 00136 OptionBase::buildoption, 00137 "Default length of a port used when the module returns an undefined " 00138 "port length (-1 in getPortLength())"); 00139 00140 declareOption(ol, "default_width", &ModuleTester::default_width, 00141 OptionBase::buildoption, 00142 "Default width of a port used when the module returns an undefined " 00143 "port width (-1 in getPortWidth())"); 00144 00145 declareOption(ol, "step", &ModuleTester::step, 00146 OptionBase::buildoption, 00147 "Small offset used to modify inputs in order to estimate the\n" 00148 "gradient by finite difference."); 00149 00150 declareOption(ol, "absolute_tolerance_threshold", 00151 &ModuleTester::absolute_tolerance_threshold, 00152 OptionBase::buildoption, 00153 "Value below which we use absolute tolerance instead of relative in\n" 00154 "order to compare gradients."); 00155 00156 declareOption(ol, "absolute_tolerance", 00157 &ModuleTester::absolute_tolerance, 00158 OptionBase::buildoption, 00159 "Absolute tolerance when comparing gradients."); 00160 00161 declareOption(ol, "relative_tolerance", 00162 &ModuleTester::relative_tolerance, 00163 OptionBase::buildoption, 00164 "Relative tolerance when comparing gradients."); 00165 00166 // Now call the parent class' declareOptions 00167 inherited::declareOptions(ol); 00168 } 00169 00170 00172 // build_ // 00174 void ModuleTester::build_() 00175 { 00176 if (!module) 00177 return; 00178 00179 PP<PRandom> random_gen = new PRandom(); 00180 TVec<Mat*> fprop_data(module->nPorts()); // Input to 'fprop'. 00181 TVec<Mat*> bprop_data(module->nPorts()); // Input to 'bpropAccUpdate'. 00182 // We also use additional matrices to store gradients in order to ensure 00183 // the module is properly accumulating. 00184 TVec<Mat*> bprop_check(module->nPorts()); 00185 // Store previous fprop result in order to be able to estimate gradient. 00186 TVec<Mat*> fprop_check(module->nPorts()); 00187 // Initialize workspace for matrices. Note that we need to allocate enough 00188 // memory from start, as otherwise an append may make previous Mat* 00189 // pointers invalid. 00190 int max_mats_size = 1000; 00191 TVec<Mat> mats(max_mats_size); 00192 00193 PP<PRandom> sub_rng = NULL; 00194 int32_t default_seed = 1827; 00195 if (!module->random_gen) { 00196 // The module needs to be provided a random generator. 00197 sub_rng = new PRandom(); 00198 module->random_gen = sub_rng; 00199 module->build(); 00200 sub_rng->manual_seed(default_seed); 00201 module->forget(); 00202 } 00203 00204 bool ok = true; 00205 for (int j = 0; ok && j < seeds.length(); j++) { 00206 random_gen->manual_seed(seeds[j]); 00207 for (int i = 0; ok && i < configurations.length(); i++) { 00208 map<string, TVec<string> >& conf = configurations[i]; 00209 const TVec<string>& in_grad = conf["in_grad"]; 00210 const TVec<string>& in_nograd = conf["in_nograd"]; 00211 const TVec<string>& out_grad = conf["out_grad"]; 00212 const TVec<string>& out_nograd = conf["out_nograd"]; 00213 TVec<string> all_in(in_grad.length() + in_nograd.length()); 00214 all_in.subVec(0, in_grad.length()) << in_grad; 00215 all_in.subVec(in_grad.length(), in_nograd.length()) << in_nograd; 00216 TVec<string> all_out(out_grad.length() + out_nograd.length()); 00217 all_out.subVec(0, out_grad.length()) << out_grad; 00218 all_out.subVec(out_grad.length(), out_nograd.length()) << out_nograd; 00219 mats.resize(0); 00220 // Prepare fprop data. 00221 fprop_data.fill(NULL); 00222 fprop_check.fill(NULL); 00223 for (int k = 0; k < all_in.length(); k++) { 00224 const string& port = all_in[k]; 00225 int length = module->getPortLength(port); 00226 int width = module->getPortWidth(port); 00227 if (length < 0) 00228 length = default_length; 00229 if (width < 0) 00230 width = default_width; 00231 mats.append(Mat()); 00232 PLCHECK( mats.length() <= max_mats_size ); 00233 Mat* in_k = & mats.lastElement(); 00234 fprop_data[module->getPortIndex(port)] = in_k; 00235 // Fill 'in_k' randomly. 00236 map<string, PP<VMatrix> >::iterator it = 00237 sampling_data.find(port); 00238 if (it == sampling_data.end()) { 00239 in_k->resize(length, width); 00240 if (fast_exact_is_equal(min_in, max_in)) 00241 in_k->fill(min_in); 00242 else 00243 random_gen->fill_random_uniform(*in_k, min_in, max_in); 00244 } else { 00245 PP<VMatrix> vmat = it->second; 00246 in_k->resize(vmat->length(), vmat->width()); 00247 *in_k << vmat->toMat(); 00248 } 00249 } 00250 for (int k = 0; k < all_out.length(); k++) { 00251 const string& port = all_out[k]; 00252 mats.append(Mat()); 00253 PLCHECK( mats.length() <= max_mats_size ); 00254 Mat* out_k = & mats.lastElement(); 00255 int idx = module->getPortIndex(port); 00256 fprop_data[idx] = out_k; 00257 mats.append(Mat()); 00258 PLCHECK( mats.length() <= max_mats_size ); 00259 fprop_check[idx] = & mats.lastElement(); 00260 } 00261 // Perform fprop. 00262 if (sub_rng) 00263 sub_rng->manual_seed(default_seed); 00264 module->forget(); 00265 module->fprop(fprop_data); 00266 // Debug output. 00267 string output; 00268 PStream out_s = openString(output, PStream::plearn_ascii, "w"); 00269 for (int k = 0; k < fprop_data.length(); k++) { 00270 out_s.setMode(PStream::raw_ascii); 00271 out_s << "FPROP(" + module->getPortName(k) + "):\n"; 00272 Mat* m = fprop_data[k]; 00273 if (!m) { 00274 out_s << "null"; 00275 } else { 00276 out_s.setMode(PStream::plearn_ascii); 00277 out_s << *m; 00278 } 00279 } 00280 out_s << endl; 00281 out_s = NULL; 00282 DBG_MODULE_LOG << output; 00283 // Prepare bprop data. 00284 bprop_data.fill(NULL); 00285 bprop_check.fill(NULL); 00286 for (int k = 0; k < in_grad.length(); k++) { 00287 const string& port = in_grad[k]; 00288 mats.append(Mat()); 00289 PLCHECK( mats.length() <= max_mats_size ); 00290 Mat* in_grad_k = & mats.lastElement(); 00291 int idx = module->getPortIndex(port); 00292 Mat* in_k = fprop_data[idx]; 00293 // We fill 'in_grad_k' with random elements to check proper 00294 // accumulation. 00295 in_grad_k->resize(in_k->length(), in_k->width()); 00296 random_gen->fill_random_uniform(*in_grad_k, -1, 1); 00297 mats.append(Mat()); 00298 PLCHECK( mats.length() <= max_mats_size ); 00299 // Do a copy of initial gradient to allow comparison later. 00300 Mat* in_check_k = & mats.lastElement(); 00301 in_check_k->resize(in_grad_k->length(), in_grad_k->width()); 00302 *in_check_k << *in_grad_k; 00303 in_grad_k->resize(0, in_grad_k->width()); 00304 bprop_data[idx] = in_grad_k; 00305 bprop_check[idx] = in_check_k; 00306 } 00307 for (int k = 0; k < out_grad.length(); k++) { 00308 const string& port = out_grad[k]; 00309 mats.append(Mat()); 00310 PLCHECK( mats.length() <= max_mats_size ); 00311 Mat* out_grad_k = & mats.lastElement(); 00312 int idx = module->getPortIndex(port); 00313 Mat* out_k = fprop_data[idx]; 00314 out_grad_k->resize(out_k->length(), out_k->width()); 00315 real min = is_missing(min_out_grad) ? min_in : min_out_grad; 00316 real max = is_missing(max_out_grad) ? max_in : max_out_grad; 00317 PLCHECK_MSG( sampling_data.find(port) == sampling_data.end(), 00318 "Specific sampling data not yet implemented for output" 00319 " gradients" ); 00320 if (fast_exact_is_equal(min, max)) 00321 // Special cast to handle in particular the case when we 00322 // want the gradient to be exactly 1 (for instance for a 00323 // cost). 00324 out_grad_k->fill(min); 00325 else 00326 random_gen->fill_random_uniform(*out_grad_k, min, max); 00327 bprop_data[idx] = out_grad_k; 00328 } 00329 // Perform bprop. 00330 if (sub_rng) 00331 sub_rng->manual_seed(default_seed); 00332 module->forget(); 00333 module->bpropAccUpdate(fprop_data, bprop_data); 00334 // Debug output. 00335 out_s = openString(output, PStream::plearn_ascii, "w"); 00336 for (int k = 0; k < bprop_data.length(); k++) { 00337 out_s.setMode(PStream::raw_ascii); 00338 out_s << "BPROP(" + module->getPortName(k) + "):\n"; 00339 Mat* m = bprop_data[k]; 00340 if (!m) { 00341 out_s << " *** NULL ***\n"; 00342 } else { 00343 out_s.setMode(PStream::plearn_ascii); 00344 out_s << *m; 00345 } 00346 } 00347 out_s << endl; 00348 out_s = NULL; 00349 DBG_MODULE_LOG << output; 00350 // Check the gradient was properly accumulated. 00351 // First compute the difference between computed gradient and the 00352 // initial value stored in the gradient matrix. 00353 for (int k = 0; k < in_grad.length(); k++) { 00354 int idx = module->getPortIndex(in_grad[k]); 00355 Mat* grad = bprop_data[idx]; 00356 if (grad) { 00357 Mat* grad_check = bprop_check[idx]; 00358 PLASSERT( grad_check ); 00359 *grad_check -= *grad; 00360 negateElements(*grad_check); 00361 } 00362 } 00363 // Then perform a new bprop pass, without accumulating. 00364 for (int k = 0; k < in_grad.length(); k++) { 00365 int idx = module->getPortIndex(in_grad[k]); 00366 bprop_data[idx]->resize(0, bprop_data[idx]->width()); 00367 } 00368 if (sub_rng) 00369 sub_rng->manual_seed(default_seed); 00370 module->forget(); // Ensure we are using same parameters. 00371 module->bpropUpdate(fprop_data, bprop_data); 00372 // Then compare 'bprop_data' and 'bprop_check'. 00373 for (int k = 0; k < in_grad.length(); k++) { 00374 int idx = module->getPortIndex(in_grad[k]); 00375 Mat* grad = bprop_data[idx]; 00376 PLASSERT( grad ); 00377 Mat* check = bprop_check[idx]; 00378 PLASSERT( check ); 00379 // TODO Using the PLearn diff mechanism would be better. 00380 for (int p = 0; p < grad->length(); p++) 00381 for (int q = 0; q < grad->width(); q++) 00382 if (!is_equal((*grad)(p,q), (*check)(p,q))) { 00383 pout << "Gradient for port '" << 00384 module->getPortName(idx) << "' was not " << 00385 "properly accumulated: " << (*grad)(p,q) << 00386 " != " << (*check)(p,q) << endl; 00387 ok = false; 00388 } 00389 } 00390 // Continue only if accumulation test passed. 00391 if (!ok) 00392 return; 00393 DBG_MODULE_LOG << "Accumulation test successful" << endl; 00394 // Verify gradient is coherent with the input, through a subtle 00395 // perturbation of said input. 00396 // Save result of fprop. 00397 for (int k = 0; k < all_out.length(); k++) { 00398 int idx = module->getPortIndex(all_out[k]); 00399 Mat* val = fprop_data[idx]; 00400 Mat* check = fprop_check[idx]; 00401 PLASSERT( val && check ); 00402 check->resize(val->length(), val->width()); 00403 *check << *val; 00404 DBG_MODULE_LOG << "Reference fprop data (" << all_out[k] << ")" 00405 << ":" << endl << *check << endl; 00406 } 00407 for (int k = 0; ok && k < in_grad.length(); k++) { 00408 int idx = module->getPortIndex(in_grad[k]); 00409 Mat* grad = bprop_data[idx]; 00410 Mat* val = fprop_data[idx]; 00411 Mat* b_check = bprop_check[idx]; 00412 PLASSERT( grad && val && b_check ); 00413 grad->fill(0); // Will be used to store estimated gradient. 00414 for (int p = 0; p < grad->length(); p++) 00415 for (int q = 0; q < grad->width(); q++) { 00416 real backup = (*val)(p, q); 00417 (*val)(p, q) += step; 00418 for (int r = 0; r < all_out.length(); r++) { 00419 int to_clear = module->getPortIndex(all_out[r]); 00420 PLASSERT( to_clear != idx ); 00421 fprop_data[to_clear]->resize(0, 0); 00422 } 00423 if (sub_rng) 00424 sub_rng->manual_seed(default_seed); 00425 module->forget(); 00426 module->fprop(fprop_data); 00427 (*val)(p, q) = backup; 00428 // Estimate gradient w.r.t. each output. 00429 for (int r = 0; r < out_grad.length(); r++) { 00430 int out_idx = module->getPortIndex(out_grad[r]); 00431 Mat* out_val = fprop_data[out_idx]; 00432 Mat* out_prev = fprop_check[out_idx]; 00433 Mat* out_grad_ = bprop_data[out_idx]; 00434 PLASSERT( out_val && out_prev && out_grad_ ); 00435 for (int oi = 0; oi < out_val->length(); oi++) 00436 for (int oj = 0; oj < out_val->width(); oj++) { 00437 real diff = (*out_val)(oi, oj) - 00438 (*out_prev)(oi, oj); 00439 (*grad)(p, q) += 00440 diff * (*out_grad_)(oi, oj) / step; 00441 DBG_MODULE_LOG << " diff = " << diff << 00442 endl << " step = " << step << endl << 00443 " out_grad = " << (*out_grad_)(oi, oj) 00444 << endl << " grad = " << (*grad)(p, q) 00445 << endl; 00446 } 00447 } 00448 } 00449 // Compare estimated and computed gradients. 00450 for (int p = 0; p < grad->length(); p++) 00451 for (int q = 0; q < grad->width(); q++) 00452 if (!is_equal((*grad)(p,q), (*b_check)(p,q), 00453 absolute_tolerance_threshold, 00454 absolute_tolerance, relative_tolerance)) { 00455 pout << "Gradient for port '" << 00456 module->getPortName(idx) << "' was not " << 00457 "properly computed: finite difference (" << 00458 (*grad)(p,q) << ") != computed (" << 00459 (*b_check)(p,q) << ")" << endl; 00460 ok = false; 00461 } else { 00462 DBG_MODULE_LOG << "Gradient for port '" << 00463 module->getPortName(idx) << "' was " << 00464 "properly computed: finite difference (" << 00465 (*grad)(p,q) << ") == computed (" << 00466 (*b_check)(p,q) << ")" << endl; 00467 } 00468 } 00469 } 00470 } 00471 if (ok) 00472 pout << "All tests passed successfully on module " << 00473 module->classname() << endl; 00474 else 00475 pout << "*** ERRROR ***" << endl; 00476 } 00477 00478 00479 } // end of namespace PLearn 00480 00481 00482 /* 00483 Local Variables: 00484 mode:c++ 00485 c-basic-offset:4 00486 c-file-style:"stroustrup" 00487 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00488 indent-tabs-mode:nil 00489 fill-column:79 00490 End: 00491 */ 00492 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :