PLearn 0.1
ModuleTester.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // ModuleTester.cc
00004 //
00005 // Copyright (C) 2007 Olivier Delalleau
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Olivier Delalleau
00036 
00040 #include "ModuleTester.h"
00041 #include <plearn/vmat/VMatrix.h>
00042 
00043 #define PL_LOG_MODULE_NAME "ModuleTester"
00044 #include <plearn/io/pl_log.h>
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 PLEARN_IMPLEMENT_OBJECT(
00050     ModuleTester,
00051     "Test an OnlineLearningModule to ensure it is properly implemented.",
00052     ""
00053 );
00054 
00055 ModuleTester::ModuleTester():
00056     seeds(TVec<int32_t>(1, int32_t(1827))),
00057     default_length(10),
00058     default_width(5),
00059     max_in(1),
00060     max_out_grad(MISSING_VALUE),
00061     min_in(0),
00062     min_out_grad(MISSING_VALUE),
00063     step(1e-6),
00064     absolute_tolerance_threshold(1),
00065     absolute_tolerance(1e-5),
00066     relative_tolerance(1e-5)
00067 {}
00068 
00069 void ModuleTester::build()
00070 {
00071     inherited::build();
00072     build_();
00073 }
00074 
00075 void ModuleTester::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00076 {
00077     inherited::makeDeepCopyFromShallowCopy(copies);
00078 
00079     // ### Call deepCopyField on all "pointer-like" fields
00080     // ### that you wish to be deepCopied rather than
00081     // ### shallow-copied.
00082     // ### ex:
00083     // deepCopyField(trainvec, copies);
00084 
00085     // ### Remove this line when you have fully implemented this method.
00086     PLERROR("ModuleTester::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00087 }
00088 
00089 void ModuleTester::declareOptions(OptionList& ol)
00090 {
00091     declareOption(ol, "module", &ModuleTester::module,
00092                   OptionBase::buildoption,
00093         "The module to be tested.");
00094 
00095     declareOption(ol, "configurations", &ModuleTester::configurations,
00096                   OptionBase::buildoption,
00097         "List of port configurations to test. Each element is a map from a\n"
00098         "string to a list of corresponding ports. This string can be one of:\n"
00099         " - 'in_grad': input ports for which a gradient must be computed\n"
00100         " - 'in_nograd': input ports for which no gradient is computed\n"
00101         " - 'out_grad': output ports for which a gradient must be provided\n"
00102         " - 'out_nograd': output ports for which no gradient is provided");
00103 
00104     declareOption(ol, "min_in", &ModuleTester::min_in,
00105                   OptionBase::buildoption,
00106         "Minimum value used when uniformly sampling input data.");
00107 
00108     declareOption(ol, "max_in", &ModuleTester::max_in,
00109                   OptionBase::buildoption,
00110         "Maximum value used when uniformly sampling input data.");
00111 
00112     declareOption(ol, "min_out_grad", &ModuleTester::min_out_grad,
00113                   OptionBase::buildoption,
00114         "Minimum value used when uniformly sampling output gradient data.\n"
00115         "If missing, then 'min_in' is used.");
00116 
00117     declareOption(ol, "max_out_grad", &ModuleTester::max_out_grad,
00118                   OptionBase::buildoption,
00119         "Maximum value used when uniformly sampling output gradient data.\n"
00120         "If missing, then 'max_in' is used.");
00121 
00122     declareOption(ol, "sampling_data", &ModuleTester::sampling_data,
00123                   OptionBase::buildoption,
00124         "A map from port names to specific data to use when sampling (either\n"
00125         "input data or output gradient data, depending on whether the port\n"
00126         "is an input or output) for this port. This mean the port data is\n"
00127         "actually not sampled, but filled with the provided VMatrix (which\n"
00128         "might be a VMatrixFromDistribution if sampling is needed).");
00129 
00130     declareOption(ol, "seeds", &ModuleTester::seeds,
00131                   OptionBase::buildoption,
00132         "Seeds used for random number generation. You can try different seeds "
00133         "if you want to test more situations.");
00134 
00135     declareOption(ol, "default_length", &ModuleTester::default_length,
00136                   OptionBase::buildoption,
00137         "Default length of a port used when the module returns an undefined "
00138         "port length (-1 in getPortLength())");
00139 
00140     declareOption(ol, "default_width", &ModuleTester::default_width,
00141                   OptionBase::buildoption,
00142         "Default width of a port used when the module returns an undefined "
00143         "port width (-1 in getPortWidth())");
00144 
00145     declareOption(ol, "step", &ModuleTester::step,
00146                   OptionBase::buildoption,
00147         "Small offset used to modify inputs in order to estimate the\n"
00148         "gradient by finite difference.");
00149 
00150     declareOption(ol, "absolute_tolerance_threshold",
00151                   &ModuleTester::absolute_tolerance_threshold,
00152                   OptionBase::buildoption,
00153         "Value below which we use absolute tolerance instead of relative in\n"
00154         "order to compare gradients.");
00155 
00156     declareOption(ol, "absolute_tolerance",
00157                   &ModuleTester::absolute_tolerance,
00158                   OptionBase::buildoption,
00159         "Absolute tolerance when comparing gradients.");
00160 
00161     declareOption(ol, "relative_tolerance",
00162                   &ModuleTester::relative_tolerance,
00163                   OptionBase::buildoption,
00164         "Relative tolerance when comparing gradients.");
00165 
00166     // Now call the parent class' declareOptions
00167     inherited::declareOptions(ol);
00168 }
00169 
00170 
00172 // build_ //
00174 void ModuleTester::build_()
00175 {
00176     if (!module)
00177         return;
00178 
00179     PP<PRandom> random_gen = new PRandom();
00180     TVec<Mat*> fprop_data(module->nPorts()); // Input to 'fprop'.
00181     TVec<Mat*> bprop_data(module->nPorts()); // Input to 'bpropAccUpdate'.
00182     // We also use additional matrices to store gradients in order to ensure
00183     // the module is properly accumulating.
00184     TVec<Mat*> bprop_check(module->nPorts());
00185     // Store previous fprop result in order to be able to estimate gradient.
00186     TVec<Mat*> fprop_check(module->nPorts());
00187     // Initialize workspace for matrices. Note that we need to allocate enough
00188     // memory from start, as otherwise an append may make previous Mat*
00189     // pointers invalid.
00190     int max_mats_size = 1000;
00191     TVec<Mat> mats(max_mats_size);
00192 
00193     PP<PRandom> sub_rng = NULL;
00194     int32_t default_seed = 1827;
00195     if (!module->random_gen) {
00196         // The module needs to be provided a random generator.
00197         sub_rng = new PRandom();
00198         module->random_gen = sub_rng;
00199         module->build();
00200         sub_rng->manual_seed(default_seed);
00201         module->forget();
00202     }
00203 
00204     bool ok = true;
00205     for (int j = 0; ok && j < seeds.length(); j++) {
00206         random_gen->manual_seed(seeds[j]);
00207         for (int i = 0; ok && i < configurations.length(); i++) {
00208             map<string, TVec<string> >& conf = configurations[i];
00209             const TVec<string>& in_grad = conf["in_grad"];
00210             const TVec<string>& in_nograd = conf["in_nograd"];
00211             const TVec<string>& out_grad = conf["out_grad"];
00212             const TVec<string>& out_nograd = conf["out_nograd"];
00213             TVec<string> all_in(in_grad.length() + in_nograd.length());
00214             all_in.subVec(0, in_grad.length()) << in_grad;
00215             all_in.subVec(in_grad.length(), in_nograd.length()) << in_nograd;
00216             TVec<string> all_out(out_grad.length() + out_nograd.length());
00217             all_out.subVec(0, out_grad.length()) << out_grad;
00218             all_out.subVec(out_grad.length(), out_nograd.length()) << out_nograd;
00219             mats.resize(0);
00220             // Prepare fprop data.
00221             fprop_data.fill(NULL);
00222             fprop_check.fill(NULL);
00223             for (int k = 0; k < all_in.length(); k++) {
00224                 const string& port = all_in[k];
00225                 int length = module->getPortLength(port);
00226                 int width = module->getPortWidth(port);
00227                 if (length < 0)
00228                     length = default_length;
00229                 if (width < 0)
00230                     width = default_width;
00231                 mats.append(Mat());
00232                 PLCHECK( mats.length() <= max_mats_size );
00233                 Mat* in_k = & mats.lastElement();
00234                 fprop_data[module->getPortIndex(port)] = in_k;
00235                 // Fill 'in_k' randomly.
00236                 map<string, PP<VMatrix> >::iterator it =
00237                     sampling_data.find(port);
00238                 if (it == sampling_data.end()) {
00239                     in_k->resize(length, width);
00240                     if (fast_exact_is_equal(min_in, max_in))
00241                         in_k->fill(min_in);
00242                     else
00243                         random_gen->fill_random_uniform(*in_k, min_in, max_in);
00244                 } else {
00245                     PP<VMatrix> vmat = it->second;
00246                     in_k->resize(vmat->length(), vmat->width());
00247                     *in_k << vmat->toMat();
00248                 }
00249             }
00250             for (int k = 0; k < all_out.length(); k++) {
00251                 const string& port = all_out[k];
00252                 mats.append(Mat());
00253                 PLCHECK( mats.length() <= max_mats_size );
00254                 Mat* out_k = & mats.lastElement();
00255                 int idx = module->getPortIndex(port);
00256                 fprop_data[idx] = out_k;
00257                 mats.append(Mat());
00258                 PLCHECK( mats.length() <= max_mats_size );
00259                 fprop_check[idx] = & mats.lastElement();
00260             }
00261             // Perform fprop.
00262             if (sub_rng)
00263                 sub_rng->manual_seed(default_seed);
00264             module->forget();
00265             module->fprop(fprop_data);
00266             // Debug output.
00267             string output;
00268             PStream out_s = openString(output, PStream::plearn_ascii, "w");
00269             for (int k = 0; k < fprop_data.length(); k++) {
00270                 out_s.setMode(PStream::raw_ascii);
00271                 out_s << "FPROP(" + module->getPortName(k) + "):\n";
00272                 Mat* m = fprop_data[k];
00273                 if (!m) {
00274                     out_s << "null";
00275                 } else {
00276                     out_s.setMode(PStream::plearn_ascii);
00277                     out_s << *m;
00278                 }
00279             }
00280             out_s << endl;
00281             out_s = NULL;
00282             DBG_MODULE_LOG << output;
00283             // Prepare bprop data.
00284             bprop_data.fill(NULL);
00285             bprop_check.fill(NULL);
00286             for (int k = 0; k < in_grad.length(); k++) {
00287                 const string& port = in_grad[k];
00288                 mats.append(Mat());
00289                 PLCHECK( mats.length() <= max_mats_size );
00290                 Mat* in_grad_k = & mats.lastElement();
00291                 int idx = module->getPortIndex(port);
00292                 Mat* in_k = fprop_data[idx];
00293                 // We fill 'in_grad_k' with random elements to check proper
00294                 // accumulation.
00295                 in_grad_k->resize(in_k->length(), in_k->width());
00296                 random_gen->fill_random_uniform(*in_grad_k, -1, 1);
00297                 mats.append(Mat());
00298                 PLCHECK( mats.length() <= max_mats_size );
00299                 // Do a copy of initial gradient to allow comparison later.
00300                 Mat* in_check_k = & mats.lastElement();
00301                 in_check_k->resize(in_grad_k->length(), in_grad_k->width());
00302                 *in_check_k << *in_grad_k;
00303                 in_grad_k->resize(0, in_grad_k->width());
00304                 bprop_data[idx] = in_grad_k;
00305                 bprop_check[idx] = in_check_k;
00306             }
00307             for (int k = 0; k < out_grad.length(); k++) {
00308                 const string& port = out_grad[k];
00309                 mats.append(Mat());
00310                 PLCHECK( mats.length() <= max_mats_size );
00311                 Mat* out_grad_k = & mats.lastElement();
00312                 int idx = module->getPortIndex(port);
00313                 Mat* out_k = fprop_data[idx];
00314                 out_grad_k->resize(out_k->length(), out_k->width());
00315                 real min = is_missing(min_out_grad) ? min_in : min_out_grad;
00316                 real max = is_missing(max_out_grad) ? max_in : max_out_grad;
00317                 PLCHECK_MSG( sampling_data.find(port) == sampling_data.end(),
00318                         "Specific sampling data not yet implemented for output"
00319                         " gradients" );
00320                 if (fast_exact_is_equal(min, max))
00321                     // Special cast to handle in particular the case when we
00322                     // want the gradient to be exactly 1 (for instance for a
00323                     // cost).
00324                     out_grad_k->fill(min);
00325                 else
00326                     random_gen->fill_random_uniform(*out_grad_k, min, max);
00327                 bprop_data[idx] = out_grad_k;
00328             }
00329             // Perform bprop.
00330             if (sub_rng)
00331                 sub_rng->manual_seed(default_seed);
00332             module->forget();
00333             module->bpropAccUpdate(fprop_data, bprop_data);
00334             // Debug output.
00335             out_s = openString(output, PStream::plearn_ascii, "w");
00336             for (int k = 0; k < bprop_data.length(); k++) {
00337                 out_s.setMode(PStream::raw_ascii);
00338                 out_s << "BPROP(" + module->getPortName(k) + "):\n";
00339                 Mat* m = bprop_data[k];
00340                 if (!m) {
00341                     out_s << "  *** NULL ***\n";
00342                 } else {
00343                     out_s.setMode(PStream::plearn_ascii);
00344                     out_s << *m;
00345                 }
00346             }
00347             out_s << endl;
00348             out_s = NULL;
00349             DBG_MODULE_LOG << output;
00350             // Check the gradient was properly accumulated.
00351             // First compute the difference between computed gradient and the
00352             // initial value stored in the gradient matrix.
00353             for (int k = 0; k < in_grad.length(); k++) {
00354                 int idx = module->getPortIndex(in_grad[k]);
00355                 Mat* grad = bprop_data[idx];
00356                 if (grad) {
00357                     Mat* grad_check = bprop_check[idx];
00358                     PLASSERT( grad_check );
00359                     *grad_check -= *grad;
00360                     negateElements(*grad_check);
00361                 }
00362             }
00363             // Then perform a new bprop pass, without accumulating.
00364             for (int k = 0; k < in_grad.length(); k++) {
00365                 int idx = module->getPortIndex(in_grad[k]);
00366                 bprop_data[idx]->resize(0, bprop_data[idx]->width());
00367             }
00368             if (sub_rng)
00369                 sub_rng->manual_seed(default_seed);
00370             module->forget(); // Ensure we are using same parameters.
00371             module->bpropUpdate(fprop_data, bprop_data);
00372             // Then compare 'bprop_data' and 'bprop_check'.
00373             for (int k = 0; k < in_grad.length(); k++) {
00374                 int idx = module->getPortIndex(in_grad[k]);
00375                 Mat* grad = bprop_data[idx];
00376                 PLASSERT( grad );
00377                 Mat* check = bprop_check[idx];
00378                 PLASSERT( check );
00379                 // TODO Using the PLearn diff mechanism would be better.
00380                 for (int p = 0; p < grad->length(); p++)
00381                     for (int q = 0; q < grad->width(); q++)
00382                         if (!is_equal((*grad)(p,q), (*check)(p,q))) {
00383                             pout << "Gradient for port '" <<
00384                                 module->getPortName(idx) << "' was not " <<
00385                                 "properly accumulated: " << (*grad)(p,q) <<
00386                                 " != " << (*check)(p,q) << endl;
00387                             ok = false;
00388                         }
00389             }
00390             // Continue only if accumulation test passed.
00391             if (!ok)
00392                 return;
00393             DBG_MODULE_LOG << "Accumulation test successful" << endl;
00394             // Verify gradient is coherent with the input, through a subtle
00395             // perturbation of said input.
00396             // Save result of fprop.
00397             for (int k = 0; k < all_out.length(); k++) {
00398                 int idx = module->getPortIndex(all_out[k]);
00399                 Mat* val = fprop_data[idx];
00400                 Mat* check = fprop_check[idx];
00401                 PLASSERT( val && check );
00402                 check->resize(val->length(), val->width());
00403                 *check << *val;
00404                 DBG_MODULE_LOG << "Reference fprop data (" << all_out[k] << ")"
00405                     << ":" << endl << *check << endl;
00406             }
00407             for (int k = 0; ok && k < in_grad.length(); k++) {
00408                 int idx = module->getPortIndex(in_grad[k]);
00409                 Mat* grad = bprop_data[idx];
00410                 Mat* val = fprop_data[idx];
00411                 Mat* b_check = bprop_check[idx];
00412                 PLASSERT( grad && val && b_check );
00413                 grad->fill(0); // Will be used to store estimated gradient.
00414                 for (int p = 0; p < grad->length(); p++)
00415                     for (int q = 0; q < grad->width(); q++) {
00416                         real backup = (*val)(p, q);
00417                         (*val)(p, q) += step;
00418                         for (int r = 0; r < all_out.length(); r++) {
00419                             int to_clear = module->getPortIndex(all_out[r]);
00420                             PLASSERT( to_clear != idx );
00421                             fprop_data[to_clear]->resize(0, 0);
00422                         }
00423                         if (sub_rng)
00424                             sub_rng->manual_seed(default_seed);
00425                         module->forget();
00426                         module->fprop(fprop_data);
00427                         (*val)(p, q) = backup;
00428                         // Estimate gradient w.r.t. each output.
00429                         for (int r = 0; r < out_grad.length(); r++) {
00430                             int out_idx = module->getPortIndex(out_grad[r]);
00431                             Mat* out_val = fprop_data[out_idx];
00432                             Mat* out_prev = fprop_check[out_idx];
00433                             Mat* out_grad_ = bprop_data[out_idx];
00434                             PLASSERT( out_val && out_prev && out_grad_ );
00435                             for (int oi = 0; oi < out_val->length(); oi++)
00436                                 for (int oj = 0; oj < out_val->width(); oj++) {
00437                                     real diff = (*out_val)(oi, oj) -
00438                                         (*out_prev)(oi, oj);
00439                                     (*grad)(p, q) +=
00440                                         diff * (*out_grad_)(oi, oj) / step;
00441                                     DBG_MODULE_LOG << "  diff = " << diff <<
00442                                         endl << "  step = " << step << endl <<
00443                                         "  out_grad = " << (*out_grad_)(oi, oj)
00444                                         << endl << "  grad = " << (*grad)(p, q)
00445                                         << endl;
00446                                 }
00447                         }
00448                     }
00449                 // Compare estimated and computed gradients.
00450                 for (int p = 0; p < grad->length(); p++)
00451                     for (int q = 0; q < grad->width(); q++)
00452                         if (!is_equal((*grad)(p,q), (*b_check)(p,q),
00453                                     absolute_tolerance_threshold,
00454                                     absolute_tolerance, relative_tolerance)) {
00455                             pout << "Gradient for port '" <<
00456                                 module->getPortName(idx) << "' was not " <<
00457                                 "properly computed: finite difference (" <<
00458                                 (*grad)(p,q) << ") != computed (" <<
00459                                 (*b_check)(p,q) << ")" << endl;
00460                             ok = false;
00461                         } else {
00462                             DBG_MODULE_LOG << "Gradient for port '" <<
00463                                 module->getPortName(idx) << "' was " <<
00464                                 "properly computed: finite difference (" <<
00465                                 (*grad)(p,q) << ") == computed (" <<
00466                                 (*b_check)(p,q) << ")" << endl;
00467                         }
00468             }
00469         }
00470     }
00471     if (ok)
00472         pout << "All tests passed successfully on module " <<
00473             module->classname() << endl;
00474     else
00475         pout << "*** ERRROR ***" << endl;
00476 }
00477 
00478 
00479 } // end of namespace PLearn
00480 
00481 
00482 /*
00483   Local Variables:
00484   mode:c++
00485   c-basic-offset:4
00486   c-file-style:"stroustrup"
00487   c-file-offsets:((innamespace . 0)(inline-open . 0))
00488   indent-tabs-mode:nil
00489   fill-column:79
00490   End:
00491 */
00492 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines