PLearn 0.1
Experimentation.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // Experimentation.cc
00004 //
00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Dan Popovici
00036 
00039 #define PL_LOG_MODULE_NAME "Experimentation"
00040 #include <plearn/io/pl_log.h>
00041 
00042 #include "Experimentation.h"
00043 #include <plearn/io/load_and_save.h>                 
00044 #include <plearn/io/fileutils.h>                     
00045 #include <plearn/math/random.h>                      
00046 #include <plearn/vmat/ExplicitSplitter.h>            
00047 #include <plearn/vmat/VariableDeletionVMatrix.h>     
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00052 PLEARN_IMPLEMENT_OBJECT(
00053     Experimentation,
00054     "Computes correlation coefficient between various discrete values and the target.",
00055     "name of the discrete variable, of the target and the values to check are options.\n"
00056 );
00057 
00059 // Experimentation //
00061 Experimentation::Experimentation()
00062 {
00063 }
00064     
00066 // declareOptions //
00068 void Experimentation::declareOptions(OptionList& ol)
00069 {
00070     declareOption(ol, "save_files", &Experimentation::save_files,
00071                   OptionBase::buildoption,
00072                   "If set to 1, save the built train and test files instead of running the experiment.");
00073     declareOption(ol, "experiment_without_missing_indicator", &Experimentation::experiment_without_missing_indicator,
00074                   OptionBase::buildoption,
00075                   "If set to 1, the missing_indicator_field_names will be excluded from the built training files.");
00076     declareOption(ol, "target_field_name", &Experimentation::target_field_name,
00077                   OptionBase::buildoption,
00078                   "The name of the field to select from the target_set as target for the built training files.");
00079     declareOption(ol, "missing_indicator_field_names", &Experimentation::missing_indicator_field_names,
00080                   OptionBase::buildoption,
00081                   "The field names of the missing indicators to exclude when we experiment without them.");
00082     declareOption(ol, "experiment_name", &Experimentation::experiment_name,
00083                   OptionBase::buildoption,
00084                   "The name of the group of experiments to conduct.");
00085     declareOption(ol, "number_of_test_samples", &Experimentation::number_of_test_samples,
00086                   OptionBase::buildoption,
00087                   "The number of test samples at the beginning of the train set.");
00088     declareOption(ol, "number_of_train_samples", &Experimentation::number_of_train_samples,
00089                   OptionBase::buildoption,
00090                   "The number of train samples in the reference set to compute the % of missing.");
00091     declareOption(ol, "reference_train_set", &Experimentation::reference_train_set,
00092                   OptionBase::buildoption,
00093                   "The train and valid set with missing values to compute the % of missing.");
00094     declareOption(ol, "target_set", &Experimentation::target_set,
00095                   OptionBase::buildoption,
00096                   "The data set with the targets corresponding to the train set.");
00097     declareOption(ol, "deletion_thresholds", &Experimentation::deletion_thresholds,
00098                   OptionBase::buildoption,
00099                   "The vector of the various deletion threshold to run this experiment with.");
00100     declareOption(ol, "experiment_directory", &Experimentation::experiment_directory,
00101                   OptionBase::buildoption,
00102                   "The path in which to build the directories for the experiment's results.");
00103     declareOption(ol, "experiment_template", &Experimentation::experiment_template,
00104                   OptionBase::buildoption,
00105                   "The template of the script to conduct the experiment.");
00106 
00107     inherited::declareOptions(ol);
00108 }
00109 
00111 // makeDeepCopyFromShallowCopy //
00113 void Experimentation::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00114 {
00115     deepCopyField(experiment_name, copies);
00116     deepCopyField(number_of_test_samples, copies);
00117     deepCopyField(number_of_train_samples, copies);
00118     deepCopyField(reference_train_set, copies);
00119     deepCopyField(target_set, copies);
00120     deepCopyField(deletion_thresholds, copies);
00121     deepCopyField(experiment_directory, copies);
00122     deepCopyField(experiment_template, copies);
00123     inherited::makeDeepCopyFromShallowCopy(copies);
00124 
00125 }
00126 
00128 // build //
00130 void Experimentation::build()
00131 {
00132     // ### Nothing to add here, simply calls build_().
00133     inherited::build();
00134     build_();
00135 }
00136 
00138 // build_ //
00140 void Experimentation::build_()
00141 {
00142     MODULE_LOG << "build_() called" << endl;
00143     if (train_set)
00144     {
00145         for (int iteration = 1; iteration <= train_set->width(); iteration++)
00146         {
00147             cout << "In Experimentation, Iteration # " << iteration << endl;
00148             experimentSetUp();
00149             train();
00150             ::PLearn::save(header_expdir + "/" + deletion_threshold_str + "/source_names.psave", source_names);
00151         }
00152         PLERROR("In Experimentation::build_() we are done here");
00153     }
00154 }
00155 
00156 void Experimentation::experimentSetUp()
00157 { 
00158     // initialize primary dataset
00159     main_row = 0;
00160     main_col = 0;
00161     main_length = train_set->length();
00162     main_width = train_set->width();
00163     main_names.resize(main_width);
00164     main_names << train_set->fieldNames();
00165     if (train_set->hasMetaDataDir()) main_metadata = train_set->getMetaDataDir();
00166     else if (experiment_directory == "") PLERROR("In Experimentation::experimentSetUp() we need one of experiment_directory or train_set->metadatadir");
00167          else main_metadata = experiment_directory;
00168     if (experiment_without_missing_indicator > 0)
00169     {
00170         fields_width = missing_indicator_field_names.size();
00171         main_fields_selected.resize(main_width - fields_width);
00172         for (fields_col = 0; fields_col < fields_width; fields_col++)
00173         {
00174             for (main_col = 0; main_col < main_width; main_col++)
00175             {
00176                 if (missing_indicator_field_names[fields_col] == main_names[main_col]) break;
00177             }
00178             if (main_col >= main_width) PLERROR("In Experimentation::experimentSetUp() no field with this name in input dataset: %", (missing_indicator_field_names[fields_col]).c_str());
00179         }
00180         main_fields_selected_col = 0;
00181         for (main_col = 0; main_col < main_width; main_col++)
00182         {
00183             for (fields_col = 0; fields_col < fields_width; fields_col++)
00184             {
00185                 if (missing_indicator_field_names[fields_col] == main_names[main_col]) break;
00186             }
00187             if (fields_col < fields_width) continue;
00188             main_fields_selected[main_fields_selected_col] = main_names[main_col];
00189             main_fields_selected_col += 1;
00190         }
00191     }
00192     
00193     // initialize target dataset
00194     target_row = 0;
00195     target_col = 0;
00196     target_length = target_set->length();
00197     target_width = target_set->width();
00198     target_names.resize(target_width);
00199     target_names << target_set->fieldNames();
00200     if (target_length != main_length) PLERROR("In Experimentation::experimentSetUp() target and main train datasets should have equal length");
00201     for (target_col = 0; target_col < target_width; target_col++)
00202     {
00203         if (target_field_name == target_names[target_col]) break;
00204     }
00205     if (target_col >= target_width) PLERROR("In Experimentation::experimentSetUp() no field with this name in target dataset: %", target_field_name.c_str());
00206     
00207     // initialize the header file
00208     cout << "initialize the header file" << endl;
00209     reference_train_set->lockMetaDataDir();
00210     if (experiment_directory == "") header_expdir = main_metadata + "/Experiment/" + experiment_name;
00211     else header_expdir = experiment_directory;
00212     header_expdir += "/" + target_field_name;
00213     if (experiment_without_missing_indicator > 0) header_expdir += "/no_ind/";
00214     else header_expdir += "/ind/";
00215     header_file_name = header_expdir + "header.pmat";
00216     if (deletion_thresholds.length() <= 0)
00217     {
00218         deletion_thresholds.resize(20);
00219         for (int header_col = 0; header_col < 20; header_col++) deletion_thresholds[header_col] = (real) to_deal_with_next / 20.0;
00220     } 
00221     header_width = deletion_thresholds.length();
00222     header_record.resize(header_width);
00223     if (!isfile(header_file_name)) createHeaderFile();
00224     else getHeaderRecord();
00225     
00226     // choose deletion threshold to experiment with
00227     cout << "choose deletion threshold to experiment with" << endl;
00228     to_deal_with_total = 0;
00229     to_deal_with_next = -1;
00230     for (int header_col = 0; header_col < header_width; header_col++)
00231     {
00232         if (header_record[header_col] != 0.0) continue;
00233         to_deal_with_total += 1;
00234         if (to_deal_with_next < 0) to_deal_with_next = header_col;
00235     }
00236     if (to_deal_with_next < 0)
00237     {
00238         reference_train_set->unlockMetaDataDir();
00239         reviewGlobalStats();
00240         PLERROR("In Experimentation::experimentSetUp() we are done here");
00241     }
00242     deletion_threshold = deletion_thresholds[to_deal_with_next];
00243     deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4);
00244     cout << "total number of thresholds left to deal with: " << to_deal_with_total << endl;
00245     cout << "next thresholds to deal with: " << deletion_threshold << endl;
00246     updateHeaderRecord(to_deal_with_next);
00247     reference_train_set->unlockMetaDataDir();
00248     
00249     // build the train and test sets
00250     setSourceDataset();
00251     cout << "source data set width: " << source_set->width() << endl;
00252     main_input.resize(source_set->width());
00253     source_names.resize(source_set->width());
00254     source_names << source_set->fieldNames();
00255     source_names.resize(source_set->width() + 1);
00256     source_names[source_set->width()] = target_field_name;
00257     
00258     // load test data set
00259     ProgressBar* pb = 0;
00260     test_length = number_of_test_samples;
00261     test_width = source_set->width() + 1;
00262     test_file = new MemoryVMatrix(test_length, test_width);
00263     test_file->defineSizes(test_width - 1, 1, 0);
00264     test_record.resize(test_width);
00265     pb = new ProgressBar( "loading the test file for threshold: " + deletion_threshold_str, test_length);
00266     for (main_row = 0; main_row < test_length; main_row++)
00267     {
00268         source_set->getRow(main_row, main_input);
00269         for (main_col = 0; main_col < source_set->width(); main_col++) test_record[main_col] = main_input[main_col];
00270         test_record[source_set->width()] = target_set->get(main_row, target_col);
00271         test_file->putRow(main_row, test_record);
00272         pb->update( main_row );
00273     }
00274     delete pb;
00275     
00276     // load training and validation data set
00277     train_valid_length = main_length - test_length;
00278     train_valid_width = source_set->width() + 1;
00279     train_valid_file = new MemoryVMatrix(train_valid_length, train_valid_width);
00280     train_valid_file->defineSizes(train_valid_width - 1, 1, 0);
00281     train_valid_record.resize(train_valid_width);
00282     pb = new ProgressBar( "loading the training and validation file for threshold: " + deletion_threshold_str, train_valid_length);
00283     for (main_row = test_length; main_row < main_length; main_row++)
00284     {
00285         source_set->getRow(main_row, main_input);
00286         for (main_col = 0; main_col < source_set->width(); main_col++) train_valid_record[main_col] = main_input[main_col];
00287         train_valid_record[source_set->width()] = target_set->get(main_row, target_col);
00288         train_valid_file->putRow(main_row - test_length, train_valid_record);
00289         pb->update( main_row - test_length );
00290     }
00291     delete pb;
00292     
00293     // save files if requested
00294     if (save_files <= 0) return;
00295     VMat save_test = new FileVMatrix(header_expdir + "/" + deletion_threshold_str + "/test.pmat", test_length, test_width);
00296     save_test->declareFieldNames(source_names);
00297     pb = new ProgressBar( "saving the test file for threshold: " + deletion_threshold_str, test_length);
00298     for (main_row = 0; main_row < test_length; main_row++)
00299     {
00300         test_file->getRow(main_row, test_record);
00301         save_test->putRow(main_row, test_record);
00302         pb->update( main_row );
00303     }
00304     delete pb;
00305     VMat save_train_valid = new FileVMatrix(header_expdir + "/" + deletion_threshold_str + "/train_valid.pmat", train_valid_length, train_valid_width);
00306     save_train_valid->declareFieldNames(source_names);
00307     pb = new ProgressBar( "saving the training and validation file for threshold: " + deletion_threshold_str, train_valid_length);
00308     for (main_row = 0; main_row < train_valid_length; main_row++)
00309     {
00310         train_valid_file->getRow(main_row, train_valid_record);
00311         save_train_valid->putRow(main_row, train_valid_record);
00312         pb->update( main_row );
00313     }
00314     delete pb;
00315     PLERROR("In Experimentation::experimentSetUp() we are done here");
00316 }
00317 
00318 void Experimentation::createHeaderFile()
00319 { 
00320     header_record.clear();
00321     header_names.resize(header_width);
00322     for (int header_col = 0; header_col < header_width; header_col++) 
00323         header_names[header_col] = tostring(deletion_thresholds[header_col] + 0.005).substr(0,4);
00324     header_file = new FileVMatrix(header_file_name, 1, header_names);
00325     header_file->putRow(0, header_record);
00326 }
00327 
00328 void Experimentation::getHeaderRecord()
00329 { 
00330     header_file = new FileVMatrix(header_file_name, true);
00331     if (header_width != header_file->width()) 
00332         PLERROR("In Experimentation::getHeaderRecord() the existing header file does not match the deletion_thresholds width)");
00333     header_names = header_file->fieldNames();
00334     for (int header_col = 0; header_col < header_width; header_col++) 
00335         if (header_names[header_col] != tostring(deletion_thresholds[header_col] + 0.005).substr(0,4))
00336             PLERROR("In Experimentation::getHeaderRecord() the existing header file names does not match the deletion_thresholds values)");;
00337     header_file->getRow(0, header_record);
00338 }
00339 
00340 void Experimentation::updateHeaderRecord(int var_col)
00341 { 
00342     header_file->put(0, var_col, 1.0);
00343     header_file->flush();
00344 }
00345 
00346 void Experimentation::setSourceDataset()
00347 {
00348     VMat selected_train_set = train_set;
00349     VMat selected_reference_set= reference_train_set;
00350     if (experiment_without_missing_indicator > 0)
00351     {
00352             SelectColumnsVMatrix* new_train_set = new SelectColumnsVMatrix();
00353             new_train_set->source = train_set;
00354             new_train_set->fields = main_fields_selected;
00355             selected_train_set = new_train_set;
00356             selected_train_set->build();
00357             selected_train_set->defineSizes(selected_train_set->width(), 0, 0);
00358             SelectColumnsVMatrix* new_reference_set = new SelectColumnsVMatrix();
00359             new_reference_set->source = reference_train_set;
00360             new_reference_set->fields = main_fields_selected;
00361             selected_reference_set = new_reference_set;
00362             selected_reference_set->build();
00363             selected_reference_set->defineSizes(selected_reference_set->width(), 0, 0);
00364     }
00365     if (deletion_threshold <= 0.0)
00366     {
00367         source_set = selected_train_set;
00368         return;
00369     }
00370     VariableDeletionVMatrix* new_set = new VariableDeletionVMatrix();
00371     // VMat: The data set with all variables to select the columns from.
00372     new_set->complete_dataset = selected_train_set;
00373     // VMat: The train set in which to compute the percentage of missing values.
00374     new_set->train_set = selected_reference_set;
00375     // double: The percentage of non-missing values for a variable above which, the variable will be selected.
00376     new_set->deletion_threshold = deletion_threshold;
00377     // bool: If set to 1, the columns with constant non-missing values will be removed.
00378     new_set->remove_columns_with_constant_value = 1;
00379     // double: If equal to zero, all the train samples are used to calculated the percentages and constant values.
00380     // If it is a fraction between 0 and 1, this proportion of the samples will be used.
00381     // If greater or equal to 1, the integer portion will be interpreted as the number of samples to use.
00382     new_set->number_of_train_samples = number_of_train_samples;
00383     // int: The row at which, to start to calculate the percentages and constant values.
00384     new_set->start_row = 0;
00385     source_set = new_set;
00386     source_set->build();
00387 }
00388 
00389 void Experimentation::reviewGlobalStats()
00390 {
00391     cout << "There is no more variable to deal with." << endl;
00392     bool missing_results_file = false;
00393     for (int header_col = 0; header_col < header_width; header_col++)
00394     {
00395         deletion_threshold = deletion_thresholds[header_col];
00396         deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4);
00397         PPath expdir = header_expdir + "/" + deletion_threshold_str;
00398         PPath train_valid_results_file_name = expdir + "/Split0/LearnerExpdir/Strat0results.pmat";
00399         PPath test_results_file_name = expdir + "/global_stats.pmat";
00400         PPath source_names_file_name = expdir + "/source_names.psave";
00401         if (!isfile(train_valid_results_file_name))
00402         {
00403             cout << "Missing training and validation results for threshold " << deletion_threshold_str << endl;
00404             missing_results_file = true;
00405         }
00406         if (!isfile(test_results_file_name))
00407         {
00408             cout << "Missing test results for threshold " << deletion_threshold_str << endl;
00409             missing_results_file = true;
00410         }
00411         if (!isfile(source_names_file_name))
00412         {
00413             cout << "Missing variable selected saved file for threshold " << deletion_threshold_str << endl;
00414             missing_results_file = true;
00415         }
00416     }
00417     if (missing_results_file) return;
00418     cout << endl << endl;
00419     cout << "Results for experiment " << experiment_name << endl;
00420     cout << "       The file used for this experiment was " << main_metadata << endl;
00421     cout << "       The target used was " << target_field_name << endl;
00422     if (experiment_without_missing_indicator > 0) cout << "       The experiment was carried without missing indicators" << endl;
00423     else cout << "       The experiment was carried with missing indicators" << endl;
00424     cout << endl << endl;
00425     cout << "           number                                                                  " << endl;
00426     cout << "             of                                                                    " << endl;
00427     cout << " deletion variable   weigth    train    valid     test     test       std      test" << endl;
00428     cout << "threshold selected    decay     mse      mse      mse      cse       error     cle " << endl;
00429     cout << endl;
00430     cout << fixed << showpoint;
00431     real best_valid_mse_threshold = -1.0;
00432     real best_valid_mse_value;
00433     for (int header_col = 0; header_col < header_width; header_col++)
00434     {
00435         deletion_threshold = deletion_thresholds[header_col];
00436         deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4);
00437         PPath expdir = header_expdir + "/" + deletion_threshold_str;
00438         PPath train_valid_results_file_name = expdir + "/Split0/LearnerExpdir/Strat0results.pmat";
00439         PPath test_results_file_name = expdir + "/global_stats.pmat";
00440         PPath source_names_file_name = expdir + "/source_names.psave";
00441         ::PLearn::load(source_names_file_name, source_names);
00442         VMat train_valid_results_file = new FileVMatrix(train_valid_results_file_name);
00443         VMat test_results_file = new FileVMatrix(test_results_file_name);
00444         int train_valid_last_row = train_valid_results_file->length() - 1;
00445         real weight_decay = train_valid_results_file->get(train_valid_last_row, 2);
00446         real train_mse =    train_valid_results_file->get(train_valid_last_row, 3);
00447         real valid_mse =    train_valid_results_file->get(train_valid_last_row, 4);
00448         if (best_valid_mse_threshold < 0.0)
00449         {
00450             best_valid_mse_threshold = deletion_threshold;
00451             best_valid_mse_value = valid_mse;
00452         }
00453         else if (valid_mse < best_valid_mse_value)
00454         {
00455             best_valid_mse_threshold = deletion_threshold;
00456             best_valid_mse_value = valid_mse;
00457         }
00458         real test_mse =     test_results_file->get(1, 0);
00459         real test_cse =     test_results_file->get(1, 2);
00460         real test_cse_std = test_results_file->get(1, 3);
00461         real test_cle =     test_results_file->get(1, 4);
00462         cout << setiosflags(ios::right) << setw(9) << deletion_threshold_str << "   "
00463              << setw(4) << source_names.size() << "    "
00464              << setw(6) << weight_decay << " "
00465              << setw(6) << train_mse << " "
00466              << setw(6) << valid_mse << " "
00467              << setw(6) << test_mse << " "
00468              << setw(6) << test_cse << "+/-"
00469              << setw(6) << test_cse_std << " "
00470              << setw(6) << test_cle << endl;
00471     }
00472     cout << endl << endl;
00473     cout << "       Based on the validation mse, the suggested threshold is " << best_valid_mse_threshold << endl;
00474     cout << endl << endl;
00475 }
00476 
00477 void Experimentation::train()
00478 {
00479     PP<ExplicitSplitter> explicit_splitter = new ExplicitSplitter();
00480     explicit_splitter->splitsets.resize(1,2);
00481     explicit_splitter->splitsets(0,0) = train_valid_file;
00482     explicit_splitter->splitsets(0,1) = test_file;
00483     experiment = ::PLearn::deepCopy(experiment_template);
00484     experiment->setOption("expdir", header_expdir + "/" + deletion_threshold_str);
00485     experiment->splitter = new ExplicitSplitter();
00486     experiment->splitter = explicit_splitter;
00487     experiment->build();
00488     Vec results = experiment->perform(true);
00489 }
00490 
00491 int Experimentation::outputsize() const {return 0;}
00492 void Experimentation::computeOutput(const Vec&, Vec&) const {}
00493 void Experimentation::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {}
00494 TVec<string> Experimentation::getTestCostNames() const
00495 {
00496     TVec<string> result;
00497     result.append( "MSE" );
00498     return result;
00499 }
00500 TVec<string> Experimentation::getTrainCostNames() const
00501 {
00502     TVec<string> result;
00503     result.append( "MSE" );
00504     return result;
00505 }
00506 
00507 } // end of namespace PLearn
00508 
00509 
00510 /*
00511   Local Variables:
00512   mode:c++
00513   c-basic-offset:4
00514   c-file-style:"stroustrup"
00515   c-file-offsets:((innamespace . 0)(inline-open . 0))
00516   indent-tabs-mode:nil
00517   fill-column:79
00518   End:
00519 */
00520 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines