PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // Experimentation.cc 00004 // 00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Dan Popovici 00036 00039 #define PL_LOG_MODULE_NAME "Experimentation" 00040 #include <plearn/io/pl_log.h> 00041 00042 #include "Experimentation.h" 00043 #include <plearn/io/load_and_save.h> 00044 #include <plearn/io/fileutils.h> 00045 #include <plearn/math/random.h> 00046 #include <plearn/vmat/ExplicitSplitter.h> 00047 #include <plearn/vmat/VariableDeletionVMatrix.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00052 PLEARN_IMPLEMENT_OBJECT( 00053 Experimentation, 00054 "Computes correlation coefficient between various discrete values and the target.", 00055 "name of the discrete variable, of the target and the values to check are options.\n" 00056 ); 00057 00059 // Experimentation // 00061 Experimentation::Experimentation() 00062 { 00063 } 00064 00066 // declareOptions // 00068 void Experimentation::declareOptions(OptionList& ol) 00069 { 00070 declareOption(ol, "save_files", &Experimentation::save_files, 00071 OptionBase::buildoption, 00072 "If set to 1, save the built train and test files instead of running the experiment."); 00073 declareOption(ol, "experiment_without_missing_indicator", &Experimentation::experiment_without_missing_indicator, 00074 OptionBase::buildoption, 00075 "If set to 1, the missing_indicator_field_names will be excluded from the built training files."); 00076 declareOption(ol, "target_field_name", &Experimentation::target_field_name, 00077 OptionBase::buildoption, 00078 "The name of the field to select from the target_set as target for the built training files."); 00079 declareOption(ol, "missing_indicator_field_names", &Experimentation::missing_indicator_field_names, 00080 OptionBase::buildoption, 00081 "The field names of the missing indicators to exclude when we experiment without them."); 00082 declareOption(ol, "experiment_name", &Experimentation::experiment_name, 00083 OptionBase::buildoption, 00084 "The name of the group of experiments to conduct."); 00085 declareOption(ol, "number_of_test_samples", &Experimentation::number_of_test_samples, 00086 OptionBase::buildoption, 00087 "The number of test samples at the beginning of the train set."); 00088 declareOption(ol, "number_of_train_samples", &Experimentation::number_of_train_samples, 00089 OptionBase::buildoption, 00090 "The number of train samples in the reference set to compute the % of missing."); 00091 declareOption(ol, "reference_train_set", &Experimentation::reference_train_set, 00092 OptionBase::buildoption, 00093 "The train and valid set with missing values to compute the % of missing."); 00094 declareOption(ol, "target_set", &Experimentation::target_set, 00095 OptionBase::buildoption, 00096 "The data set with the targets corresponding to the train set."); 00097 declareOption(ol, "deletion_thresholds", &Experimentation::deletion_thresholds, 00098 OptionBase::buildoption, 00099 "The vector of the various deletion threshold to run this experiment with."); 00100 declareOption(ol, "experiment_directory", &Experimentation::experiment_directory, 00101 OptionBase::buildoption, 00102 "The path in which to build the directories for the experiment's results."); 00103 declareOption(ol, "experiment_template", &Experimentation::experiment_template, 00104 OptionBase::buildoption, 00105 "The template of the script to conduct the experiment."); 00106 00107 inherited::declareOptions(ol); 00108 } 00109 00111 // makeDeepCopyFromShallowCopy // 00113 void Experimentation::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00114 { 00115 deepCopyField(experiment_name, copies); 00116 deepCopyField(number_of_test_samples, copies); 00117 deepCopyField(number_of_train_samples, copies); 00118 deepCopyField(reference_train_set, copies); 00119 deepCopyField(target_set, copies); 00120 deepCopyField(deletion_thresholds, copies); 00121 deepCopyField(experiment_directory, copies); 00122 deepCopyField(experiment_template, copies); 00123 inherited::makeDeepCopyFromShallowCopy(copies); 00124 00125 } 00126 00128 // build // 00130 void Experimentation::build() 00131 { 00132 // ### Nothing to add here, simply calls build_(). 00133 inherited::build(); 00134 build_(); 00135 } 00136 00138 // build_ // 00140 void Experimentation::build_() 00141 { 00142 MODULE_LOG << "build_() called" << endl; 00143 if (train_set) 00144 { 00145 for (int iteration = 1; iteration <= train_set->width(); iteration++) 00146 { 00147 cout << "In Experimentation, Iteration # " << iteration << endl; 00148 experimentSetUp(); 00149 train(); 00150 ::PLearn::save(header_expdir + "/" + deletion_threshold_str + "/source_names.psave", source_names); 00151 } 00152 PLERROR("In Experimentation::build_() we are done here"); 00153 } 00154 } 00155 00156 void Experimentation::experimentSetUp() 00157 { 00158 // initialize primary dataset 00159 main_row = 0; 00160 main_col = 0; 00161 main_length = train_set->length(); 00162 main_width = train_set->width(); 00163 main_names.resize(main_width); 00164 main_names << train_set->fieldNames(); 00165 if (train_set->hasMetaDataDir()) main_metadata = train_set->getMetaDataDir(); 00166 else if (experiment_directory == "") PLERROR("In Experimentation::experimentSetUp() we need one of experiment_directory or train_set->metadatadir"); 00167 else main_metadata = experiment_directory; 00168 if (experiment_without_missing_indicator > 0) 00169 { 00170 fields_width = missing_indicator_field_names.size(); 00171 main_fields_selected.resize(main_width - fields_width); 00172 for (fields_col = 0; fields_col < fields_width; fields_col++) 00173 { 00174 for (main_col = 0; main_col < main_width; main_col++) 00175 { 00176 if (missing_indicator_field_names[fields_col] == main_names[main_col]) break; 00177 } 00178 if (main_col >= main_width) PLERROR("In Experimentation::experimentSetUp() no field with this name in input dataset: %", (missing_indicator_field_names[fields_col]).c_str()); 00179 } 00180 main_fields_selected_col = 0; 00181 for (main_col = 0; main_col < main_width; main_col++) 00182 { 00183 for (fields_col = 0; fields_col < fields_width; fields_col++) 00184 { 00185 if (missing_indicator_field_names[fields_col] == main_names[main_col]) break; 00186 } 00187 if (fields_col < fields_width) continue; 00188 main_fields_selected[main_fields_selected_col] = main_names[main_col]; 00189 main_fields_selected_col += 1; 00190 } 00191 } 00192 00193 // initialize target dataset 00194 target_row = 0; 00195 target_col = 0; 00196 target_length = target_set->length(); 00197 target_width = target_set->width(); 00198 target_names.resize(target_width); 00199 target_names << target_set->fieldNames(); 00200 if (target_length != main_length) PLERROR("In Experimentation::experimentSetUp() target and main train datasets should have equal length"); 00201 for (target_col = 0; target_col < target_width; target_col++) 00202 { 00203 if (target_field_name == target_names[target_col]) break; 00204 } 00205 if (target_col >= target_width) PLERROR("In Experimentation::experimentSetUp() no field with this name in target dataset: %", target_field_name.c_str()); 00206 00207 // initialize the header file 00208 cout << "initialize the header file" << endl; 00209 reference_train_set->lockMetaDataDir(); 00210 if (experiment_directory == "") header_expdir = main_metadata + "/Experiment/" + experiment_name; 00211 else header_expdir = experiment_directory; 00212 header_expdir += "/" + target_field_name; 00213 if (experiment_without_missing_indicator > 0) header_expdir += "/no_ind/"; 00214 else header_expdir += "/ind/"; 00215 header_file_name = header_expdir + "header.pmat"; 00216 if (deletion_thresholds.length() <= 0) 00217 { 00218 deletion_thresholds.resize(20); 00219 for (int header_col = 0; header_col < 20; header_col++) deletion_thresholds[header_col] = (real) to_deal_with_next / 20.0; 00220 } 00221 header_width = deletion_thresholds.length(); 00222 header_record.resize(header_width); 00223 if (!isfile(header_file_name)) createHeaderFile(); 00224 else getHeaderRecord(); 00225 00226 // choose deletion threshold to experiment with 00227 cout << "choose deletion threshold to experiment with" << endl; 00228 to_deal_with_total = 0; 00229 to_deal_with_next = -1; 00230 for (int header_col = 0; header_col < header_width; header_col++) 00231 { 00232 if (header_record[header_col] != 0.0) continue; 00233 to_deal_with_total += 1; 00234 if (to_deal_with_next < 0) to_deal_with_next = header_col; 00235 } 00236 if (to_deal_with_next < 0) 00237 { 00238 reference_train_set->unlockMetaDataDir(); 00239 reviewGlobalStats(); 00240 PLERROR("In Experimentation::experimentSetUp() we are done here"); 00241 } 00242 deletion_threshold = deletion_thresholds[to_deal_with_next]; 00243 deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4); 00244 cout << "total number of thresholds left to deal with: " << to_deal_with_total << endl; 00245 cout << "next thresholds to deal with: " << deletion_threshold << endl; 00246 updateHeaderRecord(to_deal_with_next); 00247 reference_train_set->unlockMetaDataDir(); 00248 00249 // build the train and test sets 00250 setSourceDataset(); 00251 cout << "source data set width: " << source_set->width() << endl; 00252 main_input.resize(source_set->width()); 00253 source_names.resize(source_set->width()); 00254 source_names << source_set->fieldNames(); 00255 source_names.resize(source_set->width() + 1); 00256 source_names[source_set->width()] = target_field_name; 00257 00258 // load test data set 00259 ProgressBar* pb = 0; 00260 test_length = number_of_test_samples; 00261 test_width = source_set->width() + 1; 00262 test_file = new MemoryVMatrix(test_length, test_width); 00263 test_file->defineSizes(test_width - 1, 1, 0); 00264 test_record.resize(test_width); 00265 pb = new ProgressBar( "loading the test file for threshold: " + deletion_threshold_str, test_length); 00266 for (main_row = 0; main_row < test_length; main_row++) 00267 { 00268 source_set->getRow(main_row, main_input); 00269 for (main_col = 0; main_col < source_set->width(); main_col++) test_record[main_col] = main_input[main_col]; 00270 test_record[source_set->width()] = target_set->get(main_row, target_col); 00271 test_file->putRow(main_row, test_record); 00272 pb->update( main_row ); 00273 } 00274 delete pb; 00275 00276 // load training and validation data set 00277 train_valid_length = main_length - test_length; 00278 train_valid_width = source_set->width() + 1; 00279 train_valid_file = new MemoryVMatrix(train_valid_length, train_valid_width); 00280 train_valid_file->defineSizes(train_valid_width - 1, 1, 0); 00281 train_valid_record.resize(train_valid_width); 00282 pb = new ProgressBar( "loading the training and validation file for threshold: " + deletion_threshold_str, train_valid_length); 00283 for (main_row = test_length; main_row < main_length; main_row++) 00284 { 00285 source_set->getRow(main_row, main_input); 00286 for (main_col = 0; main_col < source_set->width(); main_col++) train_valid_record[main_col] = main_input[main_col]; 00287 train_valid_record[source_set->width()] = target_set->get(main_row, target_col); 00288 train_valid_file->putRow(main_row - test_length, train_valid_record); 00289 pb->update( main_row - test_length ); 00290 } 00291 delete pb; 00292 00293 // save files if requested 00294 if (save_files <= 0) return; 00295 VMat save_test = new FileVMatrix(header_expdir + "/" + deletion_threshold_str + "/test.pmat", test_length, test_width); 00296 save_test->declareFieldNames(source_names); 00297 pb = new ProgressBar( "saving the test file for threshold: " + deletion_threshold_str, test_length); 00298 for (main_row = 0; main_row < test_length; main_row++) 00299 { 00300 test_file->getRow(main_row, test_record); 00301 save_test->putRow(main_row, test_record); 00302 pb->update( main_row ); 00303 } 00304 delete pb; 00305 VMat save_train_valid = new FileVMatrix(header_expdir + "/" + deletion_threshold_str + "/train_valid.pmat", train_valid_length, train_valid_width); 00306 save_train_valid->declareFieldNames(source_names); 00307 pb = new ProgressBar( "saving the training and validation file for threshold: " + deletion_threshold_str, train_valid_length); 00308 for (main_row = 0; main_row < train_valid_length; main_row++) 00309 { 00310 train_valid_file->getRow(main_row, train_valid_record); 00311 save_train_valid->putRow(main_row, train_valid_record); 00312 pb->update( main_row ); 00313 } 00314 delete pb; 00315 PLERROR("In Experimentation::experimentSetUp() we are done here"); 00316 } 00317 00318 void Experimentation::createHeaderFile() 00319 { 00320 header_record.clear(); 00321 header_names.resize(header_width); 00322 for (int header_col = 0; header_col < header_width; header_col++) 00323 header_names[header_col] = tostring(deletion_thresholds[header_col] + 0.005).substr(0,4); 00324 header_file = new FileVMatrix(header_file_name, 1, header_names); 00325 header_file->putRow(0, header_record); 00326 } 00327 00328 void Experimentation::getHeaderRecord() 00329 { 00330 header_file = new FileVMatrix(header_file_name, true); 00331 if (header_width != header_file->width()) 00332 PLERROR("In Experimentation::getHeaderRecord() the existing header file does not match the deletion_thresholds width)"); 00333 header_names = header_file->fieldNames(); 00334 for (int header_col = 0; header_col < header_width; header_col++) 00335 if (header_names[header_col] != tostring(deletion_thresholds[header_col] + 0.005).substr(0,4)) 00336 PLERROR("In Experimentation::getHeaderRecord() the existing header file names does not match the deletion_thresholds values)");; 00337 header_file->getRow(0, header_record); 00338 } 00339 00340 void Experimentation::updateHeaderRecord(int var_col) 00341 { 00342 header_file->put(0, var_col, 1.0); 00343 header_file->flush(); 00344 } 00345 00346 void Experimentation::setSourceDataset() 00347 { 00348 VMat selected_train_set = train_set; 00349 VMat selected_reference_set= reference_train_set; 00350 if (experiment_without_missing_indicator > 0) 00351 { 00352 SelectColumnsVMatrix* new_train_set = new SelectColumnsVMatrix(); 00353 new_train_set->source = train_set; 00354 new_train_set->fields = main_fields_selected; 00355 selected_train_set = new_train_set; 00356 selected_train_set->build(); 00357 selected_train_set->defineSizes(selected_train_set->width(), 0, 0); 00358 SelectColumnsVMatrix* new_reference_set = new SelectColumnsVMatrix(); 00359 new_reference_set->source = reference_train_set; 00360 new_reference_set->fields = main_fields_selected; 00361 selected_reference_set = new_reference_set; 00362 selected_reference_set->build(); 00363 selected_reference_set->defineSizes(selected_reference_set->width(), 0, 0); 00364 } 00365 if (deletion_threshold <= 0.0) 00366 { 00367 source_set = selected_train_set; 00368 return; 00369 } 00370 VariableDeletionVMatrix* new_set = new VariableDeletionVMatrix(); 00371 // VMat: The data set with all variables to select the columns from. 00372 new_set->complete_dataset = selected_train_set; 00373 // VMat: The train set in which to compute the percentage of missing values. 00374 new_set->train_set = selected_reference_set; 00375 // double: The percentage of non-missing values for a variable above which, the variable will be selected. 00376 new_set->deletion_threshold = deletion_threshold; 00377 // bool: If set to 1, the columns with constant non-missing values will be removed. 00378 new_set->remove_columns_with_constant_value = 1; 00379 // double: If equal to zero, all the train samples are used to calculated the percentages and constant values. 00380 // If it is a fraction between 0 and 1, this proportion of the samples will be used. 00381 // If greater or equal to 1, the integer portion will be interpreted as the number of samples to use. 00382 new_set->number_of_train_samples = number_of_train_samples; 00383 // int: The row at which, to start to calculate the percentages and constant values. 00384 new_set->start_row = 0; 00385 source_set = new_set; 00386 source_set->build(); 00387 } 00388 00389 void Experimentation::reviewGlobalStats() 00390 { 00391 cout << "There is no more variable to deal with." << endl; 00392 bool missing_results_file = false; 00393 for (int header_col = 0; header_col < header_width; header_col++) 00394 { 00395 deletion_threshold = deletion_thresholds[header_col]; 00396 deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4); 00397 PPath expdir = header_expdir + "/" + deletion_threshold_str; 00398 PPath train_valid_results_file_name = expdir + "/Split0/LearnerExpdir/Strat0results.pmat"; 00399 PPath test_results_file_name = expdir + "/global_stats.pmat"; 00400 PPath source_names_file_name = expdir + "/source_names.psave"; 00401 if (!isfile(train_valid_results_file_name)) 00402 { 00403 cout << "Missing training and validation results for threshold " << deletion_threshold_str << endl; 00404 missing_results_file = true; 00405 } 00406 if (!isfile(test_results_file_name)) 00407 { 00408 cout << "Missing test results for threshold " << deletion_threshold_str << endl; 00409 missing_results_file = true; 00410 } 00411 if (!isfile(source_names_file_name)) 00412 { 00413 cout << "Missing variable selected saved file for threshold " << deletion_threshold_str << endl; 00414 missing_results_file = true; 00415 } 00416 } 00417 if (missing_results_file) return; 00418 cout << endl << endl; 00419 cout << "Results for experiment " << experiment_name << endl; 00420 cout << " The file used for this experiment was " << main_metadata << endl; 00421 cout << " The target used was " << target_field_name << endl; 00422 if (experiment_without_missing_indicator > 0) cout << " The experiment was carried without missing indicators" << endl; 00423 else cout << " The experiment was carried with missing indicators" << endl; 00424 cout << endl << endl; 00425 cout << " number " << endl; 00426 cout << " of " << endl; 00427 cout << " deletion variable weigth train valid test test std test" << endl; 00428 cout << "threshold selected decay mse mse mse cse error cle " << endl; 00429 cout << endl; 00430 cout << fixed << showpoint; 00431 real best_valid_mse_threshold = -1.0; 00432 real best_valid_mse_value; 00433 for (int header_col = 0; header_col < header_width; header_col++) 00434 { 00435 deletion_threshold = deletion_thresholds[header_col]; 00436 deletion_threshold_str = tostring(deletion_threshold + 0.005).substr(0,4); 00437 PPath expdir = header_expdir + "/" + deletion_threshold_str; 00438 PPath train_valid_results_file_name = expdir + "/Split0/LearnerExpdir/Strat0results.pmat"; 00439 PPath test_results_file_name = expdir + "/global_stats.pmat"; 00440 PPath source_names_file_name = expdir + "/source_names.psave"; 00441 ::PLearn::load(source_names_file_name, source_names); 00442 VMat train_valid_results_file = new FileVMatrix(train_valid_results_file_name); 00443 VMat test_results_file = new FileVMatrix(test_results_file_name); 00444 int train_valid_last_row = train_valid_results_file->length() - 1; 00445 real weight_decay = train_valid_results_file->get(train_valid_last_row, 2); 00446 real train_mse = train_valid_results_file->get(train_valid_last_row, 3); 00447 real valid_mse = train_valid_results_file->get(train_valid_last_row, 4); 00448 if (best_valid_mse_threshold < 0.0) 00449 { 00450 best_valid_mse_threshold = deletion_threshold; 00451 best_valid_mse_value = valid_mse; 00452 } 00453 else if (valid_mse < best_valid_mse_value) 00454 { 00455 best_valid_mse_threshold = deletion_threshold; 00456 best_valid_mse_value = valid_mse; 00457 } 00458 real test_mse = test_results_file->get(1, 0); 00459 real test_cse = test_results_file->get(1, 2); 00460 real test_cse_std = test_results_file->get(1, 3); 00461 real test_cle = test_results_file->get(1, 4); 00462 cout << setiosflags(ios::right) << setw(9) << deletion_threshold_str << " " 00463 << setw(4) << source_names.size() << " " 00464 << setw(6) << weight_decay << " " 00465 << setw(6) << train_mse << " " 00466 << setw(6) << valid_mse << " " 00467 << setw(6) << test_mse << " " 00468 << setw(6) << test_cse << "+/-" 00469 << setw(6) << test_cse_std << " " 00470 << setw(6) << test_cle << endl; 00471 } 00472 cout << endl << endl; 00473 cout << " Based on the validation mse, the suggested threshold is " << best_valid_mse_threshold << endl; 00474 cout << endl << endl; 00475 } 00476 00477 void Experimentation::train() 00478 { 00479 PP<ExplicitSplitter> explicit_splitter = new ExplicitSplitter(); 00480 explicit_splitter->splitsets.resize(1,2); 00481 explicit_splitter->splitsets(0,0) = train_valid_file; 00482 explicit_splitter->splitsets(0,1) = test_file; 00483 experiment = ::PLearn::deepCopy(experiment_template); 00484 experiment->setOption("expdir", header_expdir + "/" + deletion_threshold_str); 00485 experiment->splitter = new ExplicitSplitter(); 00486 experiment->splitter = explicit_splitter; 00487 experiment->build(); 00488 Vec results = experiment->perform(true); 00489 } 00490 00491 int Experimentation::outputsize() const {return 0;} 00492 void Experimentation::computeOutput(const Vec&, Vec&) const {} 00493 void Experimentation::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {} 00494 TVec<string> Experimentation::getTestCostNames() const 00495 { 00496 TVec<string> result; 00497 result.append( "MSE" ); 00498 return result; 00499 } 00500 TVec<string> Experimentation::getTrainCostNames() const 00501 { 00502 TVec<string> result; 00503 result.append( "MSE" ); 00504 return result; 00505 } 00506 00507 } // end of namespace PLearn 00508 00509 00510 /* 00511 Local Variables: 00512 mode:c++ 00513 c-basic-offset:4 00514 c-file-style:"stroustrup" 00515 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00516 indent-tabs-mode:nil 00517 fill-column:79 00518 End: 00519 */ 00520 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :