PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // Preprocessing.cc 00004 // 00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Dan Popovici 00036 00039 #define PL_LOG_MODULE_NAME "Preprocessing" 00040 00041 #include "Preprocessing.h" 00042 #include <plearn/io/pl_log.h> 00043 #include <plearn/io/load_and_save.h> 00044 #include <plearn/io/fileutils.h> 00045 #include <plearn/math/random.h> 00046 #include <plearn/vmat/ExplicitSplitter.h> 00047 #include <plearn/vmat/VariableDeletionVMatrix.h> 00048 #include <plearn/vmat/BootstrapVMatrix.h> 00049 00050 namespace PLearn { 00051 using namespace std; 00052 00053 PLEARN_IMPLEMENT_OBJECT( 00054 Preprocessing, 00055 "Computes correlation coefficient between various discrete values and the target.", 00056 "name of the discrete variable, of the target and the values to check are options.\n" 00057 ); 00058 00060 // Preprocessing // 00062 Preprocessing::Preprocessing() 00063 { 00064 } 00065 00067 // declareOptions // 00069 void Preprocessing::declareOptions(OptionList& ol) 00070 { 00071 declareOption(ol, "test_set", &Preprocessing::test_set, 00072 OptionBase::buildoption, 00073 "The test data set.\n"); 00074 declareOption(ol, "unknown_set", &Preprocessing::unknown_set, 00075 OptionBase::buildoption, 00076 "The unknown data set.\n"); 00077 declareOption(ol, "compute_target_learner_template", &Preprocessing::compute_target_learner_template, 00078 OptionBase::buildoption, 00079 "The template of the script to generate the class target.\n"); 00080 declareOption(ol, "fix_binary_variables_template", &Preprocessing::fix_binary_variables_template, 00081 OptionBase::buildoption, 00082 "The template of the script to fix the binary variables.\n"); 00083 declareOption(ol, "imputation_spec", &Preprocessing::imputation_spec, 00084 OptionBase::buildoption, 00085 "Pairs of instruction of the form field_name : mean | median | mode.\n"); 00086 declareOption(ol, "discrete_variable_instructions", &Preprocessing::discrete_variable_instructions, 00087 OptionBase::buildoption, 00088 "The instructions to dichotomize the variables in the form of field_name : TVec<pair>.\n" 00089 "The pairs are values from : to, each creating a 0, 1 variable.\n" 00090 "Variables with no specification will be kept as_is.\n"); 00091 declareOption(ol, "selected_variables_for_input", &Preprocessing::selected_variables_for_input, 00092 OptionBase::buildoption, 00093 "The list of variables selected as input vector.\n"); 00094 declareOption(ol, "selected_variables_for_target", &Preprocessing::selected_variables_for_target, 00095 OptionBase::buildoption, 00096 "The list of variables selected as target vector.\n"); 00097 declareOption(ol, "inputs_excluded_from_gaussianization", &Preprocessing::inputs_excluded_from_gaussianization, 00098 OptionBase::buildoption, 00099 "The list of input variables excluded from the gaussianization step.\n"); 00100 declareOption(ol, "targets_excluded_from_gaussianization", &Preprocessing::targets_excluded_from_gaussianization, 00101 OptionBase::buildoption, 00102 "The list of target variables excluded from the gaussianization step.\n"); 00103 00104 inherited::declareOptions(ol); 00105 } 00106 00108 // makeDeepCopyFromShallowCopy // 00110 void Preprocessing::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00111 { 00112 deepCopyField(test_set, copies); 00113 deepCopyField(unknown_set, copies); 00114 deepCopyField(compute_target_learner_template, copies); 00115 deepCopyField(fix_binary_variables_template, copies); 00116 deepCopyField(imputation_spec, copies); 00117 deepCopyField(discrete_variable_instructions, copies); 00118 deepCopyField(selected_variables_for_input, copies); 00119 deepCopyField(selected_variables_for_target, copies); 00120 deepCopyField(inputs_excluded_from_gaussianization, copies); 00121 deepCopyField(targets_excluded_from_gaussianization, copies); 00122 inherited::makeDeepCopyFromShallowCopy(copies); 00123 00124 } 00125 00127 // build // 00129 void Preprocessing::build() 00130 { 00131 // ### Nothing to add here, simply calls build_(). 00132 inherited::build(); 00133 build_(); 00134 } 00135 00137 // build_ // 00139 void Preprocessing::build_() 00140 { 00141 MODULE_LOG << "build_() called" << endl; 00142 if (train_set) 00143 { 00144 manageTrainTestUnknownSets(); 00145 PLERROR("In Preprocessing: Everything completed successfuly, we are done here"); 00146 } 00147 } 00148 00149 void Preprocessing::manageTrainTestUnknownSets() 00150 { 00151 00152 // defining all the variables for the train set 00153 PPath output_path; 00154 PPath train_with_class_target_file_name; 00155 VMat train_with_class_target_file; 00156 PP<ComputeDond2Target> compute_target_learner; 00157 VMat train_shuffled_file; 00158 PPath train_with_binary_fixed_file_name; 00159 VMat train_with_binary_fixed_file; 00160 PP<FixDond2BinaryVariables> fix_binary_variables_learner; 00161 PPath train_with_ind_file_name; 00162 VMat train_with_ind_vmat; 00163 VMat train_with_ind_file; 00164 Vec train_with_ind_vector; 00165 VMat mean_median_mode_with_ind_file; 00166 PPath train_with_dichotomies_file_name; 00167 VMat train_with_dichotomies_file; 00168 PPath mean_median_mode_with_dichotmies_file_name; 00169 VMat mean_median_mode_with_dichotmies_file; 00170 PP<DichotomizeDond2DiscreteVariables> dichotomize_discrete_variables_learner; 00171 SelectColumnsVMatrix* train_with_selected_columns_vmatrix; 00172 VMat train_with_selected_columns_vmat; 00173 VMat mean_median_mode_with_selected_columns_vmat; 00174 VMat train_gaussianized_vmat; 00175 GaussianizeVMatrix* mean_median_mode_gaussianized_vmatrix; 00176 VMat mean_median_mode_gaussianized_vmat; 00177 PPath train_input_preprocessed_file_name; 00178 VMat train_input_preprocessed_file; 00179 Vec train_input_preprocessed_vector; 00180 PPath mean_median_mode_input_preprocessed_file_name; 00181 VMat mean_median_mode_input_preprocessed_file; 00182 Vec mean_median_mode_input_preprocessed_vector; 00183 SelectColumnsVMatrix* train_target_with_selected_columns_vmatrix; 00184 VMat train_target_with_selected_columns_vmat; 00185 GaussianizeVMatrix* train_target_gaussianized_vmatrix; 00186 VMat train_target_gaussianized_vmat; 00187 PPath train_target_preprocessed_file_name; 00188 VMat train_target_preprocessed_file; 00189 Vec train_target_preprocessed_vector; 00190 ProgressBar* pb = 0; 00191 00192 // managing the train set 00193 cout << "In Preprocessing: we start by formatting the training set" << endl; 00194 cout << endl << "****** STEP 1 ******" << endl; 00195 cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl; 00196 cout << "It uses ComputeDond2Target to transform base_train.pmat into step1_train_with_class_target.pmat" << endl; 00197 output_path = expdir+"step1_train_with_class_target"; 00198 cout << "output_path" << output_path; 00199 train_with_class_target_file_name = output_path + ".pmat"; 00200 if (isfile(train_with_class_target_file_name)) 00201 { 00202 train_with_class_target_file = new FileVMatrix(train_with_class_target_file_name); 00203 train_with_class_target_file->defineSizes(train_with_class_target_file->width(), 0, 0); 00204 cout << train_with_class_target_file_name << " already exist, we are skipping this step." << endl; 00205 } 00206 else 00207 { 00208 compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template); 00209 compute_target_learner->unknown_sales = 0; 00210 compute_target_learner->output_path = output_path; 00211 compute_target_learner->setTrainingSet(train_set, true); 00212 train_with_class_target_file = compute_target_learner->getOutputFile(); 00213 } 00214 cout << endl << "****** STEP 2 ******" << endl; 00215 cout << "This step shuffles the training set to get training data in random order." << endl; 00216 cout << "It uses BootstrapVMatrix to transform step1_train_with_class_target.pmat" << endl; 00217 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 3" << endl; 00218 output_path = expdir+"step3_train_with_binary_fixed"; 00219 train_with_binary_fixed_file_name = output_path + ".pmat"; 00220 if (isfile(train_with_binary_fixed_file_name)) 00221 { 00222 cout << train_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl; 00223 } 00224 else 00225 { 00226 BootstrapVMatrix* train_shufffled_vmatrix = new BootstrapVMatrix(); 00227 train_shufffled_vmatrix->shuffle = 1; 00228 train_shufffled_vmatrix->frac = 1.0; 00229 train_shufffled_vmatrix->own_seed = 123456; 00230 train_shufffled_vmatrix->source = train_with_class_target_file; 00231 train_shufffled_vmatrix->build(); 00232 train_shuffled_file = train_shufffled_vmatrix; 00233 } 00234 cout << endl << "****** STEP 3 ******" << endl; 00235 cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl; 00236 cout << "This step uses FixDond2BinaryVariables to create step3_train_with_binary_fixed.pmat with 0-1 binary variables." << endl; 00237 if (isfile(train_with_binary_fixed_file_name)) 00238 { 00239 train_with_binary_fixed_file = new FileVMatrix(train_with_binary_fixed_file_name); 00240 train_with_binary_fixed_file->defineSizes(train_with_binary_fixed_file->width(), 0, 0); 00241 cout << train_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl; 00242 } 00243 else 00244 { 00245 fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template); 00246 fix_binary_variables_learner->output_path = output_path; 00247 fix_binary_variables_learner->setTrainingSet(train_shuffled_file, true); 00248 train_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile(); 00249 } 00250 cout << endl << "****** STEP 4 ******" << endl; 00251 cout << "This step adds missing indicators variables to each variable with missing values." << endl; 00252 cout << "It uses MissingIndicatorVMatrix to transform step3_train_with_binary_fixed.pmat" << endl; 00253 cout << "The resulting vitual view is stored in step4_train_with_ind.pmat." << endl; 00254 train_with_ind_file_name = "step4_train_with_ind.pmat"; 00255 if (isfile(train_with_ind_file_name)) 00256 { 00257 train_with_ind_file = new FileVMatrix(train_with_ind_file_name); 00258 train_with_ind_file->defineSizes(train_with_ind_file->width(), 0, 0); 00259 cout << train_with_ind_file_name << " already exist, we are skipping this step." << endl; 00260 } 00261 else 00262 { 00263 MissingIndicatorVMatrix* train_with_ind_vmatrix = new MissingIndicatorVMatrix(); 00264 train_with_ind_vmatrix->source = train_with_binary_fixed_file; 00265 train_with_ind_vmatrix->train_set = train_with_binary_fixed_file; 00266 train_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0; 00267 train_with_ind_vmatrix->build(); 00268 train_with_ind_vmat = train_with_ind_vmatrix; 00269 train_with_ind_file = new FileVMatrix(train_with_ind_file_name, train_with_ind_vmat->length(), train_with_ind_vmat->fieldNames()); 00270 train_with_ind_file->defineSizes(train_with_ind_vmat->inputsize(), train_with_ind_vmat->targetsize(), train_with_ind_vmat->weightsize()); 00271 pb = new ProgressBar("Saving the train file with missing indicators", train_with_ind_vmat->length()); 00272 train_with_ind_vector.resize(train_with_ind_vmat->width()); 00273 for (int train_with_ind_row = 0; train_with_ind_row < train_with_ind_vmat->length(); train_with_ind_row++) 00274 { 00275 train_with_ind_vmat->getRow(train_with_ind_row, train_with_ind_vector); 00276 train_with_ind_file->putRow(train_with_ind_row, train_with_ind_vector); 00277 pb->update( train_with_ind_row ); 00278 } 00279 delete pb; 00280 } 00281 cout << endl << "****** STEP 5 ******" << endl; 00282 cout << "This step computes the mean, median and mode vectors on step4_train_with_ind.pmat." << endl; 00283 cout << "The vectors are kept in the mean_median_mode_file.pmat of the metadata." << endl; 00284 cout << "It uses MeanMedianModeImputationVMatrix to do that" << endl; 00285 cout << "The resulting vitual view is not used." << endl; 00286 cout << "But the mean, median and mode vectors have to go thru the same transformation than the training file" << endl; 00287 cout << "from here on to the end of the preprocessing steps.." << endl; 00288 { 00289 MeanMedianModeImputationVMatrix* train_with_imp_vmatrix = new MeanMedianModeImputationVMatrix(); 00290 train_with_imp_vmatrix->source = train_with_ind_file; 00291 train_with_imp_vmatrix->train_set = train_with_ind_file; 00292 train_with_imp_vmatrix->number_of_train_samples_to_use = 30000.0; 00293 train_with_imp_vmatrix->imputation_spec = imputation_spec; 00294 train_with_imp_vmatrix->build(); 00295 mean_median_mode_with_ind_file = train_with_imp_vmatrix->getMeanMedianModeFile(); 00296 } 00297 cout << endl << "****** STEP 6 ******" << endl; 00298 cout << "This steps generates as many dichotomized variables as there are significant code values." << endl; 00299 cout << "It uses DichotomizeDond2DiscreteVariables to transform step4_train_with_ind.pmat into step6_train_with_dichotomies.pmat" << endl; 00300 output_path = expdir+"step6_train_with_dichotomies"; 00301 train_with_dichotomies_file_name = output_path + ".pmat"; 00302 if (isfile(train_with_dichotomies_file_name)) 00303 { 00304 train_with_dichotomies_file = new FileVMatrix(train_with_dichotomies_file_name); 00305 train_with_dichotomies_file->defineSizes(train_with_dichotomies_file->width(), 0, 0); 00306 cout << train_with_dichotomies_file_name << " already exist, we are skipping this step." << endl; 00307 } 00308 else 00309 { 00310 dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables(); 00311 dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions; 00312 dichotomize_discrete_variables_learner->output_path = output_path; 00313 dichotomize_discrete_variables_learner->setTrainingSet(train_with_ind_file, true); 00314 train_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile(); 00315 } 00316 cout << endl << "****** STEP 7 ******" << endl; 00317 cout << "This steps does the same thing to the mean, median and mode vectors." << endl; 00318 cout << "It uses DichotomizeDond2DiscreteVariables to transform step4_train_with_ind.pmat.metadata/mean_median_mode_file.pmat " 00319 << "into step6_train_with_dichotomies.pmat.metadata/mean_median_mode_file.pmat" << endl; 00320 output_path = expdir+train_with_dichotomies_file_name + ".metadata/mean_median_mode_file"; 00321 mean_median_mode_with_dichotmies_file_name = output_path + ".pmat"; 00322 if (isfile(mean_median_mode_with_dichotmies_file_name)) 00323 { 00324 mean_median_mode_with_dichotmies_file = new FileVMatrix(mean_median_mode_with_dichotmies_file_name); 00325 mean_median_mode_with_dichotmies_file->defineSizes(mean_median_mode_with_dichotmies_file->width(), 0, 0); 00326 cout << mean_median_mode_with_dichotmies_file_name << " already exist, we are skipping this step." << endl; 00327 } 00328 else 00329 { 00330 dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables(); 00331 dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions; 00332 dichotomize_discrete_variables_learner->output_path = output_path; 00333 dichotomize_discrete_variables_learner->setTrainingSet(mean_median_mode_with_ind_file, true); 00334 mean_median_mode_with_dichotmies_file = dichotomize_discrete_variables_learner->getOutputFile(); 00335 } 00336 cout << endl << "****** STEP 8 ******" << endl; 00337 cout << "This step select the desired columns from the training set to create the input records." << endl; 00338 cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat" << endl; 00339 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl; 00340 output_path = expdir+"final_train_input_preprocessed"; 00341 train_input_preprocessed_file_name = output_path + ".pmat"; 00342 if (isfile(train_input_preprocessed_file_name)) 00343 { 00344 cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00345 } 00346 else 00347 { 00348 train_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00349 train_with_selected_columns_vmatrix->source = train_with_dichotomies_file; 00350 train_with_selected_columns_vmatrix->fields_partial_match = 0; 00351 train_with_selected_columns_vmatrix->extend_with_missing = 0; 00352 train_with_selected_columns_vmatrix->fields = selected_variables_for_input; 00353 train_with_selected_columns_vmatrix->build(); 00354 train_with_selected_columns_vmatrix->defineSizes(train_with_selected_columns_vmatrix->width(), 0, 0); 00355 train_with_selected_columns_vmat = train_with_selected_columns_vmatrix; 00356 } 00357 cout << endl << "****** STEP 9 ******" << endl; 00358 cout << "This step does the same thing to the mean, median and mode vectors." << endl; 00359 cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat.metadata/mean_median_mode_file.pmat" << endl; 00360 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 11" << endl; 00361 output_path = expdir+train_input_preprocessed_file_name + ".metadata/mean_median_mode_file"; 00362 mean_median_mode_input_preprocessed_file_name = output_path + ".pmat"; 00363 if (isfile(mean_median_mode_input_preprocessed_file_name)) 00364 { 00365 cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00366 } 00367 else 00368 { 00369 SelectColumnsVMatrix* mean_median_mode_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00370 mean_median_mode_with_selected_columns_vmatrix->source = mean_median_mode_with_dichotmies_file; 00371 mean_median_mode_with_selected_columns_vmatrix->fields_partial_match = 0; 00372 mean_median_mode_with_selected_columns_vmatrix->extend_with_missing = 0; 00373 mean_median_mode_with_selected_columns_vmatrix->fields = selected_variables_for_input; 00374 mean_median_mode_with_selected_columns_vmatrix->build(); 00375 mean_median_mode_with_selected_columns_vmatrix->defineSizes(mean_median_mode_with_selected_columns_vmatrix->width(), 0, 0); 00376 mean_median_mode_with_selected_columns_vmat = mean_median_mode_with_selected_columns_vmatrix; 00377 } 00378 cout << endl << "****** STEP 10 ******" << endl; 00379 cout << "This gaussianizes the input records." << endl; 00380 cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl; 00381 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl; 00382 if (isfile(train_input_preprocessed_file_name)) 00383 { 00384 cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00385 } 00386 else 00387 { 00388 GaussianizeVMatrix* train_gaussianized_vmatrix = new GaussianizeVMatrix(); 00389 train_gaussianized_vmatrix->source = train_with_selected_columns_vmat; 00390 train_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat; 00391 train_gaussianized_vmatrix->threshold_ratio = 1; 00392 train_gaussianized_vmatrix->gaussianize_input = 1; 00393 train_gaussianized_vmatrix->gaussianize_target = 0; 00394 train_gaussianized_vmatrix->gaussianize_weight = 0; 00395 train_gaussianized_vmatrix->gaussianize_extra = 0; 00396 train_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization; 00397 train_gaussianized_vmatrix->build(); 00398 train_gaussianized_vmat = train_gaussianized_vmatrix; 00399 } 00400 cout << endl << "****** STEP 11 ******" << endl; 00401 cout << "This step does the same thing to the mean, median and mode vectors." << endl; 00402 cout << "It uses the vmat from step 9" << endl; 00403 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 13" << endl; 00404 if (isfile(mean_median_mode_input_preprocessed_file_name)) 00405 { 00406 cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00407 } 00408 else 00409 { 00410 mean_median_mode_gaussianized_vmatrix = new GaussianizeVMatrix(); 00411 mean_median_mode_gaussianized_vmatrix->source = mean_median_mode_with_selected_columns_vmat; 00412 mean_median_mode_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat; 00413 mean_median_mode_gaussianized_vmatrix->threshold_ratio = 1; 00414 mean_median_mode_gaussianized_vmatrix->gaussianize_input = 1; 00415 mean_median_mode_gaussianized_vmatrix->gaussianize_target = 0; 00416 mean_median_mode_gaussianized_vmatrix->gaussianize_weight = 0; 00417 mean_median_mode_gaussianized_vmatrix->gaussianize_extra = 0; 00418 mean_median_mode_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization; 00419 mean_median_mode_gaussianized_vmatrix->build(); 00420 mean_median_mode_gaussianized_vmat = mean_median_mode_gaussianized_vmatrix; 00421 } 00422 cout << endl << "****** STEP 12 ******" << endl; 00423 cout << "Finaly, the preprocessed input vectors are store on disk." << endl; 00424 cout << "The vmat from step 10 is converted to final_train_input_preprocessed.pmat." << endl; 00425 if (isfile(train_input_preprocessed_file_name)) 00426 { 00427 cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00428 } 00429 else 00430 { 00431 train_input_preprocessed_file = new FileVMatrix(train_input_preprocessed_file_name, train_gaussianized_vmat->length(), train_gaussianized_vmat->fieldNames()); 00432 train_input_preprocessed_file->defineSizes(train_gaussianized_vmat->inputsize(), train_gaussianized_vmat->targetsize(), train_gaussianized_vmat->weightsize()); 00433 pb = new ProgressBar("Saving the final train preprocessed input records", train_gaussianized_vmat->length()); 00434 train_input_preprocessed_vector.resize(train_gaussianized_vmat->width()); 00435 for (int train_gaussianized_row = 0; train_gaussianized_row < train_gaussianized_vmat->length(); train_gaussianized_row++) 00436 { 00437 train_gaussianized_vmat->getRow(train_gaussianized_row, train_input_preprocessed_vector); 00438 train_input_preprocessed_file->putRow(train_gaussianized_row, train_input_preprocessed_vector); 00439 pb->update( train_gaussianized_row ); 00440 } 00441 delete pb; 00442 } 00443 cout << endl << "****** STEP 13 ******" << endl; 00444 cout << "And we do the same for the mean, median and mode vectors." << endl; 00445 cout << "The vmat from step 11 is converted to final_train_input_preprocessed.pmat.metadata/men_median_mode_file.pmat" << endl; 00446 if (isfile(mean_median_mode_input_preprocessed_file_name)) 00447 { 00448 cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00449 } 00450 else 00451 { 00452 mean_median_mode_input_preprocessed_file = 00453 new FileVMatrix(mean_median_mode_input_preprocessed_file_name, mean_median_mode_gaussianized_vmat->length(), mean_median_mode_gaussianized_vmat->fieldNames()); 00454 mean_median_mode_input_preprocessed_file->defineSizes(mean_median_mode_gaussianized_vmat->inputsize(), 00455 mean_median_mode_gaussianized_vmat->targetsize(), mean_median_mode_gaussianized_vmat->weightsize()); 00456 pb = new ProgressBar("Saving the final mean,median and mode preprocessed input vectors", mean_median_mode_gaussianized_vmat->length()); 00457 mean_median_mode_input_preprocessed_vector.resize(mean_median_mode_gaussianized_vmat->width()); 00458 for (int mean_median_mode_gaussianized_row = 0; mean_median_mode_gaussianized_row < mean_median_mode_gaussianized_vmat->length(); mean_median_mode_gaussianized_row++) 00459 { 00460 mean_median_mode_gaussianized_vmat->getRow(mean_median_mode_gaussianized_row, mean_median_mode_input_preprocessed_vector); 00461 mean_median_mode_input_preprocessed_file->putRow(mean_median_mode_gaussianized_row, mean_median_mode_input_preprocessed_vector); 00462 pb->update( mean_median_mode_gaussianized_row ); 00463 } 00464 delete pb; 00465 } 00466 cout << endl << "****** STEP 14 ******" << endl; 00467 cout << "This step select the desired columns from the training set to create the target records." << endl; 00468 cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat" << endl; 00469 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 15" << endl; 00470 output_path = expdir+"final_train_target_preprocessed"; 00471 train_target_preprocessed_file_name = output_path + ".pmat"; 00472 if (isfile(train_target_preprocessed_file_name)) 00473 { 00474 cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00475 } 00476 else 00477 { 00478 train_target_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00479 train_target_with_selected_columns_vmatrix->source = train_with_dichotomies_file; 00480 train_target_with_selected_columns_vmatrix->fields_partial_match = 0; 00481 train_target_with_selected_columns_vmatrix->extend_with_missing = 0; 00482 train_target_with_selected_columns_vmatrix->fields = selected_variables_for_target; 00483 train_target_with_selected_columns_vmatrix->build(); 00484 train_target_with_selected_columns_vmatrix->defineSizes(train_target_with_selected_columns_vmatrix->width(), 0, 0); 00485 train_target_with_selected_columns_vmat = train_target_with_selected_columns_vmatrix; 00486 } 00487 cout << endl << "****** STEP 15 ******" << endl; 00488 cout << "This gaussianizes the input records." << endl; 00489 cout << "It uses GaussianizeVMatrix to transform the vmat from step 14" << endl; 00490 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 16" << endl; 00491 if (isfile(train_target_preprocessed_file_name)) 00492 { 00493 cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00494 } 00495 else 00496 { 00497 train_target_gaussianized_vmatrix = new GaussianizeVMatrix(); 00498 train_target_gaussianized_vmatrix->source = train_target_with_selected_columns_vmat; 00499 train_target_gaussianized_vmatrix->train_source = train_target_with_selected_columns_vmat; 00500 train_target_gaussianized_vmatrix->threshold_ratio = 1; 00501 train_target_gaussianized_vmatrix->gaussianize_input = 1; 00502 train_target_gaussianized_vmatrix->gaussianize_target = 0; 00503 train_target_gaussianized_vmatrix->gaussianize_weight = 0; 00504 train_target_gaussianized_vmatrix->gaussianize_extra = 0; 00505 train_target_gaussianized_vmatrix->excluded_fields = targets_excluded_from_gaussianization;; 00506 train_target_gaussianized_vmatrix->build(); 00507 train_target_gaussianized_vmat = train_target_gaussianized_vmatrix; 00508 } 00509 cout << endl << "****** STEP 16 ******" << endl; 00510 cout << "Finaly, the preprocessed input vectors are store on disk." << endl; 00511 cout << "The vmat from step 15 is converted to final_train_target_preprocessed.pmat." << endl; 00512 if (isfile(train_target_preprocessed_file_name)) 00513 { 00514 cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00515 } 00516 else 00517 { 00518 train_target_preprocessed_file = new FileVMatrix(train_target_preprocessed_file_name, train_target_gaussianized_vmat->length(), 00519 train_target_gaussianized_vmat->fieldNames()); 00520 train_target_preprocessed_file->defineSizes(train_target_gaussianized_vmat->inputsize(), train_target_gaussianized_vmat->targetsize(), 00521 train_target_gaussianized_vmat->weightsize()); 00522 pb = new ProgressBar("Saving the final train preprocessed target records", train_target_gaussianized_vmat->length()); 00523 train_target_preprocessed_vector.resize(train_target_gaussianized_vmat->width()); 00524 for (int train_gaussianized_row = 0; train_gaussianized_row < train_target_gaussianized_vmat->length(); train_gaussianized_row++) 00525 { 00526 train_target_gaussianized_vmat->getRow(train_gaussianized_row, train_target_preprocessed_vector); 00527 train_target_preprocessed_file->putRow(train_gaussianized_row, train_target_preprocessed_vector); 00528 pb->update( train_gaussianized_row ); 00529 } 00530 delete pb; 00531 } 00532 00533 // defining all the variables for the test set 00534 PPath test_with_class_target_file_name; 00535 VMat test_with_class_target_file; 00536 PPath test_with_binary_fixed_file_name; 00537 VMat test_with_binary_fixed_file; 00538 PPath test_with_ind_file_name; 00539 MissingIndicatorVMatrix* test_with_ind_vmatrix; 00540 VMat test_with_ind_vmat; 00541 PPath test_with_dichotomies_file_name; 00542 VMat test_with_dichotomies_file; 00543 SelectColumnsVMatrix* test_with_selected_columns_vmatrix; 00544 VMat test_with_selected_columns_vmat; 00545 GaussianizeVMatrix* test_gaussianized_vmatrix; 00546 VMat test_gaussianized_vmat; 00547 PPath test_input_preprocessed_file_name; 00548 VMat test_input_preprocessed_file; 00549 Vec test_input_preprocessed_vector; 00550 SelectColumnsVMatrix* test_target_with_selected_columns_vmatrix; 00551 VMat test_target_with_selected_columns_vmat; 00552 GaussianizeVMatrix* test_target_gaussianized_vmatrix; 00553 VMat test_target_gaussianized_vmat; 00554 PPath test_target_preprocessed_file_name; 00555 VMat test_target_preprocessed_file; 00556 Vec test_target_preprocessed_vector; 00557 00558 // managing the test set 00559 cout << endl << "********************" << endl; 00560 cout << "In Preprocessing: now, we format the test set" << endl; 00561 cout << endl << "****** STEP 1 ******" << endl; 00562 cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl; 00563 cout << "It uses ComputeDond2Target to transform base_test.pmat into step1_test_with_class_target.pmat" << endl; 00564 output_path = expdir+"step1_test_with_class_target"; 00565 test_with_class_target_file_name = output_path + ".pmat"; 00566 if (isfile(test_with_class_target_file_name)) 00567 { 00568 test_with_class_target_file = new FileVMatrix(test_with_class_target_file_name); 00569 test_with_class_target_file->defineSizes(test_with_class_target_file->width(), 0, 0); 00570 cout << test_with_class_target_file_name << " already exist, we are skipping this step." << endl; 00571 } 00572 else 00573 { 00574 compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template); 00575 compute_target_learner->unknown_sales = 0; 00576 compute_target_learner->output_path = output_path; 00577 compute_target_learner->setTrainingSet(test_set, true); 00578 test_with_class_target_file = compute_target_learner->getOutputFile(); 00579 } 00580 cout << endl << "****** STEP 2 ******" << endl; 00581 cout << "Shuffling is not required for the test set, skipped." << endl; 00582 cout << endl << "****** STEP 3 ******" << endl; 00583 cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl; 00584 cout << "This step uses FixDond2BinaryVariables to create step3_test_with_binary_fixed.pmat with 0-1 binary variables." << endl; 00585 output_path = expdir+"step3_test_with_binary_fixed"; 00586 test_with_binary_fixed_file_name = output_path + ".pmat"; 00587 if (isfile(test_with_binary_fixed_file_name)) 00588 { 00589 test_with_binary_fixed_file = new FileVMatrix(test_with_binary_fixed_file_name); 00590 test_with_binary_fixed_file->defineSizes(test_with_binary_fixed_file->width(), 0, 0); 00591 cout << test_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl; 00592 } 00593 else 00594 { 00595 fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template); 00596 fix_binary_variables_learner->output_path = output_path; 00597 fix_binary_variables_learner->setTrainingSet(test_with_class_target_file, true); 00598 test_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile(); 00599 } 00600 cout << endl << "****** STEP 4 ******" << endl; 00601 cout << "This step adds missing indicators variables to each variable with missing values." << endl; 00602 cout << "It uses MissingIndicatorVMatrix to transform step3_test_with_binary_fixed.pmat" << endl; 00603 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 6" << endl; 00604 output_path = expdir+"step6_test_with_dichotomies"; 00605 test_with_dichotomies_file_name = output_path + ".pmat"; 00606 if (isfile(test_with_dichotomies_file_name)) 00607 { 00608 cout << test_with_dichotomies_file_name << " already exist, we are skipping this step." << endl; 00609 } 00610 else 00611 { 00612 test_with_ind_vmatrix = new MissingIndicatorVMatrix(); 00613 test_with_ind_vmatrix->source = test_with_binary_fixed_file; 00614 test_with_ind_vmatrix->train_set = train_with_binary_fixed_file; 00615 test_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0; 00616 test_with_ind_vmatrix->build(); 00617 test_with_ind_vmat = test_with_ind_vmatrix; 00618 } 00619 cout << endl << "****** STEP 5 ******" << endl; 00620 cout << "Computing mean, median and mode is not required for the test set, skipped." << endl; 00621 cout << endl << "****** STEP 6 ******" << endl; 00622 cout << "This steps generates as many dichotomized variables as there are significant code values." << endl; 00623 cout << "It uses DichotomizeDond2DiscreteVariables to transform the vmat from step 4 into step6_test_with_dichotomies.pmat" << endl; 00624 if (isfile(test_with_dichotomies_file_name)) 00625 { 00626 test_with_dichotomies_file = new FileVMatrix(test_with_dichotomies_file_name); 00627 test_with_dichotomies_file->defineSizes(test_with_dichotomies_file->width(), 0, 0); 00628 cout << test_with_dichotomies_file_name << " already exist, we are skipping this step." << endl; 00629 } 00630 else 00631 { 00632 dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables(); 00633 dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions; 00634 dichotomize_discrete_variables_learner->output_path = output_path; 00635 dichotomize_discrete_variables_learner->setTrainingSet(test_with_ind_vmat, true); 00636 test_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile(); 00637 } 00638 cout << endl << "****** STEP 7 ******" << endl; 00639 cout << "Dichotomizing mean median and mode is not required for the test set, skipped." << endl; 00640 cout << endl << "****** STEP 8 ******" << endl; 00641 cout << "This step select the desired columns from the test set to create the input records." << endl; 00642 cout << "It uses SelectColumnsVMatrix to transform step6_test_with_dichotomies.pmat" << endl; 00643 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl; 00644 output_path = expdir+"final_test_input_preprocessed"; 00645 test_input_preprocessed_file_name = output_path + ".pmat"; 00646 if (isfile(test_input_preprocessed_file_name)) 00647 { 00648 cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00649 } 00650 else 00651 { 00652 test_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00653 test_with_selected_columns_vmatrix->source = test_with_dichotomies_file; 00654 test_with_selected_columns_vmatrix->fields_partial_match = 0; 00655 test_with_selected_columns_vmatrix->extend_with_missing = 0; 00656 test_with_selected_columns_vmatrix->fields = selected_variables_for_input; 00657 test_with_selected_columns_vmatrix->build(); 00658 test_with_selected_columns_vmatrix->defineSizes(test_with_selected_columns_vmatrix->width(), 0, 0); 00659 test_with_selected_columns_vmat = test_with_selected_columns_vmatrix; 00660 } 00661 cout << endl << "****** STEP 9 ******" << endl; 00662 cout << "Selecting variables for the mean, median and mode is not required for the test set, skipped." << endl; 00663 cout << endl << "****** STEP 10 ******" << endl; 00664 cout << "This gaussianizes the input records." << endl; 00665 cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl; 00666 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl; 00667 if (isfile(test_input_preprocessed_file_name)) 00668 { 00669 cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00670 } 00671 else 00672 { 00673 test_gaussianized_vmatrix = new GaussianizeVMatrix(); 00674 test_gaussianized_vmatrix->source = test_with_selected_columns_vmat; 00675 test_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat; 00676 test_gaussianized_vmatrix->threshold_ratio = 1; 00677 test_gaussianized_vmatrix->gaussianize_input = 1; 00678 test_gaussianized_vmatrix->gaussianize_target = 0; 00679 test_gaussianized_vmatrix->gaussianize_weight = 0; 00680 test_gaussianized_vmatrix->gaussianize_extra = 0; 00681 test_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization; 00682 test_gaussianized_vmatrix->build(); 00683 test_gaussianized_vmat = test_gaussianized_vmatrix; 00684 } 00685 cout << endl << "****** STEP 11 ******" << endl; 00686 cout << "Gaussianizing the mean, meadian and mode is not required for the test set, skipped." << endl; 00687 cout << endl << "****** STEP 12 ******" << endl; 00688 cout << "Finaly, the preprocessed input vectors are store on disk." << endl; 00689 cout << "The vmat from step 10 is converted to final_test_input_preprocessed.pmat." << endl; 00690 if (isfile(test_input_preprocessed_file_name)) 00691 { 00692 cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00693 } 00694 else 00695 { 00696 test_input_preprocessed_file = new FileVMatrix(test_input_preprocessed_file_name, test_gaussianized_vmat->length(), test_gaussianized_vmat->fieldNames()); 00697 test_input_preprocessed_file->defineSizes(test_gaussianized_vmat->inputsize(), test_gaussianized_vmat->targetsize(), test_gaussianized_vmat->weightsize()); 00698 pb = new ProgressBar("Saving the final test preprocessed input records", test_gaussianized_vmat->length()); 00699 test_input_preprocessed_vector.resize(test_gaussianized_vmat->width()); 00700 for (int test_gaussianized_row = 0; test_gaussianized_row < test_gaussianized_vmat->length(); test_gaussianized_row++) 00701 { 00702 test_gaussianized_vmat->getRow(test_gaussianized_row, test_input_preprocessed_vector); 00703 test_input_preprocessed_file->putRow(test_gaussianized_row, test_input_preprocessed_vector); 00704 pb->update( test_gaussianized_row ); 00705 } 00706 delete pb; 00707 } 00708 cout << endl << "****** STEP 13 ******" << endl; 00709 cout << "Saving the final mean, median and mode is not required for the test set, skipped." << endl; 00710 cout << endl << "****** STEP 14 ******" << endl; 00711 cout << "This step select the desired columns from the testing set to create the target records." << endl; 00712 cout << "It uses SelectColumnsVMatrix to transform step6_test_with_dichotomies.pmat" << endl; 00713 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 15" << endl; 00714 output_path = expdir+"final_test_target_preprocessed"; 00715 test_target_preprocessed_file_name = output_path + ".pmat"; 00716 if (isfile(test_target_preprocessed_file_name)) 00717 { 00718 cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00719 } 00720 else 00721 { 00722 test_target_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00723 test_target_with_selected_columns_vmatrix->source = test_with_dichotomies_file; 00724 test_target_with_selected_columns_vmatrix->fields_partial_match = 0; 00725 test_target_with_selected_columns_vmatrix->extend_with_missing = 0; 00726 test_target_with_selected_columns_vmatrix->fields = selected_variables_for_target; 00727 test_target_with_selected_columns_vmatrix->build(); 00728 test_target_with_selected_columns_vmatrix->defineSizes(test_target_with_selected_columns_vmatrix->width(), 0, 0); 00729 test_target_with_selected_columns_vmat = test_target_with_selected_columns_vmatrix; 00730 } 00731 cout << endl << "****** STEP 15 ******" << endl; 00732 cout << "This gaussianizes the input records." << endl; 00733 cout << "It uses GaussianizeVMatrix to transform the vmat from step 14" << endl; 00734 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 16" << endl; 00735 if (isfile(test_target_preprocessed_file_name)) 00736 { 00737 cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00738 } 00739 else 00740 { 00741 test_target_gaussianized_vmatrix = new GaussianizeVMatrix(); 00742 test_target_gaussianized_vmatrix->source = test_target_with_selected_columns_vmat; 00743 test_target_gaussianized_vmatrix->train_source = train_target_with_selected_columns_vmat; 00744 test_target_gaussianized_vmatrix->threshold_ratio = 1; 00745 test_target_gaussianized_vmatrix->gaussianize_input = 1; 00746 test_target_gaussianized_vmatrix->gaussianize_target = 0; 00747 test_target_gaussianized_vmatrix->gaussianize_weight = 0; 00748 test_target_gaussianized_vmatrix->gaussianize_extra = 0; 00749 test_target_gaussianized_vmatrix->excluded_fields = targets_excluded_from_gaussianization;; 00750 test_target_gaussianized_vmatrix->build(); 00751 test_target_gaussianized_vmat = test_target_gaussianized_vmatrix; 00752 } 00753 cout << endl << "****** STEP 16 ******" << endl; 00754 cout << "Finaly, the preprocessed input vectors are store on disk." << endl; 00755 cout << "The vmat from step 15 is converted to final_test_target_preprocessed.pmat." << endl; 00756 if (isfile(test_target_preprocessed_file_name)) 00757 { 00758 cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00759 } 00760 else 00761 { 00762 test_target_preprocessed_file = new FileVMatrix(test_target_preprocessed_file_name, test_target_gaussianized_vmat->length(), 00763 test_target_gaussianized_vmat->fieldNames()); 00764 test_target_preprocessed_file->defineSizes(test_target_gaussianized_vmat->inputsize(), test_target_gaussianized_vmat->targetsize(), 00765 test_target_gaussianized_vmat->weightsize()); 00766 pb = new ProgressBar("Saving the final test preprocessed target records", test_target_gaussianized_vmat->length()); 00767 test_target_preprocessed_vector.resize(test_target_gaussianized_vmat->width()); 00768 for (int test_gaussianized_row = 0; test_gaussianized_row < test_target_gaussianized_vmat->length(); test_gaussianized_row++) 00769 { 00770 test_target_gaussianized_vmat->getRow(test_gaussianized_row, test_target_preprocessed_vector); 00771 test_target_preprocessed_file->putRow(test_gaussianized_row, test_target_preprocessed_vector); 00772 pb->update( test_gaussianized_row ); 00773 } 00774 delete pb; 00775 } 00776 00777 // defining all the variables for the unknown set 00778 PPath unknown_with_class_target_file_name; 00779 VMat unknown_with_class_target_file; 00780 PPath unknown_with_binary_fixed_file_name; 00781 VMat unknown_with_binary_fixed_file; 00782 PPath unknown_with_ind_file_name; 00783 MissingIndicatorVMatrix* unknown_with_ind_vmatrix; 00784 VMat unknown_with_ind_vmat; 00785 PPath unknown_with_dichotomies_file_name; 00786 VMat unknown_with_dichotomies_file; 00787 SelectColumnsVMatrix* unknown_with_selected_columns_vmatrix; 00788 VMat unknown_with_selected_columns_vmat; 00789 GaussianizeVMatrix* unknown_gaussianized_vmatrix; 00790 VMat unknown_gaussianized_vmat; 00791 PPath unknown_input_preprocessed_file_name; 00792 VMat unknown_input_preprocessed_file; 00793 Vec unknown_input_preprocessed_vector; 00794 00795 // managing the unknown set 00796 cout << endl << "********************" << endl; 00797 cout << "In Preprocessing: finally, we format the unknown set" << endl; 00798 cout << endl << "****** STEP 1 ******" << endl; 00799 cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl; 00800 cout << "It uses ComputeDond2Target to transform base_unknown.pmat into step1_unknown_with_class_target.pmat" << endl; 00801 output_path = expdir+"step1_unknown_with_class_target"; 00802 unknown_with_class_target_file_name = output_path + ".pmat"; 00803 if (isfile(unknown_with_class_target_file_name)) 00804 { 00805 unknown_with_class_target_file = new FileVMatrix(unknown_with_class_target_file_name); 00806 unknown_with_class_target_file->defineSizes(unknown_with_class_target_file->width(), 0, 0); 00807 cout << unknown_with_class_target_file_name << " already exist, we are skipping this step." << endl; 00808 } 00809 else 00810 { 00811 compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template); 00812 compute_target_learner->unknown_sales = 1; 00813 compute_target_learner->output_path = output_path; 00814 compute_target_learner->setTrainingSet(unknown_set, true); 00815 unknown_with_class_target_file = compute_target_learner->getOutputFile(); 00816 } 00817 cout << endl << "****** STEP 2 ******" << endl; 00818 cout << "Shuffling is not required for the unknown set, skipped." << endl; 00819 cout << endl << "****** STEP 3 ******" << endl; 00820 cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl; 00821 cout << "This step uses FixDond2BinaryVariables to create step3_unknown_with_binary_fixed.pmat with 0-1 binary variables." << endl; 00822 output_path = expdir+"step3_unknown_with_binary_fixed"; 00823 unknown_with_binary_fixed_file_name = output_path + ".pmat"; 00824 if (isfile(unknown_with_binary_fixed_file_name)) 00825 { 00826 unknown_with_binary_fixed_file = new FileVMatrix(unknown_with_binary_fixed_file_name); 00827 unknown_with_binary_fixed_file->defineSizes(unknown_with_binary_fixed_file->width(), 0, 0); 00828 cout << unknown_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl; 00829 } 00830 else 00831 { 00832 fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template); 00833 fix_binary_variables_learner->output_path = output_path; 00834 fix_binary_variables_learner->setTrainingSet(unknown_with_class_target_file, true); 00835 unknown_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile(); 00836 } 00837 cout << endl << "****** STEP 4 ******" << endl; 00838 cout << "This step adds missing indicators variables to each variable with missing values." << endl; 00839 cout << "It uses MissingIndicatorVMatrix to transform step3_unknown_with_binary_fixed.pmat" << endl; 00840 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 6" << endl; 00841 output_path = expdir+"step6_unknown_with_dichotomies"; 00842 unknown_with_dichotomies_file_name = output_path + ".pmat"; 00843 if (isfile(unknown_with_dichotomies_file_name)) 00844 { 00845 cout << unknown_with_dichotomies_file_name << " already exist, we are skipping this step." << endl; 00846 } 00847 else 00848 { 00849 unknown_with_ind_vmatrix = new MissingIndicatorVMatrix(); 00850 unknown_with_ind_vmatrix->source = unknown_with_binary_fixed_file; 00851 unknown_with_ind_vmatrix->train_set = train_with_binary_fixed_file; 00852 unknown_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0; 00853 unknown_with_ind_vmatrix->build(); 00854 unknown_with_ind_vmat = unknown_with_ind_vmatrix; 00855 } 00856 cout << endl << "****** STEP 5 ******" << endl; 00857 cout << "Computing mean, median and mode is not required for the unknown set, skipped." << endl; 00858 cout << endl << "****** STEP 6 ******" << endl; 00859 cout << "This steps generates as many dichotomized variables as there are significant code values." << endl; 00860 cout << "It uses DichotomizeDond2DiscreteVariables to transform the vmat from step 4 into step6_unknown_with_dichotomies.pmat" << endl; 00861 if (isfile(unknown_with_dichotomies_file_name)) 00862 { 00863 unknown_with_dichotomies_file = new FileVMatrix(unknown_with_dichotomies_file_name); 00864 unknown_with_dichotomies_file->defineSizes(unknown_with_dichotomies_file->width(), 0, 0); 00865 cout << unknown_with_dichotomies_file_name << " already exist, we are skipping this step." << endl; 00866 } 00867 else 00868 { 00869 dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables(); 00870 dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions; 00871 dichotomize_discrete_variables_learner->output_path = output_path; 00872 dichotomize_discrete_variables_learner->setTrainingSet(unknown_with_ind_vmat, true); 00873 unknown_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile(); 00874 } 00875 cout << endl << "****** STEP 7 ******" << endl; 00876 cout << "Dichotomizing mean median and mode is not required for the unknown set, skipped." << endl; 00877 cout << endl << "****** STEP 8 ******" << endl; 00878 cout << "This step select the desired columns from the unknown set to create the input records." << endl; 00879 cout << "It uses SelectColumnsVMatrix to transform step6_unknown_with_dichotomies.pmat" << endl; 00880 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl; 00881 output_path = expdir+"final_unknown_input_preprocessed"; 00882 unknown_input_preprocessed_file_name = output_path + ".pmat"; 00883 if (isfile(unknown_input_preprocessed_file_name)) 00884 { 00885 cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00886 } 00887 else 00888 { 00889 unknown_with_selected_columns_vmatrix = new SelectColumnsVMatrix(); 00890 unknown_with_selected_columns_vmatrix->source = unknown_with_dichotomies_file; 00891 unknown_with_selected_columns_vmatrix->fields_partial_match = 0; 00892 unknown_with_selected_columns_vmatrix->extend_with_missing = 0; 00893 unknown_with_selected_columns_vmatrix->fields = selected_variables_for_input; 00894 unknown_with_selected_columns_vmatrix->build(); 00895 unknown_with_selected_columns_vmatrix->defineSizes(unknown_with_selected_columns_vmatrix->width(), 0, 0); 00896 unknown_with_selected_columns_vmat = unknown_with_selected_columns_vmatrix; 00897 } 00898 cout << endl << "****** STEP 9 ******" << endl; 00899 cout << "Selecting variables for the mean, median and mode is not required for the unknown set, skipped." << endl; 00900 cout << endl << "****** STEP 10 ******" << endl; 00901 cout << "This gaussianizes the input records." << endl; 00902 cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl; 00903 cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl; 00904 if (isfile(unknown_input_preprocessed_file_name)) 00905 { 00906 cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00907 } 00908 else 00909 { 00910 unknown_gaussianized_vmatrix = new GaussianizeVMatrix(); 00911 unknown_gaussianized_vmatrix->source = unknown_with_selected_columns_vmat; 00912 unknown_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat; 00913 unknown_gaussianized_vmatrix->threshold_ratio = 1; 00914 unknown_gaussianized_vmatrix->gaussianize_input = 1; 00915 unknown_gaussianized_vmatrix->gaussianize_target = 0; 00916 unknown_gaussianized_vmatrix->gaussianize_weight = 0; 00917 unknown_gaussianized_vmatrix->gaussianize_extra = 0; 00918 unknown_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization; 00919 unknown_gaussianized_vmatrix->build(); 00920 unknown_gaussianized_vmat = unknown_gaussianized_vmatrix; 00921 } 00922 cout << endl << "****** STEP 11 ******" << endl; 00923 cout << "Gaussianizing the mean, meadian and mode is not required for the unknown set, skipped." << endl; 00924 cout << endl << "****** STEP 12 ******" << endl; 00925 cout << "Finaly, the preprocessed input vectors are store on disk." << endl; 00926 cout << "The vmat from step 10 is converted to final_unknown_input_preprocessed.pmat." << endl; 00927 if (isfile(unknown_input_preprocessed_file_name)) 00928 { 00929 cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl; 00930 } 00931 else 00932 { 00933 unknown_input_preprocessed_file = new FileVMatrix(unknown_input_preprocessed_file_name, unknown_gaussianized_vmat->length(), unknown_gaussianized_vmat->fieldNames()); 00934 unknown_input_preprocessed_file->defineSizes(unknown_gaussianized_vmat->inputsize(), unknown_gaussianized_vmat->targetsize(), unknown_gaussianized_vmat->weightsize()); 00935 pb = new ProgressBar("Saving the final unknown preprocessed input records", unknown_gaussianized_vmat->length()); 00936 unknown_input_preprocessed_vector.resize(unknown_gaussianized_vmat->width()); 00937 for (int unknown_gaussianized_row = 0; unknown_gaussianized_row < unknown_gaussianized_vmat->length(); unknown_gaussianized_row++) 00938 { 00939 unknown_gaussianized_vmat->getRow(unknown_gaussianized_row, unknown_input_preprocessed_vector); 00940 unknown_input_preprocessed_file->putRow(unknown_gaussianized_row, unknown_input_preprocessed_vector); 00941 pb->update( unknown_gaussianized_row ); 00942 } 00943 delete pb; 00944 } 00945 cout << endl << "****** STEP 13 ******" << endl; 00946 cout << "Saving the final mean, median and mode is not required for the unknown set, skipped." << endl; 00947 cout << endl << "****** STEP 14 ******" << endl; 00948 cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl; 00949 cout << endl << "****** STEP 15 ******" << endl; 00950 cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl; 00951 cout << endl << "****** STEP 16 ******" << endl; 00952 cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl; 00953 } 00954 00955 void Preprocessing::train() 00956 { 00957 } 00958 00959 int Preprocessing::outputsize() const {return 0;} 00960 void Preprocessing::computeOutput(const Vec&, Vec&) const {} 00961 void Preprocessing::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {} 00962 TVec<string> Preprocessing::getTestCostNames() const 00963 { 00964 TVec<string> result; 00965 result.append( "MSE" ); 00966 return result; 00967 } 00968 TVec<string> Preprocessing::getTrainCostNames() const 00969 { 00970 TVec<string> result; 00971 result.append( "MSE" ); 00972 return result; 00973 } 00974 00975 } // end of namespace PLearn 00976 00977 00978 /* 00979 Local Variables: 00980 mode:c++ 00981 c-basic-offset:4 00982 c-file-style:"stroustrup" 00983 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00984 indent-tabs-mode:nil 00985 fill-column:79 00986 End: 00987 */ 00988 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :