PLearn 0.1
Preprocessing.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // Preprocessing.cc
00004 //
00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Dan Popovici
00036 
00039 #define PL_LOG_MODULE_NAME "Preprocessing"
00040 
00041 #include "Preprocessing.h"
00042 #include <plearn/io/pl_log.h>
00043 #include <plearn/io/load_and_save.h>                 
00044 #include <plearn/io/fileutils.h>                     
00045 #include <plearn/math/random.h>                      
00046 #include <plearn/vmat/ExplicitSplitter.h>            
00047 #include <plearn/vmat/VariableDeletionVMatrix.h>     
00048 #include <plearn/vmat/BootstrapVMatrix.h>            
00049 
00050 namespace PLearn {
00051 using namespace std;
00052 
00053 PLEARN_IMPLEMENT_OBJECT(
00054     Preprocessing,
00055     "Computes correlation coefficient between various discrete values and the target.",
00056     "name of the discrete variable, of the target and the values to check are options.\n"
00057 );
00058 
00060 // Preprocessing //
00062 Preprocessing::Preprocessing()
00063 {
00064 }
00065     
00067 // declareOptions //
00069 void Preprocessing::declareOptions(OptionList& ol)
00070 {
00071     declareOption(ol, "test_set", &Preprocessing::test_set,
00072                   OptionBase::buildoption,
00073                   "The test data set.\n");
00074     declareOption(ol, "unknown_set", &Preprocessing::unknown_set,
00075                   OptionBase::buildoption,
00076                   "The unknown data set.\n");
00077     declareOption(ol, "compute_target_learner_template", &Preprocessing::compute_target_learner_template,
00078                   OptionBase::buildoption,
00079                   "The template of the script to generate the class target.\n");
00080     declareOption(ol, "fix_binary_variables_template", &Preprocessing::fix_binary_variables_template,
00081                   OptionBase::buildoption,
00082                   "The template of the script to fix the binary variables.\n");
00083     declareOption(ol, "imputation_spec", &Preprocessing::imputation_spec,
00084                   OptionBase::buildoption,
00085                   "Pairs of instruction of the form field_name : mean | median | mode.\n");
00086     declareOption(ol, "discrete_variable_instructions", &Preprocessing::discrete_variable_instructions,
00087                   OptionBase::buildoption,
00088                   "The instructions to dichotomize the variables in the form of field_name : TVec<pair>.\n"
00089                   "The pairs are values from : to, each creating a 0, 1 variable.\n"
00090                   "Variables with no specification will be kept as_is.\n");
00091     declareOption(ol, "selected_variables_for_input", &Preprocessing::selected_variables_for_input,
00092                   OptionBase::buildoption,
00093                   "The list of variables selected as input vector.\n");
00094     declareOption(ol, "selected_variables_for_target", &Preprocessing::selected_variables_for_target,
00095                   OptionBase::buildoption,
00096                   "The list of variables selected as target vector.\n");
00097     declareOption(ol, "inputs_excluded_from_gaussianization", &Preprocessing::inputs_excluded_from_gaussianization,
00098                   OptionBase::buildoption,
00099                   "The list of input variables excluded from the gaussianization step.\n");
00100     declareOption(ol, "targets_excluded_from_gaussianization", &Preprocessing::targets_excluded_from_gaussianization,
00101                   OptionBase::buildoption,
00102                   "The list of target variables excluded from the gaussianization step.\n");
00103 
00104     inherited::declareOptions(ol);
00105 }
00106 
00108 // makeDeepCopyFromShallowCopy //
00110 void Preprocessing::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00111 {
00112     deepCopyField(test_set, copies);
00113     deepCopyField(unknown_set, copies);
00114     deepCopyField(compute_target_learner_template, copies);
00115     deepCopyField(fix_binary_variables_template, copies);
00116     deepCopyField(imputation_spec, copies);
00117     deepCopyField(discrete_variable_instructions, copies);
00118     deepCopyField(selected_variables_for_input, copies);
00119     deepCopyField(selected_variables_for_target, copies);
00120     deepCopyField(inputs_excluded_from_gaussianization, copies);
00121     deepCopyField(targets_excluded_from_gaussianization, copies);
00122     inherited::makeDeepCopyFromShallowCopy(copies);
00123 
00124 }
00125 
00127 // build //
00129 void Preprocessing::build()
00130 {
00131     // ### Nothing to add here, simply calls build_().
00132     inherited::build();
00133     build_();
00134 }
00135 
00137 // build_ //
00139 void Preprocessing::build_()
00140 {
00141     MODULE_LOG << "build_() called" << endl;
00142     if (train_set)
00143     {
00144         manageTrainTestUnknownSets();
00145         PLERROR("In Preprocessing: Everything completed successfuly, we are done here");
00146     }
00147 }
00148 
00149 void Preprocessing::manageTrainTestUnknownSets()
00150 {
00151 
00152     // defining all the variables for the train set
00153     PPath                                 output_path;
00154     PPath                                 train_with_class_target_file_name;
00155     VMat                                  train_with_class_target_file;
00156     PP<ComputeDond2Target>                compute_target_learner;
00157     VMat                                  train_shuffled_file;
00158     PPath                                 train_with_binary_fixed_file_name;
00159     VMat                                  train_with_binary_fixed_file;
00160     PP<FixDond2BinaryVariables>           fix_binary_variables_learner;
00161     PPath                                 train_with_ind_file_name;
00162     VMat                                  train_with_ind_vmat;
00163     VMat                                  train_with_ind_file;
00164     Vec                                   train_with_ind_vector;
00165     VMat                                  mean_median_mode_with_ind_file;
00166     PPath                                 train_with_dichotomies_file_name;
00167     VMat                                  train_with_dichotomies_file;
00168     PPath                                 mean_median_mode_with_dichotmies_file_name;
00169     VMat                                  mean_median_mode_with_dichotmies_file;
00170     PP<DichotomizeDond2DiscreteVariables> dichotomize_discrete_variables_learner;
00171     SelectColumnsVMatrix*                 train_with_selected_columns_vmatrix;
00172     VMat                                  train_with_selected_columns_vmat;
00173     VMat                                  mean_median_mode_with_selected_columns_vmat;
00174     VMat                                  train_gaussianized_vmat;
00175     GaussianizeVMatrix*                   mean_median_mode_gaussianized_vmatrix;
00176     VMat                                  mean_median_mode_gaussianized_vmat;
00177     PPath                                 train_input_preprocessed_file_name;
00178     VMat                                  train_input_preprocessed_file;
00179     Vec                                   train_input_preprocessed_vector;
00180     PPath                                 mean_median_mode_input_preprocessed_file_name;
00181     VMat                                  mean_median_mode_input_preprocessed_file;
00182     Vec                                   mean_median_mode_input_preprocessed_vector;
00183     SelectColumnsVMatrix*                 train_target_with_selected_columns_vmatrix;
00184     VMat                                  train_target_with_selected_columns_vmat;
00185     GaussianizeVMatrix*                   train_target_gaussianized_vmatrix;
00186     VMat                                  train_target_gaussianized_vmat;
00187     PPath                                 train_target_preprocessed_file_name;
00188     VMat                                  train_target_preprocessed_file;
00189     Vec                                   train_target_preprocessed_vector;
00190     ProgressBar*                          pb = 0;
00191     
00192     // managing the train set
00193     cout << "In Preprocessing: we start by formatting the training set" << endl;
00194     cout << endl << "****** STEP 1 ******" << endl;
00195     cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl;
00196     cout << "It uses ComputeDond2Target to transform base_train.pmat into step1_train_with_class_target.pmat" << endl;
00197     output_path = expdir+"step1_train_with_class_target";
00198     cout << "output_path" << output_path;
00199     train_with_class_target_file_name = output_path + ".pmat";
00200     if (isfile(train_with_class_target_file_name))
00201     {
00202         train_with_class_target_file = new FileVMatrix(train_with_class_target_file_name);
00203         train_with_class_target_file->defineSizes(train_with_class_target_file->width(), 0, 0);
00204         cout << train_with_class_target_file_name << " already exist, we are skipping this step." << endl;
00205     }
00206     else 
00207     {
00208         compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template);
00209         compute_target_learner->unknown_sales = 0;
00210         compute_target_learner->output_path = output_path;
00211         compute_target_learner->setTrainingSet(train_set, true);
00212         train_with_class_target_file = compute_target_learner->getOutputFile();
00213     }
00214     cout << endl << "****** STEP 2 ******" << endl;
00215     cout << "This step shuffles the training set to get training data in random order." << endl;
00216     cout << "It uses BootstrapVMatrix to transform step1_train_with_class_target.pmat" << endl;
00217     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 3" << endl;
00218     output_path = expdir+"step3_train_with_binary_fixed";
00219     train_with_binary_fixed_file_name = output_path + ".pmat";
00220     if (isfile(train_with_binary_fixed_file_name))
00221     {
00222          cout << train_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl;
00223     }
00224     else 
00225     {
00226         BootstrapVMatrix* train_shufffled_vmatrix = new BootstrapVMatrix();
00227         train_shufffled_vmatrix->shuffle = 1;
00228         train_shufffled_vmatrix->frac = 1.0;
00229         train_shufffled_vmatrix->own_seed = 123456;
00230         train_shufffled_vmatrix->source = train_with_class_target_file;
00231         train_shufffled_vmatrix->build();
00232         train_shuffled_file = train_shufffled_vmatrix;
00233     }
00234     cout << endl << "****** STEP 3 ******" << endl;
00235     cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl;
00236     cout << "This step uses FixDond2BinaryVariables to create step3_train_with_binary_fixed.pmat with 0-1 binary variables." << endl;
00237     if (isfile(train_with_binary_fixed_file_name))
00238     {
00239         train_with_binary_fixed_file = new FileVMatrix(train_with_binary_fixed_file_name);
00240         train_with_binary_fixed_file->defineSizes(train_with_binary_fixed_file->width(), 0, 0);
00241         cout << train_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl;
00242     }
00243     else 
00244     {
00245         fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template);
00246         fix_binary_variables_learner->output_path = output_path;
00247         fix_binary_variables_learner->setTrainingSet(train_shuffled_file, true);
00248         train_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile();
00249     }
00250     cout << endl << "****** STEP 4 ******" << endl;
00251     cout << "This step adds missing indicators variables to each variable with missing values." << endl;
00252     cout << "It uses MissingIndicatorVMatrix to transform step3_train_with_binary_fixed.pmat" << endl;
00253     cout << "The resulting vitual view is stored in step4_train_with_ind.pmat." << endl;
00254     train_with_ind_file_name = "step4_train_with_ind.pmat";
00255     if (isfile(train_with_ind_file_name))
00256     {
00257         train_with_ind_file = new FileVMatrix(train_with_ind_file_name);
00258         train_with_ind_file->defineSizes(train_with_ind_file->width(), 0, 0);
00259         cout << train_with_ind_file_name << " already exist, we are skipping this step." << endl;
00260     }
00261     else 
00262     {
00263         MissingIndicatorVMatrix* train_with_ind_vmatrix = new MissingIndicatorVMatrix();
00264         train_with_ind_vmatrix->source = train_with_binary_fixed_file;
00265         train_with_ind_vmatrix->train_set = train_with_binary_fixed_file;
00266         train_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0;
00267         train_with_ind_vmatrix->build();
00268         train_with_ind_vmat = train_with_ind_vmatrix;
00269         train_with_ind_file = new FileVMatrix(train_with_ind_file_name, train_with_ind_vmat->length(), train_with_ind_vmat->fieldNames());
00270         train_with_ind_file->defineSizes(train_with_ind_vmat->inputsize(), train_with_ind_vmat->targetsize(), train_with_ind_vmat->weightsize());
00271         pb = new ProgressBar("Saving the train file with missing indicators", train_with_ind_vmat->length());
00272         train_with_ind_vector.resize(train_with_ind_vmat->width());
00273         for (int train_with_ind_row = 0; train_with_ind_row < train_with_ind_vmat->length(); train_with_ind_row++)
00274         {
00275             train_with_ind_vmat->getRow(train_with_ind_row, train_with_ind_vector);
00276             train_with_ind_file->putRow(train_with_ind_row, train_with_ind_vector);
00277             pb->update( train_with_ind_row );
00278         }
00279         delete pb;
00280     }
00281     cout << endl << "****** STEP 5 ******" << endl;
00282     cout << "This step computes the mean, median and mode vectors on step4_train_with_ind.pmat." << endl;
00283     cout << "The vectors are kept in the mean_median_mode_file.pmat of the metadata." << endl;
00284     cout << "It uses MeanMedianModeImputationVMatrix to do that" << endl;
00285     cout << "The resulting vitual view is not used." << endl;
00286     cout << "But the mean, median and mode vectors have to go thru the same transformation than the training file" << endl;
00287     cout << "from here on to the end of the preprocessing steps.." << endl;
00288     { 
00289         MeanMedianModeImputationVMatrix* train_with_imp_vmatrix = new MeanMedianModeImputationVMatrix();
00290         train_with_imp_vmatrix->source = train_with_ind_file;
00291         train_with_imp_vmatrix->train_set = train_with_ind_file;
00292         train_with_imp_vmatrix->number_of_train_samples_to_use = 30000.0;
00293         train_with_imp_vmatrix->imputation_spec = imputation_spec;
00294         train_with_imp_vmatrix->build();
00295         mean_median_mode_with_ind_file = train_with_imp_vmatrix->getMeanMedianModeFile();
00296     }
00297     cout << endl << "****** STEP 6 ******" << endl;
00298     cout << "This steps generates as many dichotomized variables as there are significant code values." << endl;
00299     cout << "It uses DichotomizeDond2DiscreteVariables to transform step4_train_with_ind.pmat into step6_train_with_dichotomies.pmat" << endl;
00300     output_path = expdir+"step6_train_with_dichotomies";
00301     train_with_dichotomies_file_name = output_path + ".pmat";
00302     if (isfile(train_with_dichotomies_file_name))
00303     {
00304         train_with_dichotomies_file = new FileVMatrix(train_with_dichotomies_file_name);
00305         train_with_dichotomies_file->defineSizes(train_with_dichotomies_file->width(), 0, 0);
00306         cout << train_with_dichotomies_file_name << " already exist, we are skipping this step." << endl;
00307     }
00308     else 
00309     {
00310         dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables();
00311         dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions;
00312         dichotomize_discrete_variables_learner->output_path = output_path;
00313         dichotomize_discrete_variables_learner->setTrainingSet(train_with_ind_file, true);
00314         train_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile();
00315     }
00316     cout << endl << "****** STEP 7 ******" << endl;
00317     cout << "This steps does the same thing to the mean, median and mode vectors." << endl;
00318     cout << "It uses DichotomizeDond2DiscreteVariables to transform step4_train_with_ind.pmat.metadata/mean_median_mode_file.pmat "
00319          << "into step6_train_with_dichotomies.pmat.metadata/mean_median_mode_file.pmat" << endl;
00320     output_path = expdir+train_with_dichotomies_file_name + ".metadata/mean_median_mode_file";
00321     mean_median_mode_with_dichotmies_file_name = output_path + ".pmat";
00322     if (isfile(mean_median_mode_with_dichotmies_file_name))
00323     {
00324         mean_median_mode_with_dichotmies_file = new FileVMatrix(mean_median_mode_with_dichotmies_file_name);
00325         mean_median_mode_with_dichotmies_file->defineSizes(mean_median_mode_with_dichotmies_file->width(), 0, 0);
00326         cout << mean_median_mode_with_dichotmies_file_name << " already exist, we are skipping this step." << endl;
00327     }
00328     else 
00329     {
00330         dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables();
00331         dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions;
00332         dichotomize_discrete_variables_learner->output_path = output_path;
00333         dichotomize_discrete_variables_learner->setTrainingSet(mean_median_mode_with_ind_file, true);
00334         mean_median_mode_with_dichotmies_file = dichotomize_discrete_variables_learner->getOutputFile();
00335     }
00336     cout << endl << "****** STEP 8 ******" << endl;
00337     cout << "This step select the desired columns from the training set to create the input records." << endl;
00338     cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat" << endl;
00339     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl;
00340     output_path = expdir+"final_train_input_preprocessed";
00341     train_input_preprocessed_file_name = output_path + ".pmat";
00342     if (isfile(train_input_preprocessed_file_name))
00343     {
00344         cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00345     }
00346     else 
00347     {
00348         train_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00349         train_with_selected_columns_vmatrix->source = train_with_dichotomies_file;
00350         train_with_selected_columns_vmatrix->fields_partial_match = 0;
00351         train_with_selected_columns_vmatrix->extend_with_missing = 0;
00352         train_with_selected_columns_vmatrix->fields = selected_variables_for_input;
00353         train_with_selected_columns_vmatrix->build();
00354         train_with_selected_columns_vmatrix->defineSizes(train_with_selected_columns_vmatrix->width(), 0, 0);
00355         train_with_selected_columns_vmat = train_with_selected_columns_vmatrix;
00356     }
00357     cout << endl << "****** STEP 9 ******" << endl;
00358     cout << "This step does the same thing to the mean, median and mode vectors." << endl;
00359     cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat.metadata/mean_median_mode_file.pmat" << endl;
00360     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 11" << endl;
00361     output_path = expdir+train_input_preprocessed_file_name + ".metadata/mean_median_mode_file";
00362     mean_median_mode_input_preprocessed_file_name = output_path + ".pmat";
00363     if (isfile(mean_median_mode_input_preprocessed_file_name))
00364     {
00365         cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00366     }
00367     else 
00368     {
00369         SelectColumnsVMatrix* mean_median_mode_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00370         mean_median_mode_with_selected_columns_vmatrix->source = mean_median_mode_with_dichotmies_file;
00371         mean_median_mode_with_selected_columns_vmatrix->fields_partial_match = 0;
00372         mean_median_mode_with_selected_columns_vmatrix->extend_with_missing = 0;
00373         mean_median_mode_with_selected_columns_vmatrix->fields = selected_variables_for_input;
00374         mean_median_mode_with_selected_columns_vmatrix->build();
00375         mean_median_mode_with_selected_columns_vmatrix->defineSizes(mean_median_mode_with_selected_columns_vmatrix->width(), 0, 0);
00376         mean_median_mode_with_selected_columns_vmat = mean_median_mode_with_selected_columns_vmatrix;
00377     }
00378     cout << endl << "****** STEP 10 ******" << endl;
00379     cout << "This gaussianizes the input records." << endl;
00380     cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl;
00381     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl;
00382     if (isfile(train_input_preprocessed_file_name))
00383     {
00384         cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00385     }
00386     else 
00387     {
00388         GaussianizeVMatrix* train_gaussianized_vmatrix = new GaussianizeVMatrix();
00389         train_gaussianized_vmatrix->source = train_with_selected_columns_vmat;
00390         train_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat;
00391         train_gaussianized_vmatrix->threshold_ratio = 1;
00392         train_gaussianized_vmatrix->gaussianize_input = 1;
00393         train_gaussianized_vmatrix->gaussianize_target = 0;
00394         train_gaussianized_vmatrix->gaussianize_weight = 0;
00395         train_gaussianized_vmatrix->gaussianize_extra = 0;
00396         train_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization;
00397         train_gaussianized_vmatrix->build();
00398         train_gaussianized_vmat = train_gaussianized_vmatrix;
00399     }
00400     cout << endl << "****** STEP 11 ******" << endl;
00401     cout << "This step does the same thing to the mean, median and mode vectors." << endl;
00402     cout << "It uses the vmat from step 9" << endl;
00403     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 13" << endl;
00404     if (isfile(mean_median_mode_input_preprocessed_file_name))
00405     {
00406         cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00407     }
00408     else 
00409     {
00410         mean_median_mode_gaussianized_vmatrix = new GaussianizeVMatrix();
00411         mean_median_mode_gaussianized_vmatrix->source = mean_median_mode_with_selected_columns_vmat;
00412         mean_median_mode_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat;
00413         mean_median_mode_gaussianized_vmatrix->threshold_ratio = 1;
00414         mean_median_mode_gaussianized_vmatrix->gaussianize_input = 1;
00415         mean_median_mode_gaussianized_vmatrix->gaussianize_target = 0;
00416         mean_median_mode_gaussianized_vmatrix->gaussianize_weight = 0;
00417         mean_median_mode_gaussianized_vmatrix->gaussianize_extra = 0;
00418         mean_median_mode_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization;
00419         mean_median_mode_gaussianized_vmatrix->build();
00420         mean_median_mode_gaussianized_vmat = mean_median_mode_gaussianized_vmatrix;
00421     }
00422     cout << endl << "****** STEP 12 ******" << endl;
00423     cout << "Finaly, the preprocessed input vectors are store on disk." << endl;
00424     cout << "The vmat from step 10 is converted to final_train_input_preprocessed.pmat." << endl;
00425     if (isfile(train_input_preprocessed_file_name))
00426     {
00427         cout << train_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00428     }
00429     else 
00430     {
00431         train_input_preprocessed_file = new FileVMatrix(train_input_preprocessed_file_name, train_gaussianized_vmat->length(), train_gaussianized_vmat->fieldNames());
00432         train_input_preprocessed_file->defineSizes(train_gaussianized_vmat->inputsize(), train_gaussianized_vmat->targetsize(), train_gaussianized_vmat->weightsize());
00433         pb = new ProgressBar("Saving the final train preprocessed input records", train_gaussianized_vmat->length());
00434         train_input_preprocessed_vector.resize(train_gaussianized_vmat->width());
00435         for (int train_gaussianized_row = 0; train_gaussianized_row < train_gaussianized_vmat->length(); train_gaussianized_row++)
00436         {
00437             train_gaussianized_vmat->getRow(train_gaussianized_row, train_input_preprocessed_vector);
00438             train_input_preprocessed_file->putRow(train_gaussianized_row, train_input_preprocessed_vector);
00439             pb->update( train_gaussianized_row );
00440         }
00441         delete pb;
00442     }
00443     cout << endl << "****** STEP 13 ******" << endl;
00444     cout << "And we do the same for the mean, median and mode vectors." << endl;
00445     cout << "The vmat from step 11 is converted to final_train_input_preprocessed.pmat.metadata/men_median_mode_file.pmat" << endl;
00446     if (isfile(mean_median_mode_input_preprocessed_file_name))
00447     {
00448         cout << mean_median_mode_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00449     }
00450     else 
00451     {
00452         mean_median_mode_input_preprocessed_file = 
00453             new FileVMatrix(mean_median_mode_input_preprocessed_file_name, mean_median_mode_gaussianized_vmat->length(), mean_median_mode_gaussianized_vmat->fieldNames());
00454         mean_median_mode_input_preprocessed_file->defineSizes(mean_median_mode_gaussianized_vmat->inputsize(),
00455                                                               mean_median_mode_gaussianized_vmat->targetsize(), mean_median_mode_gaussianized_vmat->weightsize());
00456         pb = new ProgressBar("Saving the final mean,median and mode preprocessed input vectors", mean_median_mode_gaussianized_vmat->length());
00457         mean_median_mode_input_preprocessed_vector.resize(mean_median_mode_gaussianized_vmat->width());
00458         for (int mean_median_mode_gaussianized_row = 0; mean_median_mode_gaussianized_row < mean_median_mode_gaussianized_vmat->length(); mean_median_mode_gaussianized_row++)
00459         {
00460             mean_median_mode_gaussianized_vmat->getRow(mean_median_mode_gaussianized_row, mean_median_mode_input_preprocessed_vector);
00461             mean_median_mode_input_preprocessed_file->putRow(mean_median_mode_gaussianized_row, mean_median_mode_input_preprocessed_vector);
00462             pb->update( mean_median_mode_gaussianized_row );
00463         }
00464         delete pb;
00465     }
00466     cout << endl << "****** STEP 14 ******" << endl;
00467     cout << "This step select the desired columns from the training set to create the target records." << endl;
00468     cout << "It uses SelectColumnsVMatrix to transform step6_train_with_dichotomies.pmat" << endl;
00469     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 15" << endl;
00470     output_path = expdir+"final_train_target_preprocessed";
00471     train_target_preprocessed_file_name = output_path + ".pmat";
00472     if (isfile(train_target_preprocessed_file_name))
00473     {
00474         cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00475     }
00476     else 
00477     {
00478         train_target_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00479         train_target_with_selected_columns_vmatrix->source = train_with_dichotomies_file;
00480         train_target_with_selected_columns_vmatrix->fields_partial_match = 0;
00481         train_target_with_selected_columns_vmatrix->extend_with_missing = 0;
00482         train_target_with_selected_columns_vmatrix->fields = selected_variables_for_target;
00483         train_target_with_selected_columns_vmatrix->build();
00484         train_target_with_selected_columns_vmatrix->defineSizes(train_target_with_selected_columns_vmatrix->width(), 0, 0);
00485         train_target_with_selected_columns_vmat = train_target_with_selected_columns_vmatrix;
00486     }
00487     cout << endl << "****** STEP 15 ******" << endl;
00488     cout << "This gaussianizes the input records." << endl;
00489     cout << "It uses GaussianizeVMatrix to transform the vmat from step 14" << endl;
00490     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 16" << endl;
00491     if (isfile(train_target_preprocessed_file_name))
00492     {
00493         cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00494     }
00495     else 
00496     {
00497         train_target_gaussianized_vmatrix = new GaussianizeVMatrix();
00498         train_target_gaussianized_vmatrix->source = train_target_with_selected_columns_vmat;
00499         train_target_gaussianized_vmatrix->train_source = train_target_with_selected_columns_vmat;
00500         train_target_gaussianized_vmatrix->threshold_ratio = 1;
00501         train_target_gaussianized_vmatrix->gaussianize_input = 1;
00502         train_target_gaussianized_vmatrix->gaussianize_target = 0;
00503         train_target_gaussianized_vmatrix->gaussianize_weight = 0;
00504         train_target_gaussianized_vmatrix->gaussianize_extra = 0;
00505         train_target_gaussianized_vmatrix->excluded_fields = targets_excluded_from_gaussianization;;
00506         train_target_gaussianized_vmatrix->build();
00507         train_target_gaussianized_vmat = train_target_gaussianized_vmatrix;
00508     }
00509     cout << endl << "****** STEP 16 ******" << endl;
00510     cout << "Finaly, the preprocessed input vectors are store on disk." << endl;
00511     cout << "The vmat from step 15 is converted to final_train_target_preprocessed.pmat." << endl;
00512     if (isfile(train_target_preprocessed_file_name))
00513     {
00514         cout << train_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00515     }
00516     else 
00517     {
00518         train_target_preprocessed_file = new FileVMatrix(train_target_preprocessed_file_name, train_target_gaussianized_vmat->length(),
00519                                                          train_target_gaussianized_vmat->fieldNames());
00520         train_target_preprocessed_file->defineSizes(train_target_gaussianized_vmat->inputsize(), train_target_gaussianized_vmat->targetsize(),
00521                                                    train_target_gaussianized_vmat->weightsize());
00522         pb = new ProgressBar("Saving the final train preprocessed target records", train_target_gaussianized_vmat->length());
00523         train_target_preprocessed_vector.resize(train_target_gaussianized_vmat->width());
00524         for (int train_gaussianized_row = 0; train_gaussianized_row < train_target_gaussianized_vmat->length(); train_gaussianized_row++)
00525         {
00526             train_target_gaussianized_vmat->getRow(train_gaussianized_row, train_target_preprocessed_vector);
00527             train_target_preprocessed_file->putRow(train_gaussianized_row, train_target_preprocessed_vector);
00528             pb->update( train_gaussianized_row );
00529         }
00530         delete pb;
00531     }
00532     
00533     // defining all the variables for the test set
00534     PPath                                 test_with_class_target_file_name;
00535     VMat                                  test_with_class_target_file;
00536     PPath                                 test_with_binary_fixed_file_name;
00537     VMat                                  test_with_binary_fixed_file;
00538     PPath                                 test_with_ind_file_name;
00539     MissingIndicatorVMatrix*              test_with_ind_vmatrix;
00540     VMat                                  test_with_ind_vmat;
00541     PPath                                 test_with_dichotomies_file_name;
00542     VMat                                  test_with_dichotomies_file;
00543     SelectColumnsVMatrix*                 test_with_selected_columns_vmatrix;
00544     VMat                                  test_with_selected_columns_vmat;
00545     GaussianizeVMatrix*                   test_gaussianized_vmatrix;
00546     VMat                                  test_gaussianized_vmat;
00547     PPath                                 test_input_preprocessed_file_name;
00548     VMat                                  test_input_preprocessed_file;
00549     Vec                                   test_input_preprocessed_vector;
00550     SelectColumnsVMatrix*                 test_target_with_selected_columns_vmatrix;
00551     VMat                                  test_target_with_selected_columns_vmat;
00552     GaussianizeVMatrix*                   test_target_gaussianized_vmatrix;
00553     VMat                                  test_target_gaussianized_vmat;
00554     PPath                                 test_target_preprocessed_file_name;
00555     VMat                                  test_target_preprocessed_file;
00556     Vec                                   test_target_preprocessed_vector;
00557     
00558     // managing the test set
00559     cout << endl << "********************" << endl;
00560     cout << "In Preprocessing: now, we format the test set" << endl;
00561     cout << endl << "****** STEP 1 ******" << endl;
00562     cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl;
00563     cout << "It uses ComputeDond2Target to transform base_test.pmat into step1_test_with_class_target.pmat" << endl;
00564     output_path = expdir+"step1_test_with_class_target";
00565     test_with_class_target_file_name = output_path + ".pmat";
00566     if (isfile(test_with_class_target_file_name))
00567     {
00568         test_with_class_target_file = new FileVMatrix(test_with_class_target_file_name);
00569         test_with_class_target_file->defineSizes(test_with_class_target_file->width(), 0, 0);
00570         cout << test_with_class_target_file_name << " already exist, we are skipping this step." << endl;
00571     }
00572     else 
00573     {
00574         compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template);
00575         compute_target_learner->unknown_sales = 0;
00576         compute_target_learner->output_path = output_path;
00577         compute_target_learner->setTrainingSet(test_set, true);
00578         test_with_class_target_file = compute_target_learner->getOutputFile();
00579     }
00580     cout << endl << "****** STEP 2 ******" << endl;
00581     cout << "Shuffling is not required for the test set, skipped." << endl;
00582     cout << endl << "****** STEP 3 ******" << endl;
00583     cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl;
00584     cout << "This step uses FixDond2BinaryVariables to create step3_test_with_binary_fixed.pmat with 0-1 binary variables." << endl;
00585     output_path = expdir+"step3_test_with_binary_fixed";
00586     test_with_binary_fixed_file_name = output_path + ".pmat";
00587     if (isfile(test_with_binary_fixed_file_name))
00588     {
00589         test_with_binary_fixed_file = new FileVMatrix(test_with_binary_fixed_file_name);
00590         test_with_binary_fixed_file->defineSizes(test_with_binary_fixed_file->width(), 0, 0);
00591         cout << test_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl;
00592     }
00593     else 
00594     {
00595         fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template);
00596         fix_binary_variables_learner->output_path = output_path;
00597         fix_binary_variables_learner->setTrainingSet(test_with_class_target_file, true);
00598         test_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile();
00599     }
00600     cout << endl << "****** STEP 4 ******" << endl;
00601     cout << "This step adds missing indicators variables to each variable with missing values." << endl;
00602     cout << "It uses MissingIndicatorVMatrix to transform step3_test_with_binary_fixed.pmat" << endl;
00603     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 6" << endl;
00604     output_path = expdir+"step6_test_with_dichotomies";
00605     test_with_dichotomies_file_name = output_path + ".pmat";
00606     if (isfile(test_with_dichotomies_file_name))
00607     {
00608         cout << test_with_dichotomies_file_name << " already exist, we are skipping this step." << endl;
00609     }
00610     else 
00611     {
00612         test_with_ind_vmatrix = new MissingIndicatorVMatrix();
00613         test_with_ind_vmatrix->source = test_with_binary_fixed_file;
00614         test_with_ind_vmatrix->train_set = train_with_binary_fixed_file;
00615         test_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0;
00616         test_with_ind_vmatrix->build();
00617         test_with_ind_vmat = test_with_ind_vmatrix;
00618     }
00619     cout << endl << "****** STEP 5 ******" << endl;
00620     cout << "Computing mean, median and mode is not required for the test set, skipped." << endl;
00621     cout << endl << "****** STEP 6 ******" << endl;
00622     cout << "This steps generates as many dichotomized variables as there are significant code values." << endl;
00623     cout << "It uses DichotomizeDond2DiscreteVariables to transform the vmat from step 4 into step6_test_with_dichotomies.pmat" << endl;
00624     if (isfile(test_with_dichotomies_file_name))
00625     {
00626         test_with_dichotomies_file = new FileVMatrix(test_with_dichotomies_file_name);
00627         test_with_dichotomies_file->defineSizes(test_with_dichotomies_file->width(), 0, 0);
00628         cout << test_with_dichotomies_file_name << " already exist, we are skipping this step." << endl;
00629     }
00630     else 
00631     {
00632         dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables();
00633         dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions;
00634         dichotomize_discrete_variables_learner->output_path = output_path;
00635         dichotomize_discrete_variables_learner->setTrainingSet(test_with_ind_vmat, true);
00636         test_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile();
00637     }
00638     cout << endl << "****** STEP 7 ******" << endl;
00639     cout << "Dichotomizing mean median and mode is not required for the test set, skipped." << endl;
00640     cout << endl << "****** STEP 8 ******" << endl;
00641     cout << "This step select the desired columns from the test set to create the input records." << endl;
00642     cout << "It uses SelectColumnsVMatrix to transform step6_test_with_dichotomies.pmat" << endl;
00643     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl;
00644     output_path = expdir+"final_test_input_preprocessed";
00645     test_input_preprocessed_file_name = output_path + ".pmat";
00646     if (isfile(test_input_preprocessed_file_name))
00647     {
00648         cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00649     }
00650     else 
00651     {
00652         test_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00653         test_with_selected_columns_vmatrix->source = test_with_dichotomies_file;
00654         test_with_selected_columns_vmatrix->fields_partial_match = 0;
00655         test_with_selected_columns_vmatrix->extend_with_missing = 0;
00656         test_with_selected_columns_vmatrix->fields = selected_variables_for_input;
00657         test_with_selected_columns_vmatrix->build();
00658         test_with_selected_columns_vmatrix->defineSizes(test_with_selected_columns_vmatrix->width(), 0, 0);
00659         test_with_selected_columns_vmat = test_with_selected_columns_vmatrix;
00660     }
00661     cout << endl << "****** STEP 9 ******" << endl;
00662     cout << "Selecting variables for the mean, median and mode is not required for the test set, skipped." << endl;
00663     cout << endl << "****** STEP 10 ******" << endl;
00664     cout << "This gaussianizes the input records." << endl;
00665     cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl;
00666     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl;
00667     if (isfile(test_input_preprocessed_file_name))
00668     {
00669         cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00670     }
00671     else 
00672     {
00673         test_gaussianized_vmatrix = new GaussianizeVMatrix();
00674         test_gaussianized_vmatrix->source = test_with_selected_columns_vmat;
00675         test_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat;
00676         test_gaussianized_vmatrix->threshold_ratio = 1;
00677         test_gaussianized_vmatrix->gaussianize_input = 1;
00678         test_gaussianized_vmatrix->gaussianize_target = 0;
00679         test_gaussianized_vmatrix->gaussianize_weight = 0;
00680         test_gaussianized_vmatrix->gaussianize_extra = 0;
00681         test_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization;
00682         test_gaussianized_vmatrix->build();
00683         test_gaussianized_vmat = test_gaussianized_vmatrix;
00684     }
00685     cout << endl << "****** STEP 11 ******" << endl;
00686     cout << "Gaussianizing the mean, meadian and mode is not required for the test set, skipped." << endl;
00687     cout << endl << "****** STEP 12 ******" << endl;
00688     cout << "Finaly, the preprocessed input vectors are store on disk." << endl;
00689     cout << "The vmat from step 10 is converted to final_test_input_preprocessed.pmat." << endl;
00690     if (isfile(test_input_preprocessed_file_name))
00691     {
00692         cout << test_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00693     }
00694     else 
00695     {
00696         test_input_preprocessed_file = new FileVMatrix(test_input_preprocessed_file_name, test_gaussianized_vmat->length(), test_gaussianized_vmat->fieldNames());
00697         test_input_preprocessed_file->defineSizes(test_gaussianized_vmat->inputsize(), test_gaussianized_vmat->targetsize(), test_gaussianized_vmat->weightsize());
00698         pb = new ProgressBar("Saving the final test preprocessed input records", test_gaussianized_vmat->length());
00699         test_input_preprocessed_vector.resize(test_gaussianized_vmat->width());
00700         for (int test_gaussianized_row = 0; test_gaussianized_row < test_gaussianized_vmat->length(); test_gaussianized_row++)
00701         {
00702             test_gaussianized_vmat->getRow(test_gaussianized_row, test_input_preprocessed_vector);
00703             test_input_preprocessed_file->putRow(test_gaussianized_row, test_input_preprocessed_vector);
00704             pb->update( test_gaussianized_row );
00705         }
00706         delete pb;
00707     }
00708     cout << endl << "****** STEP 13 ******" << endl;
00709     cout << "Saving the final mean, median and mode is not required for the test set, skipped." << endl;
00710     cout << endl << "****** STEP 14 ******" << endl;
00711     cout << "This step select the desired columns from the testing set to create the target records." << endl;
00712     cout << "It uses SelectColumnsVMatrix to transform step6_test_with_dichotomies.pmat" << endl;
00713     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 15" << endl;
00714     output_path = expdir+"final_test_target_preprocessed";
00715     test_target_preprocessed_file_name = output_path + ".pmat";
00716     if (isfile(test_target_preprocessed_file_name))
00717     {
00718         cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00719     }
00720     else 
00721     {
00722         test_target_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00723         test_target_with_selected_columns_vmatrix->source = test_with_dichotomies_file;
00724         test_target_with_selected_columns_vmatrix->fields_partial_match = 0;
00725         test_target_with_selected_columns_vmatrix->extend_with_missing = 0;
00726         test_target_with_selected_columns_vmatrix->fields = selected_variables_for_target;
00727         test_target_with_selected_columns_vmatrix->build();
00728         test_target_with_selected_columns_vmatrix->defineSizes(test_target_with_selected_columns_vmatrix->width(), 0, 0);
00729         test_target_with_selected_columns_vmat = test_target_with_selected_columns_vmatrix;
00730     }
00731     cout << endl << "****** STEP 15 ******" << endl;
00732     cout << "This gaussianizes the input records." << endl;
00733     cout << "It uses GaussianizeVMatrix to transform the vmat from step 14" << endl;
00734     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 16" << endl;
00735     if (isfile(test_target_preprocessed_file_name))
00736     {
00737         cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00738     }
00739     else 
00740     {
00741         test_target_gaussianized_vmatrix = new GaussianizeVMatrix();
00742         test_target_gaussianized_vmatrix->source = test_target_with_selected_columns_vmat;
00743         test_target_gaussianized_vmatrix->train_source = train_target_with_selected_columns_vmat;
00744         test_target_gaussianized_vmatrix->threshold_ratio = 1;
00745         test_target_gaussianized_vmatrix->gaussianize_input = 1;
00746         test_target_gaussianized_vmatrix->gaussianize_target = 0;
00747         test_target_gaussianized_vmatrix->gaussianize_weight = 0;
00748         test_target_gaussianized_vmatrix->gaussianize_extra = 0;
00749         test_target_gaussianized_vmatrix->excluded_fields = targets_excluded_from_gaussianization;;
00750         test_target_gaussianized_vmatrix->build();
00751         test_target_gaussianized_vmat = test_target_gaussianized_vmatrix;
00752     }
00753     cout << endl << "****** STEP 16 ******" << endl;
00754     cout << "Finaly, the preprocessed input vectors are store on disk." << endl;
00755     cout << "The vmat from step 15 is converted to final_test_target_preprocessed.pmat." << endl;
00756     if (isfile(test_target_preprocessed_file_name))
00757     {
00758         cout << test_target_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00759     }
00760     else 
00761     {
00762         test_target_preprocessed_file = new FileVMatrix(test_target_preprocessed_file_name, test_target_gaussianized_vmat->length(),
00763                                                          test_target_gaussianized_vmat->fieldNames());
00764         test_target_preprocessed_file->defineSizes(test_target_gaussianized_vmat->inputsize(), test_target_gaussianized_vmat->targetsize(),
00765                                                    test_target_gaussianized_vmat->weightsize());
00766         pb = new ProgressBar("Saving the final test preprocessed target records", test_target_gaussianized_vmat->length());
00767         test_target_preprocessed_vector.resize(test_target_gaussianized_vmat->width());
00768         for (int test_gaussianized_row = 0; test_gaussianized_row < test_target_gaussianized_vmat->length(); test_gaussianized_row++)
00769         {
00770             test_target_gaussianized_vmat->getRow(test_gaussianized_row, test_target_preprocessed_vector);
00771             test_target_preprocessed_file->putRow(test_gaussianized_row, test_target_preprocessed_vector);
00772             pb->update( test_gaussianized_row );
00773         }
00774         delete pb;
00775     }
00776     
00777     // defining all the variables for the unknown set
00778     PPath                                 unknown_with_class_target_file_name;
00779     VMat                                  unknown_with_class_target_file;
00780     PPath                                 unknown_with_binary_fixed_file_name;
00781     VMat                                  unknown_with_binary_fixed_file;
00782     PPath                                 unknown_with_ind_file_name;
00783     MissingIndicatorVMatrix*              unknown_with_ind_vmatrix;
00784     VMat                                  unknown_with_ind_vmat;
00785     PPath                                 unknown_with_dichotomies_file_name;
00786     VMat                                  unknown_with_dichotomies_file;
00787     SelectColumnsVMatrix*                 unknown_with_selected_columns_vmatrix;
00788     VMat                                  unknown_with_selected_columns_vmat;
00789     GaussianizeVMatrix*                   unknown_gaussianized_vmatrix;
00790     VMat                                  unknown_gaussianized_vmat;
00791     PPath                                 unknown_input_preprocessed_file_name;
00792     VMat                                  unknown_input_preprocessed_file;
00793     Vec                                   unknown_input_preprocessed_vector;
00794     
00795     // managing the unknown set
00796     cout << endl << "********************" << endl;
00797     cout << "In Preprocessing: finally, we format the unknown set" << endl;
00798     cout << endl << "****** STEP 1 ******" << endl;
00799     cout << "The first step groups variables by type, skips untrustworthy variables, and generate class targets" << endl;
00800     cout << "It uses ComputeDond2Target to transform base_unknown.pmat into step1_unknown_with_class_target.pmat" << endl;
00801     output_path = expdir+"step1_unknown_with_class_target";
00802     unknown_with_class_target_file_name = output_path + ".pmat";
00803     if (isfile(unknown_with_class_target_file_name))
00804     {
00805         unknown_with_class_target_file = new FileVMatrix(unknown_with_class_target_file_name);
00806         unknown_with_class_target_file->defineSizes(unknown_with_class_target_file->width(), 0, 0);
00807         cout << unknown_with_class_target_file_name << " already exist, we are skipping this step." << endl;
00808     }
00809     else 
00810     {
00811         compute_target_learner = ::PLearn::deepCopy(compute_target_learner_template);
00812         compute_target_learner->unknown_sales = 1;
00813         compute_target_learner->output_path = output_path;
00814         compute_target_learner->setTrainingSet(unknown_set, true);
00815         unknown_with_class_target_file = compute_target_learner->getOutputFile();
00816     }
00817     cout << endl << "****** STEP 2 ******" << endl;
00818     cout << "Shuffling is not required for the unknown set, skipped." << endl;
00819     cout << endl << "****** STEP 3 ******" << endl;
00820     cout << "For strictly binary variables, various situations arise: zero or non-zero, missing or not-missing, a given value or not, etc..." << endl;
00821     cout << "This step uses FixDond2BinaryVariables to create step3_unknown_with_binary_fixed.pmat with 0-1 binary variables." << endl;
00822     output_path = expdir+"step3_unknown_with_binary_fixed";
00823     unknown_with_binary_fixed_file_name = output_path + ".pmat";
00824     if (isfile(unknown_with_binary_fixed_file_name))
00825     {
00826         unknown_with_binary_fixed_file = new FileVMatrix(unknown_with_binary_fixed_file_name);
00827         unknown_with_binary_fixed_file->defineSizes(unknown_with_binary_fixed_file->width(), 0, 0);
00828         cout << unknown_with_binary_fixed_file_name << " already exist, we are skipping this step." << endl;
00829     }
00830     else 
00831     {
00832         fix_binary_variables_learner = ::PLearn::deepCopy(fix_binary_variables_template);
00833         fix_binary_variables_learner->output_path = output_path;
00834         fix_binary_variables_learner->setTrainingSet(unknown_with_class_target_file, true);
00835         unknown_with_binary_fixed_file = fix_binary_variables_learner->getOutputFile();
00836     }
00837     cout << endl << "****** STEP 4 ******" << endl;
00838     cout << "This step adds missing indicators variables to each variable with missing values." << endl;
00839     cout << "It uses MissingIndicatorVMatrix to transform step3_unknown_with_binary_fixed.pmat" << endl;
00840     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 6" << endl;
00841     output_path = expdir+"step6_unknown_with_dichotomies";
00842     unknown_with_dichotomies_file_name = output_path + ".pmat";
00843     if (isfile(unknown_with_dichotomies_file_name))
00844     {
00845         cout << unknown_with_dichotomies_file_name << " already exist, we are skipping this step." << endl;
00846     }
00847     else 
00848     {
00849         unknown_with_ind_vmatrix = new MissingIndicatorVMatrix();
00850         unknown_with_ind_vmatrix->source = unknown_with_binary_fixed_file;
00851         unknown_with_ind_vmatrix->train_set = train_with_binary_fixed_file;
00852         unknown_with_ind_vmatrix->number_of_train_samples_to_use = 30000.0;
00853         unknown_with_ind_vmatrix->build();
00854         unknown_with_ind_vmat = unknown_with_ind_vmatrix;
00855     }
00856     cout << endl << "****** STEP 5 ******" << endl;
00857     cout << "Computing mean, median and mode is not required for the unknown set, skipped." << endl;
00858     cout << endl << "****** STEP 6 ******" << endl;
00859     cout << "This steps generates as many dichotomized variables as there are significant code values." << endl;
00860     cout << "It uses DichotomizeDond2DiscreteVariables to transform the vmat from step 4 into step6_unknown_with_dichotomies.pmat" << endl;
00861     if (isfile(unknown_with_dichotomies_file_name))
00862     {
00863         unknown_with_dichotomies_file = new FileVMatrix(unknown_with_dichotomies_file_name);
00864         unknown_with_dichotomies_file->defineSizes(unknown_with_dichotomies_file->width(), 0, 0);
00865         cout << unknown_with_dichotomies_file_name << " already exist, we are skipping this step." << endl;
00866     }
00867     else 
00868     {
00869         dichotomize_discrete_variables_learner = new DichotomizeDond2DiscreteVariables();
00870         dichotomize_discrete_variables_learner->discrete_variable_instructions = discrete_variable_instructions;
00871         dichotomize_discrete_variables_learner->output_path = output_path;
00872         dichotomize_discrete_variables_learner->setTrainingSet(unknown_with_ind_vmat, true);
00873         unknown_with_dichotomies_file = dichotomize_discrete_variables_learner->getOutputFile();
00874     }
00875     cout << endl << "****** STEP 7 ******" << endl;
00876     cout << "Dichotomizing mean median and mode is not required for the unknown set, skipped." << endl;
00877     cout << endl << "****** STEP 8 ******" << endl;
00878     cout << "This step select the desired columns from the unknown set to create the input records." << endl;
00879     cout << "It uses SelectColumnsVMatrix to transform step6_unknown_with_dichotomies.pmat" << endl;
00880     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 10" << endl;
00881     output_path = expdir+"final_unknown_input_preprocessed";
00882     unknown_input_preprocessed_file_name = output_path + ".pmat";
00883     if (isfile(unknown_input_preprocessed_file_name))
00884     {
00885         cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00886     }
00887     else 
00888     {
00889         unknown_with_selected_columns_vmatrix = new SelectColumnsVMatrix();
00890         unknown_with_selected_columns_vmatrix->source = unknown_with_dichotomies_file;
00891         unknown_with_selected_columns_vmatrix->fields_partial_match = 0;
00892         unknown_with_selected_columns_vmatrix->extend_with_missing = 0;
00893         unknown_with_selected_columns_vmatrix->fields = selected_variables_for_input;
00894         unknown_with_selected_columns_vmatrix->build();
00895         unknown_with_selected_columns_vmatrix->defineSizes(unknown_with_selected_columns_vmatrix->width(), 0, 0);
00896         unknown_with_selected_columns_vmat = unknown_with_selected_columns_vmatrix;
00897     }
00898     cout << endl << "****** STEP 9 ******" << endl;
00899     cout << "Selecting variables for the mean, median and mode is not required for the unknown set, skipped." << endl;
00900     cout << endl << "****** STEP 10 ******" << endl;
00901     cout << "This gaussianizes the input records." << endl;
00902     cout << "It uses GaussianizeVMatrix to transform the vmat from step 8" << endl;
00903     cout << "The resulting vitual view is not stored on disk, it is fed as input to step 12" << endl;
00904     if (isfile(unknown_input_preprocessed_file_name))
00905     {
00906         cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00907     }
00908     else 
00909     {
00910         unknown_gaussianized_vmatrix = new GaussianizeVMatrix();
00911         unknown_gaussianized_vmatrix->source = unknown_with_selected_columns_vmat;
00912         unknown_gaussianized_vmatrix->train_source = train_with_selected_columns_vmat;
00913         unknown_gaussianized_vmatrix->threshold_ratio = 1;
00914         unknown_gaussianized_vmatrix->gaussianize_input = 1;
00915         unknown_gaussianized_vmatrix->gaussianize_target = 0;
00916         unknown_gaussianized_vmatrix->gaussianize_weight = 0;
00917         unknown_gaussianized_vmatrix->gaussianize_extra = 0;
00918         unknown_gaussianized_vmatrix->excluded_fields = inputs_excluded_from_gaussianization;
00919         unknown_gaussianized_vmatrix->build();
00920         unknown_gaussianized_vmat = unknown_gaussianized_vmatrix;
00921     }
00922     cout << endl << "****** STEP 11 ******" << endl;
00923     cout << "Gaussianizing the mean, meadian and mode is not required for the unknown set, skipped." << endl;
00924     cout << endl << "****** STEP 12 ******" << endl;
00925     cout << "Finaly, the preprocessed input vectors are store on disk." << endl;
00926     cout << "The vmat from step 10 is converted to final_unknown_input_preprocessed.pmat." << endl;
00927     if (isfile(unknown_input_preprocessed_file_name))
00928     {
00929         cout << unknown_input_preprocessed_file_name << " already exist, we are skipping this step." << endl;
00930     }
00931     else 
00932     {
00933         unknown_input_preprocessed_file = new FileVMatrix(unknown_input_preprocessed_file_name, unknown_gaussianized_vmat->length(), unknown_gaussianized_vmat->fieldNames());
00934         unknown_input_preprocessed_file->defineSizes(unknown_gaussianized_vmat->inputsize(), unknown_gaussianized_vmat->targetsize(), unknown_gaussianized_vmat->weightsize());
00935         pb = new ProgressBar("Saving the final unknown preprocessed input records", unknown_gaussianized_vmat->length());
00936         unknown_input_preprocessed_vector.resize(unknown_gaussianized_vmat->width());
00937         for (int unknown_gaussianized_row = 0; unknown_gaussianized_row < unknown_gaussianized_vmat->length(); unknown_gaussianized_row++)
00938         {
00939             unknown_gaussianized_vmat->getRow(unknown_gaussianized_row, unknown_input_preprocessed_vector);
00940             unknown_input_preprocessed_file->putRow(unknown_gaussianized_row, unknown_input_preprocessed_vector);
00941             pb->update( unknown_gaussianized_row );
00942         }
00943         delete pb;
00944     }
00945     cout << endl << "****** STEP 13 ******" << endl;
00946     cout << "Saving the final mean, median and mode is not required for the unknown set, skipped." << endl;
00947     cout << endl << "****** STEP 14 ******" << endl;
00948     cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl;
00949     cout << endl << "****** STEP 15 ******" << endl;
00950     cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl;
00951     cout << endl << "****** STEP 16 ******" << endl;
00952     cout << "Saving the final target preprocessed records is not required for the unknown set, skipped." << endl;
00953 }
00954 
00955 void Preprocessing::train()
00956 {
00957 }
00958 
00959 int Preprocessing::outputsize() const {return 0;}
00960 void Preprocessing::computeOutput(const Vec&, Vec&) const {}
00961 void Preprocessing::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {}
00962 TVec<string> Preprocessing::getTestCostNames() const
00963 {
00964     TVec<string> result;
00965     result.append( "MSE" );
00966     return result;
00967 }
00968 TVec<string> Preprocessing::getTrainCostNames() const
00969 {
00970     TVec<string> result;
00971     result.append( "MSE" );
00972     return result;
00973 }
00974 
00975 } // end of namespace PLearn
00976 
00977 
00978 /*
00979   Local Variables:
00980   mode:c++
00981   c-basic-offset:4
00982   c-file-style:"stroustrup"
00983   c-file-offsets:((innamespace . 0)(inline-open . 0))
00984   indent-tabs-mode:nil
00985   fill-column:79
00986   End:
00987 */
00988 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines