PLearn 0.1
MergeDond2Files.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // MergeDond2Files.cc
00004 //
00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Dan Popovici
00036 
00039 #define PL_LOG_MODULE_NAME "MergeDond2Files"
00040 
00041 #include "MergeDond2Files.h"
00042 #include <plearn/io/pl_log.h>
00043 
00044 namespace PLearn {
00045 using namespace std;
00046 
00047 PLEARN_IMPLEMENT_OBJECT(
00048     MergeDond2Files,
00049     "Merge two files on specified keys.",
00050     "If more than 1 matching record is found in the secondary datatset,\n"
00051     "mean, mode or variable presence will be appended based on variable merge instruction.\n"
00052     "a variable from the secondary dataset may also be skipped.\n"
00053 );
00054 
00056 // MergeDond2Files //
00058 MergeDond2Files::MergeDond2Files()
00059 {
00060 }
00061     
00063 // declareOptions //
00065 void MergeDond2Files::declareOptions(OptionList& ol)
00066 {
00067     declareOption(ol, "external_dataset", &MergeDond2Files::external_dataset,
00068                   OptionBase::buildoption,
00069                   "The secondary dataset to merge with the main one.\n"
00070                   "The main one is provided as the train_set.");
00071 
00072     declareOption(ol, "missing_instructions", &MergeDond2Files::missing_instructions,
00073                   OptionBase::buildoption,
00074                   "The variable missing regeneration instructions in the form of pairs field : instruction.\n"
00075                   "Supported instructions are skip, as_is, zero_is_missing, 2436935_is_missing, present.");
00076 
00077     declareOption(ol, "merge_instructions", &MergeDond2Files::merge_instructions,
00078                   OptionBase::buildoption,
00079                   "The variable merge instructions in the form of pairs field : instruction.\n"
00080                   "Supported instructions are skip, mean, mode, present.");
00081 
00082     declareOption(ol, "merge_path", &MergeDond2Files::merge_path,
00083                   OptionBase::buildoption,
00084                   "The root name of merge files to be created.\n"
00085                   "3 files will be created: a root_train.pmat, a root_test.pmat and a root_uknown.pmat");
00086 
00087     declareOption(ol, "sec_key", &MergeDond2Files::sec_key,
00088                   OptionBase::buildoption,
00089                   "The column of the merge key in the secondary dataset.");
00090 
00091     declareOption(ol, "main_key", &MergeDond2Files::main_key,
00092                   OptionBase::buildoption,
00093                   "The column of the merge key in the main dataset.");
00094 
00095     declareOption(ol, "train_ind", &MergeDond2Files::train_ind,
00096                   OptionBase::buildoption,
00097                   "The column of the indicator of a train record in the main dataset.");
00098 
00099     declareOption(ol, "test_ind", &MergeDond2Files::test_ind,
00100                   OptionBase::buildoption,
00101                   "The column of the indicator of a test record in the main dataset.");
00102 
00103     declareOption(ol, "train_file", &MergeDond2Files::train_file,
00104                   OptionBase::learntoption,
00105                   "The train file created.");
00106 
00107     declareOption(ol, "test_file", &MergeDond2Files::test_file,
00108                   OptionBase::learntoption,
00109                   "The test file created.");
00110 
00111     declareOption(ol, "unknown_file", &MergeDond2Files::unknown_file,
00112                   OptionBase::learntoption,
00113                   "The unknown target file created.");
00114 
00115 
00116     inherited::declareOptions(ol);
00117 }
00118 
00120 // makeDeepCopyFromShallowCopy //
00122 void MergeDond2Files::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00123 {
00124     deepCopyField(external_dataset, copies);
00125     deepCopyField(missing_instructions, copies);
00126     deepCopyField(merge_instructions, copies);
00127     deepCopyField(merge_path, copies);
00128     deepCopyField(sec_key, copies);
00129     deepCopyField(main_key, copies);
00130     deepCopyField(train_ind, copies);
00131     deepCopyField(test_ind, copies);
00132     deepCopyField(train_file, copies);
00133     deepCopyField(test_file, copies);
00134     deepCopyField(unknown_file, copies);
00135     inherited::makeDeepCopyFromShallowCopy(copies);
00136 
00137 }
00138 
00140 // build //
00142 void MergeDond2Files::build()
00143 {
00144     // ### Nothing to add here, simply calls build_().
00145     inherited::build();
00146     build_();
00147 }
00148 
00150 // build_ //
00152 void MergeDond2Files::build_()
00153 {
00154     MODULE_LOG << "build_() called" << endl;
00155     if (train_set)
00156     {
00157         mergeFiles();
00158         PLERROR("MergeDond2Files: we are done here");
00159     }
00160 }
00161 
00162 void MergeDond2Files::mergeFiles()
00163 {
00164     // initialization with merge instructions
00165     sec_row = 0;
00166     sec_length = external_dataset->length();
00167     sec_width = external_dataset->width();
00168     sec_names.resize(sec_width);
00169     sec_ins.resize(sec_width);
00170     sec_input.resize(sec_width);
00171     extension_width = 0;
00172     sec_names << external_dataset->fieldNames();
00173     for (int sec_col = 0; sec_col < sec_width; sec_col++)
00174     {
00175         sec_ins[sec_col] = "mean";
00176     }
00177     for (int ins_col = 0; ins_col < merge_instructions.size(); ins_col++)
00178     {
00179         int sec_col;
00180         for (sec_col = 0; sec_col < sec_width; sec_col++)
00181         {
00182             if (merge_instructions[ins_col].first == sec_names[sec_col]) break;
00183         }
00184         if (sec_col >= sec_width) PLERROR("In MergeDond2Files: no field with this name in external_dataset data set: %s", (merge_instructions[ins_col].first).c_str());
00185         if (merge_instructions[ins_col].second == "skip") sec_ins[sec_col] = "skip";
00186         else if (merge_instructions[ins_col].second == "mean") sec_ins[sec_col] = "mean";
00187         else if (merge_instructions[ins_col].second == "mode") sec_ins[sec_col] = "mode";
00188         else if (merge_instructions[ins_col].second == "present") sec_ins[sec_col] = "present";
00189         else PLERROR("In MergeDond2Files: unsupported merge instruction: %s", (merge_instructions[ins_col].second).c_str());
00190         if (sec_ins[sec_col] != "skip") extension_width += 1;
00191     }
00192     extension_pos.resize(sec_width);
00193     extension_names.resize(extension_width);
00194     for (int ext_col = 0, sec_col = 0; sec_col < sec_width; sec_col++)
00195     {
00196         if (sec_ins[sec_col] == "skip")
00197         {
00198             extension_pos[sec_col] = -1; 
00199         }
00200         else
00201         {
00202             extension_pos[sec_col] = ext_col;
00203             extension_names[ext_col] = sec_names[sec_col];
00204             ext_col += 1;
00205         }
00206     }
00207     sec_values.resize(extension_width, 10);
00208     sec_value_cnt.resize(extension_width, 10);
00209     sec_value_ind.resize(extension_width);
00210     sec_values.clear();
00211     sec_value_cnt.clear();
00212     sec_value_ind.clear();
00213     external_dataset->getRow(sec_row, sec_input);
00214     
00215     // initialize primary dataset
00216     main_row = 0;
00217     main_length = train_set->length();
00218     main_width = train_set->width();
00219     main_input.resize(main_width);
00220     main_names.resize(main_width);
00221     main_ins.resize(main_width);
00222     main_names << train_set->fieldNames();
00223     primary_width = 0;
00224     for (int main_col = 0; main_col < main_width; main_col++)
00225     {
00226         main_ins[main_col] = "as_is";
00227     }
00228     for (int ins_col = 0; ins_col < missing_instructions.size(); ins_col++)
00229     {
00230         int main_col = 0;
00231         for (main_col = 0; main_col < main_width; main_col++)
00232         {
00233             if (missing_instructions[ins_col].first == main_names[main_col]) break;
00234         }
00235         if (main_col >= main_width) PLERROR("In MergeDond2Files: no field with this name in external_dataset data set: %s", (missing_instructions[ins_col].first).c_str());
00236         if (missing_instructions[ins_col].second == "skip") main_ins[main_col] = "skip";
00237         else if (missing_instructions[ins_col].second == "as_is") main_ins[main_col] = "as_is";
00238         else if (missing_instructions[ins_col].second == "zero_is_missing") main_ins[main_col] = "zero_is_missing";
00239         else if (missing_instructions[ins_col].second == "2436935_is_missing") main_ins[main_col] = "2436935_is_missing";
00240         else if (missing_instructions[ins_col].second == "present") main_ins[main_col] = "present";
00241         else PLERROR("In MergeDond2Files: unsupported merge instruction: %s", (missing_instructions[ins_col].second).c_str());
00242         if (main_ins[main_col] != "skip") primary_width += 1;
00243     }
00244     primary_names.resize(primary_width);
00245     for (int main_col = 0, prim_col = 0; main_col < main_width; main_col++)
00246     {
00247         if (main_ins[main_col] != "skip")
00248         {
00249             primary_names[prim_col] = main_names[main_col];
00250             prim_col += 1;
00251         }
00252     }
00253     
00254     // initialize output datasets
00255     merge_col = 0;
00256     merge_width = primary_width + extension_width;
00257     merge_output.resize(merge_width);
00258     merge_names.resize(merge_width);
00259     for (int prim_col = 0; prim_col < primary_width; prim_col++)
00260     {
00261        merge_names[merge_col] = primary_names[prim_col];
00262        merge_col +=1;
00263     }
00264     for (int ext_col = 0; ext_col < extension_width; ext_col++)
00265     {
00266        merge_names[merge_col] = extension_names[ext_col];
00267        merge_col +=1;
00268     }
00269     train_length = 0;
00270     test_length = 0;
00271     unknown_length = 0;
00272     ProgressBar* pb = 0;
00273     pb = new ProgressBar( "Counting the number of records in the train, test and unknown datasets", main_length);
00274     for (main_row = 0; main_row < main_length; main_row++)
00275     {
00276         train_set->getRow(main_row, main_input);
00277         if (main_input[train_ind] > 0.0) train_length += 1;
00278         else if (main_input[test_ind] > 0.0) test_length += 1;
00279         else unknown_length += 1;
00280         pb->update( main_row );
00281     }
00282     delete pb;
00283     train_file = new FileVMatrix(merge_path + "_train.pmat", train_length, merge_width);
00284     test_file = new FileVMatrix(merge_path + "_test.pmat", test_length, merge_width);
00285     unknown_file = new FileVMatrix(merge_path + "_unknown.pmat", unknown_length, merge_width);
00286     train_file->declareFieldNames(merge_names);
00287     test_file->declareFieldNames(merge_names);
00288     unknown_file->declareFieldNames(merge_names);
00289     train_row = 0;
00290     test_row = 0;
00291     unknown_row = 0;
00292     
00293     //Now, we can merge
00294     pb = new ProgressBar( "Merging primary and secondary datasets with instructions", main_length);
00295     for (main_row = 0; main_row < main_length; main_row++)
00296     {
00297         train_set->getRow(main_row, main_input);
00298         if (sec_row >= sec_length)
00299         {
00300             combineAndPut();
00301             continue;
00302         }
00303         while (sec_input[sec_key] < main_input[main_key])
00304         {
00305             sec_row += 1;
00306             if (sec_row >= sec_length) break;   
00307             external_dataset->getRow(sec_row, sec_input);
00308         }
00309         while (sec_input[sec_key] == main_input[main_key])
00310         {
00311             accumulateVec();
00312             sec_row += 1;
00313             if (sec_row >= sec_length) break;   
00314             external_dataset->getRow(sec_row, sec_input);
00315         }
00316         combineAndPut();
00317         pb->update( main_row );
00318     }
00319     delete pb;
00320 }
00321 
00322 void MergeDond2Files::accumulateVec()
00323 {
00324     for (int sec_col = 0; sec_col < sec_width; sec_col++)
00325     {
00326         if (is_missing(sec_input[sec_col])) continue;
00327         else if (sec_ins[sec_col] == "skip") continue;
00328         int ext_col = extension_pos[sec_col];
00329         if (sec_ins[sec_col] == "mean")
00330         {
00331             sec_values(ext_col, 0) += sec_input[sec_col];
00332             sec_value_cnt(ext_col, 0) += 1.0;
00333         }
00334         else if (sec_ins[sec_col] == "mode")
00335         {
00336             sec_value_found = false;
00337             for (int sec_value_col = 0; sec_value_col < sec_value_ind[sec_col]; sec_value_col++)
00338             {
00339                 if (sec_values(ext_col, sec_value_col) == sec_input[sec_col])
00340                 {
00341                     sec_value_found = true;
00342                     sec_value_cnt(ext_col, sec_value_col) += 1;
00343                     break;
00344                 }
00345             }
00346             if (!sec_value_found)
00347             {
00348                 if (sec_value_ind[sec_col] >= 10)
00349                 {
00350                     cout << "MergeDond2Files: main file row: " << main_row << " external file row: " << sec_row << endl;
00351                     PLERROR("MergeDond2Files: we have exceeded the capacity of the value MAT.");
00352                 }
00353                 sec_values(ext_col, sec_value_ind[sec_col]) = sec_input[sec_col];
00354                 sec_value_ind[sec_col] += 1;
00355             }
00356         }
00357         else if (sec_ins[sec_col] == "present")
00358         {
00359             sec_value_cnt(ext_col, 0) = 1.0;
00360         }
00361     }
00362 }
00363 
00364 void MergeDond2Files::combineAndPut()
00365 {
00366     merge_col = 0;
00367     for (int main_col = 0; main_col < main_width; main_col++)
00368     {
00369         if (main_ins[main_col] == "skip") continue;
00370         else if (main_ins[main_col] == "as_is")
00371         {
00372             merge_output[merge_col] = main_input[main_col];
00373         }
00374         else if (main_ins[main_col] == "zero_is_missing")
00375         {
00376             if (main_input[main_col] == 0.0) merge_output[merge_col] = MISSING_VALUE;
00377             else merge_output[merge_col] = main_input[main_col];
00378         }
00379         else if (main_ins[main_col] == "2436935_is_missing")
00380         {
00381             if (main_input[main_col] == 2436935.0) merge_output[merge_col] = MISSING_VALUE;
00382             else merge_output[merge_col] = main_input[main_col];
00383         }
00384         else if (main_ins[main_col] == "present")
00385         {
00386             if (is_missing(main_input[main_col])) merge_output[merge_col] = 0.0;
00387             else merge_output[merge_col] = 1.0;
00388         }
00389         merge_col +=1;
00390     }
00391     for (int sec_col = 0; sec_col < sec_width; sec_col++)
00392     {
00393         if (sec_ins[sec_col] == "skip") continue;
00394         int ext_col = extension_pos[sec_col];
00395         if (sec_ins[sec_col] == "mean")
00396         {
00397             if (sec_value_cnt(ext_col, 0) <= 0.0)  merge_output[merge_col] = MISSING_VALUE;
00398             else merge_output[merge_col] = sec_values(ext_col, 0) / sec_value_cnt(ext_col, 0);
00399         }
00400         else if (sec_ins[sec_col] == "mode")
00401         {
00402             if (sec_value_ind[sec_col] <= 0.0)
00403             {
00404                 merge_output[merge_col] = MISSING_VALUE;
00405                 continue;
00406             }
00407             merge_output[merge_col] = sec_values(ext_col, 0);
00408             sec_value_count_max = sec_value_cnt(ext_col, 0);
00409             for (int sec_value_col = 1; sec_value_col < sec_value_ind[sec_col]; sec_value_col++)
00410             {
00411                 if (sec_value_cnt(ext_col, sec_value_col) >= sec_value_count_max)
00412                 {
00413                     merge_output[merge_col] = sec_values(ext_col, sec_value_col);
00414                     sec_value_count_max = sec_value_cnt(ext_col, sec_value_col);
00415                 }
00416             }
00417         }
00418         else if (sec_ins[sec_col] == "present")
00419         {
00420             merge_output[merge_col] = sec_value_cnt(ext_col, 0);
00421         }
00422         merge_col +=1;
00423     }
00424     if (main_input[train_ind] > 0.0)
00425     {
00426         train_file->putRow(train_row, merge_output);
00427         train_row += 1;
00428     }
00429     else if (main_input[test_ind] > 0.0)
00430     {
00431         test_file->putRow(test_row, merge_output);
00432         test_row += 1;
00433     }
00434     else
00435     {
00436         unknown_file->putRow(unknown_row, merge_output);
00437         unknown_row += 1;
00438     }
00439     sec_values.clear();
00440     sec_value_cnt.clear();
00441     sec_value_ind.clear();
00442 }
00443 
00444 int MergeDond2Files::outputsize() const {return 0;}
00445 void MergeDond2Files::train() {}
00446 void MergeDond2Files::computeOutput(const Vec&, Vec&) const {}
00447 void MergeDond2Files::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {}
00448 TVec<string> MergeDond2Files::getTestCostNames() const
00449 {
00450     TVec<string> result;
00451     result.append( "MSE" );
00452     return result;
00453 }
00454 TVec<string> MergeDond2Files::getTrainCostNames() const
00455 {
00456     TVec<string> result;
00457     result.append( "MSE" );
00458     return result;
00459 }
00460 
00461 } // end of namespace PLearn
00462 
00463 
00464 /*
00465   Local Variables:
00466   mode:c++
00467   c-basic-offset:4
00468   c-file-style:"stroustrup"
00469   c-file-offsets:((innamespace . 0)(inline-open . 0))
00470   indent-tabs-mode:nil
00471   fill-column:79
00472   End:
00473 */
00474 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines