PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // MergeDond2Files.cc 00004 // 00005 // Copyright (C) 2006 Dan Popovici, Pascal Lamblin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Dan Popovici 00036 00039 #define PL_LOG_MODULE_NAME "MergeDond2Files" 00040 00041 #include "MergeDond2Files.h" 00042 #include <plearn/io/pl_log.h> 00043 00044 namespace PLearn { 00045 using namespace std; 00046 00047 PLEARN_IMPLEMENT_OBJECT( 00048 MergeDond2Files, 00049 "Merge two files on specified keys.", 00050 "If more than 1 matching record is found in the secondary datatset,\n" 00051 "mean, mode or variable presence will be appended based on variable merge instruction.\n" 00052 "a variable from the secondary dataset may also be skipped.\n" 00053 ); 00054 00056 // MergeDond2Files // 00058 MergeDond2Files::MergeDond2Files() 00059 { 00060 } 00061 00063 // declareOptions // 00065 void MergeDond2Files::declareOptions(OptionList& ol) 00066 { 00067 declareOption(ol, "external_dataset", &MergeDond2Files::external_dataset, 00068 OptionBase::buildoption, 00069 "The secondary dataset to merge with the main one.\n" 00070 "The main one is provided as the train_set."); 00071 00072 declareOption(ol, "missing_instructions", &MergeDond2Files::missing_instructions, 00073 OptionBase::buildoption, 00074 "The variable missing regeneration instructions in the form of pairs field : instruction.\n" 00075 "Supported instructions are skip, as_is, zero_is_missing, 2436935_is_missing, present."); 00076 00077 declareOption(ol, "merge_instructions", &MergeDond2Files::merge_instructions, 00078 OptionBase::buildoption, 00079 "The variable merge instructions in the form of pairs field : instruction.\n" 00080 "Supported instructions are skip, mean, mode, present."); 00081 00082 declareOption(ol, "merge_path", &MergeDond2Files::merge_path, 00083 OptionBase::buildoption, 00084 "The root name of merge files to be created.\n" 00085 "3 files will be created: a root_train.pmat, a root_test.pmat and a root_uknown.pmat"); 00086 00087 declareOption(ol, "sec_key", &MergeDond2Files::sec_key, 00088 OptionBase::buildoption, 00089 "The column of the merge key in the secondary dataset."); 00090 00091 declareOption(ol, "main_key", &MergeDond2Files::main_key, 00092 OptionBase::buildoption, 00093 "The column of the merge key in the main dataset."); 00094 00095 declareOption(ol, "train_ind", &MergeDond2Files::train_ind, 00096 OptionBase::buildoption, 00097 "The column of the indicator of a train record in the main dataset."); 00098 00099 declareOption(ol, "test_ind", &MergeDond2Files::test_ind, 00100 OptionBase::buildoption, 00101 "The column of the indicator of a test record in the main dataset."); 00102 00103 declareOption(ol, "train_file", &MergeDond2Files::train_file, 00104 OptionBase::learntoption, 00105 "The train file created."); 00106 00107 declareOption(ol, "test_file", &MergeDond2Files::test_file, 00108 OptionBase::learntoption, 00109 "The test file created."); 00110 00111 declareOption(ol, "unknown_file", &MergeDond2Files::unknown_file, 00112 OptionBase::learntoption, 00113 "The unknown target file created."); 00114 00115 00116 inherited::declareOptions(ol); 00117 } 00118 00120 // makeDeepCopyFromShallowCopy // 00122 void MergeDond2Files::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00123 { 00124 deepCopyField(external_dataset, copies); 00125 deepCopyField(missing_instructions, copies); 00126 deepCopyField(merge_instructions, copies); 00127 deepCopyField(merge_path, copies); 00128 deepCopyField(sec_key, copies); 00129 deepCopyField(main_key, copies); 00130 deepCopyField(train_ind, copies); 00131 deepCopyField(test_ind, copies); 00132 deepCopyField(train_file, copies); 00133 deepCopyField(test_file, copies); 00134 deepCopyField(unknown_file, copies); 00135 inherited::makeDeepCopyFromShallowCopy(copies); 00136 00137 } 00138 00140 // build // 00142 void MergeDond2Files::build() 00143 { 00144 // ### Nothing to add here, simply calls build_(). 00145 inherited::build(); 00146 build_(); 00147 } 00148 00150 // build_ // 00152 void MergeDond2Files::build_() 00153 { 00154 MODULE_LOG << "build_() called" << endl; 00155 if (train_set) 00156 { 00157 mergeFiles(); 00158 PLERROR("MergeDond2Files: we are done here"); 00159 } 00160 } 00161 00162 void MergeDond2Files::mergeFiles() 00163 { 00164 // initialization with merge instructions 00165 sec_row = 0; 00166 sec_length = external_dataset->length(); 00167 sec_width = external_dataset->width(); 00168 sec_names.resize(sec_width); 00169 sec_ins.resize(sec_width); 00170 sec_input.resize(sec_width); 00171 extension_width = 0; 00172 sec_names << external_dataset->fieldNames(); 00173 for (int sec_col = 0; sec_col < sec_width; sec_col++) 00174 { 00175 sec_ins[sec_col] = "mean"; 00176 } 00177 for (int ins_col = 0; ins_col < merge_instructions.size(); ins_col++) 00178 { 00179 int sec_col; 00180 for (sec_col = 0; sec_col < sec_width; sec_col++) 00181 { 00182 if (merge_instructions[ins_col].first == sec_names[sec_col]) break; 00183 } 00184 if (sec_col >= sec_width) PLERROR("In MergeDond2Files: no field with this name in external_dataset data set: %s", (merge_instructions[ins_col].first).c_str()); 00185 if (merge_instructions[ins_col].second == "skip") sec_ins[sec_col] = "skip"; 00186 else if (merge_instructions[ins_col].second == "mean") sec_ins[sec_col] = "mean"; 00187 else if (merge_instructions[ins_col].second == "mode") sec_ins[sec_col] = "mode"; 00188 else if (merge_instructions[ins_col].second == "present") sec_ins[sec_col] = "present"; 00189 else PLERROR("In MergeDond2Files: unsupported merge instruction: %s", (merge_instructions[ins_col].second).c_str()); 00190 if (sec_ins[sec_col] != "skip") extension_width += 1; 00191 } 00192 extension_pos.resize(sec_width); 00193 extension_names.resize(extension_width); 00194 for (int ext_col = 0, sec_col = 0; sec_col < sec_width; sec_col++) 00195 { 00196 if (sec_ins[sec_col] == "skip") 00197 { 00198 extension_pos[sec_col] = -1; 00199 } 00200 else 00201 { 00202 extension_pos[sec_col] = ext_col; 00203 extension_names[ext_col] = sec_names[sec_col]; 00204 ext_col += 1; 00205 } 00206 } 00207 sec_values.resize(extension_width, 10); 00208 sec_value_cnt.resize(extension_width, 10); 00209 sec_value_ind.resize(extension_width); 00210 sec_values.clear(); 00211 sec_value_cnt.clear(); 00212 sec_value_ind.clear(); 00213 external_dataset->getRow(sec_row, sec_input); 00214 00215 // initialize primary dataset 00216 main_row = 0; 00217 main_length = train_set->length(); 00218 main_width = train_set->width(); 00219 main_input.resize(main_width); 00220 main_names.resize(main_width); 00221 main_ins.resize(main_width); 00222 main_names << train_set->fieldNames(); 00223 primary_width = 0; 00224 for (int main_col = 0; main_col < main_width; main_col++) 00225 { 00226 main_ins[main_col] = "as_is"; 00227 } 00228 for (int ins_col = 0; ins_col < missing_instructions.size(); ins_col++) 00229 { 00230 int main_col = 0; 00231 for (main_col = 0; main_col < main_width; main_col++) 00232 { 00233 if (missing_instructions[ins_col].first == main_names[main_col]) break; 00234 } 00235 if (main_col >= main_width) PLERROR("In MergeDond2Files: no field with this name in external_dataset data set: %s", (missing_instructions[ins_col].first).c_str()); 00236 if (missing_instructions[ins_col].second == "skip") main_ins[main_col] = "skip"; 00237 else if (missing_instructions[ins_col].second == "as_is") main_ins[main_col] = "as_is"; 00238 else if (missing_instructions[ins_col].second == "zero_is_missing") main_ins[main_col] = "zero_is_missing"; 00239 else if (missing_instructions[ins_col].second == "2436935_is_missing") main_ins[main_col] = "2436935_is_missing"; 00240 else if (missing_instructions[ins_col].second == "present") main_ins[main_col] = "present"; 00241 else PLERROR("In MergeDond2Files: unsupported merge instruction: %s", (missing_instructions[ins_col].second).c_str()); 00242 if (main_ins[main_col] != "skip") primary_width += 1; 00243 } 00244 primary_names.resize(primary_width); 00245 for (int main_col = 0, prim_col = 0; main_col < main_width; main_col++) 00246 { 00247 if (main_ins[main_col] != "skip") 00248 { 00249 primary_names[prim_col] = main_names[main_col]; 00250 prim_col += 1; 00251 } 00252 } 00253 00254 // initialize output datasets 00255 merge_col = 0; 00256 merge_width = primary_width + extension_width; 00257 merge_output.resize(merge_width); 00258 merge_names.resize(merge_width); 00259 for (int prim_col = 0; prim_col < primary_width; prim_col++) 00260 { 00261 merge_names[merge_col] = primary_names[prim_col]; 00262 merge_col +=1; 00263 } 00264 for (int ext_col = 0; ext_col < extension_width; ext_col++) 00265 { 00266 merge_names[merge_col] = extension_names[ext_col]; 00267 merge_col +=1; 00268 } 00269 train_length = 0; 00270 test_length = 0; 00271 unknown_length = 0; 00272 ProgressBar* pb = 0; 00273 pb = new ProgressBar( "Counting the number of records in the train, test and unknown datasets", main_length); 00274 for (main_row = 0; main_row < main_length; main_row++) 00275 { 00276 train_set->getRow(main_row, main_input); 00277 if (main_input[train_ind] > 0.0) train_length += 1; 00278 else if (main_input[test_ind] > 0.0) test_length += 1; 00279 else unknown_length += 1; 00280 pb->update( main_row ); 00281 } 00282 delete pb; 00283 train_file = new FileVMatrix(merge_path + "_train.pmat", train_length, merge_width); 00284 test_file = new FileVMatrix(merge_path + "_test.pmat", test_length, merge_width); 00285 unknown_file = new FileVMatrix(merge_path + "_unknown.pmat", unknown_length, merge_width); 00286 train_file->declareFieldNames(merge_names); 00287 test_file->declareFieldNames(merge_names); 00288 unknown_file->declareFieldNames(merge_names); 00289 train_row = 0; 00290 test_row = 0; 00291 unknown_row = 0; 00292 00293 //Now, we can merge 00294 pb = new ProgressBar( "Merging primary and secondary datasets with instructions", main_length); 00295 for (main_row = 0; main_row < main_length; main_row++) 00296 { 00297 train_set->getRow(main_row, main_input); 00298 if (sec_row >= sec_length) 00299 { 00300 combineAndPut(); 00301 continue; 00302 } 00303 while (sec_input[sec_key] < main_input[main_key]) 00304 { 00305 sec_row += 1; 00306 if (sec_row >= sec_length) break; 00307 external_dataset->getRow(sec_row, sec_input); 00308 } 00309 while (sec_input[sec_key] == main_input[main_key]) 00310 { 00311 accumulateVec(); 00312 sec_row += 1; 00313 if (sec_row >= sec_length) break; 00314 external_dataset->getRow(sec_row, sec_input); 00315 } 00316 combineAndPut(); 00317 pb->update( main_row ); 00318 } 00319 delete pb; 00320 } 00321 00322 void MergeDond2Files::accumulateVec() 00323 { 00324 for (int sec_col = 0; sec_col < sec_width; sec_col++) 00325 { 00326 if (is_missing(sec_input[sec_col])) continue; 00327 else if (sec_ins[sec_col] == "skip") continue; 00328 int ext_col = extension_pos[sec_col]; 00329 if (sec_ins[sec_col] == "mean") 00330 { 00331 sec_values(ext_col, 0) += sec_input[sec_col]; 00332 sec_value_cnt(ext_col, 0) += 1.0; 00333 } 00334 else if (sec_ins[sec_col] == "mode") 00335 { 00336 sec_value_found = false; 00337 for (int sec_value_col = 0; sec_value_col < sec_value_ind[sec_col]; sec_value_col++) 00338 { 00339 if (sec_values(ext_col, sec_value_col) == sec_input[sec_col]) 00340 { 00341 sec_value_found = true; 00342 sec_value_cnt(ext_col, sec_value_col) += 1; 00343 break; 00344 } 00345 } 00346 if (!sec_value_found) 00347 { 00348 if (sec_value_ind[sec_col] >= 10) 00349 { 00350 cout << "MergeDond2Files: main file row: " << main_row << " external file row: " << sec_row << endl; 00351 PLERROR("MergeDond2Files: we have exceeded the capacity of the value MAT."); 00352 } 00353 sec_values(ext_col, sec_value_ind[sec_col]) = sec_input[sec_col]; 00354 sec_value_ind[sec_col] += 1; 00355 } 00356 } 00357 else if (sec_ins[sec_col] == "present") 00358 { 00359 sec_value_cnt(ext_col, 0) = 1.0; 00360 } 00361 } 00362 } 00363 00364 void MergeDond2Files::combineAndPut() 00365 { 00366 merge_col = 0; 00367 for (int main_col = 0; main_col < main_width; main_col++) 00368 { 00369 if (main_ins[main_col] == "skip") continue; 00370 else if (main_ins[main_col] == "as_is") 00371 { 00372 merge_output[merge_col] = main_input[main_col]; 00373 } 00374 else if (main_ins[main_col] == "zero_is_missing") 00375 { 00376 if (main_input[main_col] == 0.0) merge_output[merge_col] = MISSING_VALUE; 00377 else merge_output[merge_col] = main_input[main_col]; 00378 } 00379 else if (main_ins[main_col] == "2436935_is_missing") 00380 { 00381 if (main_input[main_col] == 2436935.0) merge_output[merge_col] = MISSING_VALUE; 00382 else merge_output[merge_col] = main_input[main_col]; 00383 } 00384 else if (main_ins[main_col] == "present") 00385 { 00386 if (is_missing(main_input[main_col])) merge_output[merge_col] = 0.0; 00387 else merge_output[merge_col] = 1.0; 00388 } 00389 merge_col +=1; 00390 } 00391 for (int sec_col = 0; sec_col < sec_width; sec_col++) 00392 { 00393 if (sec_ins[sec_col] == "skip") continue; 00394 int ext_col = extension_pos[sec_col]; 00395 if (sec_ins[sec_col] == "mean") 00396 { 00397 if (sec_value_cnt(ext_col, 0) <= 0.0) merge_output[merge_col] = MISSING_VALUE; 00398 else merge_output[merge_col] = sec_values(ext_col, 0) / sec_value_cnt(ext_col, 0); 00399 } 00400 else if (sec_ins[sec_col] == "mode") 00401 { 00402 if (sec_value_ind[sec_col] <= 0.0) 00403 { 00404 merge_output[merge_col] = MISSING_VALUE; 00405 continue; 00406 } 00407 merge_output[merge_col] = sec_values(ext_col, 0); 00408 sec_value_count_max = sec_value_cnt(ext_col, 0); 00409 for (int sec_value_col = 1; sec_value_col < sec_value_ind[sec_col]; sec_value_col++) 00410 { 00411 if (sec_value_cnt(ext_col, sec_value_col) >= sec_value_count_max) 00412 { 00413 merge_output[merge_col] = sec_values(ext_col, sec_value_col); 00414 sec_value_count_max = sec_value_cnt(ext_col, sec_value_col); 00415 } 00416 } 00417 } 00418 else if (sec_ins[sec_col] == "present") 00419 { 00420 merge_output[merge_col] = sec_value_cnt(ext_col, 0); 00421 } 00422 merge_col +=1; 00423 } 00424 if (main_input[train_ind] > 0.0) 00425 { 00426 train_file->putRow(train_row, merge_output); 00427 train_row += 1; 00428 } 00429 else if (main_input[test_ind] > 0.0) 00430 { 00431 test_file->putRow(test_row, merge_output); 00432 test_row += 1; 00433 } 00434 else 00435 { 00436 unknown_file->putRow(unknown_row, merge_output); 00437 unknown_row += 1; 00438 } 00439 sec_values.clear(); 00440 sec_value_cnt.clear(); 00441 sec_value_ind.clear(); 00442 } 00443 00444 int MergeDond2Files::outputsize() const {return 0;} 00445 void MergeDond2Files::train() {} 00446 void MergeDond2Files::computeOutput(const Vec&, Vec&) const {} 00447 void MergeDond2Files::computeCostsFromOutputs(const Vec&, const Vec&, const Vec&, Vec&) const {} 00448 TVec<string> MergeDond2Files::getTestCostNames() const 00449 { 00450 TVec<string> result; 00451 result.append( "MSE" ); 00452 return result; 00453 } 00454 TVec<string> MergeDond2Files::getTrainCostNames() const 00455 { 00456 TVec<string> result; 00457 result.append( "MSE" ); 00458 return result; 00459 } 00460 00461 } // end of namespace PLearn 00462 00463 00464 /* 00465 Local Variables: 00466 mode:c++ 00467 c-basic-offset:4 00468 c-file-style:"stroustrup" 00469 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00470 indent-tabs-mode:nil 00471 fill-column:79 00472 End: 00473 */ 00474 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :