PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // DatedJoinVMatrix.cc 00004 // 00005 // Copyright (C) 2004 *Yoshua Bengio* 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: DatedJoinVMatrix.cc 8617 2008-03-03 17:45:54Z nouiz $ 00037 ******************************************************* */ 00038 00039 // Authors: *Yoshua Bengio* 00040 00044 #include "DatedJoinVMatrix.h" 00045 #include <plearn/base/PDate.h> 00046 #include <plearn/math/TMat_maths.h> 00047 00048 namespace PLearn { 00049 using namespace std; 00050 00051 00052 DatedJoinVMatrix::DatedJoinVMatrix() 00053 :inherited(),master_date_field_index(-1),slave_date_interval_start_field_index(-1), 00054 slave_date_interval_end_field_index(-1), verbosity(0), output_the_slave(false), output_matching_index(false) 00055 { 00056 } 00057 00058 PLEARN_IMPLEMENT_OBJECT(DatedJoinVMatrix, 00059 "Join two vmatrices, taking into account a date field.", 00060 "The two vmatrices play an asymmetric role. They are called\n" 00061 "master and slave. The resulting vmatrix has one row for each row\n" 00062 "of the master vmatrix (or optionally of the slave vmatrix). Its\n" 00063 "columns are a concatenation of selected columns of the master vmatrix\n" 00064 "and of selected columns of the slave which 'match' according to a rule\n" 00065 "(always in the order: master fields, slave fields). Matchint is\n" 00066 "obtained using shared 'key fields'. Optionally, for matching, a date field\n" 00067 "in the master is forced to belong to a date interval in the slave,\n" 00068 "as follows: slave_date_start < master_date <= slave_date_end.\n" 00069 "If no match is found then the master (or slave) columns are left with missing values.\n" 00070 "If more than one slave row matches, then the one with the latest\n" 00071 "slave_date_start is used (and a warning is optionally issued). If\n" 00072 "no slave_date_start field is provided then no date constraint is\n" 00073 "enforced, and the last key-matching slave row is matched to a master row.\n" 00074 "An option (output_the_slave) allows to output one row for each slave row\n" 00075 "instead of the default which outputs one row for each master row.\n" 00076 "Note that if (output_the_slave) then the non-matching master rows are 'lost'\n" 00077 "whereas if (!output_the_slave) then the non-matching slave rows are 'lost'.\n" 00078 "If output_the_slave and more than one master row matches with a given slave_row\n" 00079 "then the SUM of the master fields is computed (i.e. be careful that their sum is meaningful)\n" 00080 ); 00081 00082 void DatedJoinVMatrix::getNewRow(int i, const Vec& v) const 00083 { 00084 if (!master || !slave || slave_key_indices.length()==0) // etc... 00085 PLERROR("DatedJoinVMatrix: object was not build properly!"); 00086 list<int> master_index; 00087 int slave_index=-1; 00088 if (output_the_slave) 00089 { 00090 slave_index = i; 00091 master_index = slave2master[i]; 00092 if (output_matching_index) 00093 v[0] = *(master_index.begin()); 00094 } 00095 else 00096 { 00097 master_index.push_back(i); 00098 slave_index = master2slave[i]; 00099 if (output_matching_index) 00100 v[0] = slave_index; 00101 } 00102 00103 Vec master_part = v.subVec(output_matching_index,n_master_fields); 00104 Vec slave_part = v.subVec(n_master_fields+output_matching_index,n_slave_fields); 00105 00106 if (master_index.size()>0) 00107 { 00108 list<int>::const_iterator b_it = master_index.begin(); 00109 list<int>::const_iterator e_it = master_index.end(); 00110 master_part.clear(); 00111 for (list<int>::const_iterator it=b_it;it!=e_it;++it) 00112 { 00113 // copy the master fields 00114 master->getRow(*it,master_row); 00115 if (master_field_indices.size()>0) 00116 for (int j=0;j<master_field_indices.size();j++) 00117 master_part[j] += master_row[master_field_indices[j]]; 00118 else 00119 master_part += master_row; 00120 } 00121 } 00122 else 00123 master_part.fill(MISSING_VALUE); 00124 00125 if (slave_index>=0) 00126 { 00127 // copy the slave fields 00128 slave->getRow(slave_index,slave_row); 00129 if (slave_field_indices.size()>0) 00130 for (int j=0;j<slave_field_indices.size();j++) 00131 slave_part[j] = slave_row[slave_field_indices[j]]; 00132 else 00133 slave_part << slave_row; 00134 } 00135 else 00136 slave_part.fill(MISSING_VALUE); 00137 00138 } 00139 00140 void DatedJoinVMatrix::declareOptions(OptionList& ol) 00141 { 00142 declareOption(ol, "master", &DatedJoinVMatrix::master, OptionBase::buildoption, 00143 "Master vmatrix, whose columns are directly copied in the result."); 00144 00145 declareOption(ol, "slave", &DatedJoinVMatrix::slave, OptionBase::buildoption, 00146 "Slave vmatrix, of which only some columns are copied, when the\n" 00147 "key fields and the dates match."); 00148 00149 declareOption(ol, "master_key_indices", &DatedJoinVMatrix::master_key_indices, 00150 OptionBase::buildoption, 00151 "Indices of the 'key' fields in the master vmatrix. It is not necessary\n" 00152 "to specify them if the master_key_names are given or if the slave_key_names\n" 00153 "are specified (in that case they are assumed to be the same)\n" 00154 ); 00155 00156 declareOption(ol, "master_key_names", &DatedJoinVMatrix::master_key_names, 00157 OptionBase::buildoption, 00158 "Names of the 'key' fields in the master vmatrix. They should not be\n" 00159 "specified if the master_key_indices are given directly. If not provided\n" 00160 "and if the slave_key_names are specified they are assumed to be the same.\n" 00161 ); 00162 00163 declareOption(ol, "slave_key_indices", &DatedJoinVMatrix::slave_key_indices, 00164 OptionBase::buildoption, 00165 "Indices of the 'key' fields in the slave vmatrix. It is not necessary\n" 00166 "to specify them if the slave_key_names are given or if the master_key_names\n" 00167 "are specified (in that case they are assumed to be the same)\n" 00168 ); 00169 00170 declareOption(ol, "slave_key_names", &DatedJoinVMatrix::slave_key_names, 00171 OptionBase::buildoption, 00172 "Names of the 'key' fields in the slave vmatrix. They should not be\n" 00173 "specified if the slave_key_indices are given directly. If not provided\n" 00174 "and if the master_key_names are specified they are assumed to be the same.\n" 00175 ); 00176 00177 declareOption(ol, "slave_field_indices", &DatedJoinVMatrix::slave_field_indices, 00178 OptionBase::buildoption, 00179 "Indices of the fields in the slave vmatrix to be copied in result. It is not necessary\n" 00180 "to specify them if the slave_field_names are given.\n" 00181 "N.B. IF NEITHER slave_field_indices NOR slave_field_names are given then it is assumed\n" 00182 "ALL slave fields should be copied on output.\n" 00183 ); 00184 00185 declareOption(ol, "slave_field_names", &DatedJoinVMatrix::slave_field_names, 00186 OptionBase::buildoption, 00187 "Names of the fields in the slave vmatrix to be copied in result. It is not necessary\n" 00188 "to specify them if the slave_field_indices are given.\n" 00189 "N.B. IF NEITHER slave_field_indices NOR slave_field_names are given then it is assumed\n" 00190 "ALL slave fields should be copied on output.\n" 00191 ); 00192 00193 declareOption(ol, "master_field_indices", &DatedJoinVMatrix::master_field_indices, 00194 OptionBase::buildoption, 00195 "Indices of the fields in the master vmatrix to be copied in result. It is not necessary\n" 00196 "to specify them if the slave_field_names are given.\n" 00197 "N.B. IF NEITHER master_field_indices NOR master_field_names are given then it is assumed\n" 00198 "ALL master fields should be copied on output.\n" 00199 ); 00200 00201 declareOption(ol, "master_field_names", &DatedJoinVMatrix::master_field_names, 00202 OptionBase::buildoption, 00203 "Names of the fields in the slave vmatrix to be copied in result. It is not necessary\n" 00204 "to specify them if the slave_field_indices are given.\n" 00205 "N.B. IF NEITHER master_field_indices NOR master_field_names are given then it is assumed\n" 00206 "ALL master fields should be copied on output.\n" 00207 ); 00208 00209 declareOption(ol, "master_date_field_index", &DatedJoinVMatrix::master_date_field_index, 00210 OptionBase::buildoption, 00211 "Index of the date field in the master vmatrix. Should not be specified\n" 00212 "if the master_date_field_name is given.\n" 00213 ); 00214 00215 declareOption(ol, "master_date_field_name", &DatedJoinVMatrix::master_date_field_name, 00216 OptionBase::buildoption, 00217 "Name of the date field in the master vmatrix. Should not be specified\n" 00218 "if the master_date_field_index is given.\n" 00219 ); 00220 00221 declareOption(ol, "slave_date_interval_start_field_index", 00222 &DatedJoinVMatrix::slave_date_interval_start_field_index, 00223 OptionBase::buildoption, 00224 "Index of the date interval start field in the slave vmatrix.\n" 00225 "Should not be specified if the slave_date_interval_start_field_name is given.\n" 00226 ); 00227 00228 declareOption(ol, "slave_date_interval_start_field_name", 00229 &DatedJoinVMatrix::slave_date_interval_start_field_name, 00230 OptionBase::buildoption, 00231 "Name of the date interval start field in the slave vmatrix.\n" 00232 "Should not be specified if the slave_date_interval_start_field_index is given.\n" 00233 ); 00234 00235 declareOption(ol, "slave_date_interval_end_field_index", 00236 &DatedJoinVMatrix::slave_date_interval_end_field_index, 00237 OptionBase::buildoption, 00238 "Index of the date interval end field in the slave vmatrix.\n" 00239 "Should not be specified if the slave_date_interval_end_field_name is given.\n" 00240 ); 00241 00242 declareOption(ol, "slave_date_interval_end_field_name", 00243 &DatedJoinVMatrix::slave_date_interval_end_field_name, 00244 OptionBase::buildoption, 00245 "Name of the date interval end field in the slave vmatrix.\n" 00246 "Should not be specified if the slave_date_interval_end_field_index is given.\n" 00247 ); 00248 00249 declareOption(ol, "verbosity", &DatedJoinVMatrix::verbosity, 00250 OptionBase::buildoption, 00251 "0: no warning issued,\n" 00252 "1: warning issued if more than one slave row matches,\n" 00253 "2: details about these matches are printed\n" 00254 ); 00255 00256 declareOption(ol, "output_the_slave", &DatedJoinVMatrix::output_the_slave, 00257 OptionBase::buildoption, 00258 "If true than output the SLAVE rows (with master_fields_* from matching master row)\n" 00259 "instead of the MASTER rows (with slave_fields_* from the matching slave row)\n" 00260 ); 00261 00262 declareOption(ol, "output_matching_index", &DatedJoinVMatrix::output_matching_index, 00263 OptionBase::buildoption, 00264 "If true than output an extra variable 'matching_index' which contains the row\n" 00265 "index of the matching slave row (if !output_the_slave) or matching master row\n" 00266 "if (output_the_slave).\n" 00267 ); 00268 00269 // Now call the parent class' declareOptions 00270 inherited::declareOptions(ol); 00271 } 00272 00273 void DatedJoinVMatrix::build_() 00274 { 00275 if (master && slave) // we can't really build if we don't have them 00276 { 00277 updateMtime(master); 00278 updateMtime(slave); 00279 // convert field names into indices 00280 // * get master key indices 00281 if (master_key_names.length()>0) 00282 { 00283 master_key_indices.resize(master_key_names.length()); 00284 for (int i=0;i<master_key_names.length();i++) 00285 master_key_indices[i] = master->getFieldIndex(master_key_names[i]); 00286 } 00287 else if (master_key_indices.length()==0) 00288 { 00289 if (slave_key_names.length()>0) 00290 { 00291 master_key_indices.resize(slave_key_names.length()); 00292 for (int i=0;i<slave_key_names.length();i++) 00293 master_key_indices[i] = master->getFieldIndex(slave_key_names[i]); 00294 } 00295 else PLERROR("DatedJoinVMatrix: No key names were provided and no master_key_indices were provided!"); 00296 } 00297 // * get slave key indices 00298 if (slave_key_names.length()>0) 00299 { 00300 slave_key_indices.resize(slave_key_names.length()); 00301 for (int i=0;i<slave_key_names.length();i++) 00302 slave_key_indices[i] = slave->getFieldIndex(slave_key_names[i]); 00303 } 00304 else if (slave_key_indices.length()==0) 00305 { 00306 if (master_key_names.length()>0) 00307 { 00308 slave_key_indices.resize(master_key_names.length()); 00309 for (int i=0;i<master_key_names.length();i++) 00310 slave_key_indices[i] = slave->getFieldIndex(master_key_names[i]); 00311 } 00312 else PLERROR("DatedJoinVMatrix: No key names were provided and no slave_key_indices were provided!"); 00313 } 00314 // * get slave field indices 00315 if (slave_field_names.length()>0) 00316 { 00317 slave_field_indices.resize(slave_field_names.length()); 00318 for (int i=0;i<slave_field_names.length();i++) 00319 slave_field_indices[i] = slave->getFieldIndex(slave_field_names[i]); 00320 } 00321 // * get master field indices 00322 if (master_field_names.length()>0) 00323 { 00324 master_field_indices.resize(master_field_names.length()); 00325 for (int i=0;i<master_field_names.length();i++) 00326 master_field_indices[i] = master->getFieldIndex(master_field_names[i]); 00327 } 00328 // * get master date field index 00329 if (master_date_field_name!="") 00330 master_date_field_index = master->getFieldIndex(master_date_field_name); 00331 else if (master_date_field_index<0) 00332 PLWARNING("DatedJoinVMatrix: No master_date_field_name was provided and no master_date_field_index was provided!"); 00333 // * get slave date interval start field index 00334 if (slave_date_interval_start_field_name!="") 00335 slave_date_interval_start_field_index = slave->getFieldIndex(slave_date_interval_start_field_name); 00336 else if (slave_date_interval_start_field_index<0 && master_date_field_index>=0) 00337 PLERROR("DatedJoinVMatrix: No slave_date_interval_start_field_name was provided and no slave_date_interval_start_field_index was provided!"); 00338 // * get slave date interval end field index 00339 if (slave_date_interval_end_field_name!="") 00340 slave_date_interval_end_field_index = slave->getFieldIndex(slave_date_interval_end_field_name); 00341 else if (slave_date_interval_end_field_index<0 && master_date_field_index>=0) 00342 PLERROR("DatedJoinVMatrix: No slave_date_interval_end_field_name was provided and no slave_date_interval_end_field_index was provided!"); 00343 00344 // INDEX THE SLAVE 00345 PP<ProgressBar> pb=new ProgressBar("DatedJoinVMatrix: indexing the slave.",slave.length()); 00346 key.resize(slave_key_indices.length()); 00347 slave_row.resize(slave.width()); 00348 master_row.resize(master.width()); 00349 for (int i=0;i<slave.length();i++) 00350 { 00351 slave->getRow(i,slave_row); 00352 for (int j=0;j<slave_key_indices.size();j++) 00353 key[j] = slave_row[slave_key_indices[j]]; 00354 mp.insert(make_pair(key,i)); 00355 pb->update(i); 00356 } 00357 00358 // set the width and the length 00359 if (master_field_indices.size()>0) 00360 n_master_fields = master_field_indices.size(); 00361 else 00362 n_master_fields = master->width(); 00363 if (slave_field_indices.size()>0) 00364 n_slave_fields = slave_field_indices.size(); 00365 else 00366 n_slave_fields = slave->width(); 00367 width_ = output_matching_index + n_master_fields + n_slave_fields; 00368 if (output_the_slave) 00369 length_ = slave.length(); 00370 else 00371 length_ = master.length(); 00372 00374 fieldinfos.resize(width_); 00375 Array<VMField> master_infos = master->getFieldInfos(); 00376 Array<VMField> slave_infos = slave->getFieldInfos(); 00377 if (output_matching_index) 00378 fieldinfos[0].name="matching_index"; 00379 if (master_infos.size() > 0) 00380 { 00381 if (master_field_indices.size()>0) 00382 for (int i=0; i<n_master_fields; ++i) 00383 fieldinfos[output_matching_index+i] = master_infos[master_field_indices[i]]; 00384 else 00385 for (int i=0; i<n_master_fields; ++i) 00386 fieldinfos[output_matching_index+i] = master_infos[i]; 00387 } 00388 if (slave_infos.size() > 0) 00389 { 00390 if (slave_field_indices.size()>0) 00391 for (int i=0; i<slave_field_indices.size(); ++i) 00392 { 00393 VMField f=slave_infos[slave_field_indices[i]]; 00394 if ((master_field_indices.size()==0 && master->fieldIndex(f.name)>=0) 00395 || master_field_names.contains(f.name)) 00396 f.name = "slave." + f.name; 00397 fieldinfos[output_matching_index+n_master_fields+i] = f; 00398 } 00399 else 00400 for (int i=0; i<n_slave_fields; ++i) 00401 { 00402 VMField f=slave_infos[i]; 00403 if ((master_field_indices.size()==0 && master->fieldIndex(f.name)>=0) 00404 || master_field_names.contains(f.name)) 00405 f.name = "slave." + f.name; 00406 fieldinfos[output_matching_index+n_master_fields+i] = f; 00407 } 00408 } 00409 pb=new ProgressBar("DatedJoinVMatrix: matching the master and slave.",master->length()); 00410 // pre-compute the 'match' (N.B. this is expensive...) 00411 master2slave.resize(master->length()); 00412 master2slave.fill(-1); 00413 slave2master.resize(slave->length()); 00414 for (int i=0;i<master->length();i++) 00415 { 00416 master->getRow(i,master_row); 00417 // build a key to search in the slave vmat 00418 for (int j=0;j<master_key_indices.size();j++) 00419 key[j] = master_row[master_key_indices[j]]; 00420 00421 // get the list of matching slave rows 00422 Maptype::const_iterator it,low,upp; 00423 pair<Maptype::const_iterator,Maptype::const_iterator> matches_it=mp.equal_range(key); 00424 low=matches_it.first; 00425 upp=matches_it.second; 00426 if (low!=mp.end()) 00427 { 00428 PDate master_date; 00429 if (master_date_field_index>=0) 00430 master_date = float_to_date(master_row[master_date_field_index]); 00431 PDate latest_match; 00432 int n_matches=0; 00433 static TVec<int> matches; 00434 if (verbosity>1) matches.resize(0); 00435 int matching_slave_row_index = -1; 00436 // iterate over the matching slave rows 00437 for (it=low;it!=upp;++it) 00438 { 00439 slave->getRow(it->second,slave_row); 00440 if (master_date_field_index>=0) 00441 { 00442 PDate slave_date_interval_start = float_to_date(slave_row[slave_date_interval_start_field_index]); 00443 PDate slave_date_interval_end = float_to_date(slave_row[slave_date_interval_end_field_index]); 00444 if (master_date>slave_date_interval_start && master_date<=slave_date_interval_end) 00445 { 00446 if (n_matches==0 || slave_date_interval_start > latest_match) 00447 { 00448 latest_match = slave_date_interval_start; 00449 matching_slave_row_index = it->second; 00450 } 00451 n_matches++; 00452 if (verbosity>1) matches.append(it->second); 00453 } 00454 } else // no date, the LAST one will remain 00455 { 00456 matching_slave_row_index = it->second; 00457 n_matches++; 00458 if (verbosity>1) matches.append(it->second); 00459 } 00460 } 00461 if (matching_slave_row_index>=0) 00462 { 00463 master2slave[i] = matching_slave_row_index; 00464 slave2master[matching_slave_row_index].push_back(i); 00465 } 00466 if (n_matches>1 && verbosity>0) 00467 { 00468 PLWARNING("DatedJointVMatrix:getRow(%d,.) matched more than once\n",i); 00469 if (verbosity >1) 00470 for (int j=0;j<n_matches;j++) 00471 cerr << "master row " << i << " matched slave row " << matches[j] << endl; 00472 } 00473 } 00474 pb->update(i); 00475 } 00476 } 00477 } 00478 00479 // ### Nothing to add here, simply calls build_ 00480 void DatedJoinVMatrix::build() 00481 { 00482 inherited::build(); 00483 build_(); 00484 } 00485 00486 void DatedJoinVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00487 { 00488 inherited::makeDeepCopyFromShallowCopy(copies); 00489 00490 deepCopyField(slave_row, copies); 00491 deepCopyField(key, copies); 00492 deepCopyField(master2slave, copies); 00493 deepCopyField(slave2master, copies); 00494 deepCopyField(master, copies); 00495 deepCopyField(slave, copies); 00496 deepCopyField(master_key_indices, copies); 00497 deepCopyField(slave_key_indices, copies); 00498 deepCopyField(master_key_names, copies); 00499 deepCopyField(slave_key_names, copies); 00500 deepCopyField(slave_field_indices, copies); 00501 deepCopyField(slave_field_names, copies); 00502 } 00503 00504 } // end of namespace PLearn 00505 00506 00507 /* 00508 Local Variables: 00509 mode:c++ 00510 c-basic-offset:4 00511 c-file-style:"stroustrup" 00512 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00513 indent-tabs-mode:nil 00514 fill-column:79 00515 End: 00516 */ 00517 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :