PLearn 0.1
DatedJoinVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // DatedJoinVMatrix.cc
00004 //
00005 // Copyright (C) 2004 *Yoshua Bengio*
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: DatedJoinVMatrix.cc 8617 2008-03-03 17:45:54Z nouiz $
00037  ******************************************************* */
00038 
00039 // Authors: *Yoshua Bengio*
00040 
00044 #include "DatedJoinVMatrix.h"
00045 #include <plearn/base/PDate.h>
00046 #include <plearn/math/TMat_maths.h>
00047 
00048 namespace PLearn {
00049 using namespace std;
00050 
00051 
00052 DatedJoinVMatrix::DatedJoinVMatrix()
00053     :inherited(),master_date_field_index(-1),slave_date_interval_start_field_index(-1),
00054      slave_date_interval_end_field_index(-1), verbosity(0), output_the_slave(false), output_matching_index(false)
00055 {
00056 }
00057 
00058 PLEARN_IMPLEMENT_OBJECT(DatedJoinVMatrix,
00059                         "Join two vmatrices, taking into account a date field.",
00060                         "The two vmatrices play an asymmetric role. They are called\n"
00061                         "master and slave. The resulting vmatrix has one row for each row\n"
00062                         "of the master vmatrix (or optionally of the slave vmatrix). Its\n"
00063                         "columns are a concatenation of selected columns of the master vmatrix\n"
00064                         "and of selected columns of the slave which 'match' according to a rule\n"
00065                         "(always in the order: master fields, slave fields). Matchint is\n"
00066                         "obtained using shared 'key fields'. Optionally, for matching, a date field\n"
00067                         "in the master is forced to belong to a date interval in the slave,\n"
00068                         "as follows: slave_date_start < master_date <= slave_date_end.\n"
00069                         "If no match is found then the master (or slave) columns are left with missing values.\n"
00070                         "If more than one slave row matches, then the one with the latest\n"
00071                         "slave_date_start is used (and a warning is optionally issued). If\n"
00072                         "no slave_date_start field is provided then no date constraint is\n"
00073                         "enforced, and the last key-matching slave row is matched to a master row.\n"
00074                         "An option (output_the_slave) allows to output one row for each slave row\n"
00075                         "instead of the default which outputs one row for each master row.\n"
00076                         "Note that if (output_the_slave) then the non-matching master rows are 'lost'\n"
00077                         "whereas if (!output_the_slave) then the non-matching slave rows are 'lost'.\n"
00078                         "If output_the_slave and more than one master row matches with a given slave_row\n"
00079                         "then the SUM of the master fields is computed (i.e. be careful that their sum is meaningful)\n"
00080     );
00081 
00082 void DatedJoinVMatrix::getNewRow(int i, const Vec& v) const
00083 {
00084     if (!master || !slave || slave_key_indices.length()==0) // etc...
00085         PLERROR("DatedJoinVMatrix: object was not build properly!");
00086     list<int> master_index;
00087     int slave_index=-1;
00088     if (output_the_slave)
00089     {
00090         slave_index = i;
00091         master_index = slave2master[i];
00092         if (output_matching_index)
00093             v[0] = *(master_index.begin());
00094     }
00095     else
00096     {
00097         master_index.push_back(i);
00098         slave_index = master2slave[i];
00099         if (output_matching_index)
00100             v[0] = slave_index;
00101     }
00102 
00103     Vec master_part = v.subVec(output_matching_index,n_master_fields);
00104     Vec slave_part = v.subVec(n_master_fields+output_matching_index,n_slave_fields);
00105 
00106     if (master_index.size()>0)
00107     {
00108         list<int>::const_iterator b_it = master_index.begin();
00109         list<int>::const_iterator e_it = master_index.end();
00110         master_part.clear();
00111         for (list<int>::const_iterator it=b_it;it!=e_it;++it)
00112         {
00113             // copy the master fields
00114             master->getRow(*it,master_row);
00115             if (master_field_indices.size()>0)
00116                 for (int j=0;j<master_field_indices.size();j++)
00117                     master_part[j] += master_row[master_field_indices[j]];
00118             else
00119                 master_part += master_row;
00120         }
00121     }
00122     else
00123         master_part.fill(MISSING_VALUE);
00124 
00125     if (slave_index>=0)
00126     {
00127         // copy the slave fields
00128         slave->getRow(slave_index,slave_row);
00129         if (slave_field_indices.size()>0)
00130             for (int j=0;j<slave_field_indices.size();j++)
00131                 slave_part[j] = slave_row[slave_field_indices[j]];
00132         else
00133             slave_part << slave_row;
00134     }
00135     else
00136         slave_part.fill(MISSING_VALUE);
00137 
00138 }
00139 
00140 void DatedJoinVMatrix::declareOptions(OptionList& ol)
00141 {
00142     declareOption(ol, "master", &DatedJoinVMatrix::master, OptionBase::buildoption,
00143                   "Master vmatrix, whose columns are directly copied in the result.");
00144 
00145     declareOption(ol, "slave", &DatedJoinVMatrix::slave, OptionBase::buildoption,
00146                   "Slave vmatrix, of which only some columns are copied, when the\n"
00147                   "key fields and the dates match.");
00148 
00149     declareOption(ol, "master_key_indices", &DatedJoinVMatrix::master_key_indices,
00150                   OptionBase::buildoption,
00151                   "Indices of the 'key' fields in the master vmatrix. It is not necessary\n"
00152                   "to specify them if the master_key_names are given or if the slave_key_names\n"
00153                   "are specified (in that case they are assumed to be the same)\n"
00154         );
00155 
00156     declareOption(ol, "master_key_names", &DatedJoinVMatrix::master_key_names,
00157                   OptionBase::buildoption,
00158                   "Names of the 'key' fields in the master vmatrix. They should not be\n"
00159                   "specified if the master_key_indices are given directly. If not provided\n"
00160                   "and if the slave_key_names are specified they are assumed to be the same.\n"
00161         );
00162 
00163     declareOption(ol, "slave_key_indices", &DatedJoinVMatrix::slave_key_indices,
00164                   OptionBase::buildoption,
00165                   "Indices of the 'key' fields in the slave vmatrix. It is not necessary\n"
00166                   "to specify them if the slave_key_names are given or if the master_key_names\n"
00167                   "are specified (in that case they are assumed to be the same)\n"
00168         );
00169 
00170     declareOption(ol, "slave_key_names", &DatedJoinVMatrix::slave_key_names,
00171                   OptionBase::buildoption,
00172                   "Names of the 'key' fields in the slave vmatrix. They should not be\n"
00173                   "specified if the slave_key_indices are given directly. If not provided\n"
00174                   "and if the master_key_names are specified they are assumed to be the same.\n"
00175         );
00176 
00177     declareOption(ol, "slave_field_indices", &DatedJoinVMatrix::slave_field_indices,
00178                   OptionBase::buildoption,
00179                   "Indices of the fields in the slave vmatrix to be copied in result. It is not necessary\n"
00180                   "to specify them if the slave_field_names are given.\n"
00181                   "N.B. IF NEITHER slave_field_indices NOR slave_field_names are given then it is assumed\n"
00182                   "ALL slave fields should be copied on output.\n"
00183         );
00184 
00185     declareOption(ol, "slave_field_names", &DatedJoinVMatrix::slave_field_names,
00186                   OptionBase::buildoption,
00187                   "Names of the fields in the slave vmatrix to be copied in result. It is not necessary\n"
00188                   "to specify them if the slave_field_indices are given.\n"
00189                   "N.B. IF NEITHER slave_field_indices NOR slave_field_names are given then it is assumed\n"
00190                   "ALL slave fields should be copied on output.\n"
00191         );
00192 
00193     declareOption(ol, "master_field_indices", &DatedJoinVMatrix::master_field_indices,
00194                   OptionBase::buildoption,
00195                   "Indices of the fields in the master vmatrix to be copied in result. It is not necessary\n"
00196                   "to specify them if the slave_field_names are given.\n"
00197                   "N.B. IF NEITHER master_field_indices NOR master_field_names are given then it is assumed\n"
00198                   "ALL master fields should be copied on output.\n"
00199         );
00200 
00201     declareOption(ol, "master_field_names", &DatedJoinVMatrix::master_field_names,
00202                   OptionBase::buildoption,
00203                   "Names of the fields in the slave vmatrix to be copied in result. It is not necessary\n"
00204                   "to specify them if the slave_field_indices are given.\n"
00205                   "N.B. IF NEITHER master_field_indices NOR master_field_names are given then it is assumed\n"
00206                   "ALL master fields should be copied on output.\n"
00207         );
00208 
00209     declareOption(ol, "master_date_field_index", &DatedJoinVMatrix::master_date_field_index,
00210                   OptionBase::buildoption,
00211                   "Index of the date field in the master vmatrix. Should not be specified\n"
00212                   "if the master_date_field_name is given.\n"
00213         );
00214 
00215     declareOption(ol, "master_date_field_name", &DatedJoinVMatrix::master_date_field_name,
00216                   OptionBase::buildoption,
00217                   "Name of the date field in the master vmatrix. Should not be specified\n"
00218                   "if the master_date_field_index is given.\n"
00219         );
00220 
00221     declareOption(ol, "slave_date_interval_start_field_index",
00222                   &DatedJoinVMatrix::slave_date_interval_start_field_index,
00223                   OptionBase::buildoption,
00224                   "Index of the date interval start field in the slave vmatrix.\n"
00225                   "Should not be specified if the slave_date_interval_start_field_name is given.\n"
00226         );
00227 
00228     declareOption(ol, "slave_date_interval_start_field_name",
00229                   &DatedJoinVMatrix::slave_date_interval_start_field_name,
00230                   OptionBase::buildoption,
00231                   "Name of the date interval start field in the slave vmatrix.\n"
00232                   "Should not be specified if the slave_date_interval_start_field_index is given.\n"
00233         );
00234 
00235     declareOption(ol, "slave_date_interval_end_field_index",
00236                   &DatedJoinVMatrix::slave_date_interval_end_field_index,
00237                   OptionBase::buildoption,
00238                   "Index of the date interval end field in the slave vmatrix.\n"
00239                   "Should not be specified if the slave_date_interval_end_field_name is given.\n"
00240         );
00241 
00242     declareOption(ol, "slave_date_interval_end_field_name",
00243                   &DatedJoinVMatrix::slave_date_interval_end_field_name,
00244                   OptionBase::buildoption,
00245                   "Name of the date interval end field in the slave vmatrix.\n"
00246                   "Should not be specified if the slave_date_interval_end_field_index is given.\n"
00247         );
00248 
00249     declareOption(ol, "verbosity", &DatedJoinVMatrix::verbosity,
00250                   OptionBase::buildoption,
00251                   "0: no warning issued,\n"
00252                   "1: warning issued if more than one slave row matches,\n"
00253                   "2: details about these matches are printed\n"
00254         );
00255 
00256     declareOption(ol, "output_the_slave", &DatedJoinVMatrix::output_the_slave,
00257                   OptionBase::buildoption,
00258                   "If true than output the SLAVE rows (with master_fields_* from matching master row)\n"
00259                   "instead of the MASTER rows (with slave_fields_* from the matching slave row)\n"
00260         );
00261 
00262     declareOption(ol, "output_matching_index", &DatedJoinVMatrix::output_matching_index,
00263                   OptionBase::buildoption,
00264                   "If true than output an extra variable 'matching_index' which contains the row\n"
00265                   "index of the matching slave row (if !output_the_slave) or matching master row\n"
00266                   "if (output_the_slave).\n"
00267         );
00268 
00269     // Now call the parent class' declareOptions
00270     inherited::declareOptions(ol);
00271 }
00272 
00273 void DatedJoinVMatrix::build_()
00274 {
00275     if (master && slave) // we can't really build if we don't have them
00276     {
00277         updateMtime(master);
00278         updateMtime(slave);
00279         // convert field names into indices
00280         // * get master key indices
00281         if (master_key_names.length()>0)
00282         {
00283             master_key_indices.resize(master_key_names.length());
00284             for (int i=0;i<master_key_names.length();i++)
00285                 master_key_indices[i] = master->getFieldIndex(master_key_names[i]);
00286         }
00287         else if (master_key_indices.length()==0)
00288         {
00289             if (slave_key_names.length()>0)
00290             {
00291                 master_key_indices.resize(slave_key_names.length());
00292                 for (int i=0;i<slave_key_names.length();i++)
00293                     master_key_indices[i] = master->getFieldIndex(slave_key_names[i]);
00294             }
00295             else PLERROR("DatedJoinVMatrix: No key names were provided and no master_key_indices were provided!");
00296         }
00297         // * get slave key indices
00298         if (slave_key_names.length()>0)
00299         {
00300             slave_key_indices.resize(slave_key_names.length());
00301             for (int i=0;i<slave_key_names.length();i++)
00302                 slave_key_indices[i] = slave->getFieldIndex(slave_key_names[i]);
00303         }
00304         else if (slave_key_indices.length()==0)
00305         {
00306             if (master_key_names.length()>0)
00307             {
00308                 slave_key_indices.resize(master_key_names.length());
00309                 for (int i=0;i<master_key_names.length();i++)
00310                     slave_key_indices[i] = slave->getFieldIndex(master_key_names[i]);
00311             }
00312             else PLERROR("DatedJoinVMatrix: No key names were provided and no slave_key_indices were provided!");
00313         }
00314         // * get slave field indices
00315         if (slave_field_names.length()>0)
00316         {
00317             slave_field_indices.resize(slave_field_names.length());
00318             for (int i=0;i<slave_field_names.length();i++)
00319                 slave_field_indices[i] = slave->getFieldIndex(slave_field_names[i]);
00320         }
00321         // * get master field indices
00322         if (master_field_names.length()>0)
00323         {
00324             master_field_indices.resize(master_field_names.length());
00325             for (int i=0;i<master_field_names.length();i++)
00326                 master_field_indices[i] = master->getFieldIndex(master_field_names[i]);
00327         }
00328         // * get master date field index
00329         if (master_date_field_name!="")
00330             master_date_field_index = master->getFieldIndex(master_date_field_name);
00331         else if (master_date_field_index<0)
00332             PLWARNING("DatedJoinVMatrix: No master_date_field_name was provided and no master_date_field_index was provided!");
00333         // * get slave date interval start field index
00334         if (slave_date_interval_start_field_name!="")
00335             slave_date_interval_start_field_index = slave->getFieldIndex(slave_date_interval_start_field_name);
00336         else if (slave_date_interval_start_field_index<0 && master_date_field_index>=0)
00337             PLERROR("DatedJoinVMatrix: No slave_date_interval_start_field_name was provided and no slave_date_interval_start_field_index was provided!");
00338         // * get slave date interval end field index
00339         if (slave_date_interval_end_field_name!="")
00340             slave_date_interval_end_field_index = slave->getFieldIndex(slave_date_interval_end_field_name);
00341         else if (slave_date_interval_end_field_index<0 && master_date_field_index>=0)
00342             PLERROR("DatedJoinVMatrix: No slave_date_interval_end_field_name was provided and no slave_date_interval_end_field_index was provided!");
00343 
00344         // INDEX THE SLAVE
00345         PP<ProgressBar> pb=new ProgressBar("DatedJoinVMatrix: indexing the slave.",slave.length());
00346         key.resize(slave_key_indices.length());
00347         slave_row.resize(slave.width());
00348         master_row.resize(master.width());
00349         for (int i=0;i<slave.length();i++)
00350         {
00351             slave->getRow(i,slave_row);
00352             for (int j=0;j<slave_key_indices.size();j++)
00353                 key[j] = slave_row[slave_key_indices[j]];
00354             mp.insert(make_pair(key,i));
00355             pb->update(i);
00356         }
00357 
00358         // set the width and the length
00359         if (master_field_indices.size()>0)
00360             n_master_fields = master_field_indices.size();
00361         else
00362             n_master_fields = master->width();
00363         if (slave_field_indices.size()>0)
00364             n_slave_fields = slave_field_indices.size();
00365         else
00366             n_slave_fields = slave->width();
00367         width_ = output_matching_index + n_master_fields + n_slave_fields;
00368         if (output_the_slave)
00369             length_ = slave.length();
00370         else
00371             length_ = master.length();
00372 
00374         fieldinfos.resize(width_);
00375         Array<VMField> master_infos = master->getFieldInfos();
00376         Array<VMField> slave_infos = slave->getFieldInfos();
00377         if (output_matching_index)
00378             fieldinfos[0].name="matching_index";
00379         if (master_infos.size() > 0)
00380         {
00381             if (master_field_indices.size()>0)
00382                 for (int i=0; i<n_master_fields; ++i)
00383                     fieldinfos[output_matching_index+i] = master_infos[master_field_indices[i]];
00384             else
00385                 for (int i=0; i<n_master_fields; ++i)
00386                     fieldinfos[output_matching_index+i] = master_infos[i];
00387         }
00388         if (slave_infos.size() > 0)
00389         {
00390             if (slave_field_indices.size()>0)
00391                 for (int i=0; i<slave_field_indices.size(); ++i)
00392                 {
00393                     VMField f=slave_infos[slave_field_indices[i]];
00394                     if ((master_field_indices.size()==0 && master->fieldIndex(f.name)>=0)
00395                         || master_field_names.contains(f.name))
00396                         f.name = "slave." + f.name;
00397                     fieldinfos[output_matching_index+n_master_fields+i] = f;
00398                 }
00399             else
00400                 for (int i=0; i<n_slave_fields; ++i)
00401                 {
00402                     VMField f=slave_infos[i];
00403                     if ((master_field_indices.size()==0 && master->fieldIndex(f.name)>=0)
00404                         || master_field_names.contains(f.name))
00405                         f.name = "slave." + f.name;
00406                     fieldinfos[output_matching_index+n_master_fields+i] = f;
00407                 }
00408         }
00409         pb=new ProgressBar("DatedJoinVMatrix: matching the master and slave.",master->length());
00410         // pre-compute the 'match' (N.B. this is expensive...)
00411         master2slave.resize(master->length());
00412         master2slave.fill(-1);
00413         slave2master.resize(slave->length());
00414         for (int i=0;i<master->length();i++)
00415         {
00416             master->getRow(i,master_row);
00417             // build a key to search in the slave vmat
00418             for (int j=0;j<master_key_indices.size();j++)
00419                 key[j] = master_row[master_key_indices[j]];
00420 
00421             // get the list of matching slave rows
00422             Maptype::const_iterator it,low,upp;
00423             pair<Maptype::const_iterator,Maptype::const_iterator> matches_it=mp.equal_range(key);
00424             low=matches_it.first;
00425             upp=matches_it.second;
00426             if (low!=mp.end())
00427             {
00428                 PDate master_date;
00429                 if (master_date_field_index>=0)
00430                     master_date = float_to_date(master_row[master_date_field_index]);
00431                 PDate latest_match;
00432                 int n_matches=0;
00433                 static TVec<int> matches;
00434                 if (verbosity>1) matches.resize(0);
00435                 int matching_slave_row_index = -1;
00436                 // iterate over the matching slave rows
00437                 for (it=low;it!=upp;++it)
00438                 {
00439                     slave->getRow(it->second,slave_row);
00440                     if (master_date_field_index>=0)
00441                     {
00442                         PDate slave_date_interval_start = float_to_date(slave_row[slave_date_interval_start_field_index]);
00443                         PDate slave_date_interval_end = float_to_date(slave_row[slave_date_interval_end_field_index]);
00444                         if (master_date>slave_date_interval_start && master_date<=slave_date_interval_end)
00445                         {
00446                             if (n_matches==0 || slave_date_interval_start > latest_match)
00447                             {
00448                                 latest_match = slave_date_interval_start;
00449                                 matching_slave_row_index = it->second;
00450                             }
00451                             n_matches++;
00452                             if (verbosity>1) matches.append(it->second);
00453                         }
00454                     } else // no date, the LAST one will remain
00455                     {
00456                         matching_slave_row_index = it->second;
00457                         n_matches++;
00458                         if (verbosity>1) matches.append(it->second);
00459                     }
00460                 }
00461                 if (matching_slave_row_index>=0)
00462                 {
00463                     master2slave[i] = matching_slave_row_index;
00464                     slave2master[matching_slave_row_index].push_back(i);
00465                 }
00466                 if (n_matches>1 && verbosity>0)
00467                 {
00468                     PLWARNING("DatedJointVMatrix:getRow(%d,.) matched more than once\n",i);
00469                     if (verbosity >1)
00470                         for (int j=0;j<n_matches;j++)
00471                             cerr << "master row " << i << " matched slave row " << matches[j] << endl;
00472                 }
00473             }
00474             pb->update(i);
00475         }
00476     }
00477 }
00478 
00479 // ### Nothing to add here, simply calls build_
00480 void DatedJoinVMatrix::build()
00481 {
00482     inherited::build();
00483     build_();
00484 }
00485 
00486 void DatedJoinVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00487 {
00488     inherited::makeDeepCopyFromShallowCopy(copies);
00489 
00490     deepCopyField(slave_row, copies);
00491     deepCopyField(key, copies);
00492     deepCopyField(master2slave, copies);
00493     deepCopyField(slave2master, copies);
00494     deepCopyField(master, copies);
00495     deepCopyField(slave, copies);
00496     deepCopyField(master_key_indices, copies);
00497     deepCopyField(slave_key_indices, copies);
00498     deepCopyField(master_key_names, copies);
00499     deepCopyField(slave_key_names, copies);
00500     deepCopyField(slave_field_indices, copies);
00501     deepCopyField(slave_field_names, copies);
00502 }
00503 
00504 } // end of namespace PLearn
00505 
00506 
00507 /*
00508   Local Variables:
00509   mode:c++
00510   c-basic-offset:4
00511   c-file-style:"stroustrup"
00512   c-file-offsets:((innamespace . 0)(inline-open . 0))
00513   indent-tabs-mode:nil
00514   fill-column:79
00515   End:
00516 */
00517 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines