PLearn 0.1
ConcatRowsVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 //
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 //
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 //
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 //
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 //
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 //
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 /* *******************************************************
00038  * $Id: ConcatRowsVMatrix.cc 9774 2008-12-11 21:06:26Z nouiz $
00039  ******************************************************* */
00040 
00041 #include "ConcatRowsVMatrix.h"
00042 #include "SelectColumnsVMatrix.h"
00043 
00044 namespace PLearn {
00045 using namespace std;
00046 
00049 PLEARN_IMPLEMENT_OBJECT(
00050     ConcatRowsVMatrix,
00051     "Concatenates the rows of a number of VMats.",
00052     "If the VMats do not have the same fields, it is possible to either keep\n"
00053     "only the common fields ('only_common_fields' = 1), or to keep them all\n"
00054     "and add missing values accordingly ('fill_missing' = 1).\n"
00055     "If none of these options is set, the VMats are assumed to have the same\n"
00056     "fields.\n"
00057     );
00058 
00059 
00061 // ConcatRowsVMatrix //
00063 ConcatRowsVMatrix::ConcatRowsVMatrix(TVec<VMat> the_sources)
00064     : sources(the_sources),
00065       fill_missing(false),
00066       fully_check_mappings(false),
00067       only_common_fields(false),
00068       m_last_vmat_index(-1),
00069       m_last_vmat_startrow(-1),
00070       m_last_vmat_lastrow(-1)
00071 {
00072     if (sources.size() > 0)
00073         build_();
00074 }
00075 
00076 ConcatRowsVMatrix::ConcatRowsVMatrix(VMat d1, VMat d2,
00077                                      bool the_fully_check_mapping)
00078     : sources(2),
00079       fill_missing(false),
00080       fully_check_mappings(the_fully_check_mapping),
00081       only_common_fields(false),
00082       m_last_vmat_index(-1),
00083       m_last_vmat_startrow(-1),
00084       m_last_vmat_lastrow(-1)
00085 {
00086     sources[0] = d1;
00087     sources[1] = d2;
00088     build_();
00089 }
00090 
00092 // declareOptions //
00094 void ConcatRowsVMatrix::declareOptions(OptionList &ol)
00095 {
00096     declareOption(ol, "array", &ConcatRowsVMatrix::sources,
00097                   (OptionBase::learntoption | OptionBase::nosave),
00098                   "DEPRECATED - Use 'sources' instead.");
00099 
00100     declareOption(ol, "sources", &ConcatRowsVMatrix::sources,
00101                   OptionBase::buildoption,
00102                   "The VMat to concatenate (horizontally).");
00103 
00104     declareOption(ol, "fill_missing", &ConcatRowsVMatrix::fill_missing,
00105                   OptionBase::buildoption,
00106                   "If set to 1, then all fields will be kept, and in each VMat"
00107                   " missing fields\n"
00108                   "will be filled with missing values.\n");
00109 
00110     declareOption(ol, "fully_check_mappings",
00111                   &ConcatRowsVMatrix::fully_check_mappings,
00112                   OptionBase::buildoption,
00113                   "If set to 1, then columns for which there is a string <->"
00114                   " real mapping\n"
00115                   "will be examined to ensure that no numerical data in a VMat"
00116                   " conflicts with\n"
00117                   "a mapping in another VMat.\n");
00118 
00119     declareOption(ol, "only_common_fields",
00120                   &ConcatRowsVMatrix::only_common_fields,
00121                   OptionBase::buildoption,
00122                   "If set to 1, then only the fields whose names are common to"
00123                   " all VMat\n"
00124                   "in 'sources' will be kept (and reordered if needed).\n");
00125 
00126     inherited::declareOptions(ol);
00127 }
00128 
00130 // build //
00132 void ConcatRowsVMatrix::build()
00133 {
00134     inherited::build();
00135     build_();
00136 }
00137 
00139 // build_ //
00141 void ConcatRowsVMatrix::build_()
00142 {
00143     int n = sources.size();
00144     if (n < 1)
00145         // No sources provided yet: nothing to do.
00146         return;
00147     for(int i=0;i<sources.size();i++)
00148         updateMtime(sources[i]);
00149     if (only_common_fields && fill_missing)
00150         PLERROR("In ConcatRowsVMatrix::build_ - You can't set both 'only_common_fields' and 'fill_missing'");
00151 
00152     // Get the fields info.
00153     fieldinfos = sources[0]->getFieldInfos();
00154     if (only_common_fields) {
00155         findCommonFields();
00156     } else if (fill_missing) {
00157         findAllFields();
00158     } else {
00159         to_concat = sources;
00160     }
00161 
00162     // Set length_ and width_.
00163     recomputeDimensions();
00164 
00165     // Note that the 4 lines below will overwrite any provided sizes.
00166     if(inputsize_<0 && targetsize_<0 && weightsize_<0){
00167         inputsize_ = to_concat[0]->inputsize();
00168         targetsize_ = to_concat[0]->targetsize();
00169         weightsize_ = to_concat[0]->weightsize();
00170         extrasize_ = to_concat[0]->extrasize();
00171         if(fieldinfos.size()!=to_concat[0].width())
00172             PLERROR("In ConcatRowsVMatrix::build_() - We override "
00173                       "inputsize, targetsize and weightsize with the value"
00174                       " of sources[0] and this cause inconsistency");
00175     }
00176     computeMissingSizeValue();
00177 
00178     // Make sure mappings are correct.
00179     ensureMappingsConsistency();
00180     if (fully_check_mappings)
00181         fullyCheckMappings();
00182     if (need_fix_mappings && !fully_check_mappings)
00183         PLWARNING("In ConcatRowsVMatrix::build_ - Mappings need to be fixed, but you did not set 'fully_check_mappings' to true, this might be dangerous");
00184 
00185     // Reset last-used cache for completeness
00186     m_last_vmat_index = m_last_vmat_startrow = m_last_vmat_lastrow = -1;
00187 }
00188 
00190 // dot //
00192 real ConcatRowsVMatrix::dot(int i1, int i2, int inputsize) const
00193 {
00194     int whichvm1, rowofvm1;
00195     getpositions(i1,whichvm1,rowofvm1);
00196     int whichvm2, rowofvm2;
00197     getpositions(i2,whichvm2,rowofvm2);
00198     if(whichvm1==whichvm2 && !need_fix_mappings)
00199         return to_concat[whichvm1]->dot(rowofvm1, rowofvm2, inputsize);
00200     else
00201         return VMatrix::dot(i1,i2,inputsize);
00202 }
00203 
00204 real ConcatRowsVMatrix::dot(int i, const Vec& v) const
00205 {
00206     if (!need_fix_mappings) {
00207         int whichvm, rowofvm;
00208         getpositions(i,whichvm,rowofvm);
00209         return to_concat[whichvm]->dot(rowofvm,v);
00210     }
00211     else
00212         return VMatrix::dot(i, v);
00213 }
00214 
00216 // ensureMappingsConsistency //
00218 void ConcatRowsVMatrix::ensureMappingsConsistency()
00219 {
00220     // Make sure the string mappings are consistent.
00221     // For this, we start from the mappings of the first VMat, and add missing
00222     // mappings obtained from the other VMats. If another VMat has a different
00223     // mapping for the same string, we remember it in order to fix the output
00224     // when a getxxxx() method is called.
00225     need_fix_mappings = false;
00226     copyStringMappingsFrom(to_concat[0]);
00227     map<string, real> other_map;
00228     map<string, real>* cur_map;
00229     map<string, real>::iterator it, find_map, jt;
00230     bool report_progress = false; // Turn it on for debugging.
00231     PP<ProgressBar> pb;
00232     if (report_progress)
00233         pb = new ProgressBar("Checking mappings consistency", width());
00234     for (int j = 0; j < width(); j++) {
00235         cur_map = &map_sr[j];
00236         // Find the maximum mapping value for this column.
00237         real max = -REAL_MAX;
00238         for (jt = cur_map->begin(); jt != cur_map->end(); jt++)
00239             if (jt->second > max)
00240                 max = jt->second;
00241         for (int i = 1; i < to_concat.length(); i++) {
00242             other_map = to_concat[i]->getStringToRealMapping(j);
00243             for(it = other_map.begin(); it != other_map.end(); it++) {
00244                 find_map = cur_map->find(it->first);
00245                 if (find_map != cur_map->end()) {
00246                     // The string mapped in VMat 'i' is also mapped in our current mapping.
00247                     if (!fast_exact_is_equal(find_map->second, it->second)) {
00248                         // But the same string is not mapped to the same value.
00249                         // This needs to be fixed.
00250                         /*cout << find_map->first << endl;
00251                           cout << find_map->second << endl;*/
00252                         need_fix_mappings = true;
00253                         fixed_mappings.resize(to_concat.length(), width());
00254                         fixed_mappings(i, j)[it->second] = find_map->second;
00255                     }
00256                 } else {
00257                     // The string mapped in VMat 'i' is not mapped in our current mapping.
00258                     // We need to add this mapping. But we must make sure there is no
00259                     // existing mapping to the same real number.
00260                     real new_map_val = it->second;
00261                     if (getValString(j, it->second) != "") {
00262                         // There is already a mapping to this real number.
00263                         need_fix_mappings = true;
00264                         fixed_mappings.resize(to_concat.length(), width());
00265                         // We pick for the number the maximum of the current mapped numbers, +1.
00266                         max++;
00267                         fixed_mappings(i, j)[it->second] = max;
00268                         new_map_val = max;
00269                     } else {
00270                         // There is no mapping to this real number, it is thus ok to add it.
00271                         if (new_map_val > max)
00272                             max = new_map_val;
00273                     }
00274                     addStringMapping(j, it->first, new_map_val);
00275                 }
00276             }
00277         }
00278         if (report_progress)
00279             pb->update(j + 1);
00280     }
00281 }
00282 
00284 // findAllFields //
00286 void ConcatRowsVMatrix::findAllFields()
00287 {
00288     // At this point, fieldinfos already contains the fields of the first
00289     // concatenated VMat.
00290     bool report_progress = true;
00291     TVec<VMField> other_fields;
00292     PP<ProgressBar> pb;
00293     int count;
00294     if (report_progress) {
00295         count = 0;
00296         for (int i = 1; i < sources.length(); i++)
00297             count += sources[i]->getFieldInfos().length();
00298         pb = new ProgressBar("Computing the whole set of fields", count);
00299     }
00300     count = 0;
00301     for (int i = 1; i < sources.length(); i++) {
00302         other_fields = sources[i]->getFieldInfos();
00303         for (int j = 0; j < other_fields.length(); j++) {
00304             bool present_already = false;
00305             for (int k = 0; !present_already && k < fieldinfos.length(); k++)
00306                 if (fieldinfos[k].name == other_fields[j].name)
00307                     present_already = true;
00308             if (!present_already) {
00309                 // We need to add this field.
00310                 fieldinfos.append(other_fields[j]);
00311             }
00312             if (report_progress)
00313                 pb->update(++count);
00314         }
00315     }
00316     // Create the vector of field names.
00317     TVec<string> fnames(fieldinfos.length());
00318     for (int i = 0; i < fieldinfos.length(); i++)
00319         fnames[i] = fieldinfos[i].name;
00320     // Now fill 'to_concat' with the corresponding VMats.
00321     to_concat.resize(sources.length());
00322     for (int i = 0; i < sources.length(); i++) {
00323         TVec<string> source_fnames=sources[i]->fieldNames();
00324         if(fnames!=source_fnames){
00325             to_concat[i] =
00326                 new SelectColumnsVMatrix(sources[i], fnames, true);
00327         } else
00328             to_concat[i] = sources[i];
00329     }
00330 }
00331 
00333 // findCommonFields //
00335 void ConcatRowsVMatrix::findCommonFields()
00336 {
00337     // Find out which fields need to be kept.
00338     TVec<VMField> final_fields(fieldinfos.length());
00339     final_fields << fieldinfos;
00340     TVec<VMField> other_fields;
00341     TVec<VMField> tmp(final_fields.length());
00342     for (int i = 1; i < sources.length(); i++) {
00343         map<string, bool> can_be_kept;
00344         other_fields = sources[i]->getFieldInfos();
00345         for (int j = 0; j < other_fields.length(); j++) {
00346             can_be_kept[other_fields[j].name] = true;
00347         }
00348         tmp.resize(0);
00349         for (int j = 0; j < final_fields.length(); j++)
00350             if (can_be_kept.count(final_fields[j].name) > 0)
00351                 tmp.append(final_fields[j]);
00352         final_fields.resize(tmp.length());
00353         final_fields << tmp;
00354     }
00355     fieldinfos.resize(final_fields.length());
00356     fieldinfos << final_fields;
00357     // Get the corresponding field names.
00358     TVec<string> final_fieldnames(final_fields.length());
00359     for (int i = 0; i < final_fields.length(); i++)
00360         final_fieldnames[i] = final_fields[i].name;
00361     // Now fill 'to_concat' with the selected columns of each VMat in 'sources'
00362     to_concat.resize(sources.length());
00363     for (int i = 0; i < sources.length(); i++)
00364         to_concat[i] = new SelectColumnsVMatrix(sources[i], final_fieldnames);
00365 }
00366 
00368 // fullyCheckMappings //
00370 void ConcatRowsVMatrix::fullyCheckMappings(bool report_progress)
00371 {
00372     Vec row(width());
00373     TVec<int> max(width());
00374     PP<ProgressBar> pb;
00375     if (report_progress)
00376         pb = new ProgressBar("Full check of string mappings", length());
00377     int count = 0;
00378     for (int i = 0; i < to_concat.length(); i++) {
00379         for (int j = 0; j < to_concat[i]->length(); j++) {
00380             to_concat[i]->getRow(j, row);
00381             for (int k = 0; k < width(); k++) {
00382                 if (!is_missing(row[k]) && map_sr[k].size() > 0) {
00383                     // There is a string mapping for this column.
00384                     // Note that if the value is missing, we should not look for a mapping
00385                     // from this value, because it would find it even if it does not exist
00386                     // (see the STL map documentation to understand why this happens).
00387                     if (to_concat[i]->getValString(k, row[k]) == "") {
00388                         // But it is a numerical value for the i-th VMat.
00389                         if (map_rs[k].find(row[k]) != map_rs[k].end()) {
00390                             // And, unfortunately, we have used this numerical value in
00391                             // the column string mapping. It could be fixed, but
00392                             // this would be pretty annoying, thus we just raise an error.
00393                             PLERROR("In ConcatRowsVMatrix::fullyCheckMappings - In column %s of concatenated VMat number %d, the row %d (%s) contains a numerical value (%f) that is used in a string mapping (mapped to %s)", getFieldInfos(k).name.c_str(), i, j, to_concat[i]->fieldName(j).c_str(), row[k], map_rs[k][row[k]].c_str());
00394                         }
00395                     }
00396                 }
00397             }
00398             if (report_progress)
00399                 pb->update(++count);
00400         }
00401     }
00402 }
00403 
00405 // get //
00407 real ConcatRowsVMatrix::get(int i, int j) const
00408 {
00409     int whichvm, rowofvm;
00410     getpositions(i,whichvm,rowofvm);
00411     if (!need_fix_mappings || fixed_mappings(whichvm, j).size() == 0)
00412         return to_concat[whichvm]->get(rowofvm,j);
00413     else {
00414         real val = to_concat[whichvm]->get(rowofvm,j);
00415         if (!is_missing(val)) {
00416             map<real, real>::iterator it = fixed_mappings(whichvm, j).find(val);
00417             if (it != fixed_mappings(whichvm, j).end()) {
00418                 // We need to modify this value.
00419                 return it->second;
00420             }
00421         }
00422         return val;
00423     }
00424 }
00425 
00427 // getSubRow //
00429 void ConcatRowsVMatrix::getSubRow(int i, int j, Vec v) const
00430 {
00431     static map<real, real> fixed;
00432     static map<real, real>::iterator it;
00433     int whichvm, rowofvm;
00434     getpositions(i,whichvm,rowofvm);
00435     to_concat[whichvm]->getSubRow(rowofvm, j, v);
00436     if (need_fix_mappings) {
00437         for (int k = j; k < j + v.length(); k++) {
00438             fixed = fixed_mappings(whichvm, k);
00439             if (!is_missing(v[k-j])) {
00440                 it = fixed.find(v[k -j]);
00441                 if (it != fixed.end())
00442                     v[k - j] = it->second;
00443             }
00444         }
00445     }
00446 }
00447 
00449 // makeDeepCopyFromShallowCopy //
00451 void ConcatRowsVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) {
00452     inherited::makeDeepCopyFromShallowCopy(copies);
00453     deepCopyField(sources, copies);
00454     deepCopyField(to_concat, copies);
00455     deepCopyField(fixed_mappings, copies);
00456 }
00457 
00459 // putMat //
00461 void ConcatRowsVMatrix::putMat(int i, int j, Mat m) {
00462     int whichvm, rowofvm;
00463     for (int row = 0; row < length(); row++) {
00464         getpositions(row + i, whichvm, rowofvm);
00465         to_concat[whichvm]->putSubRow(rowofvm, j, m(row));
00466     }
00467 }
00468 
00470 // recomputDimensions //
00472 void ConcatRowsVMatrix::recomputeDimensions() {
00473     width_ = to_concat[0]->width();
00474     length_ = 0;
00475     for (int i=0; i<to_concat.length(); i++) {
00476         if (to_concat[i]->width() != width_)
00477             PLERROR("ConcatRowsVMatrix: underlying-VMat %d has %d width, while"
00478                     " 0-th has %d", i, sources[i]->width(), width_);
00479         length_ += to_concat[i]->length();
00480     }
00481 }
00482 
00484 // reset_dimensions //
00486 void ConcatRowsVMatrix::reset_dimensions() {
00487     for (int i=0;i<to_concat.size();i++)
00488         to_concat[i]->reset_dimensions();
00489     recomputeDimensions();
00490 }
00491 
00493 // getPositionsAux //
00495 void ConcatRowsVMatrix::getPositionsAux(int i, int& whichvm, int& rowofvm) const
00496 {
00497     int pos = 0;
00498     int k=0;
00499     while(i>=pos+to_concat[k]->length())
00500     {
00501         pos += to_concat[k]->length();
00502         k++;
00503     }
00504 
00505     // Update cache of last-used VMat
00506     m_last_vmat_index    = k;
00507     m_last_vmat_startrow = pos;
00508     m_last_vmat_lastrow  = pos + to_concat[k]->length() - 1;
00509     
00510     whichvm = k;
00511     rowofvm = i-pos;
00512 }
00513 
00514 
00515 } // end of namespace PLearn
00516 
00517 
00518 /*
00519   Local Variables:
00520   mode:c++
00521   c-basic-offset:4
00522   c-file-style:"stroustrup"
00523   c-file-offsets:((innamespace . 0)(inline-open . 0))
00524   indent-tabs-mode:nil
00525   fill-column:79
00526   End:
00527 */
00528 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines