PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 /* ******************************************************* 00038 * $Id: ConcatRowsVMatrix.cc 9774 2008-12-11 21:06:26Z nouiz $ 00039 ******************************************************* */ 00040 00041 #include "ConcatRowsVMatrix.h" 00042 #include "SelectColumnsVMatrix.h" 00043 00044 namespace PLearn { 00045 using namespace std; 00046 00049 PLEARN_IMPLEMENT_OBJECT( 00050 ConcatRowsVMatrix, 00051 "Concatenates the rows of a number of VMats.", 00052 "If the VMats do not have the same fields, it is possible to either keep\n" 00053 "only the common fields ('only_common_fields' = 1), or to keep them all\n" 00054 "and add missing values accordingly ('fill_missing' = 1).\n" 00055 "If none of these options is set, the VMats are assumed to have the same\n" 00056 "fields.\n" 00057 ); 00058 00059 00061 // ConcatRowsVMatrix // 00063 ConcatRowsVMatrix::ConcatRowsVMatrix(TVec<VMat> the_sources) 00064 : sources(the_sources), 00065 fill_missing(false), 00066 fully_check_mappings(false), 00067 only_common_fields(false), 00068 m_last_vmat_index(-1), 00069 m_last_vmat_startrow(-1), 00070 m_last_vmat_lastrow(-1) 00071 { 00072 if (sources.size() > 0) 00073 build_(); 00074 } 00075 00076 ConcatRowsVMatrix::ConcatRowsVMatrix(VMat d1, VMat d2, 00077 bool the_fully_check_mapping) 00078 : sources(2), 00079 fill_missing(false), 00080 fully_check_mappings(the_fully_check_mapping), 00081 only_common_fields(false), 00082 m_last_vmat_index(-1), 00083 m_last_vmat_startrow(-1), 00084 m_last_vmat_lastrow(-1) 00085 { 00086 sources[0] = d1; 00087 sources[1] = d2; 00088 build_(); 00089 } 00090 00092 // declareOptions // 00094 void ConcatRowsVMatrix::declareOptions(OptionList &ol) 00095 { 00096 declareOption(ol, "array", &ConcatRowsVMatrix::sources, 00097 (OptionBase::learntoption | OptionBase::nosave), 00098 "DEPRECATED - Use 'sources' instead."); 00099 00100 declareOption(ol, "sources", &ConcatRowsVMatrix::sources, 00101 OptionBase::buildoption, 00102 "The VMat to concatenate (horizontally)."); 00103 00104 declareOption(ol, "fill_missing", &ConcatRowsVMatrix::fill_missing, 00105 OptionBase::buildoption, 00106 "If set to 1, then all fields will be kept, and in each VMat" 00107 " missing fields\n" 00108 "will be filled with missing values.\n"); 00109 00110 declareOption(ol, "fully_check_mappings", 00111 &ConcatRowsVMatrix::fully_check_mappings, 00112 OptionBase::buildoption, 00113 "If set to 1, then columns for which there is a string <->" 00114 " real mapping\n" 00115 "will be examined to ensure that no numerical data in a VMat" 00116 " conflicts with\n" 00117 "a mapping in another VMat.\n"); 00118 00119 declareOption(ol, "only_common_fields", 00120 &ConcatRowsVMatrix::only_common_fields, 00121 OptionBase::buildoption, 00122 "If set to 1, then only the fields whose names are common to" 00123 " all VMat\n" 00124 "in 'sources' will be kept (and reordered if needed).\n"); 00125 00126 inherited::declareOptions(ol); 00127 } 00128 00130 // build // 00132 void ConcatRowsVMatrix::build() 00133 { 00134 inherited::build(); 00135 build_(); 00136 } 00137 00139 // build_ // 00141 void ConcatRowsVMatrix::build_() 00142 { 00143 int n = sources.size(); 00144 if (n < 1) 00145 // No sources provided yet: nothing to do. 00146 return; 00147 for(int i=0;i<sources.size();i++) 00148 updateMtime(sources[i]); 00149 if (only_common_fields && fill_missing) 00150 PLERROR("In ConcatRowsVMatrix::build_ - You can't set both 'only_common_fields' and 'fill_missing'"); 00151 00152 // Get the fields info. 00153 fieldinfos = sources[0]->getFieldInfos(); 00154 if (only_common_fields) { 00155 findCommonFields(); 00156 } else if (fill_missing) { 00157 findAllFields(); 00158 } else { 00159 to_concat = sources; 00160 } 00161 00162 // Set length_ and width_. 00163 recomputeDimensions(); 00164 00165 // Note that the 4 lines below will overwrite any provided sizes. 00166 if(inputsize_<0 && targetsize_<0 && weightsize_<0){ 00167 inputsize_ = to_concat[0]->inputsize(); 00168 targetsize_ = to_concat[0]->targetsize(); 00169 weightsize_ = to_concat[0]->weightsize(); 00170 extrasize_ = to_concat[0]->extrasize(); 00171 if(fieldinfos.size()!=to_concat[0].width()) 00172 PLERROR("In ConcatRowsVMatrix::build_() - We override " 00173 "inputsize, targetsize and weightsize with the value" 00174 " of sources[0] and this cause inconsistency"); 00175 } 00176 computeMissingSizeValue(); 00177 00178 // Make sure mappings are correct. 00179 ensureMappingsConsistency(); 00180 if (fully_check_mappings) 00181 fullyCheckMappings(); 00182 if (need_fix_mappings && !fully_check_mappings) 00183 PLWARNING("In ConcatRowsVMatrix::build_ - Mappings need to be fixed, but you did not set 'fully_check_mappings' to true, this might be dangerous"); 00184 00185 // Reset last-used cache for completeness 00186 m_last_vmat_index = m_last_vmat_startrow = m_last_vmat_lastrow = -1; 00187 } 00188 00190 // dot // 00192 real ConcatRowsVMatrix::dot(int i1, int i2, int inputsize) const 00193 { 00194 int whichvm1, rowofvm1; 00195 getpositions(i1,whichvm1,rowofvm1); 00196 int whichvm2, rowofvm2; 00197 getpositions(i2,whichvm2,rowofvm2); 00198 if(whichvm1==whichvm2 && !need_fix_mappings) 00199 return to_concat[whichvm1]->dot(rowofvm1, rowofvm2, inputsize); 00200 else 00201 return VMatrix::dot(i1,i2,inputsize); 00202 } 00203 00204 real ConcatRowsVMatrix::dot(int i, const Vec& v) const 00205 { 00206 if (!need_fix_mappings) { 00207 int whichvm, rowofvm; 00208 getpositions(i,whichvm,rowofvm); 00209 return to_concat[whichvm]->dot(rowofvm,v); 00210 } 00211 else 00212 return VMatrix::dot(i, v); 00213 } 00214 00216 // ensureMappingsConsistency // 00218 void ConcatRowsVMatrix::ensureMappingsConsistency() 00219 { 00220 // Make sure the string mappings are consistent. 00221 // For this, we start from the mappings of the first VMat, and add missing 00222 // mappings obtained from the other VMats. If another VMat has a different 00223 // mapping for the same string, we remember it in order to fix the output 00224 // when a getxxxx() method is called. 00225 need_fix_mappings = false; 00226 copyStringMappingsFrom(to_concat[0]); 00227 map<string, real> other_map; 00228 map<string, real>* cur_map; 00229 map<string, real>::iterator it, find_map, jt; 00230 bool report_progress = false; // Turn it on for debugging. 00231 PP<ProgressBar> pb; 00232 if (report_progress) 00233 pb = new ProgressBar("Checking mappings consistency", width()); 00234 for (int j = 0; j < width(); j++) { 00235 cur_map = &map_sr[j]; 00236 // Find the maximum mapping value for this column. 00237 real max = -REAL_MAX; 00238 for (jt = cur_map->begin(); jt != cur_map->end(); jt++) 00239 if (jt->second > max) 00240 max = jt->second; 00241 for (int i = 1; i < to_concat.length(); i++) { 00242 other_map = to_concat[i]->getStringToRealMapping(j); 00243 for(it = other_map.begin(); it != other_map.end(); it++) { 00244 find_map = cur_map->find(it->first); 00245 if (find_map != cur_map->end()) { 00246 // The string mapped in VMat 'i' is also mapped in our current mapping. 00247 if (!fast_exact_is_equal(find_map->second, it->second)) { 00248 // But the same string is not mapped to the same value. 00249 // This needs to be fixed. 00250 /*cout << find_map->first << endl; 00251 cout << find_map->second << endl;*/ 00252 need_fix_mappings = true; 00253 fixed_mappings.resize(to_concat.length(), width()); 00254 fixed_mappings(i, j)[it->second] = find_map->second; 00255 } 00256 } else { 00257 // The string mapped in VMat 'i' is not mapped in our current mapping. 00258 // We need to add this mapping. But we must make sure there is no 00259 // existing mapping to the same real number. 00260 real new_map_val = it->second; 00261 if (getValString(j, it->second) != "") { 00262 // There is already a mapping to this real number. 00263 need_fix_mappings = true; 00264 fixed_mappings.resize(to_concat.length(), width()); 00265 // We pick for the number the maximum of the current mapped numbers, +1. 00266 max++; 00267 fixed_mappings(i, j)[it->second] = max; 00268 new_map_val = max; 00269 } else { 00270 // There is no mapping to this real number, it is thus ok to add it. 00271 if (new_map_val > max) 00272 max = new_map_val; 00273 } 00274 addStringMapping(j, it->first, new_map_val); 00275 } 00276 } 00277 } 00278 if (report_progress) 00279 pb->update(j + 1); 00280 } 00281 } 00282 00284 // findAllFields // 00286 void ConcatRowsVMatrix::findAllFields() 00287 { 00288 // At this point, fieldinfos already contains the fields of the first 00289 // concatenated VMat. 00290 bool report_progress = true; 00291 TVec<VMField> other_fields; 00292 PP<ProgressBar> pb; 00293 int count; 00294 if (report_progress) { 00295 count = 0; 00296 for (int i = 1; i < sources.length(); i++) 00297 count += sources[i]->getFieldInfos().length(); 00298 pb = new ProgressBar("Computing the whole set of fields", count); 00299 } 00300 count = 0; 00301 for (int i = 1; i < sources.length(); i++) { 00302 other_fields = sources[i]->getFieldInfos(); 00303 for (int j = 0; j < other_fields.length(); j++) { 00304 bool present_already = false; 00305 for (int k = 0; !present_already && k < fieldinfos.length(); k++) 00306 if (fieldinfos[k].name == other_fields[j].name) 00307 present_already = true; 00308 if (!present_already) { 00309 // We need to add this field. 00310 fieldinfos.append(other_fields[j]); 00311 } 00312 if (report_progress) 00313 pb->update(++count); 00314 } 00315 } 00316 // Create the vector of field names. 00317 TVec<string> fnames(fieldinfos.length()); 00318 for (int i = 0; i < fieldinfos.length(); i++) 00319 fnames[i] = fieldinfos[i].name; 00320 // Now fill 'to_concat' with the corresponding VMats. 00321 to_concat.resize(sources.length()); 00322 for (int i = 0; i < sources.length(); i++) { 00323 TVec<string> source_fnames=sources[i]->fieldNames(); 00324 if(fnames!=source_fnames){ 00325 to_concat[i] = 00326 new SelectColumnsVMatrix(sources[i], fnames, true); 00327 } else 00328 to_concat[i] = sources[i]; 00329 } 00330 } 00331 00333 // findCommonFields // 00335 void ConcatRowsVMatrix::findCommonFields() 00336 { 00337 // Find out which fields need to be kept. 00338 TVec<VMField> final_fields(fieldinfos.length()); 00339 final_fields << fieldinfos; 00340 TVec<VMField> other_fields; 00341 TVec<VMField> tmp(final_fields.length()); 00342 for (int i = 1; i < sources.length(); i++) { 00343 map<string, bool> can_be_kept; 00344 other_fields = sources[i]->getFieldInfos(); 00345 for (int j = 0; j < other_fields.length(); j++) { 00346 can_be_kept[other_fields[j].name] = true; 00347 } 00348 tmp.resize(0); 00349 for (int j = 0; j < final_fields.length(); j++) 00350 if (can_be_kept.count(final_fields[j].name) > 0) 00351 tmp.append(final_fields[j]); 00352 final_fields.resize(tmp.length()); 00353 final_fields << tmp; 00354 } 00355 fieldinfos.resize(final_fields.length()); 00356 fieldinfos << final_fields; 00357 // Get the corresponding field names. 00358 TVec<string> final_fieldnames(final_fields.length()); 00359 for (int i = 0; i < final_fields.length(); i++) 00360 final_fieldnames[i] = final_fields[i].name; 00361 // Now fill 'to_concat' with the selected columns of each VMat in 'sources' 00362 to_concat.resize(sources.length()); 00363 for (int i = 0; i < sources.length(); i++) 00364 to_concat[i] = new SelectColumnsVMatrix(sources[i], final_fieldnames); 00365 } 00366 00368 // fullyCheckMappings // 00370 void ConcatRowsVMatrix::fullyCheckMappings(bool report_progress) 00371 { 00372 Vec row(width()); 00373 TVec<int> max(width()); 00374 PP<ProgressBar> pb; 00375 if (report_progress) 00376 pb = new ProgressBar("Full check of string mappings", length()); 00377 int count = 0; 00378 for (int i = 0; i < to_concat.length(); i++) { 00379 for (int j = 0; j < to_concat[i]->length(); j++) { 00380 to_concat[i]->getRow(j, row); 00381 for (int k = 0; k < width(); k++) { 00382 if (!is_missing(row[k]) && map_sr[k].size() > 0) { 00383 // There is a string mapping for this column. 00384 // Note that if the value is missing, we should not look for a mapping 00385 // from this value, because it would find it even if it does not exist 00386 // (see the STL map documentation to understand why this happens). 00387 if (to_concat[i]->getValString(k, row[k]) == "") { 00388 // But it is a numerical value for the i-th VMat. 00389 if (map_rs[k].find(row[k]) != map_rs[k].end()) { 00390 // And, unfortunately, we have used this numerical value in 00391 // the column string mapping. It could be fixed, but 00392 // this would be pretty annoying, thus we just raise an error. 00393 PLERROR("In ConcatRowsVMatrix::fullyCheckMappings - In column %s of concatenated VMat number %d, the row %d (%s) contains a numerical value (%f) that is used in a string mapping (mapped to %s)", getFieldInfos(k).name.c_str(), i, j, to_concat[i]->fieldName(j).c_str(), row[k], map_rs[k][row[k]].c_str()); 00394 } 00395 } 00396 } 00397 } 00398 if (report_progress) 00399 pb->update(++count); 00400 } 00401 } 00402 } 00403 00405 // get // 00407 real ConcatRowsVMatrix::get(int i, int j) const 00408 { 00409 int whichvm, rowofvm; 00410 getpositions(i,whichvm,rowofvm); 00411 if (!need_fix_mappings || fixed_mappings(whichvm, j).size() == 0) 00412 return to_concat[whichvm]->get(rowofvm,j); 00413 else { 00414 real val = to_concat[whichvm]->get(rowofvm,j); 00415 if (!is_missing(val)) { 00416 map<real, real>::iterator it = fixed_mappings(whichvm, j).find(val); 00417 if (it != fixed_mappings(whichvm, j).end()) { 00418 // We need to modify this value. 00419 return it->second; 00420 } 00421 } 00422 return val; 00423 } 00424 } 00425 00427 // getSubRow // 00429 void ConcatRowsVMatrix::getSubRow(int i, int j, Vec v) const 00430 { 00431 static map<real, real> fixed; 00432 static map<real, real>::iterator it; 00433 int whichvm, rowofvm; 00434 getpositions(i,whichvm,rowofvm); 00435 to_concat[whichvm]->getSubRow(rowofvm, j, v); 00436 if (need_fix_mappings) { 00437 for (int k = j; k < j + v.length(); k++) { 00438 fixed = fixed_mappings(whichvm, k); 00439 if (!is_missing(v[k-j])) { 00440 it = fixed.find(v[k -j]); 00441 if (it != fixed.end()) 00442 v[k - j] = it->second; 00443 } 00444 } 00445 } 00446 } 00447 00449 // makeDeepCopyFromShallowCopy // 00451 void ConcatRowsVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) { 00452 inherited::makeDeepCopyFromShallowCopy(copies); 00453 deepCopyField(sources, copies); 00454 deepCopyField(to_concat, copies); 00455 deepCopyField(fixed_mappings, copies); 00456 } 00457 00459 // putMat // 00461 void ConcatRowsVMatrix::putMat(int i, int j, Mat m) { 00462 int whichvm, rowofvm; 00463 for (int row = 0; row < length(); row++) { 00464 getpositions(row + i, whichvm, rowofvm); 00465 to_concat[whichvm]->putSubRow(rowofvm, j, m(row)); 00466 } 00467 } 00468 00470 // recomputDimensions // 00472 void ConcatRowsVMatrix::recomputeDimensions() { 00473 width_ = to_concat[0]->width(); 00474 length_ = 0; 00475 for (int i=0; i<to_concat.length(); i++) { 00476 if (to_concat[i]->width() != width_) 00477 PLERROR("ConcatRowsVMatrix: underlying-VMat %d has %d width, while" 00478 " 0-th has %d", i, sources[i]->width(), width_); 00479 length_ += to_concat[i]->length(); 00480 } 00481 } 00482 00484 // reset_dimensions // 00486 void ConcatRowsVMatrix::reset_dimensions() { 00487 for (int i=0;i<to_concat.size();i++) 00488 to_concat[i]->reset_dimensions(); 00489 recomputeDimensions(); 00490 } 00491 00493 // getPositionsAux // 00495 void ConcatRowsVMatrix::getPositionsAux(int i, int& whichvm, int& rowofvm) const 00496 { 00497 int pos = 0; 00498 int k=0; 00499 while(i>=pos+to_concat[k]->length()) 00500 { 00501 pos += to_concat[k]->length(); 00502 k++; 00503 } 00504 00505 // Update cache of last-used VMat 00506 m_last_vmat_index = k; 00507 m_last_vmat_startrow = pos; 00508 m_last_vmat_lastrow = pos + to_concat[k]->length() - 1; 00509 00510 whichvm = k; 00511 rowofvm = i-pos; 00512 } 00513 00514 00515 } // end of namespace PLearn 00516 00517 00518 /* 00519 Local Variables: 00520 mode:c++ 00521 c-basic-offset:4 00522 c-file-style:"stroustrup" 00523 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00524 indent-tabs-mode:nil 00525 fill-column:79 00526 End: 00527 */ 00528 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :