PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // FilteredVMatrix.cc 00004 // 00005 // Copyright (C) 2003 Pascal Vincent 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: FilteredVMatrix.cc 9525 2008-10-06 15:11:48Z tihocan $ 00037 ******************************************************* */ 00038 00039 // Authors: Pascal Vincent 00040 00043 #include "FilteredVMatrix.h" 00044 #include <plearn/base/ProgressBar.h> 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 PLEARN_IMPLEMENT_OBJECT( 00050 FilteredVMatrix, 00051 "A filtered view of its source VMatrix.", 00052 "The filter is an expression in VPL language.\n" 00053 "If a metadata directory is provided, the filtered indexes are saved in\n" 00054 "this directory. Otherwise the filtered indexes are re-computed at each\n" 00055 "call to build()." 00056 ); 00057 00059 // FilteredVMatrix // 00061 FilteredVMatrix::FilteredVMatrix(): 00062 build_complete(false), 00063 allow_repeat_rows(false), 00064 repeat_id_field_name(""), 00065 repeat_count_field_name(""), 00066 warn_no_metadatadir(false), 00067 report_progress(true) 00068 {} 00069 00070 FilteredVMatrix::FilteredVMatrix( VMat the_source, const string& program_string, 00071 const PPath& the_metadatadir, bool the_report_progress, 00072 bool allow_repeat_rows_, 00073 const string& repeat_id_field_name_, 00074 const string& repeat_count_field_name_, 00075 bool call_build_): 00076 inherited(the_source, call_build_), 00077 allow_repeat_rows(allow_repeat_rows_), 00078 repeat_id_field_name(repeat_id_field_name_), 00079 repeat_count_field_name(repeat_count_field_name_), 00080 report_progress(the_report_progress), 00081 prg(program_string) 00082 { 00083 // Note that although VMatrix::build_ would be tempted to call 00084 // setMetaDataDir when inherited(the_source, true) is called above (if 00085 // call_build_ is true), the metadatadir is only set below, so it will not 00086 // happen. 00087 metadatadir = the_metadatadir; 00088 00089 if (call_build_) 00090 build_(); 00091 } 00092 00094 // computeFilteredIndices // 00096 void FilteredVMatrix::computeFilteredIndices() 00097 { 00098 int l = source.length(); 00099 Vec result(1); 00100 PP<ProgressBar> pb; 00101 if (report_progress) 00102 pb = new ProgressBar("Filtering source vmat", l); 00103 mem_indices.resize(0); 00104 for(int i=0; i<l; i++) 00105 { 00106 if (report_progress) 00107 pb->update(i); 00108 program.run(i,result); 00109 if(!allow_repeat_rows) 00110 { 00111 if(!fast_exact_is_equal(result[0], 0)) 00112 mem_indices.append(i); 00113 } 00114 else 00115 for(int x = int(round(result[0])); x > 0; --x) 00116 mem_indices.append(i); 00117 } 00118 length_ = mem_indices.length(); 00119 } 00120 00122 // openIndex // 00124 void FilteredVMatrix::openIndex() 00125 { 00126 PLASSERT(hasMetaDataDir()); 00127 00128 PPath idxfname = getMetaDataDir() / "filtered.idx"; 00129 if(!force_mkdir(getMetaDataDir())) 00130 PLERROR("In FilteredVMatrix::openIndex - Could not create " 00131 "directory %s", getMetaDataDir().absolute().c_str()); 00132 00133 lockMetaDataDir(); 00134 try{ 00135 if(isUpToDate(idxfname)) 00136 indexes.open(idxfname.absolute()); 00137 else // let's (re)create the index 00138 { 00139 computeFilteredIndices(); 00140 rm(idxfname); // force remove it 00141 indexes.open(idxfname.absolute(), true); 00142 for (int i = 0; i < mem_indices.length(); i++) 00143 indexes.append(mem_indices[i]); 00144 indexes.close(); 00145 indexes.open(idxfname.absolute()); 00146 mem_indices = TVec<int>(); // Free memory. 00147 } 00148 }catch(const PLearnError& e){ 00149 unlockMetaDataDir(); 00150 //we erase the file if we are creating it 00151 // as it can be partilly saved. 00152 if(!isUpToDate(idxfname) && isfile(idxfname)) 00153 rm(idxfname); 00154 throw e; 00155 } 00156 unlockMetaDataDir(); 00157 length_ = indexes.length(); 00158 } 00159 00161 // setMetaDataDir // 00163 void FilteredVMatrix::setMetaDataDir(const PPath& the_metadatadir) 00164 { 00165 inherited::setMetaDataDir(the_metadatadir); 00166 if (build_complete) { 00167 // Only call openIndex() if the build has been completed, 00168 // otherwise the filtering program will not be ready yet. 00169 openIndex(); 00170 // We call 'setMetaInfoFromSource' only after the index file has been 00171 // correctly read. 00172 setMetaInfoFromSource(); 00173 } 00174 } 00175 00177 // getNewRow // 00179 void FilteredVMatrix::getNewRow(int i, const Vec& v) const 00180 { 00181 if (mem_indices.isEmpty() && indexes.length() == -1) 00182 PLERROR("In FilteredVMatrix::getNewRow - Filtered indices are not\n" 00183 "set yet."); 00184 00185 int j= source->width(); 00186 int idx = mem_indices.isEmpty() ? indexes[i] : mem_indices[i]; 00187 00188 source->getRow(idx, v.subVec(0, j)); 00189 00190 if(!repeat_id_field_name.empty()) 00191 { 00192 int k= 1; 00193 while(k <= i && (mem_indices.isEmpty() 00194 ? indexes[i]==indexes[i-k] 00195 : mem_indices[i] == mem_indices[i-k])) 00196 ++k; 00197 v[j++]= static_cast<real>(k-1); 00198 } 00199 00200 if(!repeat_count_field_name.empty()) 00201 { 00202 int k0= 1; 00203 while(k0 <= i && (mem_indices.isEmpty() 00204 ? indexes[i]==indexes[i-k0] 00205 : mem_indices[i] == mem_indices[i-k0])) 00206 ++k0; 00207 --k0; 00208 int k1= 1; 00209 while(k1+i < length() && (mem_indices.isEmpty() 00210 ? indexes[i]==indexes[i+k1] 00211 : mem_indices[i] == mem_indices[i+k1])) 00212 ++k1; 00213 v[j++]= static_cast<real>(k0+k1); 00214 } 00215 00216 } 00217 00219 // declareOptions // 00221 void FilteredVMatrix::declareOptions(OptionList& ol) 00222 { 00223 declareOption(ol, "prg", &FilteredVMatrix::prg, OptionBase::buildoption, 00224 "The VPL code that should produce a single scalar, indicating whether \n" 00225 "we should keep the line (if the produced scalar is non zero) or throw it away (if it's zero)"); 00226 00227 declareOption(ol, "report_progress", &FilteredVMatrix::report_progress, OptionBase::buildoption, 00228 "Whether to report the filtering progress or not."); 00229 00230 declareOption(ol, "allow_repeat_rows", &FilteredVMatrix::allow_repeat_rows, OptionBase::buildoption, 00231 "When true, the result of the program indicates the number of times this row should be repeated.\n" 00232 "Simple filtering when false."); 00233 00234 declareOption(ol, "repeat_id_field_name", &FilteredVMatrix::repeat_id_field_name, OptionBase::buildoption, 00235 "Field name for the repetition id (0, 1, ..., n-1). No field is added if empty."); 00236 00237 declareOption(ol, "repeat_count_field_name", &FilteredVMatrix::repeat_count_field_name, OptionBase::buildoption, 00238 "Field name for the number of repetitions (n). No field is added if empty."); 00239 00240 declareOption(ol, "warn_no_metadatadir", 00241 &FilteredVMatrix::warn_no_metadatadir, 00242 OptionBase::buildoption, 00243 "If True, generate a warning when there is no metadatadir.\n" 00244 "This can be useful as if there is no metadatadir we will recompute\n" 00245 "filtered indices every time."); 00246 00247 // Now call the parent class' declareOptions 00248 inherited::declareOptions(ol); 00249 00250 declareOption(ol, "length", &FilteredVMatrix::length_, 00251 OptionBase::nosave, 00252 "The number of example. Computed each time from source."); 00253 00254 redeclareOption(ol, "inputsize", &FilteredVMatrix::inputsize_, 00255 OptionBase::nosave, 00256 "Taken from source in FilteredVMatrix."); 00257 00258 redeclareOption(ol, "targetsize", &FilteredVMatrix::targetsize_, 00259 OptionBase::nosave, 00260 "Taken from source in FilteredVMatrix."); 00261 00262 redeclareOption(ol, "weightsize", &FilteredVMatrix::weightsize_, 00263 OptionBase::nosave, 00264 "Taken from source in FilteredVMatrix."); 00265 00266 redeclareOption(ol, "extrasize", &FilteredVMatrix::extrasize_, 00267 OptionBase::nosave, 00268 "Taken from source in FilteredVMatrix."); 00269 00270 redeclareOption(ol, "width", &FilteredVMatrix::width_, 00271 OptionBase::nosave, 00272 "Taken from source in FilteredVMatrix."); 00273 } 00274 00276 // build_ // 00278 void FilteredVMatrix::build_() 00279 { 00280 if (source) { 00281 updateMtime(source); 00282 vector<string> fieldnames; 00283 program.setSource(source); 00284 program.compileString(prg,fieldnames); 00285 build_complete = true; 00286 if (hasMetaDataDir()) 00287 // Calling setMetaDataDir() will compute indices and save them 00288 // in the give metadata directory. 00289 setMetaDataDir(getMetaDataDir()); 00290 else { 00291 // Compute selected indices in memory only. 00292 if(warn_no_metadatadir) 00293 PLWARNING("In FilteredVMatrix::build_() - there is no " 00294 "metadatadir: recomputing filtered indices"); 00295 computeFilteredIndices(); 00296 } 00297 00298 Array<VMField> finfos= source->getFieldInfos().copy(); 00299 00300 if(!repeat_id_field_name.empty()) 00301 { 00302 finfos.append(VMField(repeat_id_field_name)); 00303 if(0 < width_) 00304 ++width_; 00305 } 00306 00307 if(!repeat_count_field_name.empty()) 00308 { 00309 finfos.append(VMField(repeat_count_field_name)); 00310 if(0 < width_) 00311 ++width_; 00312 } 00313 00314 setFieldInfos(finfos); 00315 setMetaInfoFromSource(); 00316 } else 00317 length_ = width_ = 0; 00318 } 00319 00321 // build // 00323 void FilteredVMatrix::build() 00324 { 00325 build_complete = false; 00326 inherited::build(); 00327 build_(); 00328 } 00329 00331 // makeDeepCopyFromShallowCopy // 00333 void FilteredVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00334 { 00335 inherited::makeDeepCopyFromShallowCopy(copies); 00336 deepCopyField(mem_indices, copies); 00337 } 00338 00339 } // end of namespace PLearn 00340 00341 00342 /* 00343 Local Variables: 00344 mode:c++ 00345 c-basic-offset:4 00346 c-file-style:"stroustrup" 00347 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00348 indent-tabs-mode:nil 00349 fill-column:79 00350 End: 00351 */ 00352 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :