PLearn 0.1
FilteredVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // FilteredVMatrix.cc
00004 //
00005 // Copyright (C) 2003 Pascal Vincent
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: FilteredVMatrix.cc 9525 2008-10-06 15:11:48Z tihocan $
00037  ******************************************************* */
00038 
00039 // Authors: Pascal Vincent
00040 
00043 #include "FilteredVMatrix.h"
00044 #include <plearn/base/ProgressBar.h>
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 PLEARN_IMPLEMENT_OBJECT(
00050     FilteredVMatrix,
00051     "A filtered view of its source VMatrix.",
00052     "The filter is an expression in VPL language.\n"
00053     "If a metadata directory is provided, the filtered indexes are saved in\n"
00054     "this directory. Otherwise the filtered indexes are re-computed at each\n"
00055     "call to build()."
00056 );
00057 
00059 // FilteredVMatrix //
00061 FilteredVMatrix::FilteredVMatrix():
00062     build_complete(false),
00063     allow_repeat_rows(false),
00064     repeat_id_field_name(""),
00065     repeat_count_field_name(""),
00066     warn_no_metadatadir(false),
00067     report_progress(true)
00068 {}
00069 
00070 FilteredVMatrix::FilteredVMatrix( VMat the_source, const string& program_string,
00071                                   const PPath& the_metadatadir, bool the_report_progress,
00072                                   bool allow_repeat_rows_, 
00073                                   const string& repeat_id_field_name_,
00074                                   const string& repeat_count_field_name_,
00075                                   bool call_build_):
00076     inherited(the_source, call_build_),
00077     allow_repeat_rows(allow_repeat_rows_),
00078     repeat_id_field_name(repeat_id_field_name_),
00079     repeat_count_field_name(repeat_count_field_name_),
00080     report_progress(the_report_progress),
00081     prg(program_string)
00082 {
00083     // Note that although VMatrix::build_ would be tempted to call
00084     // setMetaDataDir when inherited(the_source, true) is called above (if
00085     // call_build_ is true), the metadatadir is only set below, so it will not
00086     // happen.
00087     metadatadir = the_metadatadir;
00088 
00089     if (call_build_)
00090         build_();
00091 }
00092 
00094 // computeFilteredIndices //
00096 void FilteredVMatrix::computeFilteredIndices()
00097 {
00098     int l = source.length();
00099     Vec result(1);
00100     PP<ProgressBar> pb;
00101     if (report_progress)
00102         pb = new ProgressBar("Filtering source vmat", l);
00103     mem_indices.resize(0);
00104     for(int i=0; i<l; i++)
00105     {
00106         if (report_progress)
00107             pb->update(i);
00108         program.run(i,result);
00109         if(!allow_repeat_rows)
00110         {
00111             if(!fast_exact_is_equal(result[0], 0))
00112                 mem_indices.append(i);
00113         }
00114         else
00115             for(int x = int(round(result[0])); x > 0; --x)
00116                 mem_indices.append(i);
00117     }
00118     length_ = mem_indices.length();
00119 }
00120 
00122 // openIndex //
00124 void FilteredVMatrix::openIndex()
00125 {
00126     PLASSERT(hasMetaDataDir());
00127 
00128     PPath idxfname = getMetaDataDir() / "filtered.idx";
00129     if(!force_mkdir(getMetaDataDir()))
00130         PLERROR("In FilteredVMatrix::openIndex - Could not create "
00131                 "directory %s", getMetaDataDir().absolute().c_str());
00132 
00133     lockMetaDataDir();
00134     try{
00135         if(isUpToDate(idxfname))
00136             indexes.open(idxfname.absolute());
00137         else  // let's (re)create the index
00138         {
00139             computeFilteredIndices();
00140             rm(idxfname);       // force remove it
00141             indexes.open(idxfname.absolute(), true);
00142             for (int i = 0; i < mem_indices.length(); i++)
00143                 indexes.append(mem_indices[i]);
00144             indexes.close();
00145             indexes.open(idxfname.absolute());
00146             mem_indices = TVec<int>();  // Free memory.
00147         }
00148     }catch(const PLearnError& e){
00149         unlockMetaDataDir();
00150         //we erase the file if we are creating it
00151         // as it can be partilly saved.
00152         if(!isUpToDate(idxfname) && isfile(idxfname))
00153             rm(idxfname);
00154         throw e;
00155     }
00156     unlockMetaDataDir();
00157     length_ = indexes.length();
00158 }
00159 
00161 // setMetaDataDir //
00163 void FilteredVMatrix::setMetaDataDir(const PPath& the_metadatadir)
00164 {
00165     inherited::setMetaDataDir(the_metadatadir);
00166     if (build_complete) {
00167         // Only call openIndex() if the build has been completed,
00168         // otherwise the filtering program will not be ready yet.
00169         openIndex();
00170         // We call 'setMetaInfoFromSource' only after the index file has been
00171         // correctly read.
00172         setMetaInfoFromSource();
00173     }
00174 }
00175 
00177 // getNewRow //
00179 void FilteredVMatrix::getNewRow(int i, const Vec& v) const
00180 {
00181     if (mem_indices.isEmpty() && indexes.length() == -1)
00182         PLERROR("In FilteredVMatrix::getNewRow - Filtered indices are not\n"
00183                 "set yet.");
00184 
00185     int j= source->width();
00186     int idx = mem_indices.isEmpty() ? indexes[i] : mem_indices[i];
00187 
00188     source->getRow(idx, v.subVec(0, j));
00189 
00190     if(!repeat_id_field_name.empty())
00191     {
00192         int k= 1;
00193         while(k <= i && (mem_indices.isEmpty()
00194                                 ? indexes[i]==indexes[i-k]
00195                                 : mem_indices[i] == mem_indices[i-k]))
00196             ++k;
00197         v[j++]= static_cast<real>(k-1);
00198     }
00199 
00200     if(!repeat_count_field_name.empty())
00201     {
00202         int k0= 1;
00203         while(k0 <= i && (mem_indices.isEmpty()
00204                                 ? indexes[i]==indexes[i-k0]
00205                                 : mem_indices[i] == mem_indices[i-k0]))
00206             ++k0;
00207         --k0;
00208         int k1= 1;
00209         while(k1+i < length() && (mem_indices.isEmpty()
00210                                         ? indexes[i]==indexes[i+k1]
00211                                         : mem_indices[i] == mem_indices[i+k1]))
00212             ++k1;
00213         v[j++]= static_cast<real>(k0+k1);
00214     }
00215 
00216 }
00217 
00219 // declareOptions //
00221 void FilteredVMatrix::declareOptions(OptionList& ol)
00222 {
00223     declareOption(ol, "prg", &FilteredVMatrix::prg, OptionBase::buildoption,
00224                   "The VPL code that should produce a single scalar, indicating whether \n"
00225                   "we should keep the line (if the produced scalar is non zero) or throw it away (if it's zero)");
00226 
00227     declareOption(ol, "report_progress", &FilteredVMatrix::report_progress, OptionBase::buildoption,
00228                   "Whether to report the filtering progress or not.");
00229 
00230     declareOption(ol, "allow_repeat_rows", &FilteredVMatrix::allow_repeat_rows, OptionBase::buildoption,
00231                   "When true, the result of the program indicates the number of times this row should be repeated.\n"
00232                   "Simple filtering when false.");
00233 
00234     declareOption(ol, "repeat_id_field_name", &FilteredVMatrix::repeat_id_field_name, OptionBase::buildoption,
00235                   "Field name for the repetition id (0, 1, ..., n-1).  No field is added if empty.");
00236 
00237     declareOption(ol, "repeat_count_field_name", &FilteredVMatrix::repeat_count_field_name, OptionBase::buildoption,
00238                   "Field name for the number of repetitions (n).  No field is added if empty.");
00239 
00240     declareOption(ol, "warn_no_metadatadir",
00241                   &FilteredVMatrix::warn_no_metadatadir,
00242                   OptionBase::buildoption,
00243         "If True, generate a warning when there is no metadatadir.\n"
00244         "This can be useful as if there is no metadatadir we will recompute\n"
00245         "filtered indices every time.");
00246 
00247     // Now call the parent class' declareOptions
00248     inherited::declareOptions(ol);
00249 
00250     declareOption(ol, "length", &FilteredVMatrix::length_,
00251                   OptionBase::nosave,
00252                   "The number of example. Computed each time from source.");
00253 
00254     redeclareOption(ol, "inputsize", &FilteredVMatrix::inputsize_,
00255                     OptionBase::nosave,
00256                     "Taken from source in  FilteredVMatrix.");
00257     
00258     redeclareOption(ol, "targetsize", &FilteredVMatrix::targetsize_,
00259                     OptionBase::nosave,
00260                     "Taken from source in FilteredVMatrix.");
00261     
00262     redeclareOption(ol, "weightsize", &FilteredVMatrix::weightsize_,
00263                     OptionBase::nosave,
00264                     "Taken from source in FilteredVMatrix.");
00265     
00266     redeclareOption(ol, "extrasize", &FilteredVMatrix::extrasize_,
00267                     OptionBase::nosave,
00268                     "Taken from source in FilteredVMatrix.");
00269 
00270     redeclareOption(ol, "width", &FilteredVMatrix::width_,
00271                     OptionBase::nosave,
00272                     "Taken from source in FilteredVMatrix.");
00273 }
00274 
00276 // build_ //
00278 void FilteredVMatrix::build_()
00279 {
00280     if (source) {
00281         updateMtime(source);
00282         vector<string> fieldnames;
00283         program.setSource(source);
00284         program.compileString(prg,fieldnames);
00285         build_complete = true;
00286         if (hasMetaDataDir())
00287             // Calling setMetaDataDir() will compute indices and save them
00288             // in the give metadata directory.
00289             setMetaDataDir(getMetaDataDir());
00290         else {
00291             // Compute selected indices in memory only.
00292             if(warn_no_metadatadir)
00293                 PLWARNING("In FilteredVMatrix::build_() - there is no "
00294                         "metadatadir: recomputing filtered indices");
00295             computeFilteredIndices();
00296         }
00297 
00298         Array<VMField> finfos= source->getFieldInfos().copy();
00299             
00300         if(!repeat_id_field_name.empty())
00301         {
00302             finfos.append(VMField(repeat_id_field_name));
00303             if(0 < width_)
00304                 ++width_;
00305         }
00306 
00307         if(!repeat_count_field_name.empty())
00308         {
00309             finfos.append(VMField(repeat_count_field_name));
00310             if(0 < width_)
00311                 ++width_;
00312         }
00313 
00314         setFieldInfos(finfos);
00315         setMetaInfoFromSource();
00316     } else
00317         length_ = width_ = 0;
00318 }
00319 
00321 // build //
00323 void FilteredVMatrix::build()
00324 {
00325     build_complete = false;
00326     inherited::build();
00327     build_();
00328 }
00329 
00331 // makeDeepCopyFromShallowCopy //
00333 void FilteredVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00334 {
00335     inherited::makeDeepCopyFromShallowCopy(copies);
00336     deepCopyField(mem_indices, copies);
00337 }
00338 
00339 } // end of namespace PLearn
00340 
00341 
00342 /*
00343   Local Variables:
00344   mode:c++
00345   c-basic-offset:4
00346   c-file-style:"stroustrup"
00347   c-file-offsets:((innamespace . 0)(inline-open . 0))
00348   indent-tabs-mode:nil
00349   fill-column:79
00350   End:
00351 */
00352 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines