PLearn 0.1
VPLProcessor.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // VPLProcessor.cc
00004 //
00005 // Copyright (C) 2005, 2006 Pascal Vincent 
00006 // 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 // 
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 // 
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 // 
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 // 
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 // 
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************      
00036  * $Id: VPLProcessor.cc 5480 2006-05-03 18:57:39Z plearner $ 
00037  ******************************************************* */
00038 
00039 // Authors: Pascal Vincent
00040 
00043 #include <sstream>
00044 
00045 #include "VPLProcessor.h"
00046 #include <plearn/vmat/ProcessingVMatrix.h>
00047 #include <plearn/vmat/FilteredVMatrix.h>
00048 #include <plearn/base/tostring.h>
00049 
00050 namespace PLearn {
00051 using namespace std;
00052 
00053 VPLProcessor::VPLProcessor() 
00054     :orig_inputsize(-1),
00055      orig_targetsize(-1),
00056      use_filtering_prg_for_repeat(false),
00057      repeat_id_field_name(""),
00058      repeat_count_field_name("")
00059 {
00060 }
00061 
00062 PLEARN_IMPLEMENT_OBJECT(
00063     VPLProcessor,
00064     "Learner whose training-set, inputs and outputs can be pre/post-processed by VPL code",
00065     "See VMatLanguage for the definition of the allowed VPL syntax."
00066     );
00067 
00068 void VPLProcessor::declareOptions(OptionList& ol)
00069 {
00070     declareOption(ol, "filtering_prg", &VPLProcessor::filtering_prg, OptionBase::buildoption,
00071                   "Optional program string in VPL language to apply as filtering on the training VMat.\n"
00072                   "This program is to produce a single value interpreted as a boolean: only the rows for which\n"
00073                   "it evaluates to non-zero will be kept.\n"
00074                   "An empty string means NO FILTERING.");
00075 
00076     declareOption(ol, "input_prg", &VPLProcessor::input_prg, OptionBase::buildoption,
00077                   "Program string in VPL language to be applied to each raw input \n"
00078                   "to generate the new preprocessed input.\n"
00079                   "Note that names must be given to the generated values with :fieldname VPL syntax.\n"
00080                   "An empty string means NO PREPROCESSING. (initial raw input is used as is)");
00081 
00082     declareOption(ol, "target_prg", &VPLProcessor::target_prg, OptionBase::buildoption,
00083                   "Program string in VPL language to be applied to a dataset row\n"
00084                   "to generate a proper target.\n"
00085                   "Note that names must be given to the generated values with :fieldname VPL syntax.\n"
00086                   "If it's an empty string, then we'll use the original target from the data set");
00087   
00088     declareOption(ol, "weight_prg", &VPLProcessor::weight_prg, OptionBase::buildoption,
00089                   "Program string in VPL language to be applied to a dataset row\n"
00090                   "to generate a proper weight.\n"
00091                   "Note that names must be given to the generated values with :fieldname VPL syntax.\n"
00092                   "If it's an empty string, then we'll use the original weight from the data set");
00093 
00094     declareOption(ol, "extra_prg", &VPLProcessor::extra_prg, OptionBase::buildoption,
00095                   "Program string in VPL language to be applied to a dataset row\n"
00096                   "to generate proper extra fields.\n"
00097                   "Note that names must be given to the generated values with :fieldname VPL syntax.\n"
00098                   "If it's an empty string, then we'll use the original extra fields from the data set");
00099 
00100 
00101     declareOption(ol, "use_filtering_prg_for_repeat", &VPLProcessor::use_filtering_prg_for_repeat, OptionBase::buildoption,
00102                   "When true, the result of the filtering program indicates the number of times a row should be repeated (0..n).\n"
00103                   "(sets FilteredVMatrix::allow_repeat_rows.)");
00104 
00105     declareOption(ol, "repeat_id_field_name", &VPLProcessor::repeat_id_field_name, OptionBase::buildoption,
00106                   "Field name for the repetition id (0, 1, ..., n-1).  No field is added if empty.");
00107 
00108     declareOption(ol, "repeat_count_field_name", &VPLProcessor::repeat_count_field_name, OptionBase::buildoption,
00109                   "Field name for the number of repetitions (n).  No field is added if empty.");
00110 
00111     // learnt
00112 
00113     declareOption(ol, "orig_fieldnames", &VPLProcessor::orig_fieldnames, OptionBase::learntoption,
00114                   "original fieldnames of the training set");
00115     declareOption(ol, "orig_inputsize", &VPLProcessor::orig_inputsize, OptionBase::learntoption,
00116                   "original inputsize of the training set");
00117     declareOption(ol, "orig_targetsize", &VPLProcessor::orig_targetsize, OptionBase::learntoption,
00118                   "original targetsize of the training set");
00119 
00120 
00121     // Now call the parent class' declareOptions
00122     inherited::declareOptions(ol);
00123 }
00124 
00125 void VPLProcessor::build_()
00126 {
00127     // We're probably reloading a saved VPLProcessor
00128     if(train_set.isNull() && (orig_inputsize>0 || orig_targetsize>0) )
00129         initializeInputPrograms();
00130 }
00131 
00132 void VPLProcessor::build()
00133 {
00134     inherited::build();
00135     build_();
00136 }
00137 
00138 
00140 void VPLProcessor::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00141 {
00142     inherited::makeDeepCopyFromShallowCopy(copies);
00143 
00144     input_prg_.makeDeepCopyFromShallowCopy(copies);
00145 
00146     deepCopyField(input_prg_fieldnames, copies);
00147     deepCopyField(processed_input, copies);
00148     deepCopyField(orig_fieldnames, copies);
00149 }
00150 
00151 int VPLProcessor::outputsize() const
00152 {
00153     if (!input_prg.empty())
00154         return input_prg_fieldnames.length();
00155 
00156     return inputsize();
00157 }
00158 
00159 void VPLProcessor::forget()
00160 {
00161     inherited::forget();
00162     stage = 0;
00163 }
00164 
00168 void VPLProcessor::initializeInputPrograms()
00169 {
00170     if (!input_prg.empty())
00171     {
00172         input_prg_.setSourceFieldNames(orig_fieldnames.subVec(0,orig_inputsize));
00173         input_prg_.compileString(input_prg, input_prg_fieldnames);
00174     }
00175     else
00176     {
00177         input_prg_.clear();
00178         input_prg_fieldnames.resize(0);
00179     }
00180 
00181 }
00182 
00186 void VPLProcessor::setTrainingSet(VMat training_set, bool call_forget)
00187 {
00188     orig_fieldnames = training_set->fieldNames();
00189     orig_inputsize  = training_set->inputsize();
00190     orig_targetsize  = training_set->targetsize();
00191     initializeInputPrograms();
00192 
00193     VMat filtered_trainset = training_set;
00194     PPath filtered_trainset_metadatadir = getExperimentDirectory() / "filtered_train_set.metadata";
00195     if (!filtering_prg.empty())
00196         filtered_trainset = new FilteredVMatrix(training_set, filtering_prg, filtered_trainset_metadatadir, verbosity>1,
00197                                                 use_filtering_prg_for_repeat, repeat_id_field_name, repeat_count_field_name);
00198 
00199     // XXX The next line does nothing!
00200     VMat processed_trainset = new ProcessingVMatrix(filtered_trainset, input_prg, target_prg, weight_prg, extra_prg);
00201     inherited::setTrainingSet(training_set, call_forget); // will call forget if needed
00202 }
00203 
00204 VMat VPLProcessor::processDataSet(VMat dataset) const
00205 {
00206     VMat filtered_dataset = dataset;
00207     PPath filtered_dataset_metadatadir = getExperimentDirectory() / "filtered_dataset.metadata";
00208     if (!filtering_prg.empty())
00209         filtered_dataset = new FilteredVMatrix(dataset, filtering_prg, filtered_dataset_metadatadir, verbosity>1,
00210                                                use_filtering_prg_for_repeat, repeat_id_field_name, repeat_count_field_name);
00211 
00212     // Since ProcessingVMatrix produces 0 length vectors when given an empty
00213     // program (which is not the behavior that VPLProcessors is documented as
00214     // implementing), we need to replace each program that is an empty string
00215     // by a small VPL snippet that copies all the fields for the input or
00216     // target, etc.
00217 
00218     // First compute the start of each section (input, target, etc.) in the
00219     // columns of the dataset.
00220     const int start_of_targets = dataset->inputsize();
00221     const int start_of_weights = start_of_targets + dataset->targetsize();
00222     const int start_of_extras = start_of_weights + dataset->weightsize();
00223 
00224     // Now compute each processing_*_prg program.
00225     string processing_input_prg = input_prg;
00226     if (processing_input_prg.empty() && dataset->inputsize() > 0) {
00227         processing_input_prg = "[%0:%" + tostring(start_of_targets-1) + "]";
00228     }
00229     
00230     string processing_target_prg = target_prg;
00231     if (processing_target_prg.empty() && dataset->targetsize() > 0) {
00232         processing_target_prg = "[%" + tostring(start_of_targets) + ":%" +
00233             tostring(start_of_weights-1) + "]";
00234     }
00235 
00236     string processing_weight_prg = weight_prg;
00237     if (processing_weight_prg.empty() && dataset->weightsize() > 0) {
00238         processing_weight_prg = "[%" + tostring(start_of_weights) + ":%" +
00239             tostring(start_of_extras-1) + "]";
00240     }
00241 
00242     string processing_extras_prg = extra_prg;
00243     if (processing_extras_prg.empty() && dataset->extrasize() > 0) {
00244         processing_extras_prg = "[%" + tostring(start_of_extras) + ":END]";
00245     }
00246     
00247     return new ProcessingVMatrix(filtered_dataset, processing_input_prg,
00248                                  processing_target_prg, processing_weight_prg,
00249                                  processing_extras_prg);
00250 }
00251 
00252 void VPLProcessor::computeOutput(const Vec& input, Vec& output) const
00253 {
00254     output.resize(outputsize());
00255     Vec newinput = input;
00256     if (!input_prg.empty())
00257     {
00258         processed_input.resize(input_prg_fieldnames.length());
00259         input_prg_.run(input, processed_input);
00260         newinput= processed_input;
00261     }
00262 
00263     output << newinput;
00264 }
00265 
00266 void VPLProcessor::computeOutputAndCosts(const Vec& input, const Vec& target, 
00267                                                    Vec& output, Vec& costs) const
00268 { 
00269     output.resize(outputsize());
00270     costs.resize(nTestCosts());
00271 
00272     costs.fill(-1);
00273     return computeOutput(input, output);
00274 }
00275 
00276 void VPLProcessor::computeCostsFromOutputs(const Vec& input, const Vec& output, 
00277                                            const Vec& target, Vec& costs) const
00278 { 
00279     Vec nonconst_output = output; // to make the constipated compiler happy
00280     computeOutputAndCosts(input, target, nonconst_output, costs); 
00281 }
00282 
00285 TVec<string> VPLProcessor::getOutputNames() const
00286 {
00287     if (!input_prg.empty())//output_prg_)
00288         return input_prg_fieldnames;
00289 
00290     VMat trainset= getTrainingSet();
00291     if (trainset==0)
00292         PLERROR("in VPLProcessor::getOutputNames: no train set specified yet.");
00293 
00294     return trainset->inputFieldNames();
00295 }
00296 
00297 
00298 void VPLProcessor::train()
00299 {}
00300 
00301 TVec<std::string> VPLProcessor::getTestCostNames() const
00302 {return TVec<string>(0);}
00303 
00304 TVec<string> VPLProcessor::getTrainCostNames() const
00305 {return TVec<string>(0);}
00306 
00307 
00308 
00309 } // end of namespace PLearn
00310 
00311 
00312 /*
00313   Local Variables:
00314   mode:c++
00315   c-basic-offset:4
00316   c-file-style:"stroustrup"
00317   c-file-offsets:((innamespace . 0)(inline-open . 0))
00318   indent-tabs-mode:nil
00319   fill-column:79
00320   End:
00321 */
00322 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines