PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // VPLProcessor.cc 00004 // 00005 // Copyright (C) 2005, 2006 Pascal Vincent 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: VPLProcessor.cc 5480 2006-05-03 18:57:39Z plearner $ 00037 ******************************************************* */ 00038 00039 // Authors: Pascal Vincent 00040 00043 #include <sstream> 00044 00045 #include "VPLProcessor.h" 00046 #include <plearn/vmat/ProcessingVMatrix.h> 00047 #include <plearn/vmat/FilteredVMatrix.h> 00048 #include <plearn/base/tostring.h> 00049 00050 namespace PLearn { 00051 using namespace std; 00052 00053 VPLProcessor::VPLProcessor() 00054 :orig_inputsize(-1), 00055 orig_targetsize(-1), 00056 use_filtering_prg_for_repeat(false), 00057 repeat_id_field_name(""), 00058 repeat_count_field_name("") 00059 { 00060 } 00061 00062 PLEARN_IMPLEMENT_OBJECT( 00063 VPLProcessor, 00064 "Learner whose training-set, inputs and outputs can be pre/post-processed by VPL code", 00065 "See VMatLanguage for the definition of the allowed VPL syntax." 00066 ); 00067 00068 void VPLProcessor::declareOptions(OptionList& ol) 00069 { 00070 declareOption(ol, "filtering_prg", &VPLProcessor::filtering_prg, OptionBase::buildoption, 00071 "Optional program string in VPL language to apply as filtering on the training VMat.\n" 00072 "This program is to produce a single value interpreted as a boolean: only the rows for which\n" 00073 "it evaluates to non-zero will be kept.\n" 00074 "An empty string means NO FILTERING."); 00075 00076 declareOption(ol, "input_prg", &VPLProcessor::input_prg, OptionBase::buildoption, 00077 "Program string in VPL language to be applied to each raw input \n" 00078 "to generate the new preprocessed input.\n" 00079 "Note that names must be given to the generated values with :fieldname VPL syntax.\n" 00080 "An empty string means NO PREPROCESSING. (initial raw input is used as is)"); 00081 00082 declareOption(ol, "target_prg", &VPLProcessor::target_prg, OptionBase::buildoption, 00083 "Program string in VPL language to be applied to a dataset row\n" 00084 "to generate a proper target.\n" 00085 "Note that names must be given to the generated values with :fieldname VPL syntax.\n" 00086 "If it's an empty string, then we'll use the original target from the data set"); 00087 00088 declareOption(ol, "weight_prg", &VPLProcessor::weight_prg, OptionBase::buildoption, 00089 "Program string in VPL language to be applied to a dataset row\n" 00090 "to generate a proper weight.\n" 00091 "Note that names must be given to the generated values with :fieldname VPL syntax.\n" 00092 "If it's an empty string, then we'll use the original weight from the data set"); 00093 00094 declareOption(ol, "extra_prg", &VPLProcessor::extra_prg, OptionBase::buildoption, 00095 "Program string in VPL language to be applied to a dataset row\n" 00096 "to generate proper extra fields.\n" 00097 "Note that names must be given to the generated values with :fieldname VPL syntax.\n" 00098 "If it's an empty string, then we'll use the original extra fields from the data set"); 00099 00100 00101 declareOption(ol, "use_filtering_prg_for_repeat", &VPLProcessor::use_filtering_prg_for_repeat, OptionBase::buildoption, 00102 "When true, the result of the filtering program indicates the number of times a row should be repeated (0..n).\n" 00103 "(sets FilteredVMatrix::allow_repeat_rows.)"); 00104 00105 declareOption(ol, "repeat_id_field_name", &VPLProcessor::repeat_id_field_name, OptionBase::buildoption, 00106 "Field name for the repetition id (0, 1, ..., n-1). No field is added if empty."); 00107 00108 declareOption(ol, "repeat_count_field_name", &VPLProcessor::repeat_count_field_name, OptionBase::buildoption, 00109 "Field name for the number of repetitions (n). No field is added if empty."); 00110 00111 // learnt 00112 00113 declareOption(ol, "orig_fieldnames", &VPLProcessor::orig_fieldnames, OptionBase::learntoption, 00114 "original fieldnames of the training set"); 00115 declareOption(ol, "orig_inputsize", &VPLProcessor::orig_inputsize, OptionBase::learntoption, 00116 "original inputsize of the training set"); 00117 declareOption(ol, "orig_targetsize", &VPLProcessor::orig_targetsize, OptionBase::learntoption, 00118 "original targetsize of the training set"); 00119 00120 00121 // Now call the parent class' declareOptions 00122 inherited::declareOptions(ol); 00123 } 00124 00125 void VPLProcessor::build_() 00126 { 00127 // We're probably reloading a saved VPLProcessor 00128 if(train_set.isNull() && (orig_inputsize>0 || orig_targetsize>0) ) 00129 initializeInputPrograms(); 00130 } 00131 00132 void VPLProcessor::build() 00133 { 00134 inherited::build(); 00135 build_(); 00136 } 00137 00138 00140 void VPLProcessor::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00141 { 00142 inherited::makeDeepCopyFromShallowCopy(copies); 00143 00144 input_prg_.makeDeepCopyFromShallowCopy(copies); 00145 00146 deepCopyField(input_prg_fieldnames, copies); 00147 deepCopyField(processed_input, copies); 00148 deepCopyField(orig_fieldnames, copies); 00149 } 00150 00151 int VPLProcessor::outputsize() const 00152 { 00153 if (!input_prg.empty()) 00154 return input_prg_fieldnames.length(); 00155 00156 return inputsize(); 00157 } 00158 00159 void VPLProcessor::forget() 00160 { 00161 inherited::forget(); 00162 stage = 0; 00163 } 00164 00168 void VPLProcessor::initializeInputPrograms() 00169 { 00170 if (!input_prg.empty()) 00171 { 00172 input_prg_.setSourceFieldNames(orig_fieldnames.subVec(0,orig_inputsize)); 00173 input_prg_.compileString(input_prg, input_prg_fieldnames); 00174 } 00175 else 00176 { 00177 input_prg_.clear(); 00178 input_prg_fieldnames.resize(0); 00179 } 00180 00181 } 00182 00186 void VPLProcessor::setTrainingSet(VMat training_set, bool call_forget) 00187 { 00188 orig_fieldnames = training_set->fieldNames(); 00189 orig_inputsize = training_set->inputsize(); 00190 orig_targetsize = training_set->targetsize(); 00191 initializeInputPrograms(); 00192 00193 VMat filtered_trainset = training_set; 00194 PPath filtered_trainset_metadatadir = getExperimentDirectory() / "filtered_train_set.metadata"; 00195 if (!filtering_prg.empty()) 00196 filtered_trainset = new FilteredVMatrix(training_set, filtering_prg, filtered_trainset_metadatadir, verbosity>1, 00197 use_filtering_prg_for_repeat, repeat_id_field_name, repeat_count_field_name); 00198 00199 // XXX The next line does nothing! 00200 VMat processed_trainset = new ProcessingVMatrix(filtered_trainset, input_prg, target_prg, weight_prg, extra_prg); 00201 inherited::setTrainingSet(training_set, call_forget); // will call forget if needed 00202 } 00203 00204 VMat VPLProcessor::processDataSet(VMat dataset) const 00205 { 00206 VMat filtered_dataset = dataset; 00207 PPath filtered_dataset_metadatadir = getExperimentDirectory() / "filtered_dataset.metadata"; 00208 if (!filtering_prg.empty()) 00209 filtered_dataset = new FilteredVMatrix(dataset, filtering_prg, filtered_dataset_metadatadir, verbosity>1, 00210 use_filtering_prg_for_repeat, repeat_id_field_name, repeat_count_field_name); 00211 00212 // Since ProcessingVMatrix produces 0 length vectors when given an empty 00213 // program (which is not the behavior that VPLProcessors is documented as 00214 // implementing), we need to replace each program that is an empty string 00215 // by a small VPL snippet that copies all the fields for the input or 00216 // target, etc. 00217 00218 // First compute the start of each section (input, target, etc.) in the 00219 // columns of the dataset. 00220 const int start_of_targets = dataset->inputsize(); 00221 const int start_of_weights = start_of_targets + dataset->targetsize(); 00222 const int start_of_extras = start_of_weights + dataset->weightsize(); 00223 00224 // Now compute each processing_*_prg program. 00225 string processing_input_prg = input_prg; 00226 if (processing_input_prg.empty() && dataset->inputsize() > 0) { 00227 processing_input_prg = "[%0:%" + tostring(start_of_targets-1) + "]"; 00228 } 00229 00230 string processing_target_prg = target_prg; 00231 if (processing_target_prg.empty() && dataset->targetsize() > 0) { 00232 processing_target_prg = "[%" + tostring(start_of_targets) + ":%" + 00233 tostring(start_of_weights-1) + "]"; 00234 } 00235 00236 string processing_weight_prg = weight_prg; 00237 if (processing_weight_prg.empty() && dataset->weightsize() > 0) { 00238 processing_weight_prg = "[%" + tostring(start_of_weights) + ":%" + 00239 tostring(start_of_extras-1) + "]"; 00240 } 00241 00242 string processing_extras_prg = extra_prg; 00243 if (processing_extras_prg.empty() && dataset->extrasize() > 0) { 00244 processing_extras_prg = "[%" + tostring(start_of_extras) + ":END]"; 00245 } 00246 00247 return new ProcessingVMatrix(filtered_dataset, processing_input_prg, 00248 processing_target_prg, processing_weight_prg, 00249 processing_extras_prg); 00250 } 00251 00252 void VPLProcessor::computeOutput(const Vec& input, Vec& output) const 00253 { 00254 output.resize(outputsize()); 00255 Vec newinput = input; 00256 if (!input_prg.empty()) 00257 { 00258 processed_input.resize(input_prg_fieldnames.length()); 00259 input_prg_.run(input, processed_input); 00260 newinput= processed_input; 00261 } 00262 00263 output << newinput; 00264 } 00265 00266 void VPLProcessor::computeOutputAndCosts(const Vec& input, const Vec& target, 00267 Vec& output, Vec& costs) const 00268 { 00269 output.resize(outputsize()); 00270 costs.resize(nTestCosts()); 00271 00272 costs.fill(-1); 00273 return computeOutput(input, output); 00274 } 00275 00276 void VPLProcessor::computeCostsFromOutputs(const Vec& input, const Vec& output, 00277 const Vec& target, Vec& costs) const 00278 { 00279 Vec nonconst_output = output; // to make the constipated compiler happy 00280 computeOutputAndCosts(input, target, nonconst_output, costs); 00281 } 00282 00285 TVec<string> VPLProcessor::getOutputNames() const 00286 { 00287 if (!input_prg.empty())//output_prg_) 00288 return input_prg_fieldnames; 00289 00290 VMat trainset= getTrainingSet(); 00291 if (trainset==0) 00292 PLERROR("in VPLProcessor::getOutputNames: no train set specified yet."); 00293 00294 return trainset->inputFieldNames(); 00295 } 00296 00297 00298 void VPLProcessor::train() 00299 {} 00300 00301 TVec<std::string> VPLProcessor::getTestCostNames() const 00302 {return TVec<string>(0);} 00303 00304 TVec<string> VPLProcessor::getTrainCostNames() const 00305 {return TVec<string>(0);} 00306 00307 00308 00309 } // end of namespace PLearn 00310 00311 00312 /* 00313 Local Variables: 00314 mode:c++ 00315 c-basic-offset:4 00316 c-file-style:"stroustrup" 00317 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00318 indent-tabs-mode:nil 00319 fill-column:79 00320 End: 00321 */ 00322 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :