PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // ProcessDatasetVMatrix.cc 00004 // 00005 // Copyright (C) 2005 Olivier Delalleau 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $ 00037 ******************************************************* */ 00038 00039 // Authors: Olivier Delalleau 00040 00044 #include "ProcessDatasetVMatrix.h" 00045 #include <plearn/io/fileutils.h> 00046 #include <plearn/vmat/ConcatColumnsVMatrix.h> 00047 #include <plearn/vmat/FileVMatrix.h> 00048 #include <plearn/vmat/MemoryVMatrix.h> 00049 #include <plearn/vmat/ShiftAndRescaleVMatrix.h> 00050 #include <plearn/vmat/SubVMatrix.h> 00051 00052 namespace PLearn { 00053 using namespace std; 00054 00056 // ProcessDatasetVMatrix // 00058 ProcessDatasetVMatrix::ProcessDatasetVMatrix(): 00059 max_mbytes_disk (1000), 00060 max_mbytes_memory (30), 00061 duplicate ("none"), 00062 input_normalization ("none"), 00063 precompute ("auto"), 00064 target_normalization("none") 00065 {} 00066 00067 PLEARN_IMPLEMENT_OBJECT(ProcessDatasetVMatrix, 00068 "Perform some standard preprocessing over a dataset.", 00069 "" 00070 ); 00071 00073 // declareOptions // 00075 void ProcessDatasetVMatrix::declareOptions(OptionList& ol) 00076 { 00077 declareOption(ol, "source", &ProcessDatasetVMatrix::source, OptionBase::buildoption, 00078 "The underlying VMatrix."); 00079 00080 declareOption(ol, "input_normalization", &ProcessDatasetVMatrix::input_normalization, OptionBase::buildoption, 00081 "Kind of normalization performed on the input features:\n" 00082 " - 'none' : no normalization\n" 00083 " - 'standard' : rescaled to have mean = 0 and standard deviation = 1\n"); 00084 00085 declareOption(ol, "target_normalization", &ProcessDatasetVMatrix::target_normalization, OptionBase::buildoption, 00086 "Kind of normalization performed on the target features (see 'input_normalization')"); 00087 00088 declareOption(ol, "duplicate", &ProcessDatasetVMatrix::duplicate, OptionBase::buildoption, 00089 "What to do with duplicated / conflicting samples:\n" 00090 " - 'all' : nothing (all samples are kept)\n" 00091 " - 'no_same_input' : samples must have a unique input (first one is kept)\n" 00092 " - 'no_same_input_and_target' : only exact duplicates are removed (first one is kept)\n" 00093 "Note: duplicated samples will only be removed after normalization is performed."); 00094 00095 declareOption(ol, "precompute", &ProcessDatasetVMatrix::precompute, OptionBase::buildoption, 00096 "How to precompute the dataset:\n" 00097 " - 'none' : it is not precomputed\n" 00098 " - 'memory' : it is precomputed in memory\n" 00099 " - 'disk' : it is precomputed in the underlying VMat metadatadir\n" 00100 " - 'auto' : it is precomputed in memory if it takes less than 'max_mbytes' Mb,\n" 00101 " it is precomputed in the underlying VMat metadatadir if it takes\n" 00102 " less than 'max_mbytes_disk' Mb, otherwise it is not precomputed.\n"); 00103 00104 declareOption(ol, "max_mbytes_memory", &ProcessDatasetVMatrix::max_mbytes_memory, OptionBase::buildoption, 00105 "Maximum number of megabytes allowed in memory when 'precompute' is set to 'auto'"); 00106 00107 declareOption(ol, "max_mbytes_disk", &ProcessDatasetVMatrix::max_mbytes_disk, OptionBase::buildoption, 00108 "Maximum number of megabytes allowed on disk when 'precompute' is set to 'auto'"); 00109 00110 // Now call the parent class' declareOptions 00111 inherited::declareOptions(ol); 00112 00113 // Hide unused options. 00114 00115 redeclareOption(ol, "vm", &ProcessDatasetVMatrix::vm, OptionBase::nosave, 00116 "Defined at build time."); 00117 00118 } 00119 00121 // build // 00123 void ProcessDatasetVMatrix::build() 00124 { 00125 inherited::build(); 00126 build_(); 00127 } 00128 00130 // build_ // 00132 void ProcessDatasetVMatrix::build_() 00133 { 00134 if (!source) { 00135 // Empty VMatrix. 00136 vm = 0; 00137 inherited::build(); 00138 return; 00139 } 00140 if (source->inputsize() < 0 || source->targetsize() < 0 || source->weightsize() < 0) 00141 PLERROR("In ProcessDatasetVMatrix::build_ - The source VMat's sizes must be defined"); 00142 updateMtime(source); 00143 vm = source; 00144 PPath filename_base = string("processed_dataset") 00145 + "-input_normalization=" + input_normalization 00146 + "-target_normalization=" + target_normalization 00147 + "-duplicate=" + duplicate; 00148 00149 bool target_normalization_is_performed = false; 00150 if (input_normalization == "none") { 00151 } else if (input_normalization == "standard") { 00152 int n_inputs = vm->inputsize(); 00153 if (target_normalization == "standard") { 00154 target_normalization_is_performed = true; 00155 n_inputs += vm->targetsize(); 00156 } 00157 vm = new ShiftAndRescaleVMatrix(vm, n_inputs); 00158 } else 00159 PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the " 00160 "'input_normalization' option: %s", input_normalization.c_str()); 00161 00162 if (!target_normalization_is_performed) { 00163 if (target_normalization == "none") { 00164 } else if (target_normalization == "standard") { 00165 VMat noninput_part = new SubVMatrix(vm, 0, vm->inputsize(), vm->length(), 00166 vm->targetsize() + vm->weightsize()); 00167 VMat input_part = new SubVMatrix(vm, 0, 0, vm->length(), vm->inputsize()); 00168 noninput_part = new ShiftAndRescaleVMatrix(noninput_part, vm->targetsize()); 00169 VMat result = new ConcatColumnsVMatrix(input_part, noninput_part); 00170 result->defineSizes(vm->inputsize(), vm->targetsize(), vm->weightsize()); 00171 vm = result; 00172 } 00173 } 00174 00175 if (duplicate == "all") { 00176 } else if (duplicate == "no_same_input") { 00177 if (!source->hasMetaDataDir()) 00178 PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have " 00179 "a metadata directory in order to compute duplicated samples"); 00180 PPath meta = source->getMetaDataDir(); 00181 // TODO Continue here! 00182 } 00183 00184 int n = source->length(); 00185 int w = source->width(); 00186 string precomp = precompute; 00187 if (precompute == "auto") { 00188 // Need to find out whether to precompute in memory or on disk. 00189 real memory_used = n / real(1024) * w / real(1024) * sizeof(real); 00190 // pout << "Memory used: " << memory_used << " Mbs" << endl; 00191 if (memory_used <= max_mbytes_memory) 00192 precomp = "memory"; 00193 else if (memory_used <= max_mbytes_disk) 00194 precomp = "disk"; 00195 else 00196 precomp = "none"; 00197 } 00198 00199 if (precomp == "none") { 00200 } else if (precomp == "memory") { 00201 vm = new MemoryVMatrix(vm); 00202 } else if (precomp == "disk") { 00203 if (!source->hasMetaDataDir()) 00204 PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have " 00205 "a metadata directory in order to precompute on disk"); 00206 PPath metadata = source->getMetaDataDir(); 00207 PPath filename = metadata / (filename_base + ".pmat"); 00208 bool need_recompute = true; 00209 VMat old_vm; 00210 if (isfile(filename)) { 00211 old_vm = new FileVMatrix(filename); 00212 if (old_vm->length() == n && old_vm->width() == w) 00213 need_recompute = false; 00214 } 00215 if (need_recompute) { 00216 vm->savePMAT(filename); 00217 vm = new FileVMatrix(filename); 00218 } else 00219 vm = old_vm; 00220 } else 00221 PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the " 00222 "'precompute' option: '%s'", precompute.c_str()); 00223 00224 inherited::build(); 00225 } 00226 00228 // makeDeepCopyFromShallowCopy // 00230 void ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00231 { 00232 inherited::makeDeepCopyFromShallowCopy(copies); 00233 00234 // ### Call deepCopyField on all "pointer-like" fields 00235 // ### that you wish to be deepCopied rather than 00236 // ### shallow-copied. 00237 // ### ex: 00238 // deepCopyField(trainvec, copies); 00239 00240 // ### Remove this line when you have fully implemented this method. 00241 PLERROR("ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00242 } 00243 00244 } // end of namespace PLearn 00245 00246 00247 /* 00248 Local Variables: 00249 mode:c++ 00250 c-basic-offset:4 00251 c-file-style:"stroustrup" 00252 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00253 indent-tabs-mode:nil 00254 fill-column:79 00255 End: 00256 */ 00257 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :