PLearn 0.1
ProcessDatasetVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // ProcessDatasetVMatrix.cc
00004 //
00005 // Copyright (C) 2005 Olivier Delalleau
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $
00037  ******************************************************* */
00038 
00039 // Authors: Olivier Delalleau
00040 
00044 #include "ProcessDatasetVMatrix.h"
00045 #include <plearn/io/fileutils.h>                  
00046 #include <plearn/vmat/ConcatColumnsVMatrix.h>
00047 #include <plearn/vmat/FileVMatrix.h>
00048 #include <plearn/vmat/MemoryVMatrix.h>
00049 #include <plearn/vmat/ShiftAndRescaleVMatrix.h>
00050 #include <plearn/vmat/SubVMatrix.h>
00051 
00052 namespace PLearn {
00053 using namespace std;
00054 
00056 // ProcessDatasetVMatrix //
00058 ProcessDatasetVMatrix::ProcessDatasetVMatrix():
00059     max_mbytes_disk     (1000),
00060     max_mbytes_memory   (30),
00061     duplicate           ("none"),
00062     input_normalization ("none"),
00063     precompute          ("auto"),
00064     target_normalization("none")
00065 {}
00066 
00067 PLEARN_IMPLEMENT_OBJECT(ProcessDatasetVMatrix,
00068                         "Perform some standard preprocessing over a dataset.",
00069                         ""
00070     );
00071 
00073 // declareOptions //
00075 void ProcessDatasetVMatrix::declareOptions(OptionList& ol)
00076 {
00077     declareOption(ol, "source", &ProcessDatasetVMatrix::source, OptionBase::buildoption,
00078                   "The underlying VMatrix.");
00079 
00080     declareOption(ol, "input_normalization", &ProcessDatasetVMatrix::input_normalization, OptionBase::buildoption,
00081                   "Kind of normalization performed on the input features:\n"
00082                   " - 'none'      : no normalization\n"
00083                   " - 'standard'  : rescaled to have mean = 0 and standard deviation = 1\n");
00084 
00085     declareOption(ol, "target_normalization", &ProcessDatasetVMatrix::target_normalization, OptionBase::buildoption,
00086                   "Kind of normalization performed on the target features (see 'input_normalization')");
00087 
00088     declareOption(ol, "duplicate", &ProcessDatasetVMatrix::duplicate, OptionBase::buildoption,
00089                   "What to do with duplicated / conflicting samples:\n"
00090                   " - 'all'                      : nothing (all samples are kept)\n"
00091                   " - 'no_same_input'            : samples must have a unique input  (first one is kept)\n"
00092                   " - 'no_same_input_and_target' : only exact duplicates are removed (first one is kept)\n"
00093                   "Note: duplicated samples will only be removed after normalization is performed.");
00094 
00095     declareOption(ol, "precompute", &ProcessDatasetVMatrix::precompute, OptionBase::buildoption,
00096                   "How to precompute the dataset:\n"
00097                   " - 'none'   : it is not precomputed\n"
00098                   " - 'memory' : it is precomputed in memory\n"
00099                   " - 'disk'   : it is precomputed in the underlying VMat metadatadir\n"
00100                   " - 'auto'   : it is precomputed in memory if it takes less than 'max_mbytes' Mb,\n"
00101                   "              it is precomputed in the underlying VMat metadatadir if it takes\n"
00102                   "              less than 'max_mbytes_disk' Mb, otherwise it is not precomputed.\n");
00103 
00104     declareOption(ol, "max_mbytes_memory", &ProcessDatasetVMatrix::max_mbytes_memory, OptionBase::buildoption,
00105                   "Maximum number of megabytes allowed in memory when 'precompute' is set to 'auto'");
00106 
00107     declareOption(ol, "max_mbytes_disk", &ProcessDatasetVMatrix::max_mbytes_disk, OptionBase::buildoption,
00108                   "Maximum number of megabytes allowed on disk when 'precompute' is set to 'auto'");
00109 
00110     // Now call the parent class' declareOptions
00111     inherited::declareOptions(ol);
00112 
00113     // Hide unused options.
00114 
00115     redeclareOption(ol, "vm", &ProcessDatasetVMatrix::vm, OptionBase::nosave,
00116                     "Defined at build time.");
00117 
00118 }
00119 
00121 // build //
00123 void ProcessDatasetVMatrix::build()
00124 {
00125     inherited::build();
00126     build_();
00127 }
00128 
00130 // build_ //
00132 void ProcessDatasetVMatrix::build_()
00133 {
00134     if (!source) {
00135         // Empty VMatrix.
00136         vm = 0;
00137         inherited::build();
00138         return;
00139     }
00140     if (source->inputsize() < 0 || source->targetsize() < 0 || source->weightsize() < 0)
00141         PLERROR("In ProcessDatasetVMatrix::build_ - The source VMat's sizes must be defined");
00142     updateMtime(source);
00143     vm = source;
00144     PPath filename_base = string("processed_dataset")
00145         + "-input_normalization="  + input_normalization
00146         + "-target_normalization=" + target_normalization
00147         + "-duplicate="            + duplicate;
00148 
00149     bool target_normalization_is_performed = false;
00150     if (input_normalization == "none") {
00151     } else if (input_normalization == "standard") {
00152         int n_inputs = vm->inputsize();
00153         if (target_normalization == "standard") {
00154             target_normalization_is_performed = true;
00155             n_inputs += vm->targetsize();
00156         }
00157         vm = new ShiftAndRescaleVMatrix(vm, n_inputs);
00158     } else
00159         PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the "
00160                 "'input_normalization' option: %s", input_normalization.c_str());
00161 
00162     if (!target_normalization_is_performed) {
00163         if (target_normalization == "none") {
00164         } else if (target_normalization == "standard") {
00165             VMat noninput_part = new SubVMatrix(vm, 0, vm->inputsize(), vm->length(),
00166                                                 vm->targetsize() + vm->weightsize());
00167             VMat input_part = new SubVMatrix(vm, 0, 0, vm->length(), vm->inputsize());
00168             noninput_part = new ShiftAndRescaleVMatrix(noninput_part, vm->targetsize());
00169             VMat result = new ConcatColumnsVMatrix(input_part, noninput_part);
00170             result->defineSizes(vm->inputsize(), vm->targetsize(), vm->weightsize());
00171             vm = result;
00172         }
00173     }
00174 
00175     if (duplicate == "all") {
00176     } else if (duplicate == "no_same_input") {
00177         if (!source->hasMetaDataDir())
00178             PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have "
00179                     "a metadata directory in order to compute duplicated samples");
00180         PPath meta = source->getMetaDataDir();
00181         // TODO Continue here!
00182     }
00183 
00184     int n = source->length();
00185     int w = source->width();
00186     string precomp = precompute;
00187     if (precompute == "auto") {
00188         // Need to find out whether to precompute in memory or on disk.
00189         real memory_used = n / real(1024) * w / real(1024) * sizeof(real);
00190         // pout << "Memory used: " << memory_used << " Mbs" << endl;
00191         if (memory_used <= max_mbytes_memory)
00192             precomp = "memory";
00193         else if (memory_used <= max_mbytes_disk)
00194             precomp = "disk";
00195         else
00196             precomp = "none";
00197     }
00198 
00199     if (precomp == "none") {
00200     } else if (precomp == "memory") {
00201         vm = new MemoryVMatrix(vm);
00202     } else if (precomp == "disk") {
00203         if (!source->hasMetaDataDir())
00204             PLERROR("In ProcessDatasetVMatrix::build_ - The source VMatrix needs to have "
00205                     "a metadata directory in order to precompute on disk");
00206         PPath metadata = source->getMetaDataDir();
00207         PPath filename = metadata / (filename_base + ".pmat");
00208         bool need_recompute = true;
00209         VMat old_vm;
00210         if (isfile(filename)) {
00211             old_vm = new FileVMatrix(filename);
00212             if (old_vm->length() == n && old_vm->width() == w)
00213                 need_recompute = false;
00214         }
00215         if (need_recompute) {
00216             vm->savePMAT(filename);
00217             vm = new FileVMatrix(filename);
00218         } else
00219             vm = old_vm;
00220     } else
00221         PLERROR("In ProcessDatasetVMatrix::build_ - Unknown value for the "
00222                 "'precompute' option: '%s'", precompute.c_str());
00223 
00224     inherited::build();
00225 }
00226 
00228 // makeDeepCopyFromShallowCopy //
00230 void ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00231 {
00232     inherited::makeDeepCopyFromShallowCopy(copies);
00233 
00234     // ### Call deepCopyField on all "pointer-like" fields
00235     // ### that you wish to be deepCopied rather than
00236     // ### shallow-copied.
00237     // ### ex:
00238     // deepCopyField(trainvec, copies);
00239 
00240     // ### Remove this line when you have fully implemented this method.
00241     PLERROR("ProcessDatasetVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00242 }
00243 
00244 } // end of namespace PLearn
00245 
00246 
00247 /*
00248   Local Variables:
00249   mode:c++
00250   c-basic-offset:4
00251   c-file-style:"stroustrup"
00252   c-file-offsets:((innamespace . 0)(inline-open . 0))
00253   indent-tabs-mode:nil
00254   fill-column:79
00255   End:
00256 */
00257 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines