PLearn 0.1
ConditionalMeanImputationVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003 Olivier Delalleau
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************************    
00039    * $Id: ConditionalMeanImputationVMatrix.cc 3658 2005-07-06 20:30:15  Godbout $
00040    ******************************************************************* */
00041 
00042 
00043 #include "ConditionalMeanImputationVMatrix.h"
00044 #include <plearn/io/fileutils.h>              
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00051 PLEARN_IMPLEMENT_OBJECT(
00052   ConditionalMeanImputationVMatrix,
00053   "VMat class to impute the conditional mean to replace missing values in the source matrix.",
00054   "This class will replace missing values in the underlying dataset with the estimated values\n"
00055   "from a preceding machine learning step where each variable with missing value have to have.\n"
00056   "been considered as target in turns.\n"
00057   "The predictions are expected in the metadata directory of the data set.\n"
00058   );
00059 
00060 ConditionalMeanImputationVMatrix::ConditionalMeanImputationVMatrix()
00061 {
00062 }
00063 
00064 ConditionalMeanImputationVMatrix::~ConditionalMeanImputationVMatrix()
00065 {
00066 }
00067 
00068 void ConditionalMeanImputationVMatrix::declareOptions(OptionList &ol)
00069 {
00070   declareOption(ol, "condmean_dir", &ConditionalMeanImputationVMatrix::condmean_dir, OptionBase::buildoption, 
00071                 "The directory in the source metadatadir housing the variable conditional mean files.\n");
00072   declareOption(ol, "condmean", &ConditionalMeanImputationVMatrix::condmean, OptionBase::learntoption, 
00073                 "The matrix of conditional means.\n");
00074   declareOption(ol, "condmean_col_ref", &ConditionalMeanImputationVMatrix::condmean_col_ref, OptionBase::learntoption, 
00075                 "The cross reference between columns of source and condmean.\n");
00076   inherited::declareOptions(ol);
00077 }
00078 
00079 void ConditionalMeanImputationVMatrix::build()
00080 {
00081   inherited::build();
00082   build_();
00083 }
00084 
00085 void ConditionalMeanImputationVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00086 {
00087   deepCopyField(condmean_dir, copies);
00088   deepCopyField(condmean, copies);
00089   deepCopyField(condmean_col_ref, copies);
00090   inherited::makeDeepCopyFromShallowCopy(copies);
00091 }
00092 
00093 void ConditionalMeanImputationVMatrix::getExample(int i, Vec& input, Vec& target, real& weight)
00094 {
00095   source->getExample(i, input, target, weight);
00096   for (int source_col = 0; source_col < input->length(); source_col++)
00097     if (is_missing(input[source_col]) && condmean_col_ref[source_col] >= 0) input[source_col] = condmean(i, condmean_col_ref[source_col]);
00098     else if (is_missing(input[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getExample(%d,vec,vec,vec) we have a missing value in column %d that haven't been assigned a value",i,source_col);
00099 }
00100 
00101 real ConditionalMeanImputationVMatrix::get(int i, int j) const
00102 { 
00103   real variable_value = source->get(i, j);
00104   if (is_missing(variable_value) && condmean_col_ref[j] >= 0) return condmean(i, condmean_col_ref[j]);
00105   else if (is_missing(variable_value)) PLERROR("In ConditionalMeanImputationVMatrix::getExample(%d,%d) we have a missing value that haven't been assigned a value",i,j);
00106   return variable_value;
00107 }
00108 
00109 void ConditionalMeanImputationVMatrix::put(int i, int j, real value)
00110 {
00111   PLERROR("In ConditionalMeanImputationVMatrix::put not implemented");
00112 }
00113 
00114 void ConditionalMeanImputationVMatrix::getSubRow(int i, int j, Vec v) const
00115 {  
00116   source->getSubRow(i, j, v);
00117   for (int source_col = 0; source_col < v->length(); source_col++) 
00118     if (is_missing(v[source_col])) v[source_col] = condmean(i, condmean_col_ref[source_col + j]);
00119     else if (is_missing(v[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getSubRow(%d,%d,vec) we have a missing value in colomn %d that haven't been assigned a value",i,j,source_col);
00120 }
00121 
00122 void ConditionalMeanImputationVMatrix::putSubRow(int i, int j, Vec v)
00123 {
00124   PLERROR("In ConditionalMeanImputationVMatrix::putSubRow not implemented");
00125 }
00126 
00127 void ConditionalMeanImputationVMatrix::appendRow(Vec v)
00128 {
00129   PLERROR("In ConditionalMeanImputationVMatrix::appendRow not implemented");
00130 }
00131 
00132 void ConditionalMeanImputationVMatrix::insertRow(int i, Vec v)
00133 {
00134   PLERROR("In ConditionalMeanImputationVMatrix::insertRow not implemented");
00135 }
00136 
00137 void ConditionalMeanImputationVMatrix::getRow(int i, Vec v) const
00138 {  
00139   source-> getRow(i, v);
00140   for (int source_col = 0; source_col < v->length(); source_col++)
00141     if (is_missing(v[source_col]) && condmean_col_ref[source_col] >= 0) v[source_col] = condmean(i, condmean_col_ref[source_col]);
00142     else if (is_missing(v[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getRow(%d,vec) we have a missing value in column %d that haven't been assigned a value",i,source_col);
00143 }
00144 
00145 void ConditionalMeanImputationVMatrix::putRow(int i, Vec v)
00146 {
00147   PLERROR("In ConditionalMeanImputationVMatrix::putRow not implemented");
00148 }
00149 
00150 void ConditionalMeanImputationVMatrix::getColumn(int i, Vec v) const
00151 {  
00152   source-> getColumn(i, v);
00153   for (int source_row = 0; source_row < v->length(); source_row++)
00154     if (is_missing(v[source_row]) && condmean_col_ref[i] >= 0) v[source_row] = condmean(source_row, condmean_col_ref[i]);
00155     else if (is_missing(v[source_row])) PLERROR("In ConditionalMeanImputationVMatrix::getColumn(%d,vec) we have a missing value in row %d that haven't been assigned a value",i,source_row);
00156 }
00157 
00158 
00159 
00160 void ConditionalMeanImputationVMatrix::build_()
00161 {
00162     if (!source) PLERROR("In ConditionalMeanImputationVMatrix::source vmat must be supplied");
00163     loadCondMeanMatrix(); 
00164     testResultantVMatrix();
00165 }
00166 
00167 void ConditionalMeanImputationVMatrix::loadCondMeanMatrix()
00168 /*  
00169 Imputation step:
00170   count the # of variables with missing values in the train and test datasets.
00171   create a matrix in memory with this number of columns and keep cross reference of columns.
00172   at the build stage, for each variable of train and test:
00173     if # of missing = 0 there is nothing to do.
00174     look for the (cond_mean_dir (/TreeCondMean/dir/) + field_name + /Split0/test1_outputs.pmat) file in the metadatadir;
00175     add its column 0 as a column of the matrix.
00176   then, if is_missing(source[i,j]) replace it with matrix[i, cross_reference[j]]
00177 */
00178 {
00179     // initialize source dataset
00180     source_length = source->length();
00181     source_width = source->width();
00182     source_inputsize = source->inputsize();
00183     source_targetsize = source->targetsize();
00184     source_weightsize = source->weightsize();
00185     source_names.resize(source_width);
00186     source_names = source->fieldNames();
00187     source_metadata = source->getMetaDataDir();
00188     length_ = source_length;
00189     width_ = source_width;
00190     inputsize_ = source_inputsize;
00191     targetsize_ = source_targetsize;
00192     weightsize_ = source_weightsize;
00193     declareFieldNames(source_names);
00194     
00195     // count the # of variables with missing values in the source datasets.
00196     // create a matrix in memory with this number of columns and keep cross reference of columns.
00197     int count_variable_with_missing = 0;
00198     condmean_col_ref.resize(source_width);
00199     condmean_col_ref.fill(-1);
00200     for (source_col = 0; source_col < source_width; source_col++)
00201     {
00202         source_stats = source->getStats(source_col);
00203         if (source_stats.nmissing() <= 0) continue;
00204         condmean_col_ref[source_col] = count_variable_with_missing;
00205         count_variable_with_missing += 1;
00206     }
00207     condmean.resize(source_length, count_variable_with_missing);
00208     
00209     // for each variable with missing value, 
00210     // look for the (cond_mean_dir (/TreeCondMean/dir/) + field_name + /Split0/test1_outputs.pmat) file in the metadatadir;
00211     // add its column 0 as a column of the condmean matrix.
00212     for (source_col = 0; source_col < source_width; source_col++)
00213     {
00214         source_stats = source->getStats(source_col);
00215         if (source_stats.nmissing() <= 0) continue;
00216         int condmean_col = condmean_col_ref[source_col];
00217         PPath condmean_variable_file_name = source_metadata + "/" + condmean_dir + "/dir/" + source_names[source_col] + "/Split0/test1_outputs.pmat";
00218         if (!isfile(condmean_variable_file_name)) PLERROR("In ConditionalMeanImputationVMatrix::A conditional mean file(%s) was not found for variable %s",
00219                                                           condmean_variable_file_name.c_str(),source_names[source_col].c_str());
00220         VMat condmean_variable_file = new FileVMatrix(condmean_variable_file_name, false);
00221         if (condmean_variable_file->length() != source_length)
00222             PLERROR("In ConditionalMeanImputationVMatrix::Source and conditional mean file length are not equal for variable %s", source_names[source_col].c_str());
00223         for (source_row = 0; source_row < source_length; source_row++)
00224             condmean(source_row, condmean_col) = condmean_variable_file->get(source_row, 0);
00225     }
00226 }
00227 
00228 } // end of namespcae PLearn
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines