PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2003 Olivier Delalleau 00008 // 00009 // Redistribution and use in source and binary forms, with or without 00010 // modification, are permitted provided that the following conditions are met: 00011 // 00012 // 1. Redistributions of source code must retain the above copyright 00013 // notice, this list of conditions and the following disclaimer. 00014 // 00015 // 2. Redistributions in binary form must reproduce the above copyright 00016 // notice, this list of conditions and the following disclaimer in the 00017 // documentation and/or other materials provided with the distribution. 00018 // 00019 // 3. The name of the authors may not be used to endorse or promote 00020 // products derived from this software without specific prior written 00021 // permission. 00022 // 00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00033 // 00034 // This file is part of the PLearn library. For more information on the PLearn 00035 // library, go to the PLearn Web site at www.plearn.org 00036 00037 00038 /* ******************************************************************* 00039 * $Id: ConditionalMeanImputationVMatrix.cc 3658 2005-07-06 20:30:15 Godbout $ 00040 ******************************************************************* */ 00041 00042 00043 #include "ConditionalMeanImputationVMatrix.h" 00044 #include <plearn/io/fileutils.h> 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00051 PLEARN_IMPLEMENT_OBJECT( 00052 ConditionalMeanImputationVMatrix, 00053 "VMat class to impute the conditional mean to replace missing values in the source matrix.", 00054 "This class will replace missing values in the underlying dataset with the estimated values\n" 00055 "from a preceding machine learning step where each variable with missing value have to have.\n" 00056 "been considered as target in turns.\n" 00057 "The predictions are expected in the metadata directory of the data set.\n" 00058 ); 00059 00060 ConditionalMeanImputationVMatrix::ConditionalMeanImputationVMatrix() 00061 { 00062 } 00063 00064 ConditionalMeanImputationVMatrix::~ConditionalMeanImputationVMatrix() 00065 { 00066 } 00067 00068 void ConditionalMeanImputationVMatrix::declareOptions(OptionList &ol) 00069 { 00070 declareOption(ol, "condmean_dir", &ConditionalMeanImputationVMatrix::condmean_dir, OptionBase::buildoption, 00071 "The directory in the source metadatadir housing the variable conditional mean files.\n"); 00072 declareOption(ol, "condmean", &ConditionalMeanImputationVMatrix::condmean, OptionBase::learntoption, 00073 "The matrix of conditional means.\n"); 00074 declareOption(ol, "condmean_col_ref", &ConditionalMeanImputationVMatrix::condmean_col_ref, OptionBase::learntoption, 00075 "The cross reference between columns of source and condmean.\n"); 00076 inherited::declareOptions(ol); 00077 } 00078 00079 void ConditionalMeanImputationVMatrix::build() 00080 { 00081 inherited::build(); 00082 build_(); 00083 } 00084 00085 void ConditionalMeanImputationVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00086 { 00087 deepCopyField(condmean_dir, copies); 00088 deepCopyField(condmean, copies); 00089 deepCopyField(condmean_col_ref, copies); 00090 inherited::makeDeepCopyFromShallowCopy(copies); 00091 } 00092 00093 void ConditionalMeanImputationVMatrix::getExample(int i, Vec& input, Vec& target, real& weight) 00094 { 00095 source->getExample(i, input, target, weight); 00096 for (int source_col = 0; source_col < input->length(); source_col++) 00097 if (is_missing(input[source_col]) && condmean_col_ref[source_col] >= 0) input[source_col] = condmean(i, condmean_col_ref[source_col]); 00098 else if (is_missing(input[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getExample(%d,vec,vec,vec) we have a missing value in column %d that haven't been assigned a value",i,source_col); 00099 } 00100 00101 real ConditionalMeanImputationVMatrix::get(int i, int j) const 00102 { 00103 real variable_value = source->get(i, j); 00104 if (is_missing(variable_value) && condmean_col_ref[j] >= 0) return condmean(i, condmean_col_ref[j]); 00105 else if (is_missing(variable_value)) PLERROR("In ConditionalMeanImputationVMatrix::getExample(%d,%d) we have a missing value that haven't been assigned a value",i,j); 00106 return variable_value; 00107 } 00108 00109 void ConditionalMeanImputationVMatrix::put(int i, int j, real value) 00110 { 00111 PLERROR("In ConditionalMeanImputationVMatrix::put not implemented"); 00112 } 00113 00114 void ConditionalMeanImputationVMatrix::getSubRow(int i, int j, Vec v) const 00115 { 00116 source->getSubRow(i, j, v); 00117 for (int source_col = 0; source_col < v->length(); source_col++) 00118 if (is_missing(v[source_col])) v[source_col] = condmean(i, condmean_col_ref[source_col + j]); 00119 else if (is_missing(v[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getSubRow(%d,%d,vec) we have a missing value in colomn %d that haven't been assigned a value",i,j,source_col); 00120 } 00121 00122 void ConditionalMeanImputationVMatrix::putSubRow(int i, int j, Vec v) 00123 { 00124 PLERROR("In ConditionalMeanImputationVMatrix::putSubRow not implemented"); 00125 } 00126 00127 void ConditionalMeanImputationVMatrix::appendRow(Vec v) 00128 { 00129 PLERROR("In ConditionalMeanImputationVMatrix::appendRow not implemented"); 00130 } 00131 00132 void ConditionalMeanImputationVMatrix::insertRow(int i, Vec v) 00133 { 00134 PLERROR("In ConditionalMeanImputationVMatrix::insertRow not implemented"); 00135 } 00136 00137 void ConditionalMeanImputationVMatrix::getRow(int i, Vec v) const 00138 { 00139 source-> getRow(i, v); 00140 for (int source_col = 0; source_col < v->length(); source_col++) 00141 if (is_missing(v[source_col]) && condmean_col_ref[source_col] >= 0) v[source_col] = condmean(i, condmean_col_ref[source_col]); 00142 else if (is_missing(v[source_col])) PLERROR("In ConditionalMeanImputationVMatrix::getRow(%d,vec) we have a missing value in column %d that haven't been assigned a value",i,source_col); 00143 } 00144 00145 void ConditionalMeanImputationVMatrix::putRow(int i, Vec v) 00146 { 00147 PLERROR("In ConditionalMeanImputationVMatrix::putRow not implemented"); 00148 } 00149 00150 void ConditionalMeanImputationVMatrix::getColumn(int i, Vec v) const 00151 { 00152 source-> getColumn(i, v); 00153 for (int source_row = 0; source_row < v->length(); source_row++) 00154 if (is_missing(v[source_row]) && condmean_col_ref[i] >= 0) v[source_row] = condmean(source_row, condmean_col_ref[i]); 00155 else if (is_missing(v[source_row])) PLERROR("In ConditionalMeanImputationVMatrix::getColumn(%d,vec) we have a missing value in row %d that haven't been assigned a value",i,source_row); 00156 } 00157 00158 00159 00160 void ConditionalMeanImputationVMatrix::build_() 00161 { 00162 if (!source) PLERROR("In ConditionalMeanImputationVMatrix::source vmat must be supplied"); 00163 loadCondMeanMatrix(); 00164 testResultantVMatrix(); 00165 } 00166 00167 void ConditionalMeanImputationVMatrix::loadCondMeanMatrix() 00168 /* 00169 Imputation step: 00170 count the # of variables with missing values in the train and test datasets. 00171 create a matrix in memory with this number of columns and keep cross reference of columns. 00172 at the build stage, for each variable of train and test: 00173 if # of missing = 0 there is nothing to do. 00174 look for the (cond_mean_dir (/TreeCondMean/dir/) + field_name + /Split0/test1_outputs.pmat) file in the metadatadir; 00175 add its column 0 as a column of the matrix. 00176 then, if is_missing(source[i,j]) replace it with matrix[i, cross_reference[j]] 00177 */ 00178 { 00179 // initialize source dataset 00180 source_length = source->length(); 00181 source_width = source->width(); 00182 source_inputsize = source->inputsize(); 00183 source_targetsize = source->targetsize(); 00184 source_weightsize = source->weightsize(); 00185 source_names.resize(source_width); 00186 source_names = source->fieldNames(); 00187 source_metadata = source->getMetaDataDir(); 00188 length_ = source_length; 00189 width_ = source_width; 00190 inputsize_ = source_inputsize; 00191 targetsize_ = source_targetsize; 00192 weightsize_ = source_weightsize; 00193 declareFieldNames(source_names); 00194 00195 // count the # of variables with missing values in the source datasets. 00196 // create a matrix in memory with this number of columns and keep cross reference of columns. 00197 int count_variable_with_missing = 0; 00198 condmean_col_ref.resize(source_width); 00199 condmean_col_ref.fill(-1); 00200 for (source_col = 0; source_col < source_width; source_col++) 00201 { 00202 source_stats = source->getStats(source_col); 00203 if (source_stats.nmissing() <= 0) continue; 00204 condmean_col_ref[source_col] = count_variable_with_missing; 00205 count_variable_with_missing += 1; 00206 } 00207 condmean.resize(source_length, count_variable_with_missing); 00208 00209 // for each variable with missing value, 00210 // look for the (cond_mean_dir (/TreeCondMean/dir/) + field_name + /Split0/test1_outputs.pmat) file in the metadatadir; 00211 // add its column 0 as a column of the condmean matrix. 00212 for (source_col = 0; source_col < source_width; source_col++) 00213 { 00214 source_stats = source->getStats(source_col); 00215 if (source_stats.nmissing() <= 0) continue; 00216 int condmean_col = condmean_col_ref[source_col]; 00217 PPath condmean_variable_file_name = source_metadata + "/" + condmean_dir + "/dir/" + source_names[source_col] + "/Split0/test1_outputs.pmat"; 00218 if (!isfile(condmean_variable_file_name)) PLERROR("In ConditionalMeanImputationVMatrix::A conditional mean file(%s) was not found for variable %s", 00219 condmean_variable_file_name.c_str(),source_names[source_col].c_str()); 00220 VMat condmean_variable_file = new FileVMatrix(condmean_variable_file_name, false); 00221 if (condmean_variable_file->length() != source_length) 00222 PLERROR("In ConditionalMeanImputationVMatrix::Source and conditional mean file length are not equal for variable %s", source_names[source_col].c_str()); 00223 for (source_row = 0; source_row < source_length; source_row++) 00224 condmean(source_row, condmean_col) = condmean_variable_file->get(source_row, 0); 00225 } 00226 } 00227 00228 } // end of namespcae PLearn