PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal 00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux 00007 // Copyright (C) 2003 Olivier Delalleau 00008 // Copyright (C) 2007 Frederic Bastien 00009 // 00010 // Redistribution and use in source and binary forms, with or without 00011 // modification, are permitted provided that the following conditions are met: 00012 // 00013 // 1. Redistributions of source code must retain the above copyright 00014 // notice, this list of conditions and the following disclaimer. 00015 // 00016 // 2. Redistributions in binary form must reproduce the above copyright 00017 // notice, this list of conditions and the following disclaimer in the 00018 // documentation and/or other materials provided with the distribution. 00019 // 00020 // 3. The name of the authors may not be used to endorse or promote 00021 // products derived from this software without specific prior written 00022 // permission. 00023 // 00024 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00025 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00026 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00027 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00028 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00029 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00030 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00031 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00032 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00033 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00034 // 00035 // This file is part of the PLearn library. For more information on the PLearn 00036 // library, go to the PLearn Web site at www.plearn.org 00037 00038 00039 /* ******************************************************************* 00040 * $Id: NeighborhoodImputationVMatrix.cc 3658 2005-07-06 20:30:15 Godbout $ 00041 ******************************************************************* */ 00042 00043 00044 #include "ImputationVMatrix.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00051 PLEARN_IMPLEMENT_ABSTRACT_OBJECT( 00052 ImputationVMatrix, 00053 "Super-class for VMatrices that replace missing value in another one", 00054 "" 00055 ); 00056 00057 ImputationVMatrix::ImputationVMatrix(): 00058 test_level(0) 00059 { 00060 } 00061 00062 ImputationVMatrix::~ImputationVMatrix() 00063 { 00064 } 00065 00066 void ImputationVMatrix::declareOptions(OptionList &ol) 00067 { 00068 declareOption(ol, "source", &ImputationVMatrix::source, OptionBase::buildoption, 00069 "The source VMatrix with missing values that will be filled.\n"); 00070 declareOption(ol, "test_level", &ImputationVMatrix::test_level, OptionBase::buildoption, 00071 "The level of test of final matrix. 0 : no test, 1: linear in column or row test, 2: linear in cell\n"); 00072 00073 inherited::declareOptions(ol); 00074 } 00075 00076 void ImputationVMatrix::build() 00077 { 00078 inherited::build(); 00079 build_(); 00080 } 00081 00082 void ImputationVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00083 { 00084 deepCopyField(source, copies); 00085 inherited::makeDeepCopyFromShallowCopy(copies); 00086 } 00087 00088 void ImputationVMatrix::build_() 00089 { 00090 } 00091 00092 void ImputationVMatrix::testResultantVMatrix() 00093 { 00094 TVec<string> source_names(source->width()); 00095 source_names = source->fieldNames(); 00096 00097 if(test_level>=1){ 00098 for(int row=0;row<length();row++) 00099 for(int col=0;col<width();col++){ 00100 real data=get(row,col); 00101 real sourcedata=source->get(row,col); 00102 00103 //test if variable not missing are not changed. 00104 if(!is_missing(sourcedata)) 00105 if(data!=sourcedata){ 00106 PLERROR("ImputationImputations::testResultantVMatrix() data at [%d,%d] are different but the data is not missing",row,col); 00107 } 00108 00109 //test if missing variable are replaced by a value not missing. 00110 if(is_missing(data)) 00111 PLERROR("ImputationImputations::testResultantVMatrix() data at [%d,%d] in the final matrix is missing",row,col); 00112 } 00113 } 00114 if(test_level>=2){ //Must verify if source->getStats is linear 00115 getStats(); 00116 source->getStats(); 00117 //print the variable that the replacement of missing value change the mean by more then 3 times the stderr 00118 int nberr = 0; 00119 for(int col=0;col<width();col++) 00120 { 00121 real mean = field_stats[col].mean(); 00122 real smean = source->getStats(col).mean(); 00123 real sstderr = source->getStats(col).stddev()/sqrt(source->getStats(col).nnonmissing()); 00124 real val=(mean-smean)/sstderr; 00125 if(fabs(val)>3){ 00126 PLWARNING("ImputationImputations::testResultantVMatrix() the variable %d(%s) have a value of %f for abs((mean-sourcemean)/source_stderr)",col,source_names[col].c_str(),val); 00127 nberr++; 00128 } 00129 } 00130 if(nberr>0) 00131 PLWARNING("ImputationImputations::testResultantVMatrix() There have been %d variables with the mean after imputation outside of sourcemean +- 3*source_stderr",nberr); 00132 } 00133 // for(int row=0;row<length();row++) 00134 /* for(int col=0;col<width();col++) 00135 { 00136 StatsCollector sstats=source->getStats(col); 00137 StatsCollector stats=getStats(col); 00138 if(sstats.nmissing()!=0){ 00139 real sum=0; 00140 condmean_variable_file_name = source_metadata + "/" + condmean_dir + "/dir/" + source_names[col] + "/Split0/test1_outputs.pmat"; 00141 00142 if (!isfile(condmean_variable_file_name)) 00143 PLERROR("In ImputationVMatrix::A conditional mean file was not found for variable %s", source_names[col].c_str()); 00144 condmean_variable_file = new FileVMatrix(condmean_variable_file_name, false); 00145 if (condmean_variable_file->length() != source_length) 00146 PLERROR("In ImputationVMatrix::Source and conditional mean file length are not equal for variable %s", source_names[col].c_str()); 00147 for (int source_row = 0; source_row < source_length; source_row++){ 00148 real rdata = condmean_variable_file->get(source_row, 0); 00149 real data = get(source_row, col); 00150 if(!is_missing(rdata)) 00151 sum+=pow(data - rdata, 2.0); 00152 } 00153 real mse=sum/stats.nnonmissing(); 00154 real diff=mse/sstats.variance(); 00155 if(diff >0.9){ 00156 perr <<col<<" "<<diff<<endl; 00157 PLWARNING("ImputationImputations::testresultantVMatrix() the variable %d(%s) have a MSEtreecondmean(%f)/MSEmean(%f) of %f",col,source_names[col].c_str(),mse,sstats.variance(),diff); 00158 } 00159 }*/ 00160 } 00161 } // end of namespcae PLearn