PLearn 0.1
ImputationVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003 Olivier Delalleau
00008 // Copyright (C) 2007 Frederic Bastien
00009 //
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are met:
00012 // 
00013 //  1. Redistributions of source code must retain the above copyright
00014 //     notice, this list of conditions and the following disclaimer.
00015 // 
00016 //  2. Redistributions in binary form must reproduce the above copyright
00017 //     notice, this list of conditions and the following disclaimer in the
00018 //     documentation and/or other materials provided with the distribution.
00019 // 
00020 //  3. The name of the authors may not be used to endorse or promote
00021 //     products derived from this software without specific prior written
00022 //     permission.
00023 // 
00024 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00025 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00026 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00027 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00028 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00029 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00030 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00031 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00032 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00033 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00034 // 
00035 // This file is part of the PLearn library. For more information on the PLearn
00036 // library, go to the PLearn Web site at www.plearn.org
00037 
00038 
00039 /* *******************************************************************    
00040    * $Id: NeighborhoodImputationVMatrix.cc 3658 2005-07-06 20:30:15  Godbout $
00041    ******************************************************************* */
00042 
00043 
00044 #include "ImputationVMatrix.h"
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00051 PLEARN_IMPLEMENT_ABSTRACT_OBJECT(
00052   ImputationVMatrix,
00053   "Super-class for VMatrices that replace missing value in another one",
00054   ""
00055   );
00056 
00057   ImputationVMatrix::ImputationVMatrix():
00058     test_level(0)
00059 {
00060 }
00061 
00062 ImputationVMatrix::~ImputationVMatrix()
00063 {
00064 }
00065 
00066 void ImputationVMatrix::declareOptions(OptionList &ol)
00067 {
00068   declareOption(ol, "source", &ImputationVMatrix::source, OptionBase::buildoption, 
00069                 "The source VMatrix with missing values that will be filled.\n");
00070   declareOption(ol, "test_level", &ImputationVMatrix::test_level, OptionBase::buildoption, 
00071                 "The level of test of final matrix. 0 : no test, 1: linear in column or row test, 2: linear in cell\n");
00072 
00073   inherited::declareOptions(ol);
00074 }
00075 
00076 void ImputationVMatrix::build()
00077 {
00078   inherited::build();
00079   build_();
00080 }
00081 
00082 void ImputationVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00083 {
00084   deepCopyField(source, copies);
00085   inherited::makeDeepCopyFromShallowCopy(copies);
00086 }
00087 
00088 void ImputationVMatrix::build_()
00089 {
00090 }
00091 
00092 void ImputationVMatrix::testResultantVMatrix()
00093 {
00094   TVec<string> source_names(source->width());
00095   source_names = source->fieldNames();
00096   
00097   if(test_level>=1){
00098     for(int row=0;row<length();row++)
00099       for(int col=0;col<width();col++){
00100         real data=get(row,col);
00101         real sourcedata=source->get(row,col);
00102 
00103         //test if variable not missing are not changed.
00104         if(!is_missing(sourcedata))
00105           if(data!=sourcedata){
00106             PLERROR("ImputationImputations::testResultantVMatrix() data at [%d,%d] are different but the data is not missing",row,col);
00107           }
00108 
00109         //test if missing variable are replaced by a value not missing.
00110         if(is_missing(data))
00111           PLERROR("ImputationImputations::testResultantVMatrix() data at [%d,%d] in the final matrix is missing",row,col);
00112       }
00113   }
00114   if(test_level>=2){ //Must verify if source->getStats is linear
00115     getStats();
00116     source->getStats();
00117     //print the variable that the replacement of missing value change the mean by more then 3 times the stderr
00118     int nberr = 0;
00119     for(int col=0;col<width();col++)
00120       {
00121         real mean = field_stats[col].mean();
00122         real smean = source->getStats(col).mean();
00123         real sstderr = source->getStats(col).stddev()/sqrt(source->getStats(col).nnonmissing());
00124         real val=(mean-smean)/sstderr;
00125         if(fabs(val)>3){
00126           PLWARNING("ImputationImputations::testResultantVMatrix() the variable %d(%s) have a value of %f for abs((mean-sourcemean)/source_stderr)",col,source_names[col].c_str(),val);
00127           nberr++;
00128         }
00129       }
00130     if(nberr>0)
00131       PLWARNING("ImputationImputations::testResultantVMatrix() There have been %d variables with the mean after imputation outside of sourcemean +- 3*source_stderr",nberr);
00132   }
00133   //    for(int row=0;row<length();row++)
00134   /*    for(int col=0;col<width();col++)
00135         {
00136         StatsCollector sstats=source->getStats(col);
00137         StatsCollector stats=getStats(col);
00138         if(sstats.nmissing()!=0){
00139         real sum=0;
00140         condmean_variable_file_name = source_metadata + "/" + condmean_dir + "/dir/" + source_names[col] + "/Split0/test1_outputs.pmat";
00141         
00142         if (!isfile(condmean_variable_file_name))
00143         PLERROR("In ImputationVMatrix::A conditional mean file was not found for variable %s", source_names[col].c_str());
00144         condmean_variable_file = new FileVMatrix(condmean_variable_file_name, false);
00145         if (condmean_variable_file->length() != source_length)
00146         PLERROR("In ImputationVMatrix::Source and conditional mean file length are not equal for variable %s", source_names[col].c_str());
00147         for (int source_row = 0; source_row < source_length; source_row++){
00148         real rdata = condmean_variable_file->get(source_row, 0);
00149         real data = get(source_row, col);
00150         if(!is_missing(rdata))
00151         sum+=pow(data - rdata, 2.0);
00152         }
00153         real mse=sum/stats.nnonmissing();
00154         real diff=mse/sstats.variance();
00155         if(diff >0.9){
00156         perr <<col<<" "<<diff<<endl;
00157         PLWARNING("ImputationImputations::testresultantVMatrix() the variable %d(%s) have a MSEtreecondmean(%f)/MSEmean(%f) of %f",col,source_names[col].c_str(),mse,sstats.variance(),diff);
00158         }
00159         }*/
00160 }
00161 } // end of namespcae PLearn
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines