PLearn 0.1
NeighborhoodImputationVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2001 Pascal Vincent, Yoshua Bengio, Rejean Ducharme and University of Montreal
00006 // Copyright (C) 2002 Pascal Vincent, Julien Keable, Xavier Saint-Mleux
00007 // Copyright (C) 2003 Olivier Delalleau
00008 //
00009 // Redistribution and use in source and binary forms, with or without
00010 // modification, are permitted provided that the following conditions are met:
00011 // 
00012 //  1. Redistributions of source code must retain the above copyright
00013 //     notice, this list of conditions and the following disclaimer.
00014 // 
00015 //  2. Redistributions in binary form must reproduce the above copyright
00016 //     notice, this list of conditions and the following disclaimer in the
00017 //     documentation and/or other materials provided with the distribution.
00018 // 
00019 //  3. The name of the authors may not be used to endorse or promote
00020 //     products derived from this software without specific prior written
00021 //     permission.
00022 // 
00023 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00024 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00025 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00026 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00027 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00028 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00029 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00030 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00031 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00032 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00033 // 
00034 // This file is part of the PLearn library. For more information on the PLearn
00035 // library, go to the PLearn Web site at www.plearn.org
00036 
00037 
00038 /* *******************************************************************    
00039    * $Id: NeighborhoodImputationVMatrix.cc 3658 2005-07-06 20:30:15  Godbout $
00040    ******************************************************************* */
00041 
00042 
00043 #include "NeighborhoodImputationVMatrix.h"
00044 
00045 namespace PLearn {
00046 using namespace std;
00047 
00050 PLEARN_IMPLEMENT_OBJECT(
00051   NeighborhoodImputationVMatrix,
00052   "VMat class to impute the observed variable mean to replace missing values in the source matrix.",
00053   "This class will replace missing values in the underlying dataset with the mean, median or mode observed on the train set.\n"
00054   "The imputed value is based on the imputation instruction option.\n"
00055   );
00056 
00057 NeighborhoodImputationVMatrix::NeighborhoodImputationVMatrix()
00058   : count_missing_neighbors(0)
00059 {
00060 }
00061 
00062 NeighborhoodImputationVMatrix::~NeighborhoodImputationVMatrix()
00063 {
00064 }
00065 
00066 void NeighborhoodImputationVMatrix::declareOptions(OptionList &ol)
00067 {
00068   declareOption(ol, "reference_index", &NeighborhoodImputationVMatrix::reference_index, OptionBase::buildoption, 
00069                 "The set of pre-computed neighbors index.\n"
00070                 "This can be done with BallTreeNearestNeighbors.\n");
00071 
00072   declareOption(ol, "reference_with_missing", &NeighborhoodImputationVMatrix::reference_with_missing, OptionBase::buildoption, 
00073                 "The reference set corresponding to the pre-computed index with missing values.");
00074       
00075   declareOption(ol, "reference_with_covariance_preserved", &NeighborhoodImputationVMatrix::reference_with_covariance_preserved, OptionBase::buildoption, 
00076                 "The reference set corresponding to the pre-computed index with the initial imputations.");
00077 
00078   declareOption(ol, "number_of_neighbors", &NeighborhoodImputationVMatrix::number_of_neighbors, OptionBase::buildoption,
00079                 "This is usually called K, the number of neighbors to consider.\n"   
00080                 "It must be less or equal than the with of the reference index.");
00081 
00082   declareOption(ol, "count_missing_neighbors", &NeighborhoodImputationVMatrix::count_missing_neighbors, OptionBase::buildoption,
00083                 "0: (default) We do not count a neighbour with a missing value in the number of neighbors.\n"   
00084                 "1: We count a neighbour with a missing value in the number of neighbors.\n");
00085 
00086   declareOption(ol, "imputation_spec", &NeighborhoodImputationVMatrix::imputation_spec, OptionBase::buildoption,
00087                 "A vector that give for each variable the number of neighbors to use.\n"
00088                 "If a variable don't have a value, we use the value of the varialbe: number_of_neighbors.\n"
00089                 " Ex: [ varname1 : nbneighbors1, varname2 : nbneighbors2 ]\n");
00090   inherited::declareOptions(ol);
00091 }
00092 
00093 void NeighborhoodImputationVMatrix::build()
00094 {
00095   inherited::build();
00096   build_();
00097   testResultantVMatrix();
00098 }
00099 
00100 void NeighborhoodImputationVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00101 {
00102   deepCopyField(reference_index, copies);
00103   deepCopyField(reference_with_missing, copies);
00104   deepCopyField(reference_with_covariance_preserved, copies);
00105   deepCopyField(number_of_neighbors, copies);
00106   deepCopyField(count_missing_neighbors, copies);
00107   deepCopyField(imputation_spec, copies);
00108   inherited::makeDeepCopyFromShallowCopy(copies);
00109 }
00110 
00111 void NeighborhoodImputationVMatrix::getExample(int i, Vec& input, Vec& target, real& weight)
00112 {
00113   source->getExample(i, input, target, weight);
00114   for (int source_col = 0; source_col < input->length(); source_col++)
00115   {
00116     if (is_missing(input[source_col])) input[source_col] = impute(i, source_col);
00117   }  
00118 }
00119 
00120 real NeighborhoodImputationVMatrix::get(int i, int j) const
00121 { 
00122   real variable_value = source->get(i, j);
00123   if (!is_missing(variable_value)) return variable_value;
00124   return impute(i, j);
00125 }
00126 
00127 void NeighborhoodImputationVMatrix::put(int i, int j, real value)
00128 {
00129   PLERROR("In NeighborhoodImputationVMatrix::put not implemented");
00130 }
00131 
00132 void NeighborhoodImputationVMatrix::getSubRow(int i, int j, Vec v) const
00133 {  
00134   source->getSubRow(i, j, v);
00135   for (int source_col = 0; source_col < v->length(); source_col++) 
00136     if (is_missing(v[source_col])) v[source_col] = impute(i, source_col + j);
00137 }
00138 
00139 void NeighborhoodImputationVMatrix::putSubRow(int i, int j, Vec v)
00140 {
00141   PLERROR("In NeighborhoodImputationVMatrix::putSubRow not implemented");
00142 }
00143 
00144 void NeighborhoodImputationVMatrix::appendRow(Vec v)
00145 {
00146   PLERROR("In NeighborhoodImputationVMatrix::appendRow not implemented");
00147 }
00148 
00149 void NeighborhoodImputationVMatrix::insertRow(int i, Vec v)
00150 {
00151   PLERROR("In NeighborhoodImputationVMatrix::insertRow not implemented");
00152 }
00153 
00154 void NeighborhoodImputationVMatrix::getRow(int i, Vec v) const
00155 {  
00156   source-> getRow(i, v);
00157   for (int source_col = 0; source_col < v->length(); source_col++)
00158     if (is_missing(v[source_col])) v[source_col] = impute(i, source_col); 
00159 }
00160 
00161 void NeighborhoodImputationVMatrix::putRow(int i, Vec v)
00162 {
00163   PLERROR("In NeighborhoodImputationVMatrix::putRow not implemented");
00164 }
00165 
00166 void NeighborhoodImputationVMatrix::getColumn(int i, Vec v) const
00167 {  
00168   source-> getColumn(i, v);
00169   for (int source_row = 0; source_row < v->length(); source_row++)
00170     if (is_missing(v[source_row])) v[source_row] = impute(source_row, i);
00171 }
00172 
00173 void NeighborhoodImputationVMatrix::build_()
00174 {
00175     if (!source)                 PLERROR("In NeighborhoodImputationVMatrix::source with missing set must be supplied");
00176     if (!reference_index)                     PLERROR("In NeighborhoodImputationVMatrix::reference index set must be supplied");
00177     if (!reference_with_missing)              PLERROR("In NeighborhoodImputationVMatrix::reference with missing set must be supplied");
00178     if (!reference_with_covariance_preserved) PLERROR("In NeighborhoodImputationVMatrix::reference with covariance preserved must be supplied");
00179     src_length = source->length();
00180     if (src_length != reference_index->length())
00181         PLERROR("In NeighborhoodImputationVMatrix::length of the source and its index must agree, got: %i - %i", src_length, reference_index->length());
00182     ref_length = reference_with_missing->length();
00183     if (ref_length != reference_with_covariance_preserved->length())
00184         PLERROR("In NeighborhoodImputationVMatrix::length of the reference set with missing and with covariance preserved must agree, got: %i - %i",
00185                 ref_length, reference_with_covariance_preserved->length());
00186     src_width = source->width();
00187     if (src_width != reference_with_missing->width())
00188         PLERROR("In NeighborhoodImputationVMatrix::width of the source and the reference with missing must agree, got: %i - %i",
00189                 src_width, reference_with_missing->width());
00190     if (src_width != reference_with_covariance_preserved->width())
00191         PLERROR("In NeighborhoodImputationVMatrix::width of the source and the reference with covariance preserved must agree, got: %i - %i",
00192                 src_width, reference_with_covariance_preserved->width());
00193     if (number_of_neighbors < 1)
00194       PLERROR("In NeighborhoodImputationVMatrix::there must be at least 1 neighbors, got: %d",number_of_neighbors);
00195     if (number_of_neighbors > reference_index->width())
00196         PLERROR("In NeighborhoodImputationVMatrix::the index must contains at least as many reference as the specified number of neighbors, got: %i - %i",
00197                 number_of_neighbors, reference_index->width());
00198     ref_idx.resize(reference_index->length(), reference_index->width());
00199     ref_idx = reference_index->toMat();
00200     ref_mis.resize(reference_with_missing->length(), reference_with_missing->width());
00201     ref_mis = reference_with_missing->toMat();
00202     ref_cov.resize(reference_with_covariance_preserved->length(), reference_with_covariance_preserved->width());
00203     ref_cov = reference_with_covariance_preserved->toMat();
00204 
00205     length_ = src_length;
00206     width_ = src_width;
00207     inputsize_ = source->inputsize();
00208     targetsize_ = source->targetsize();
00209     weightsize_ = source->weightsize();
00210     declareFieldNames(source->fieldNames());
00211 
00212     nb_neighbors.resize(source->inputsize());
00213     nb_neighbors.fill(number_of_neighbors);
00214     for (int spec_col = 0; spec_col < imputation_spec.size(); spec_col++)
00215       {
00216         int source_col = fieldIndex(imputation_spec[spec_col].first);
00217         if (source_col < 0) 
00218           PLERROR("In NeighborhoodImputationVMatrix::build_() no field with this name in source data set: %s", (imputation_spec[spec_col].first).c_str());
00219         nb_neighbors[source_col] = imputation_spec[spec_col].second;
00220       }
00221 }
00222 real NeighborhoodImputationVMatrix::impute(int i, int j) const
00223 {
00224     int ref_row;
00225     real return_value = 0.0;
00226     int value_count = 0;
00227     int neighbors_count = 0;
00228     for (int ref_idx_col = 0; ref_idx_col < ref_idx.width() &&
00229            neighbors_count < nb_neighbors[j]; ref_idx_col++)
00230     {
00231         ref_row = (int) ref_idx(i, ref_idx_col);
00232         if (ref_row < 0 || ref_row >= ref_length)
00233             PLERROR("In NeighborhoodImputationVMatrix::invalid index reference, got: %i for sample %i", ref_row, i);
00234         if (is_missing(ref_mis(ref_row, j))){
00235           if(count_missing_neighbors)
00236             neighbors_count++;
00237           continue;
00238         }
00239         return_value += ref_mis(ref_row, j);
00240         value_count++;
00241         neighbors_count++;
00242     }
00243     if (value_count > 0) return return_value / value_count;
00244     //if all neighbors have missing value we use the inputed version
00245     //TODO put a warning
00246     return_value = 0.0;
00247     value_count = 0;
00248     for (int ref_idx_col = 0; ref_idx_col < number_of_neighbors; ref_idx_col++)
00249     {
00250         ref_row = (int) ref_idx(i, ref_idx_col);
00251         if (is_missing(ref_cov(ref_row, j)))
00252             PLERROR("In NeighborhoodImputationVMatrix::missing value found in the reference with covariance preserved at: %i , %i", ref_row, j);
00253         return_value += ref_cov(ref_row, j);
00254         value_count += 1;
00255     }
00256     return return_value / value_count;
00257 }
00258 
00259 } // end of namespcae PLearn
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines