PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // 00005 // Copyright (C) 2001,2002 Pascal Vincent 00006 // Copyright (C) 2005 University of Montreal 00007 // Copyright (C) 2007 Xavier Saint-Mleux, ApSTAT Technologies inc. 00008 // 00009 00010 // Redistribution and use in source and binary forms, with or without 00011 // modification, are permitted provided that the following conditions are met: 00012 // 00013 // 1. Redistributions of source code must retain the above copyright 00014 // notice, this list of conditions and the following disclaimer. 00015 // 00016 // 2. Redistributions in binary form must reproduce the above copyright 00017 // notice, this list of conditions and the following disclaimer in the 00018 // documentation and/or other materials provided with the distribution. 00019 // 00020 // 3. The name of the authors may not be used to endorse or promote 00021 // products derived from this software without specific prior written 00022 // permission. 00023 // 00024 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00025 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00026 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00027 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00028 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00029 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00030 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00031 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00032 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00033 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00034 // 00035 // This file is part of the PLearn library. For more information on the PLearn 00036 // library, go to the PLearn Web site at www.plearn.org 00037 00038 /* ******************************************************* 00039 * $Id: StatsCollector.h 9205 2008-07-03 16:58:51Z nouiz $ 00040 * This file is part of the PLearn library. 00041 ******************************************************* */ 00042 00043 #ifndef StatsCollector_INC 00044 #define StatsCollector_INC 00045 00046 #include <plearn/base/Object.h> 00047 #include <string> 00048 #include <plearn/base/general.h> 00049 #include <plearn/base/RealMapping.h> 00050 #include "TMat.h" 00051 00052 namespace PLearn { 00053 using namespace std; 00054 00056 class PLearnDiff; 00057 00058 class StatsCollectorCounts 00059 { 00060 public: 00061 double n; 00062 double nbelow; 00063 double sum; 00064 double sumsquare; 00065 int id; 00066 00067 StatsCollectorCounts(): 00068 n(0), nbelow(0), 00069 sum(0.), sumsquare(0.),id(0) 00070 {} 00071 00072 //to keep some compilers quiet: 00073 virtual ~StatsCollectorCounts() {} 00074 00079 virtual void merge(const StatsCollectorCounts& other) 00080 { 00081 //don't change id 00082 n+= other.n; 00083 nbelow+= other.nbelow; 00084 sum+= other.sum; 00085 sumsquare+= other.sumsquare; 00086 } 00087 }; 00088 00089 typedef pair<real,StatsCollectorCounts*> PairRealSCCType; 00090 00092 00093 inline PStream& operator>>(PStream& in, StatsCollectorCounts& c) 00094 { in >> c.n >> c.nbelow >> c.sum >> c.sumsquare >> c.id; return in; } 00095 00096 inline PStream& operator<<(PStream& out, const StatsCollectorCounts& c) 00097 { out << c.n << c.nbelow << c.sum << c.sumsquare << c.id; return out; } 00098 00099 template<class ObjectType> 00100 int diff(const string& refer, const string& other, 00101 const Option<ObjectType, StatsCollectorCounts>* opt, 00102 PLearnDiff* diffs) 00103 { 00104 StatsCollectorCounts refer_sc, other_sc; 00105 PStream in = openString(refer, PStream::plearn_ascii); 00106 in >> refer_sc; 00107 in = openString(other, PStream::plearn_ascii); 00108 in >> other_sc; 00109 int n_diffs = 0; 00110 PP<OptionBase> opt_double = new Option<ObjectType, double> 00111 ("", 0, 0, TypeTraits<double>::name(), "", "", opt->level()); 00112 PP<OptionBase> opt_int = new Option<ObjectType, int> 00113 ("", 0, 0, TypeTraits<int>::name(), "", "", opt->level()); 00114 opt_double->setOptionName(opt->optionname() + ".n"); 00115 n_diffs += opt_double->diff(tostring(refer_sc.n), 00116 tostring(other_sc.n), diffs); 00117 opt_double->setOptionName(opt->optionname() + ".nbelow"); 00118 n_diffs += opt_double->diff(tostring(refer_sc.nbelow), 00119 tostring(other_sc.nbelow), diffs); 00120 opt_double->setOptionName(opt->optionname() + ".sum"); 00121 n_diffs += opt_double->diff(tostring(refer_sc.sum), 00122 tostring(other_sc.sum), diffs); 00123 opt_double->setOptionName(opt->optionname() + ".sumsquare"); 00124 n_diffs += opt_double->diff(tostring(refer_sc.sumsquare), 00125 tostring(other_sc.sumsquare), diffs); 00126 opt_int->setOptionName(opt->optionname() + ".id"); 00127 n_diffs += opt_int->diff(tostring(refer_sc.id), 00128 tostring(other_sc.id), diffs); 00129 return n_diffs; 00130 } 00131 00132 DECLARE_SPECIALIZED_DIFF_CLASS(StatsCollectorCounts) 00133 00134 class StatsCollector: public Object 00135 { 00136 typedef Object inherited; 00137 00138 public: 00139 00140 PLEARN_DECLARE_OBJECT(StatsCollector); 00141 00142 public: 00143 00144 // ** Build options ** 00145 00153 double epsilon; 00154 00160 int maxnvalues; 00161 00172 bool no_removal_warnings; 00173 00174 // ** Learnt options ** 00175 00176 double nmissing_; 00177 double nnonmissing_; 00178 double sumsquarew_; 00179 double sum_; 00180 double sumsquare_; 00181 double sumcube_; 00182 double sumfourth_; 00183 real min_; 00184 real max_; 00185 real agemin_; 00186 real agemax_; 00187 real first_; 00188 real last_; 00189 bool more_than_maxnvalues; 00190 int binary_; 00191 int integer_; 00192 00193 map<real, StatsCollectorCounts> counts; 00194 map<int, real> count_ids; 00195 00196 protected: 00197 00199 map<real, StatsCollectorCounts> approximate_counts; 00200 00203 mutable Mat sorted_values; 00204 00206 mutable bool sorted; 00207 00208 private: 00209 00211 void build_(); 00212 00214 void calculate_binary_integer(); 00215 00216 protected: 00217 00219 static void declareOptions(OptionList& ol); 00220 00222 static void declareMethods(RemoteMethodMap& rmm); 00223 00225 void sort_values_by_magnitude() const; 00226 00229 bool storeCounts() { return (maxnvalues == -1 || maxnvalues > 0); } 00230 00231 public: 00232 00233 StatsCollector(int the_maxnvalues=0); 00234 00235 real n() const { return nmissing_ + nnonmissing_; } 00236 real nmissing() const { return nmissing_; } 00237 real nnonmissing() const { return nnonmissing_; } 00238 real sumsquarew() const { return sumsquarew_; } 00239 real sum() const { return real(nnonmissing_ > 0 00240 ? sum_ + nnonmissing_*first_ 00241 : 0); } 00242 real sumsquare() const { return real(nnonmissing_ > 0 00243 ? sumsquare_+2*first_*sum() - 00244 first_*first_*nnonmissing_ 00245 : 0); } 00246 real min() const { return min_; } 00247 real max() const { return max_; } 00248 real agemin() const { return agemin_; } 00249 real agemax() const { return agemax_; } 00250 real range() const { return max_ - min_; } 00251 real mean() const { return real(sum()/nnonmissing_); } 00257 real variance() const { return real((sumsquare_ - square(sum_)/nnonmissing_)/ 00258 (nnonmissing_ - sumsquarew_/nnonmissing_)) 00259 + epsilon; } 00260 real stddev() const { return sqrt(variance()); } 00261 real skewness() const; 00262 real kurtosis() const; 00263 real stderror() const { return sqrt(variance()/nnonmissing()); } 00264 real first_obs() const { return first_; } 00265 real last_obs() const { return last_; } 00266 real sharperatio() const; 00267 real mean_over_skewness() const { return mean()/skewness(); } 00268 real mean_over_skewness_ms() const; 00269 real mean_over_kurtosis() const { return mean()/kurtosis(); } 00270 real zstat() const { return mean()/stderror(); } 00271 real zpr1t() const; 00272 real zpr2t() const; 00273 real iqr() const { return pseudo_quantile(0.75) - pseudo_quantile(0.25); } 00274 real prr() const { return pseudo_quantile(0.99) - pseudo_quantile(0.01); } 00279 real lift(int k, int& n_pos_in_k, int n_pos_in_k_minus_1 = -1, real pos_fraction = -1) const; 00280 real nips_lift() const; 00281 00282 00283 00284 real mean_lift(real* pos_fraction = NULL) const; 00285 real prbp() const; 00286 00287 real dmode() const; 00288 Vec dmodes() const; 00289 00292 real getStat(const string& statname) const; 00293 00295 virtual void build(); 00296 00298 void forget(); 00299 00301 void update(real val, real weight = 1.0); 00302 00307 void remove_observation(real val, real weight = 1.0); 00308 00310 void finalize() {} 00311 00314 map<real, StatsCollectorCounts>* getCounts(){return &counts;} 00315 00320 map<real, StatsCollectorCounts>* getApproximateCounts(); 00321 00322 int getMaxNValues(){return maxnvalues;} 00323 00326 Mat cdf(bool normalized=true) const; 00327 00333 real pseudo_quantile(real q) const; 00334 00339 00342 00343 00344 00345 00346 00347 00348 00349 RealMapping getBinMapping(double discrete_mincount, 00350 double continuous_mincount, 00351 real tolerance=.1, 00352 TVec<double>* fcount=0) const; 00353 00354 RealMapping getAllValuesMapping(TVec<double>* fcount=0) const; 00364 RealMapping getAllValuesMapping(TVec<bool>* to_be_included, TVec<double>* fcount=0, bool ignore_other = false, real tolerance = 0) const; 00365 00366 virtual void oldwrite(ostream& out) const; 00367 /* TODO Remove (deprecated) 00368 virtual void oldread(istream& in); 00369 */ 00370 00372 virtual void newwrite(PStream& out) const; 00373 00375 virtual void merge(const StatsCollector& other); 00376 00379 bool isbinary(){return binary_;} 00380 00383 bool isinteger(){return integer_;} 00384 00387 Vec getCount(real value){ 00388 Vec v(0,5); 00389 StatsCollectorCounts c = counts[value]; 00390 00391 v.append(c.n); 00392 v.append(c.nbelow); 00393 v.append(c.sum); 00394 v.append(c.sumsquare); 00395 v.append(c.id); 00396 return v; 00397 } 00398 }; 00399 00400 DECLARE_OBJECT_PTR(StatsCollector); 00401 00405 template <> 00406 inline void deepCopyField(StatsCollector& field, CopiesMap& copies) 00407 { 00408 field.makeDeepCopyFromShallowCopy(copies); 00409 } 00410 00411 TVec<RealMapping> computeRanges(TVec<StatsCollector> stats, int discrete_mincount, int continuous_mincount); 00412 00413 } // end of namespace PLearn 00414 00415 #endif 00416 00417 00418 /* 00419 Local Variables: 00420 mode:c++ 00421 c-basic-offset:4 00422 c-file-style:"stroustrup" 00423 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00424 indent-tabs-mode:nil 00425 fill-column:79 00426 End: 00427 */ 00428 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :