PLearn 0.1
StatsCollector.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 //
00005 // Copyright (C) 2001,2002 Pascal Vincent
00006 // Copyright (C) 2005 University of Montreal
00007 // Copyright (C) 2007 Xavier Saint-Mleux, ApSTAT Technologies inc.
00008 //
00009 
00010 // Redistribution and use in source and binary forms, with or without
00011 // modification, are permitted provided that the following conditions are met:
00012 // 
00013 //  1. Redistributions of source code must retain the above copyright
00014 //     notice, this list of conditions and the following disclaimer.
00015 // 
00016 //  2. Redistributions in binary form must reproduce the above copyright
00017 //     notice, this list of conditions and the following disclaimer in the
00018 //     documentation and/or other materials provided with the distribution.
00019 // 
00020 //  3. The name of the authors may not be used to endorse or promote
00021 //     products derived from this software without specific prior written
00022 //     permission.
00023 // 
00024 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00025 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00026 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00027 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00028 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00029 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00030 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00031 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00032 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00033 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00034 // 
00035 // This file is part of the PLearn library. For more information on the PLearn
00036 // library, go to the PLearn Web site at www.plearn.org
00037  
00038 /* *******************************************************      
00039  * $Id: StatsCollector.h 9205 2008-07-03 16:58:51Z nouiz $
00040  * This file is part of the PLearn library.
00041  ******************************************************* */
00042 
00043 #ifndef StatsCollector_INC
00044 #define StatsCollector_INC
00045 
00046 #include <plearn/base/Object.h>
00047 #include <string>
00048 #include <plearn/base/general.h>
00049 #include <plearn/base/RealMapping.h>
00050 #include "TMat.h"
00051 
00052 namespace PLearn {
00053 using namespace std;
00054 
00056 class PLearnDiff;
00057 
00058 class StatsCollectorCounts
00059 {
00060 public:
00061     double n; 
00062     double nbelow; 
00063     double sum; 
00064     double sumsquare; 
00065     int id; 
00066   
00067     StatsCollectorCounts(): 
00068         n(0), nbelow(0),
00069         sum(0.), sumsquare(0.),id(0) 
00070     {}          
00071 
00072     //to keep some compilers quiet:
00073     virtual ~StatsCollectorCounts() {}
00074     
00079     virtual void merge(const StatsCollectorCounts& other)
00080     {
00081         //don't change id
00082         n+= other.n;
00083         nbelow+= other.nbelow;
00084         sum+= other.sum;
00085         sumsquare+= other.sumsquare;
00086     }
00087 };
00088 
00089 typedef pair<real,StatsCollectorCounts*> PairRealSCCType;
00090 
00092 
00093 inline PStream& operator>>(PStream& in, StatsCollectorCounts& c)
00094 { in >> c.n >> c.nbelow >> c.sum >> c.sumsquare >> c.id; return in; }
00095 
00096 inline PStream& operator<<(PStream& out, const StatsCollectorCounts& c)
00097 { out << c.n << c.nbelow << c.sum << c.sumsquare << c.id; return out; }
00098 
00099 template<class ObjectType>
00100 int diff(const string& refer, const string& other,
00101          const Option<ObjectType, StatsCollectorCounts>* opt,
00102          PLearnDiff* diffs)
00103 {
00104     StatsCollectorCounts refer_sc, other_sc;
00105     PStream in = openString(refer, PStream::plearn_ascii);
00106     in >> refer_sc;
00107     in = openString(other, PStream::plearn_ascii);
00108     in >> other_sc;
00109     int n_diffs = 0;
00110     PP<OptionBase> opt_double = new Option<ObjectType, double>
00111         ("", 0, 0, TypeTraits<double>::name(), "", "", opt->level());
00112     PP<OptionBase> opt_int = new Option<ObjectType, int>
00113         ("", 0, 0, TypeTraits<int>::name(), "", "", opt->level());
00114     opt_double->setOptionName(opt->optionname() + ".n");
00115     n_diffs +=  opt_double->diff(tostring(refer_sc.n),
00116                                  tostring(other_sc.n), diffs);
00117     opt_double->setOptionName(opt->optionname() + ".nbelow");
00118     n_diffs += opt_double->diff(tostring(refer_sc.nbelow),
00119                                 tostring(other_sc.nbelow), diffs);
00120     opt_double->setOptionName(opt->optionname() + ".sum");
00121     n_diffs += opt_double->diff(tostring(refer_sc.sum),
00122                                 tostring(other_sc.sum), diffs);
00123     opt_double->setOptionName(opt->optionname() + ".sumsquare");
00124     n_diffs += opt_double->diff(tostring(refer_sc.sumsquare),
00125                                 tostring(other_sc.sumsquare), diffs);
00126     opt_int->setOptionName(opt->optionname() + ".id");
00127     n_diffs += opt_int->diff(tostring(refer_sc.id),
00128                                 tostring(other_sc.id), diffs);
00129     return n_diffs;
00130 }
00131 
00132 DECLARE_SPECIALIZED_DIFF_CLASS(StatsCollectorCounts)
00133 
00134 class StatsCollector: public Object
00135 {
00136     typedef Object inherited;
00137 
00138 public:
00139 
00140     PLEARN_DECLARE_OBJECT(StatsCollector);
00141       
00142 public:
00143 
00144     // ** Build options **
00145 
00153     double epsilon;
00154 
00160     int maxnvalues; 
00161 
00172     bool no_removal_warnings;
00173 
00174     // ** Learnt options **
00175 
00176     double nmissing_;      
00177     double nnonmissing_;   
00178     double sumsquarew_;    
00179     double sum_;           
00180     double sumsquare_;     
00181     double sumcube_;       
00182     double sumfourth_;     
00183     real min_;             
00184     real max_;             
00185     real agemin_;          
00186     real agemax_;          
00187     real first_;           
00188     real last_;            
00189     bool more_than_maxnvalues;
00190     int binary_;           
00191     int integer_;          
00192 
00193     map<real, StatsCollectorCounts> counts; 
00194     map<int, real> count_ids;
00195 
00196 protected:
00197 
00199     map<real, StatsCollectorCounts> approximate_counts; 
00200 
00203     mutable Mat sorted_values;
00204 
00206     mutable bool sorted;
00207       
00208 private:
00209 
00211     void build_();
00212 
00214     void calculate_binary_integer();
00215 
00216 protected: 
00217 
00219     static void declareOptions(OptionList& ol);
00220 
00222     static void declareMethods(RemoteMethodMap& rmm);
00223 
00225     void sort_values_by_magnitude() const;
00226 
00229     bool storeCounts() { return (maxnvalues == -1 || maxnvalues > 0); }
00230 
00231 public:
00232 
00233     StatsCollector(int the_maxnvalues=0);
00234       
00235     real n() const { return nmissing_ + nnonmissing_; } 
00236     real nmissing() const               { return nmissing_; }
00237     real nnonmissing() const            { return nnonmissing_; }
00238     real sumsquarew() const             { return sumsquarew_; }
00239     real sum() const                    { return real(nnonmissing_ > 0
00240                                                 ? sum_ + nnonmissing_*first_
00241                                                 : 0); }
00242     real sumsquare() const              { return real(nnonmissing_ > 0
00243                                                 ? sumsquare_+2*first_*sum() -
00244                                                     first_*first_*nnonmissing_
00245                                                 : 0); }
00246     real min() const                    { return min_; }
00247     real max() const                    { return max_; }
00248     real agemin() const                 { return agemin_; }
00249     real agemax() const                 { return agemax_; }
00250     real range() const                  { return max_ - min_; }
00251     real mean() const                   { return real(sum()/nnonmissing_); }
00257     real variance() const               { return real((sumsquare_ - square(sum_)/nnonmissing_)/
00258                                                       (nnonmissing_ - sumsquarew_/nnonmissing_))
00259                                               + epsilon; }
00260     real stddev() const                 { return sqrt(variance()); }
00261     real skewness() const;
00262     real kurtosis() const;
00263     real stderror() const               { return sqrt(variance()/nnonmissing()); }
00264     real first_obs() const              { return first_; }
00265     real last_obs() const               { return last_; }
00266     real sharperatio() const;
00267     real mean_over_skewness() const     { return mean()/skewness(); }
00268     real mean_over_skewness_ms() const;        
00269     real mean_over_kurtosis() const     { return mean()/kurtosis(); }
00270     real zstat() const                  { return mean()/stderror(); }
00271     real zpr1t() const;                        
00272     real zpr2t() const;                        
00273     real iqr() const                    { return pseudo_quantile(0.75) - pseudo_quantile(0.25); }
00274     real prr() const                    { return pseudo_quantile(0.99) - pseudo_quantile(0.01); }
00279     real lift(int k, int& n_pos_in_k, int n_pos_in_k_minus_1 = -1, real pos_fraction = -1) const;
00280     real nips_lift() const;   
00281 
00282 
00283 
00284     real mean_lift(real* pos_fraction = NULL) const;
00285     real prbp() const;        
00286 
00287     real dmode() const;
00288     Vec dmodes() const;
00289 
00292     real getStat(const string& statname) const;
00293 
00295     virtual void build();
00296 
00298     void forget();
00299 
00301     void update(real val, real weight = 1.0);
00302 
00307     void remove_observation(real val, real weight = 1.0);
00308 
00310     void finalize() {}
00311 
00314     map<real, StatsCollectorCounts>* getCounts(){return &counts;}
00315 
00320     map<real, StatsCollectorCounts>* getApproximateCounts();
00321     
00322     int getMaxNValues(){return maxnvalues;}
00323 
00326     Mat cdf(bool normalized=true) const;
00327 
00333     real pseudo_quantile(real q) const;
00334 
00339 
00342 
00343 
00344 
00345 
00346 
00347 
00348 
00349     RealMapping getBinMapping(double discrete_mincount,
00350                               double continuous_mincount,
00351                               real tolerance=.1,
00352                               TVec<double>* fcount=0) const;
00353 
00354     RealMapping getAllValuesMapping(TVec<double>* fcount=0) const;
00364     RealMapping getAllValuesMapping(TVec<bool>* to_be_included, TVec<double>* fcount=0, bool ignore_other = false, real tolerance = 0) const;
00365 
00366     virtual void oldwrite(ostream& out) const;
00367     /* TODO Remove (deprecated)
00368        virtual void oldread(istream& in);
00369     */
00370 
00372     virtual void newwrite(PStream& out) const;
00373 
00375     virtual void merge(const StatsCollector& other);
00376 
00379     bool isbinary(){return binary_;}
00380 
00383     bool isinteger(){return integer_;}
00384 
00387     Vec getCount(real value){
00388         Vec v(0,5);
00389         StatsCollectorCounts c = counts[value];
00390         
00391         v.append(c.n);
00392         v.append(c.nbelow);
00393         v.append(c.sum);
00394         v.append(c.sumsquare);
00395         v.append(c.id);
00396         return v;
00397     }
00398 };
00399 
00400 DECLARE_OBJECT_PTR(StatsCollector);
00401 
00405 template <>
00406 inline void deepCopyField(StatsCollector& field, CopiesMap& copies)
00407 {
00408     field.makeDeepCopyFromShallowCopy(copies);
00409 }
00410 
00411 TVec<RealMapping> computeRanges(TVec<StatsCollector> stats, int discrete_mincount, int continuous_mincount);
00412 
00413 } // end of namespace PLearn
00414 
00415 #endif
00416 
00417 
00418 /*
00419   Local Variables:
00420   mode:c++
00421   c-basic-offset:4
00422   c-file-style:"stroustrup"
00423   c-file-offsets:((innamespace . 0)(inline-open . 0))
00424   indent-tabs-mode:nil
00425   fill-column:79
00426   End:
00427 */
00428 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines