PLearn 0.1
|
#include <VecStatsCollector.h>
Public Member Functions | |
VecStatsCollector () | |
virtual int | length () const |
int | size () const |
virtual void | build () |
simply calls inherited::build() then build_() | |
virtual void | forget () |
clears all previously accumulated statistics | |
virtual void | remote_update (const Vec &x, real weight=1.0) |
updates the statistics when seeing x The weight applies to all elements of x | |
virtual void | update (const Vec &x, real weight=1.0) |
updates the statistics when seeing x The weight applies to all elements of x | |
bool | shouldUpdateWindow (const Vec &x) |
Handling m_window_nan_code. | |
virtual void | remove_observation (const Vec &x, real weight=1.0) |
void | setFieldNames (TVec< string > the_fieldnames) |
Declares names for the columns of the vector passed to update. | |
TVec< string > | getFieldNames () const |
Returns the declared names. | |
int | getFieldNum (const string &fieldname_or_num) const |
Returns the index corresponding to a fieldname or to the fieldnum passed as a string. | |
virtual double | getStat (const string &statspec) |
Returns a particular statistic. | |
void | update (const Mat &m) |
calls update on all rows of m; weight assumed to be 1.0 for all roes | |
void | update (const Mat &m, const Vec &weights) |
calls update on all rows of m; vector of weights given, weighting each row | |
virtual void | finalize () |
finishes whatever computation are needed after all updates have been made | |
const StatsCollector & | getStats (int i) const |
returns statistics for element i | |
StatsCollector & | getStats (int i) |
returns non-const statistics for element i | |
Vec | getMean () const |
returns the empirical mean (sample average) vec | |
Vec | remote_getMean () |
Remote version of getMean. | |
void | getMean (Vec &mean) const |
Store the empirical mean in the given vec (which is resized) | |
Vec | getVariance () const |
returns the empirical variance vec | |
Vec | getStdDev () const |
returns the empirical standard deviation vec | |
Vec | getStdError () const |
returns the empirical standard deviation vec | |
const Mat & | getXtX () const |
Return X'X (note that this matrix is weighted, and the weight might be different for each element if there were missing values observed). | |
void | getCovariance (Mat &covar) const |
Covariance matrix computation. | |
Mat | getCovariance () const |
Mat | remote_getCovariance () |
Remote version of getCovariance. | |
Mat | getCorrelation () const |
returns correlation matrix | |
Vec | getAllStats (Vec &st) const |
Fills vector st with [mean, variance, stddev, min, max] (after resizing it if it had a size of 0) However the order and number may change in future versions, so it's better to first call getIndexInAllStats to get the index of a given stat. | |
Vec | getAllStats (const string &statname) const |
Call getStat() with the given statname on all the statscollectors. | |
void | getAllStats (const string &statname, Vec &result) const |
Call getStat() with the given statname on all the statscollectors, and put result in given Vec. | |
int | getIndexInAllStats (int fieldindex, const string &statname) const |
Returns the index in the vector returned by getAllStats of the stat with the given name. | |
void | append (const VecStatsCollector &vsc, const string fieldname_prefix="", const TVec< string > &new_fieldnames=TVec< string >()) |
A little magic function that appends all the StatsCollectors of an existing VecStatsCollector into this one. | |
void | remote_append (const VecStatsCollector *vsc, const string fieldname_prefix, const TVec< string > &new_fieldnames) |
remote version of append: takes pointer to other VecStatsCollector | |
virtual void | setWindowSize (int sz) |
sets the size of the observation window | |
const Mat & | getObservations () const |
const PP< ObservationWindow > | getObservationWindow () const |
virtual void | merge (VecStatsCollector &other) |
merges another VecStatsCollector into this one | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual VecStatsCollector * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
int | maxnvalues |
bool | compute_covariance |
Should we compute and keep X'.X ? (default false) | |
double | epsilon |
Small regularizing value to be added to the covariance matrix estimator, and forwarded to the enclosed vector of StatsCollector. | |
int | m_window |
If positive, the window restricts the stats computed by this FinVecStatsCollector to the last 'window' observations. | |
int | m_full_update_frequency |
If the window mechanism is used, number of updates at which a full update of the underlying StatsCollector is performed. | |
int | m_window_nan_code |
How to deal with update vectors containing NaNs with respect to the window mechanism. | |
bool | no_removal_warnings |
If the remove_observation mechanism is used (without 'full_update_frequency=1') and the removed value is equal to one of first_, last_, min_ or max_, the default behavior is to warn the user. | |
TVec< StatsCollector > | stats |
the stats for each element | |
Mat | cov |
See .cc for help. | |
Mat | sum_cross |
Mat | sum_cross_weights |
Mat | sum_cross_square_weights |
real | sum_non_missing_weights |
real | sum_non_missing_square_weights |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
static void | declareMethods (RemoteMethodMap &rmm) |
Declare the methods that are remote-callable. | |
Protected Attributes | |
map< string, int > | fieldnames_num |
Map from fieldnames to fieldnumbers, to really speed up getFieldNum which can be a speed bottleneck in some experiments. | |
TVec< string > | fieldnames |
Names of the fields of the update vector; now protected: use setFieldNames to set them! | |
PP< ObservationWindow > | m_observation_window |
Window mechanism. | |
int | m_num_incremental |
(Window mechanism) Number of incremental updates since the last update from scratch of the underlying statscollectors | |
Private Types | |
typedef Object | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Definition at line 56 of file VecStatsCollector.h.
typedef Object PLearn::VecStatsCollector::inherited [private] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 58 of file VecStatsCollector.h.
PLearn::VecStatsCollector::VecStatsCollector | ( | ) |
Definition at line 51 of file VecStatsCollector.cc.
: maxnvalues(0), compute_covariance(false), epsilon(0.0), m_window(-1), m_full_update_frequency(-1), m_window_nan_code(0), no_removal_warnings(false), // Window mechanism sum_non_missing_weights(0), sum_non_missing_square_weights(0), m_num_incremental(0) { }
string PLearn::VecStatsCollector::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
OptionList & PLearn::VecStatsCollector::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
RemoteMethodMap & PLearn::VecStatsCollector::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
Object * PLearn::VecStatsCollector::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
StaticInitializer VecStatsCollector::_static_initializer_ & PLearn::VecStatsCollector::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
void PLearn::VecStatsCollector::append | ( | const VecStatsCollector & | vsc, |
const string | fieldname_prefix = "" , |
||
const TVec< string > & | new_fieldnames = TVec<string>() |
||
) |
A little magic function that appends all the StatsCollectors of an existing VecStatsCollector into this one.
A fieldname prefix can be specified, in which case the prefix is contatenated to the existing fieldnames. Otherwise, a vector of new fieldnames can be specified (overrides the prefix). If compute_covariance=true, a block-diagonal covariance matrix is computed.
Definition at line 782 of file VecStatsCollector.cc.
References PLearn::TVec< T >::append(), compute_covariance, cov, fieldnames, PLearn::TMat< T >::fill(), i, PLearn::TMat< T >::length(), MISSING_VALUE, n, PLASSERT, PLearn::TVec< T >::resize(), setFieldNames(), PLearn::TVec< T >::size(), stats, PLearn::TMat< T >::subMat(), sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, sum_non_missing_weights, and PLearn::TMat< T >::width().
Referenced by remote_append().
{ // To avoid problems with fieldnames, ensure we don't start out with too // many fieldnames, and pad nonexistent fieldnames in *this with "" fieldnames.resize(stats.size()); for (int i=fieldnames.size(), n = stats.size() ; i<n ; ++i) fieldnames.append(""); stats.append(vsc.stats); // Take care of field names if (new_fieldnames.size() > 0) { PLASSERT( new_fieldnames.size() == vsc.stats.size() ); fieldnames.append(new_fieldnames); } else { const int n = vsc.stats.size(); PLASSERT(vsc.fieldnames.size() == n || n == 0); fieldnames.resize(fieldnames.size(), n); for (int i=0 ; i<n ; ++i) fieldnames.append(fieldname_prefix + vsc.fieldnames[i]); } setFieldNames(fieldnames); // update map // Take care of covariance matrix if (compute_covariance) { const int oldsize = cov.width(); const int vscsize = vsc.cov.width(); PLASSERT( oldsize == cov.length() && vscsize == vsc.cov.length() ); int new_n = stats.size(); Mat newcov(new_n, new_n, 0.0); Mat new_sum_cross(new_n, new_n, 0.0); Mat new_sum_cross_weights(new_n, new_n, 0.0); Mat new_sum_cross_square_weights(new_n, new_n, 0.0); newcov.subMat(0,0,oldsize,oldsize) << cov; Mat sub = new_sum_cross.subMat(0, 0, oldsize, oldsize); sub << sum_cross; sub = new_sum_cross_weights.subMat(0, 0, oldsize, oldsize); sub << sum_cross_weights; sub += sum_non_missing_weights; sum_non_missing_weights = 0; sub = new_sum_cross_square_weights.subMat(0, 0, oldsize, oldsize); sub << sum_cross_square_weights; sub += sum_non_missing_square_weights; sum_non_missing_square_weights = 0; if (vsc.compute_covariance) { newcov.subMat(oldsize,oldsize,vscsize,vscsize) << vsc.cov; sub = new_sum_cross.subMat(oldsize,oldsize,vscsize,vscsize); sub << vsc.sum_cross; sub = new_sum_cross_weights.subMat(oldsize,oldsize,vscsize,vscsize); sub << vsc.sum_cross_weights; sub += vsc.sum_non_missing_weights; sub = new_sum_cross_square_weights.subMat(oldsize,oldsize,vscsize,vscsize); sub << vsc.sum_cross_square_weights; sub += vsc.sum_non_missing_square_weights; } else { newcov.subMat(oldsize,oldsize,vscsize,vscsize).fill(MISSING_VALUE); new_sum_cross.subMat(oldsize, oldsize, vscsize, vscsize).fill(MISSING_VALUE); new_sum_cross_weights.subMat(oldsize,oldsize,vscsize,vscsize).fill(MISSING_VALUE); new_sum_cross_square_weights.subMat(oldsize,oldsize,vscsize,vscsize).fill(MISSING_VALUE); } cov = newcov; sum_cross = new_sum_cross; sum_cross_weights = new_sum_cross_weights; sum_cross_square_weights = new_sum_cross_square_weights; } }
void PLearn::VecStatsCollector::build | ( | ) | [virtual] |
simply calls inherited::build() then build_()
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 581 of file VecStatsCollector.cc.
References PLearn::Object::build(), and build_().
Referenced by PLearn::LiftStatsCollector::build(), PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), PLearn::computeMeanAndCovar(), PLearn::computeMeanAndVariance(), PLearn::computeWeightedMeanAndCovar(), PLearn::RemoveObservationTest::perform(), and PLearn::LearnerCommand::test().
{ inherited::build(); build_(); }
void PLearn::VecStatsCollector::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 565 of file VecStatsCollector.cc.
References PLearn::PP< T >::isNull(), m_observation_window, m_window, and PLERROR.
Referenced by build().
{ if(m_window > 0 || m_window == -2) { if ( m_observation_window.isNull() ) m_observation_window = new ObservationWindow(m_window); else { m_observation_window->m_window = m_window; m_observation_window->forget(); } } if( m_window_nan_code < 0 || m_window_nan_code > 2 ) PLERROR("The 'window_nan_code' option can only take values 0, 1 or 2."); }
string PLearn::VecStatsCollector::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
void PLearn::VecStatsCollector::declareMethods | ( | RemoteMethodMap & | rmm | ) | [static, protected] |
Declare the methods that are remote-callable.
Reimplemented from PLearn::Object.
Definition at line 193 of file VecStatsCollector.cc.
References PLearn::Object::_getRemoteMethodMap_(), PLearn::declareMethod(), forget(), getCorrelation(), getFieldNames(), getStat(), getStdDev(), getStdError(), getVariance(), getXtX(), PLearn::RemoteMethodMap::inherited(), length(), remote_append(), remote_getCovariance(), remote_getMean(), remote_update(), and setFieldNames().
{ // Insert a backpointer to remote methods; note that this // different than for declareOptions() rmm.inherited(inherited::_getRemoteMethodMap_()); declareMethod( rmm, "forget", &VecStatsCollector::forget, (BodyDoc("Clear all previously accumulated statistics.\n"))); declareMethod( rmm, "getStat", &VecStatsCollector::getStat, (BodyDoc("Returns a particular statistic of a particular cost.\n"), ArgDoc ("statspec", "A string that is standard statistics specification of the form ex: STAT[fieldname]\n" "or STAT[fieldnum] where STAT is one of the statistics names understood by\n" "StatsCollector::getStat. fieldnum start at 0, and fieldnames must have been\n" "registered with setFieldNames.\n"), RetDoc ("Requested statistic (a real number)."))); declareMethod( rmm, "getMean", &VecStatsCollector::remote_getMean, (BodyDoc("Return the mean of each field..\n"), RetDoc ("The vector of means for each field."))); declareMethod( rmm, "getVariance", &VecStatsCollector::getVariance, (BodyDoc("Return the vector of variances of all field..\n"), RetDoc ("The vector of variance for each field."))); declareMethod( rmm, "getStdDev", &VecStatsCollector::getStdDev, (BodyDoc("Return the vector of standard deviations of all field..\n"), RetDoc ("The vector of standard deviation for each field."))); declareMethod( rmm, "getStdError", &VecStatsCollector::getStdError, (BodyDoc("Return the vector of standard error of all field..\n"), RetDoc ("The vector of standard error for each field."))); declareMethod( rmm, "getXtX", &VecStatsCollector::getXtX, (BodyDoc(""), RetDoc ("Return the matrix XtX "))); declareMethod( rmm, "getCovariance", &VecStatsCollector::remote_getCovariance, (BodyDoc(""), RetDoc ("Returns the (centered) covariance matrix"))); declareMethod( rmm, "getCorrelation", &VecStatsCollector::getCorrelation, (BodyDoc(""), RetDoc ("Returns the correlation matrix"))); declareMethod( rmm, "setFieldNames", &VecStatsCollector::setFieldNames, (BodyDoc("Set field names.\n"), ArgDoc ("fieldnames", "A vector of strings corresponding to the names of each field" " in the VecStatsCollector.\n"))); declareMethod( rmm, "getFieldNames", &VecStatsCollector::getFieldNames, (BodyDoc("Get field names.\n"))); declareMethod( rmm, "length", &VecStatsCollector::length, (BodyDoc("Returns the number of statistics collected.\n"), RetDoc ("=stats.length()"))); declareMethod( rmm, "update", &VecStatsCollector::remote_update, (BodyDoc("Update the stats with gived data.\n"), ArgDoc ("x"," the new data\n"), ArgDoc ("weight"," the weight of the data"))); declareMethod( rmm, "append", &VecStatsCollector::remote_append, (BodyDoc("Appends all the StatsCollectors of an " "existing VecStatsCollector into this one.\n"), ArgDoc ("vsc","the other VecStatsCollector\n"), ArgDoc ("fieldname_prefix","prefix concatenated " "to the existing field names\n"), ArgDoc ("new_fieldnames","new name for appended fields (overrides prefix)\n"))); }
void PLearn::VecStatsCollector::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 73 of file VecStatsCollector.cc.
References PLearn::OptionBase::buildoption, compute_covariance, cov, PLearn::declareOption(), PLearn::Object::declareOptions(), epsilon, fieldnames, PLearn::OptionBase::learntoption, m_full_update_frequency, m_observation_window, m_window, m_window_nan_code, maxnvalues, no_removal_warnings, PLearn::OptionBase::nosave, PLearn::OptionBase::remotetransmit, stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, and sum_non_missing_weights.
Referenced by PLearn::LiftStatsCollector::declareOptions().
{ declareOption( ol, "maxnvalues", &VecStatsCollector::maxnvalues, OptionBase::buildoption, "Maximum number of different values to keep track of for each element.\n" "If -1, we will keep track of all different values.\n" "If 0, we will only keep track of global statistics.\n"); declareOption( ol, "fieldnames", &VecStatsCollector::fieldnames, OptionBase::buildoption, "Names of the fields of the vector"); declareOption( ol, "compute_covariance", &VecStatsCollector::compute_covariance, OptionBase::buildoption, "Should we compute and keep the covariance X'X ?"); declareOption( ol, "epsilon", &VecStatsCollector::epsilon, OptionBase::buildoption, "Small regularizing value to be added to the covariance matrix\n" "estimator, and forwarded to the enclosed vector of StatsCollector.\n" "This permits dividing by the standard deviation to perform a\n" "normalization, without fearing a division by zero.\n"); declareOption( ol, "window", &VecStatsCollector::m_window, OptionBase::buildoption, "If positive, the window restricts the stats computed by this\n" "VecStatsCollector to the last 'window' observations. This uses the\n" "VecStatsCollector::remove_observation mechanism.\n" "Default: -1 (all observations are considered);\n" " -2 means all observations kept in an ObservationWindow\n"); declareOption( ol, "full_update_frequency", &VecStatsCollector::m_full_update_frequency, OptionBase::buildoption, "If the window mechanism is used, number of updates at which a full\n" "update of the underlying StatsCollector is performed. A 'full update'\n" "is defined as:\n" "\n" "- 1. Calling forget()\n" "- 2. Updating the StatsCollector from all observations in the window.\n" "\n" "This is useful for two reasons: 1) when performing a remove-observation\n" "on a StatsCollector that contains a wide range of values, the\n" "accumulators for the fourth power may become negative, yielding\n" "inconsistent estimation. 2) without this option, the statistics\n" "'FIRST', 'LAST', 'MIN', 'MAX' are not updated properly in the presence\n" "of a window. To get proper estimation of these statistics, you must\n" "use the setting 'full_update_frequency=1'.\n" "\n" "Default value: -1 (never re-update the StatsCollector from scratch).\n"); declareOption( ol, "window_nan_code", &VecStatsCollector::m_window_nan_code, OptionBase::buildoption, "How to deal with update vectors containing NaNs with respect to the\n" "window mechanism.\n" "\n" "- 0: Do not check for NaNs (all updates are accounted in the window)\n" "- 1: If *all* entries of the update vector are NaNs, do not account for\n" " that observation in the window.\n" "- 2: If *any* entries of the update vector are NaNs, do not account for\n" " that observation in the window.\n" "\n" " Default: 0" ); declareOption( ol, "no_removal_warnings", &VecStatsCollector::no_removal_warnings, OptionBase::buildoption, "If the remove_observation mechanism is used (without\n" "'full_update_frequency=1') and the removed value is equal to one of\n" "first_, last_, min_ or max_, the default behavior is to warn the user.\n" "\n" "To disable this feature, set 'no_removal_warnings' to true.\n" "\n" "Default: false (0)." ); declareOption( ol, "stats", &VecStatsCollector::stats, OptionBase::learntoption, "the stats for each element"); declareOption( ol, "cov", &VecStatsCollector::cov, OptionBase::learntoption, "The uncentered and unnormalized covariance matrix (mean not subtracted): X'X"); declareOption( ol, "sum_cross", &VecStatsCollector::sum_cross, OptionBase::learntoption, "Element (i,j) is equal to the (weighted) sum of x_i when both x_i and x_j were observed"); declareOption( ol, "sum_cross_weights", &VecStatsCollector::sum_cross_weights, OptionBase::learntoption, "Element (i,j) is the sum of weights when both x_i and x_j were observed\n" "(only used when 'compute_covariance' is set to 1)\n"); declareOption( ol, "sum_cross_square_weights", &VecStatsCollector::sum_cross_square_weights, OptionBase::learntoption, "Element (i,j) is the sum of square weights when both x_i and x_j were observed\n" "(only used when 'compute_covariance' is set to 1)\n"); declareOption( ol, "sum_non_missing_weights", &VecStatsCollector::sum_non_missing_weights, OptionBase::learntoption, "Sum of weights for vectors with no missing value."); declareOption( ol, "sum_non_missing_square_weights", &VecStatsCollector::sum_non_missing_square_weights, OptionBase::learntoption, "Sum of square weights for vectors with no missing value."); declareOption( ol, "observation_window", &VecStatsCollector::m_observation_window, OptionBase::learntoption | OptionBase::nosave | OptionBase::remotetransmit, "The observation window itself."); // Now call the parent class' declareOptions inherited::declareOptions(ol); }
static const PPath& PLearn::VecStatsCollector::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 306 of file VecStatsCollector.h.
:
VecStatsCollector * PLearn::VecStatsCollector::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
void PLearn::VecStatsCollector::finalize | ( | ) | [virtual] |
finishes whatever computation are needed after all updates have been made
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 622 of file VecStatsCollector.cc.
References i, n, PLearn::TVec< T >::size(), and stats.
Referenced by PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), PLearn::computeStats(), PLearn::LiftStatsCollector::finalize(), and PLearn::NormalizationLearner::train().
void PLearn::VecStatsCollector::forget | ( | ) | [virtual] |
clears all previously accumulated statistics
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 587 of file VecStatsCollector.cc.
References cov, m_num_incremental, m_observation_window, m_window, PLearn::TMat< T >::resize(), PLearn::TVec< T >::resize(), stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, and sum_non_missing_weights.
Referenced by PLearn::BaggingLearner::computeOutput(), PLearn::computeStats(), declareMethods(), PLearn::PCA::forget(), PLearn::LiftStatsCollector::forget(), PLearn::RemoveObservationTest::perform(), remove_observation(), PLearn::DeepReconstructorNet::trainHiddenLayer(), PLearn::DeepReconstructorNet::trainSupervisedLayer(), and update().
{ stats.resize(0); cov.resize(0,0); sum_cross.resize(0,0); sum_cross_weights.resize(0,0); sum_cross_square_weights.resize(0,0); sum_non_missing_weights = 0; sum_non_missing_square_weights = 0; // Window mechanism m_num_incremental = 0; if (m_window > 0 || m_window == -2) m_observation_window->forget(); }
Fills vector st with [mean, variance, stddev, min, max] (after resizing it if it had a size of 0) However the order and number may change in future versions, so it's better to first call getIndexInAllStats to get the index of a given stat.
Referenced by PLearn::RemoveObservationTest::compareStats(), and getAllStats().
Vec PLearn::VecStatsCollector::getAllStats | ( | const string & | statname | ) | const |
Call getStat() with the given statname on all the statscollectors.
Definition at line 765 of file VecStatsCollector.cc.
References getAllStats(), n, PLearn::TVec< T >::size(), and stats.
{ const int n = stats.size(); Vec r(n); getAllStats(statname, r); return r; }
void PLearn::VecStatsCollector::getAllStats | ( | const string & | statname, |
Vec & | result | ||
) | const |
Call getStat() with the given statname on all the statscollectors, and put result in given Vec.
The vector is resized as necessary.
Definition at line 773 of file VecStatsCollector.cc.
References PLearn::StatsCollector::getStat(), getStats(), i, n, PLearn::TVec< T >::resize(), PLearn::TVec< T >::size(), and stats.
{ const int n = stats.size(); result.resize(n); for (int i=0; i<n; ++i) result[i] = getStats(i).getStat(statname); }
Mat PLearn::VecStatsCollector::getCorrelation | ( | ) | const |
returns correlation matrix
Definition at line 758 of file VecStatsCollector.cc.
References cov, PLearn::externalProduct(), getCovariance(), getStdDev(), PLearn::norm(), and PLearn::TMat< T >::width().
Referenced by declareMethods().
{ Mat norm(cov.width(),cov.width()); externalProduct(norm,getStdDev(),getStdDev()); return getCovariance()/norm; }
Mat PLearn::VecStatsCollector::getCovariance | ( | ) | const |
Definition at line 748 of file VecStatsCollector.cc.
References PLearn::covariance().
Referenced by getCorrelation().
{ Mat covariance; getCovariance(covariance); return covariance; }
void PLearn::VecStatsCollector::getCovariance | ( | Mat & | covar | ) | const |
Covariance matrix computation.
Note that the covariance is computed in order to give an unbiased estimator (under the i.i.d. assumption), so that the normalization coefficient is not exactly the sum of weights.
Definition at line 679 of file VecStatsCollector.cc.
References compute_covariance, cov, d, epsilon, PLearn::fast_exact_is_equal(), PLearn::fast_is_equal(), getMean(), i, j, PLearn::TMat< T >::length(), MISSING_VALUE, PLASSERT, PLearn::TMat< T >::resize(), PLearn::sqrt(), stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, sum_non_missing_weights, and PLearn::TMat< T >::width().
Referenced by PLearn::RemoveObservationTest::compareCovariance(), PLearn::computeMeanAndCovar(), PLearn::computeWeightedMeanAndCovar(), and PLearn::PCA::incremental_algo().
{ // Formula used to compute an unbiased estimate of the covariance (note // that it may not yield a positive semi-definite matrix). // Notations: // x(k)_i = i-th coordinate of k-th sample // sum^i_k f(i,k) = sum over k of f(i,k) for k such that // x(k)_i is not missing // sum^i,j_k f(i,j,k) = sum over k of f(i,j,k) for k such that // neither x(k)_i nor x(k)_j is missing // w(k) = weight of k-th sample // cov_i_j = sum^i,j_k w(k) x(k)_i * x(k)_j // mean_i = (sum^i_k w(k) x(k)_i) / sum^i_k w(k) // The estimator for element (i,j) of the covariance matrix is then: // covariance(i,j) = [ cov_i_j + mean_i * mean_j * sum^i,j_k w(k) // - mean_j * sum^i,j_k w(k) x(k)_i // - mean_i * sum^i,j_k w(k) x(k)_j // ] / // [ sum^i,j_k w(k) // + (sum^i,j_k w(k) - sum^i_k w(k) - sum^j_k w(k) ) // * sum^i,j_k w(k)^2 / (sum^i_k w(k) * sum^j_k w(k)) // ] // // If features i and j have never been observed simultaneously, or one of // the two features has been observed only once, then covariance(i,j) is // set to MISSING_VALUE. // The first case occurs when sum^i,j_k w(k) == 0. // The second case occurs when sum^i,j_k w(k) == sqrt(sum^i,j_k w(k)^2) // == sum^{i or j}_k w(k) Vec meanvec; PLASSERT( compute_covariance && cov.length() == cov.width() ); int d = cov.length(); getMean(meanvec); covar.resize(d,d); for(int i=0; i<d; i++) { real sum_weights_i = stats[i].nnonmissing(); for(int j=i; j<d; j++) { real sum_weights_j = stats[j].nnonmissing(); real sum_cross_weights_i_j = sum_cross_weights(i,j) + sum_non_missing_weights; real sum_cross_square_weights_i_j = sum_cross_square_weights(i,j) + sum_non_missing_square_weights; real mean_i = meanvec[i]; real mean_j = meanvec[j]; if (fast_exact_is_equal(sum_cross_weights_i_j, 0) || (fast_is_equal(sum_cross_weights_i_j, sqrt(sum_cross_square_weights_i_j)) && (fast_is_equal(sum_cross_weights_i_j, sum_weights_i) || fast_is_equal(sum_cross_weights_i_j, sum_weights_j)))) // One of the two cases described above. covar(i,j) = MISSING_VALUE; else covar(i,j) = (cov(i,j) + mean_i * mean_j * sum_cross_weights_i_j - mean_j * sum_cross(i,j) - mean_i * sum_cross(j,i) ) / ( sum_cross_weights_i_j + (sum_cross_weights_i_j - sum_weights_i - sum_weights_j) * sum_cross_square_weights_i_j / (sum_weights_i * sum_weights_j) ); if (j == i) covar(i,j) += epsilon; else covar(j,i) = covar(i,j); } } }
TVec<string> PLearn::VecStatsCollector::getFieldNames | ( | ) | const [inline] |
Returns the declared names.
Definition at line 188 of file VecStatsCollector.h.
Referenced by declareMethods(), and PLearn::StatsCommand::run().
{ return fieldnames; }
int PLearn::VecStatsCollector::getFieldNum | ( | const string & | fieldname_or_num | ) | const |
Returns the index corresponding to a fieldname or to the fieldnum passed as a string.
returns -1 if not found
Definition at line 333 of file VecStatsCollector.cc.
References fieldnames_num, PLearn::pl_isnumber(), and PLearn::toint().
Referenced by PLearn::LiftStatsCollector::build_(), and getStat().
{ map<string,int>::const_iterator it = fieldnames_num.find(fieldname_or_num); if (it == fieldnames_num.end()) { // not found if (pl_isnumber(fieldname_or_num)) return toint(fieldname_or_num); else return -1; // unknown field } else return it->second; }
Returns the index in the vector returned by getAllStats of the stat with the given name.
Currently available names are E (mean) V (variance) STDDEV MIN MAX Will throw an exception if statname is invalid
Vec PLearn::VecStatsCollector::getMean | ( | ) | const [inline] |
returns the empirical mean (sample average) vec
Definition at line 222 of file VecStatsCollector.h.
References PLearn::mean().
Referenced by PLearn::computeInputMean(), PLearn::computeMeanAndCovar(), PLearn::computeMeanAndVariance(), PLearn::computeWeightedMean(), PLearn::computeWeightedMeanAndCovar(), getCovariance(), PLearn::PCA::incremental_algo(), main(), PLearn::Experiment::run(), PLearn::NormalizationLearner::train(), PLearn::DeepReconstructorNet::trainHiddenLayer(), and PLearn::DeepReconstructorNet::trainSupervisedLayer().
void PLearn::VecStatsCollector::getMean | ( | Vec & | mean | ) | const |
Store the empirical mean in the given vec (which is resized)
Definition at line 632 of file VecStatsCollector.cc.
References PLearn::mean(), n, PLearn::TVec< T >::resize(), PLearn::TVec< T >::size(), and stats.
const Mat & PLearn::VecStatsCollector::getObservations | ( | ) | const |
Definition at line 312 of file VecStatsCollector.cc.
References m_observation_window, m_window, and PLASSERT.
{ PLASSERT( m_window > 0 ); return m_observation_window->m_observations; }
const PP< ObservationWindow > PLearn::VecStatsCollector::getObservationWindow | ( | ) | const |
Definition at line 319 of file VecStatsCollector.cc.
References m_observation_window, m_window, and PLASSERT.
{ PLASSERT( m_window > 0 ); return m_observation_window; }
OptionList & PLearn::VecStatsCollector::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
OptionMap & PLearn::VecStatsCollector::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
RemoteMethodMap & PLearn::VecStatsCollector::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 71 of file VecStatsCollector.cc.
double PLearn::VecStatsCollector::getStat | ( | const string & | statspec | ) | [virtual] |
Returns a particular statistic.
Standard statistics specifications are of the form ex: STAT[fieldname] or STAT[fieldnum] where STAT is one of the statistics names understood by StatsCollector::getStat. fieldnum start at 0, and fieldnames must have been registered with setFieldNames. Subclasses may overload this to handle more exotic statistics than the few standard ones.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 286 of file VecStatsCollector.cc.
References fieldnames, getFieldNum(), PLearn::StatsCollector::getStat(), getStats(), in, PLearn::TVec< T >::length(), MISSING_VALUE, PLearn::openString(), PLearn::PStream::plearn_ascii, PLERROR, PLearn::PStream::smartReadUntilNext(), stats, and PLearn::tostring().
Referenced by declareMethods(), and PLearn::LiftStatsCollector::getStat().
{ PStream in = openString(statspec,PStream::plearn_ascii); string statname; in.smartReadUntilNext("[", statname); string fieldname; in.smartReadUntilNext("]", fieldname); if(fieldname.empty()) PLERROR("In VecStatsCollector::getStat - the stat asked is invalid." "Parsed stat name '%s' with an empty field name.", statname.c_str()); int fieldnum = getFieldNum(fieldname); if(fieldnum<0) PLERROR("In VecStatsCollector::getStat invalid fieldname: %s;\n" "valid fieldnames are: %s",fieldname.c_str(), tostring(fieldnames).c_str()); // It could be that nothing was accumulated into the stats collector, // which is different from accessing the "wrong" field. In the first // case, return MISSING_VALUE if (stats.length() == 0) return MISSING_VALUE; return getStats(fieldnum).getStat(statname); }
StatsCollector& PLearn::VecStatsCollector::getStats | ( | int | i | ) | [inline] |
returns non-const statistics for element i
Definition at line 219 of file VecStatsCollector.h.
References i.
const StatsCollector& PLearn::VecStatsCollector::getStats | ( | int | i | ) | const [inline] |
returns statistics for element i
Definition at line 215 of file VecStatsCollector.h.
References i.
Referenced by PLearn::BaggingLearner::computeOutput(), PLearn::computeRange(), getAllStats(), getStat(), PLearn::StatsCommand::run(), and PLearn::DeepReconstructorNet::trainSupervisedLayer().
Vec PLearn::VecStatsCollector::getStdDev | ( | ) | const |
returns the empirical standard deviation vec
Definition at line 655 of file VecStatsCollector.cc.
References epsilon, n, PLearn::TVec< T >::size(), PLearn::sqrt(), stats, and PLearn::variance().
Referenced by declareMethods(), getCorrelation(), and PLearn::Experiment::run().
{ int n = stats.size(); Vec res(n); for(int k=0; k<n; k++) res[k] = sqrt(stats[k].variance() + epsilon); return res; }
Vec PLearn::VecStatsCollector::getStdError | ( | ) | const |
returns the empirical standard deviation vec
Definition at line 667 of file VecStatsCollector.cc.
References n, PLearn::TVec< T >::size(), and stats.
Referenced by declareMethods(), PLearn::Experiment::run(), PLearn::DeepReconstructorNet::trainHiddenLayer(), and PLearn::DeepReconstructorNet::trainSupervisedLayer().
{ int n = stats.size(); Vec res(n); for(int k=0; k<n; k++) res[k] = stats[k].stderror(); return res; }
Vec PLearn::VecStatsCollector::getVariance | ( | ) | const |
returns the empirical variance vec
Definition at line 643 of file VecStatsCollector.cc.
References epsilon, n, PLearn::TVec< T >::size(), stats, and PLearn::variance().
Referenced by PLearn::computeMeanAndVariance(), and declareMethods().
{ int n = stats.size(); Vec res(n); for(int k=0; k<n; k++) res[k] = stats[k].variance() + epsilon; return res; }
const Mat& PLearn::VecStatsCollector::getXtX | ( | ) | const [inline] |
Return X'X (note that this matrix is weighted, and the weight might be different for each element if there were missing values observed).
Definition at line 246 of file VecStatsCollector.h.
Referenced by declareMethods().
{ return cov; }
int PLearn::VecStatsCollector::length | ( | ) | const [virtual] |
Definition at line 281 of file VecStatsCollector.cc.
References PLearn::TVec< T >::length(), and stats.
Referenced by PLearn::RemoveObservationTest::compareCovariance(), PLearn::RemoveObservationTest::compareStats(), and declareMethods().
{ return stats.length(); }
void PLearn::VecStatsCollector::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 924 of file VecStatsCollector.cc.
References cov, PLearn::deepCopyField(), fieldnames, m_observation_window, PLearn::Object::makeDeepCopyFromShallowCopy(), stats, sum_cross, sum_cross_square_weights, and sum_cross_weights.
Referenced by PLearn::LiftStatsCollector::makeDeepCopyFromShallowCopy().
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(fieldnames, copies); deepCopyField(stats, copies); deepCopyField(cov, copies); deepCopyField(sum_cross, copies); deepCopyField(sum_cross_weights, copies); deepCopyField(sum_cross_square_weights, copies); deepCopyField(m_observation_window, copies); }
void PLearn::VecStatsCollector::merge | ( | VecStatsCollector & | other | ) | [virtual] |
merges another VecStatsCollector into this one
Definition at line 860 of file VecStatsCollector.cc.
References compute_covariance, cov, fieldnames, PLearn::TMat< T >::fill(), i, j, PLearn::TMat< T >::length(), PLearn::TVec< T >::length(), m_observation_window, m_window, maxnvalues, n, no_removal_warnings, PLASSERT_MSG, PLearn::TMat< T >::resize(), PLearn::TVec< T >::resize(), PLearn::TVec< T >::size(), stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, sum_non_missing_weights, update(), and PLearn::TMat< T >::width().
{ PLASSERT_MSG(fieldnames == other.fieldnames, "VecStatsCollector::merge : cannot merge VecStatsCollectors with different fieldnames."); if(stats.size()==0)//if this one is empty, resize stats before merging { int n= other.stats.size(); stats.resize(n); for(int k=0; k<n; k++) { // TODO It would be cool to have a simple (or automatic) mechanism // to be able to specify a different value of 'maxnvalues' for each // StatsCollector (e.g. when only one StatsCollector is meant to // compute a lift statistics). stats[k].maxnvalues = maxnvalues; stats[k].no_removal_warnings = no_removal_warnings; stats[k].forget(); } if(compute_covariance) { cov.resize(n,n); sum_cross.resize(n,n); sum_cross_weights.resize(n,n); sum_cross_square_weights.resize(n,n); cov.fill(0); sum_cross.fill(0); sum_cross_weights.fill(0); sum_cross_square_weights.fill(0); } } PLASSERT_MSG(stats.length() == other.stats.length(), "VecStatsCollector::merge : cannot merge VecStatsCollectors with different stats length."); if(m_window != 0 || other.m_window != 0) { PLASSERT_MSG(m_window != 0 && other.m_window != 0, "VecStatsCollector::merge : either none or both should have an observation window"); PP<ObservationWindow> oow= other.m_observation_window; for(int i= 0; i < oow->length(); ++i) update(oow->getObs(i), oow->getWeight(i)); return; // avoid extra indentation } for(int i= 0; i < stats.length(); ++i) stats[i].merge(other.stats[i]); if(compute_covariance) { for(int i= 0; i < cov.length(); ++i) for(int j= 0; j < cov.width(); ++j) { cov(i,j)+= other.cov(i,j); sum_cross(i,j)+= other.sum_cross(i,j); sum_cross_weights(i,j)+= other.sum_cross_weights(i,j); sum_cross_square_weights(i,j)+= other.sum_cross_square_weights(i,j); } sum_non_missing_weights+= other.sum_non_missing_weights; sum_non_missing_square_weights+= other.sum_non_missing_square_weights; } }
void PLearn::VecStatsCollector::remote_append | ( | const VecStatsCollector * | vsc, |
const string | fieldname_prefix, | ||
const TVec< string > & | new_fieldnames | ||
) |
remote version of append: takes pointer to other VecStatsCollector
Definition at line 853 of file VecStatsCollector.cc.
References append().
Referenced by declareMethods().
{ append(*vsc, fieldname_prefix, new_fieldnames); }
Mat PLearn::VecStatsCollector::remote_getCovariance | ( | ) | [inline] |
Remote version of getCovariance.
Definition at line 257 of file VecStatsCollector.h.
Referenced by declareMethods().
{ return getCovariance(); }
Vec PLearn::VecStatsCollector::remote_getMean | ( | ) | [inline] |
Remote version of getMean.
Definition at line 229 of file VecStatsCollector.h.
Referenced by declareMethods().
{ return getMean(); }
updates the statistics when seeing x The weight applies to all elements of x
Definition at line 457 of file VecStatsCollector.cc.
References update().
Referenced by declareMethods().
Update statistics as if the vectorial observation x was removed of the observation sequence.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 484 of file VecStatsCollector.cc.
References compute_covariance, cov, PLearn::externalProductScaleAcc(), PLearn::fast_exact_is_equal(), forget(), PLearn::TVec< T >::hasMissing(), i, PLearn::is_missing(), j, n, PLASSERT, PLERROR, PLearn::TVec< T >::size(), stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, and sum_non_missing_weights.
Referenced by PLearn::PCA::incremental_algo(), and update().
{ PLASSERT( stats.size() > 0 ); int n = x.size(); if(stats.size()!=n) PLERROR( "In VecStatsCollector: problem, called remove_observation with vector of length %d, " "while size of stats (and most likeley previously seen vector) is %d", n, stats.size() ); for(int k=0; k<n; k++) { real obs = x[k]; stats[k].remove_observation(obs, weight); //TBA: if ( is_equal(obs, stats[k].min_) ) //TBA: m_observation_window->columnMin(k, stats[k].min_, stats[k].agemin_); //TBA: if ( is_equal(obs, stats[k].max_) ) //TBA: m_observation_window->columnMax(k, stats[k].max_, stats[k].agemax_); } // This removes the observation x contribution to the covariance matrix. if( compute_covariance ) { if (fast_exact_is_equal(stats[0].nnonmissing(), 0)) { // We removed the last observation. It may be safer to reset everything // so that numerical approximations do not lead to negative values for // statistics that should always be positive. forget(); } else { if (x.hasMissing()) { // Slower version to handle missing values. // TODO Could certainly be optimized. real val_i, val_j; for (int i = 0; i < n; i++) { val_i = x[i]; if (!is_missing(val_i)) { for (int j = 0; j < n; j++) { val_j = x[j]; if (!is_missing(val_j)) { cov(i,j) -= weight * val_i * val_j; sum_cross(i,j) -= weight * val_i; sum_cross_weights(i,j) -= weight; sum_cross_square_weights(i,j) -= weight * weight; } } } } } else { externalProductScaleAcc(cov, x, x, -weight); sum_non_missing_weights -= weight; sum_non_missing_square_weights -= weight * weight; // TODO The two lines below could be optimized with an additional Vec // storing the sum of weighted x_i for non missing data. for (int i = 0; i < n; i++) sum_cross(i) -= weight * x[i]; } } } }
void PLearn::VecStatsCollector::setFieldNames | ( | TVec< string > | the_fieldnames | ) |
Declares names for the columns of the vector passed to update.
Definition at line 325 of file VecStatsCollector.cc.
References PLearn::TVec< T >::copy(), fieldnames, fieldnames_num, i, n, and PLearn::TVec< T >::size().
Referenced by append(), PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), PLearn::computeStats(), and declareMethods().
{ fieldnames = the_fieldnames.copy(); fieldnames_num.clear(); for (int i=0, n=fieldnames.size() ; i<n ; ++i) fieldnames_num[fieldnames[i]] = i; }
void PLearn::VecStatsCollector::setWindowSize | ( | int | sz | ) | [virtual] |
sets the size of the observation window
Definition at line 603 of file VecStatsCollector.cc.
References m_observation_window, and m_window.
{ m_window= sz; if(m_window > 0 || m_window == -2) { if(!m_observation_window) m_observation_window = new ObservationWindow(m_window); else { if(sz == -2) m_observation_window->unlimited_size= true; m_observation_window->m_window= sz; m_observation_window->forget(); } } }
Handling m_window_nan_code.
Definition at line 462 of file VecStatsCollector.cc.
References PLearn::TVec< T >::begin(), PLearn::TVec< T >::end(), PLearn::is_missing(), PLearn::TVec< T >::length(), and m_window_nan_code.
Referenced by update().
{ // Avoid dealing with missings if not necessary if ( m_window_nan_code > 0 ) { int count = 0; Vec::iterator it = x.begin(); Vec::iterator itend = x.end(); for(; it!=itend; ++it) if(is_missing(*it)) count++; if ( (m_window_nan_code == 1 && count == x.length()) || (m_window_nan_code == 2 && count > 0) ) return false; } return true; }
int PLearn::VecStatsCollector::size | ( | ) | const [inline] |
Definition at line 159 of file VecStatsCollector.h.
Referenced by PLearn::BaggingLearner::computeOutput(), and PLearn::StatsCommand::run().
{ return length(); }
calls update on all rows of m; vector of weights given, weighting each row
calls update on all rows of m, with given weight vector
Definition at line 554 of file VecStatsCollector.cc.
References i, PLearn::TMat< T >::length(), m, PLERROR, PLearn::TVec< T >::size(), and update().
{ if (m.length() != weights.size()) PLERROR("VecStatsCollector::update: matrix height (%d) " "is incompatible with weights length (%d)", m.length(), weights.size()); int l = m.length(); for(int i=0; i<l; i++) update(m(i), weights[i]); }
updates the statistics when seeing x The weight applies to all elements of x
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 350 of file VecStatsCollector.cc.
References compute_covariance, cov, epsilon, PLearn::externalProductScaleAcc(), PLearn::TMat< T >::fill(), forget(), PLearn::TVec< T >::hasMissing(), i, PLearn::is_missing(), PLearn::TVec< T >::isNotEmpty(), j, m_full_update_frequency, m_num_incremental, m_observation_window, m_window, maxnvalues, n, no_removal_warnings, PLERROR, remove_observation(), PLearn::TMat< T >::resize(), PLearn::TVec< T >::resize(), shouldUpdateWindow(), PLearn::TVec< T >::size(), stats, sum_cross, sum_cross_square_weights, sum_cross_weights, sum_non_missing_square_weights, sum_non_missing_weights, and w.
Referenced by PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), PLearn::computeInputMean(), PLearn::computeMeanAndCovar(), PLearn::computeMeanAndVariance(), PLearn::BaggingLearner::computeOutput(), PLearn::computeRange(), PLearn::computeStats(), PLearn::computeWeightedMean(), PLearn::computeWeightedMeanAndCovar(), PLearn::PCA::incremental_algo(), merge(), PLearn::OnlineGramNaturalGradientOptimizer::optimizeN(), PLearn::GradientOptimizer::optimizeN(), PLearn::ConjGradientOptimizer::optimizeN(), PLearn::AutoScaledGradientOptimizer::optimizeN(), PLearn::AdaptGradientOptimizer::optimizeN(), PLearn::RemoveObservationTest::perform(), remote_update(), PLearn::Experiment::run(), PLearn::NormalizationLearner::train(), update(), and PLearn::LiftStatsCollector::update().
{ int n = x.size(); if(stats.size()==0) { stats.resize(n); for(int k=0; k<n; k++) { // TODO It would be cool to have a simple (or automatic) mechanism // to be able to specify a different value of 'maxnvalues' for each // StatsCollector (e.g. when only one StatsCollector is meant to // compute a lift statistics). stats[k].epsilon = epsilon; stats[k].maxnvalues = maxnvalues; stats[k].no_removal_warnings = no_removal_warnings; stats[k].forget(); } if(compute_covariance) { cov.resize(n,n); sum_cross.resize(n,n); sum_cross_weights.resize(n,n); sum_cross_square_weights.resize(n,n); cov.fill(0); sum_cross.fill(0); sum_cross_weights.fill(0); sum_cross_square_weights.fill(0); } } if(stats.size()!=n) PLERROR("In VecStatsCollector::update - Called update with vector of length " "%d, while size of stats (and most likely previously seen vector) is " "%d", n, stats.size()); // Update the underlying StatsCollectors. If we use the window mechanism // and we are at a boundary given by m_full_update_frequency, perform a // full re-update of the StatsCollectors from the saved observations in the // window. if ((m_window > 0 || m_window == -2) && m_full_update_frequency > 0 && ++m_num_incremental >= m_full_update_frequency) { for (int k=0 ; k<n ; ++k) stats[k].forget(); // Drop oldest observation in window to make room for new observation: // start at t=1 for (int t=1 ; t<m_observation_window->length() ; ++t) { Vec obs = m_observation_window->getObs(t); real w = m_observation_window->getWeight(t); for (int k=0 ; k<n ; ++k) stats[k].update(obs[k], w); } m_num_incremental = 0; } // Incremental update with current observation, as usual for(int k=0; k<n; ++k) stats[k].update(x[k], weight); // Compute covariance if required if(compute_covariance) { if (x.hasMissing()) { // Slower version to handle missing values. // TODO Could certainly be optimized. real val_i, val_j; for (int i = 0; i < n; i++) { val_i = x[i]; if (!is_missing(val_i)) { for (int j = 0; j < n; j++) { val_j = x[j]; if (!is_missing(val_j)) { cov(i,j) += weight * val_i * val_j; sum_cross(i,j) += weight * val_i; sum_cross_weights(i,j) += weight; sum_cross_square_weights(i,j) += weight * weight; } } } } } else { externalProductScaleAcc(cov, x, x, weight); sum_non_missing_weights += weight; sum_non_missing_square_weights += weight * weight; // TODO The two lines below could be optimized with an additional Vec // storing the sum of weighted x_i for non missing data. for (int i = 0; i < n; i++) sum_cross(i) += weight * x[i]; } } // Window mechanism if ( (m_window > 0 || m_window == -2) && shouldUpdateWindow(x) ) { tuple<Vec, real> outdated = m_observation_window->update(x, weight); Vec& obs = get<0>(outdated); real w = get<1>(outdated); // If m_num_incremental==0, we just re-updated the StatsCollectors from // scratch. In this case, don't call remove_observation. if(obs.isNotEmpty() && m_window > 0 && (m_full_update_frequency <= 0 || m_num_incremental > 0)) { remove_observation(obs, w); } } }
void PLearn::VecStatsCollector::update | ( | const Mat & | m | ) |
calls update on all rows of m; weight assumed to be 1.0 for all roes
calls update on all rows of m
Definition at line 546 of file VecStatsCollector.cc.
References i, PLearn::TMat< T >::length(), m, and update().
Reimplemented from PLearn::Object.
Reimplemented in PLearn::LiftStatsCollector.
Definition at line 306 of file VecStatsCollector.h.
Should we compute and keep X'.X ? (default false)
Definition at line 68 of file VecStatsCollector.h.
Referenced by append(), PLearn::PCA::build_(), PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), PLearn::computeInputMeanAndCovar(), PLearn::computeMeanAndCovar(), PLearn::computeWeightedMeanAndCovar(), declareOptions(), getCovariance(), merge(), PLearn::RemoveObservationTest::perform(), remove_observation(), and update().
See .cc for help.
Definition at line 142 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCorrelation(), getCovariance(), makeDeepCopyFromShallowCopy(), merge(), remove_observation(), and update().
Small regularizing value to be added to the covariance matrix estimator, and forwarded to the enclosed vector of StatsCollector.
This permits dividing by the standard deviation to perform a normalization, without fearing a division by zero.
Definition at line 76 of file VecStatsCollector.h.
Referenced by PLearn::computeInputMeanAndVariance(), PLearn::computeMeanAndCovar(), PLearn::computeMeanAndVariance(), PLearn::computeWeightedMeanAndCovar(), declareOptions(), getCovariance(), getStdDev(), getVariance(), and update().
TVec<string> PLearn::VecStatsCollector::fieldnames [protected] |
Names of the fields of the update vector; now protected: use setFieldNames to set them!
Definition at line 315 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), getStat(), makeDeepCopyFromShallowCopy(), merge(), and setFieldNames().
map<string,int> PLearn::VecStatsCollector::fieldnames_num [protected] |
Map from fieldnames to fieldnumbers, to really speed up getFieldNum which can be a speed bottleneck in some experiments.
Definition at line 306 of file VecStatsCollector.h.
Referenced by getFieldNum(), and setFieldNames().
If the window mechanism is used, number of updates at which a full update of the underlying StatsCollector is performed.
A 'full update' is defined as:
This is useful for two reasons: 1) when performing a remove-observation on a StatsCollector that contains a wide range of values, the accumulators for the fourth power may become negative, yielding inconsistent estimation. 2) without this option, the statistics 'FIRST', 'LAST', 'MIN', 'MAX' are not updated properly in the presence of a window. To get proper estimation of these statistics, you must use the setting 'full_update_frequency=1'.
Default value: -1 (never re-update the StatsCollector from scratch).
Definition at line 106 of file VecStatsCollector.h.
Referenced by declareOptions(), and update().
int PLearn::VecStatsCollector::m_num_incremental [protected] |
(Window mechanism) Number of incremental updates since the last update from scratch of the underlying statscollectors
Definition at line 322 of file VecStatsCollector.h.
Window mechanism.
Definition at line 318 of file VecStatsCollector.h.
Referenced by build_(), declareOptions(), forget(), getObservations(), getObservationWindow(), makeDeepCopyFromShallowCopy(), merge(), setWindowSize(), and update().
If positive, the window restricts the stats computed by this FinVecStatsCollector to the last 'window' observations.
This uses the VecStatsCollector::remove_observation mechanism; but see 'full_update_frequency' below.
Default: -1 (all observations are considered).
Definition at line 86 of file VecStatsCollector.h.
Referenced by build_(), declareOptions(), forget(), getObservations(), getObservationWindow(), merge(), setWindowSize(), and update().
How to deal with update vectors containing NaNs with respect to the window mechanism.
Default: 0
Definition at line 120 of file VecStatsCollector.h.
Referenced by declareOptions(), and shouldUpdateWindow().
Definition at line 65 of file VecStatsCollector.h.
Referenced by PLearn::DeepReconstructorNet::computeAndSaveLayerActivationStats(), declareOptions(), merge(), and update().
If the remove_observation mechanism is used (without 'full_update_frequency=1') and the removed value is equal to one of first_, last_, min_ or max_, the default behavior is to warn the user.
To disable this feature, set 'no_removal_warnings' to true.
Default: false (0).
Definition at line 131 of file VecStatsCollector.h.
Referenced by PLearn::PCA::build_(), declareOptions(), merge(), PLearn::RemoveObservationTest::perform(), and update().
the stats for each element
Definition at line 139 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), finalize(), forget(), getAllStats(), getCovariance(), getMean(), getStat(), getStdDev(), getStdError(), getVariance(), length(), makeDeepCopyFromShallowCopy(), merge(), remove_observation(), PLearn::NormalizationLearner::train(), and update().
Definition at line 144 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCovariance(), makeDeepCopyFromShallowCopy(), merge(), remove_observation(), and update().
Definition at line 146 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCovariance(), makeDeepCopyFromShallowCopy(), merge(), remove_observation(), and update().
Definition at line 145 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCovariance(), makeDeepCopyFromShallowCopy(), merge(), remove_observation(), and update().
Definition at line 148 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCovariance(), merge(), remove_observation(), and update().
Definition at line 147 of file VecStatsCollector.h.
Referenced by append(), declareOptions(), forget(), getCovariance(), merge(), remove_observation(), and update().