PLearn 0.1
|
#include <LiftStatsCollector.h>
Public Member Functions | |
LiftStatsCollector () | |
virtual void | forget () |
clears all previously accumulated statistics | |
virtual void | update (const Vec &x, real weight=1.0) |
updates the statistics when seeing x The weight applies to all elements of x | |
virtual void | finalize () |
This finalize override makes sure only the n_samples_to_keep samples from the matrix n_first_updates with the highest output are left. | |
virtual double | getStat (const string &statspec) |
In addition to the regular VecStatsCollector statistics, we understand specific lift statistics (see the .cc). | |
virtual void | remove_observation (const Vec &x, real weight=1.0) |
Overridden because it is not supported in this VecStatsCollector. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual LiftStatsCollector * | deepCopy (CopiesMap &copies) const |
virtual void | build () |
simply calls inherited::build() then build_() | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
string | lift_file |
real | lift_fraction |
bool | opposite_lift |
string | output_column |
string | roc_file |
Vec | roc_fractions |
int | sign_trick |
int | target_column |
int | verbosity |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
real | computeAUC () |
Return the AUC statistic. | |
real | computeLift () |
Return the LIFT statistic. | |
real | computeLiftMax () |
Return the LIFT_MAX statistic. | |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Protected Attributes | |
int | count_fin |
Vec | roc_values |
Mat | n_first_updates |
Matrix storing the output and target of the samples with highest output, as well as all the other data retrieved since the last call to finalize. | |
bool | is_finalized |
Set to true after each call to finalize(). | |
int | nstored |
Number of examples stored in the n_first_updates matrix. | |
int | nsamples |
Number of samples seen. | |
int | npos |
Number of positive examples that are not retained in the ones with the highest output (that is to say the ones in n_first_updates). | |
int | n_samples_to_keep |
Number of examples to keep (nsamples * lift_fraction). | |
int | output_column_index |
Index of the output column. | |
Private Types | |
typedef VecStatsCollector | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Definition at line 52 of file LiftStatsCollector.h.
typedef VecStatsCollector PLearn::LiftStatsCollector::inherited [private] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 57 of file LiftStatsCollector.h.
PLearn::LiftStatsCollector::LiftStatsCollector | ( | ) |
Definition at line 55 of file LiftStatsCollector.cc.
: inherited(), count_fin(0), is_finalized(false), nstored(0), nsamples(0), npos(0), output_column_index(0), lift_file(""), lift_fraction(0.1), opposite_lift(false), output_column(""), roc_file(""), sign_trick(0), target_column(1), verbosity(0) { }
string PLearn::LiftStatsCollector::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
OptionList & PLearn::LiftStatsCollector::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
RemoteMethodMap & PLearn::LiftStatsCollector::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
Object * PLearn::LiftStatsCollector::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
StaticInitializer LiftStatsCollector::_static_initializer_ & PLearn::LiftStatsCollector::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
void PLearn::LiftStatsCollector::build | ( | ) | [virtual] |
simply calls inherited::build() then build_()
Reimplemented from PLearn::VecStatsCollector.
Definition at line 153 of file LiftStatsCollector.cc.
References PLearn::VecStatsCollector::build(), and build_().
{ inherited::build(); build_(); }
void PLearn::LiftStatsCollector::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 162 of file LiftStatsCollector.cc.
References PLearn::VecStatsCollector::getFieldNum(), i, output_column, and output_column_index.
Referenced by build().
{ /* PLDEPRECATED("The 'LiftStatsCollector class is now deprecated: one should " "instead use a standard 'VecStatsCollector', that is now able" " to compute lift statistics"); */ if (output_column != "") { int i = this->getFieldNum(output_column); if (i >= 0) { output_column_index = i; } else { // Not found. output_column_index = 0; } } else { output_column_index = 0; } }
string PLearn::LiftStatsCollector::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
real PLearn::LiftStatsCollector::computeAUC | ( | ) | [protected] |
Return the AUC statistic.
Definition at line 185 of file LiftStatsCollector.cc.
References PLearn::TVec< T >::append(), finalize(), i, is_finalized, PLearn::TVec< T >::length(), n, opposite_lift, PLERROR, PLearn::TVec< T >::resize(), roc_fractions, and roc_values.
Referenced by getStat().
{ if (!is_finalized) finalize(); // Compute statistics. int n = roc_fractions.length(); if (n <= 0) PLERROR("In LiftStatsCollector::computeAUC - You need to use the 'roc_fractions' option if you want to compute the AUC"); real previous_val = 0; real previous_fraction = 0; real auc = 0; roc_fractions.append(1); // Make sure we take into account the whole curve. roc_values.append(1); n++; for (int i = 0; i < n; i++) { real mean_val = (roc_values[i] + previous_val) / 2; real interval_width = roc_fractions[i] - previous_fraction; auc += interval_width * mean_val; previous_val = roc_values[i]; previous_fraction = roc_fractions[i]; } // Remove appended '1'. roc_fractions.resize(roc_fractions.length() - 1); roc_values.resize(roc_values.length() - 1); if (opposite_lift) return -auc; else return auc; }
real PLearn::LiftStatsCollector::computeLift | ( | ) | [protected] |
Return the LIFT statistic.
Definition at line 217 of file LiftStatsCollector.cc.
References PLearn::TMat< T >::column(), PLearn::endl(), finalize(), is_finalized, lift_fraction, n_first_updates, n_samples_to_keep, npos, nsamples, nstored, opposite_lift, output_column, sign_trick, PLearn::sum(), target_column, and verbosity.
Referenced by getStat().
{ if (!is_finalized) finalize(); // Compute statistics. int npos_in_n_first = (int) sum(n_first_updates.column(1)); real first_samples_perf = npos_in_n_first/ (real) n_samples_to_keep; real targets_perf = (npos_in_n_first + npos) / (real) nsamples; real lift = first_samples_perf/targets_perf*100.0; if (verbosity >= 10) { cout << "LiftStatsCollector : is_finalized=" << is_finalized << ", nstored=" << nstored << ", nsamples=" << nsamples << ", npos=" << npos << ", n_samples_to_keep=" << n_samples_to_keep << ", lift_fraction=" << lift_fraction << ", output_column=" << output_column << ", sign_trick=" << sign_trick << ", target_column=" << target_column << ", verbosity= " << verbosity << endl; } if (verbosity >= 2) { cout << "There is a total of " << npos_in_n_first + npos << " positive examples to discover." << endl; cout << "The learner found " << npos_in_n_first << " of them in the fraction considered (" << lift_fraction << ")." << endl; } if (opposite_lift == 1) { return -lift; } return lift; }
real PLearn::LiftStatsCollector::computeLiftMax | ( | ) | [protected] |
Return the LIFT_MAX statistic.
Definition at line 249 of file LiftStatsCollector.cc.
References PLearn::TMat< T >::column(), finalize(), is_finalized, MIN, n_first_updates, n_samples_to_keep, npos, nsamples, and PLearn::sum().
Referenced by getStat().
{ if (!is_finalized) finalize(); int npos_in_n_first = (int) sum(n_first_updates.column(1)); real nones = npos_in_n_first + npos; real max_first_samples_perf = MIN(nones,(real)n_samples_to_keep) / (real) n_samples_to_keep; real targets_perf = (npos_in_n_first + npos) / (real) nsamples; real max_lift = max_first_samples_perf/targets_perf*100.0; return max_lift; }
void PLearn::LiftStatsCollector::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 103 of file LiftStatsCollector.cc.
References PLearn::OptionBase::buildoption, count_fin, PLearn::declareOption(), PLearn::VecStatsCollector::declareOptions(), PLearn::OptionBase::learntoption, lift_file, lift_fraction, opposite_lift, output_column, roc_file, roc_fractions, roc_values, sign_trick, target_column, and verbosity.
{ declareOption(ol, "lift_fraction", &LiftStatsCollector::lift_fraction, OptionBase::buildoption, " the % of samples to consider (default = 0.1)\n"); declareOption(ol, "opposite_lift", &LiftStatsCollector::opposite_lift, OptionBase::buildoption, " If set to 1, the LIFT stat will return -LIFT, so that it can be considered as a cost (default = 0)\n" " Similarly, the AUC stat will return -AUC."); declareOption(ol, "output_column", &LiftStatsCollector::output_column, OptionBase::buildoption, " the name of the column in which is the output value (the default value, \"\", assumes it is the first column))\n"); declareOption(ol, "sign_trick", &LiftStatsCollector::sign_trick, OptionBase::buildoption, " if set to 1, then you won't have to specify a target column: if the output is\n" " negative, the target will be assumed to be 0, and 1 otherwise - and in both cases\n" " we only consider the absolute value of the output\n" " (default = 0)\n" ); declareOption(ol, "target_column", &LiftStatsCollector::target_column, OptionBase::buildoption, " the column in which is the target value (default = 1)\n"); declareOption(ol, "verbosity", &LiftStatsCollector::verbosity, OptionBase::buildoption, " to be set >= 2 in order to display more info (default = 0)\n"); declareOption(ol, "roc_file", &LiftStatsCollector::roc_file, OptionBase::buildoption, "If provided, the points of the ROC curve computed for different fractions (see\n" "'roc_fractions') will be appended in ASCII format to the given file."); declareOption(ol, "lift_file", &LiftStatsCollector::lift_file, OptionBase::buildoption, "If provided, the lifts computed for different fractions (see 'roc_fractions')\n" "will be appended in ASCII format to the given file."); declareOption(ol, "roc_fractions", &LiftStatsCollector::roc_fractions, OptionBase::buildoption, "(Ordered) fractions used to compute and save points in the ROC curve, or additional lifts."); declareOption(ol, "count_fin", &LiftStatsCollector::count_fin, OptionBase::learntoption, " the number of times finalize() has been called since the last forget()"); declareOption(ol, "roc_values", &LiftStatsCollector::roc_values, OptionBase::learntoption, " The values of the ROC curve, evaluated at the 'roc_fractions' points."); // Now call the parent class' declareOptions inherited::declareOptions(ol); }
static const PPath& PLearn::LiftStatsCollector::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 159 of file LiftStatsCollector.h.
LiftStatsCollector * PLearn::LiftStatsCollector::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
void PLearn::LiftStatsCollector::finalize | ( | ) | [virtual] |
This finalize override makes sure only the n_samples_to_keep samples from the matrix n_first_updates with the highest output are left.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 264 of file LiftStatsCollector.cc.
References count_fin, PLearn::endl(), PLearn::VecStatsCollector::finalize(), i, is_finalized, PLearn::TMat< T >::length(), PLearn::TVec< T >::length(), lift_file, lift_fraction, n_first_updates, n_samples_to_keep, npos, nsamples, nstored, PLearn::TVec< T >::resize(), PLearn::TMat< T >::resize(), roc_file, roc_fractions, roc_values, PLearn::selectAndOrder(), PLearn::sortRows(), PLearn::sum(), PLearn::tostring(), verbosity, and PLearn::TMat< T >::width().
Referenced by computeAUC(), computeLift(), and computeLiftMax().
{ n_first_updates.resize(nstored,2); // get rid of the extra space allocated. n_samples_to_keep = int(lift_fraction*nsamples); if (nstored > n_samples_to_keep) { // If not, then no change has to be made to n_first_updates. // Compute additional lifts if required. if (roc_fractions.length() > 0) { // Copy data to make sure we do not change anything. Mat data(n_first_updates.length(), n_first_updates.width()); data << n_first_updates; sortRows(data, 0, false); // Create result file if does not exist already. string command; if (roc_file != "") { command = "touch " + roc_file; // cout << "Command: " << command << endl; system(command.c_str()); } if (lift_file != "") { command = "touch " + lift_file; // cout << "Command: " << command << endl; system(command.c_str()); } // Compute lifts. int nones = npos + int(sum(data.column(1)) + 1e-3); real frac_pos = nones / real(nsamples * 100); int lift_index = 0; int count_pos = 0; int sample_index = 0; string result_roc = ""; string result_lift = ""; roc_values.resize(roc_fractions.length()); while (lift_index < roc_fractions.length()) { while (sample_index < real(nsamples) * roc_fractions[lift_index]) { if (data(sample_index, 1) == 1) count_pos++; sample_index++; } real lift_value = real(count_pos) / real(sample_index) / frac_pos; real roc_value = real(count_pos) / real(nones); roc_values[lift_index] = roc_value; lift_index++; result_roc += tostring(roc_value) + "\t"; result_lift += tostring(lift_value) + "\t"; } // Save the lifts in the given file. // We only save if the number of samples seen is > 1, because it may happen // that when using an HyperLearner, train statistics are computed, and we // could have nsamples == 1. if (lift_file != "" && nsamples > 1) { command = "echo " + result_lift + " >> " + lift_file; // cout << "Command: " << command << endl; system(command.c_str()); } if (roc_file != "" && nsamples > 1) { command = "echo " + result_roc + " >> " + roc_file; // cout << "Command: " << command << endl; system(command.c_str()); } } // Make sure the highest ouputs are in the last n_samples_to_keep elements // of n_first_updates. if (n_samples_to_keep > 0) { selectAndOrder(n_first_updates, nstored - n_samples_to_keep); } // Count the number of positive examples in the lowest outputs. for (int i = 0; i < nstored - n_samples_to_keep; i++) { if (n_first_updates(i,1) == 1) { npos++; } } // Clear the lowest outputs, that are now useless. for (int i = 0; i < n_samples_to_keep; i++) { n_first_updates(i,0) = n_first_updates(i + nstored - n_samples_to_keep, 0); n_first_updates(i,1) = n_first_updates(i + nstored - n_samples_to_keep, 1); } n_first_updates.resize(n_samples_to_keep, 2); nstored = n_samples_to_keep; } inherited::finalize(); is_finalized = true; count_fin++; if (verbosity >= 10) { cout << "Called finalized " << count_fin << " times" << endl; } }
void PLearn::LiftStatsCollector::forget | ( | ) | [virtual] |
clears all previously accumulated statistics
Reimplemented from PLearn::VecStatsCollector.
Definition at line 362 of file LiftStatsCollector.cc.
References count_fin, PLearn::VecStatsCollector::forget(), is_finalized, n_first_updates, npos, nsamples, nstored, PLearn::TVec< T >::resize(), PLearn::TMat< T >::resize(), and roc_values.
{ is_finalized = false; nstored = 0; npos = 0; nsamples = 0; n_first_updates.resize(0,0); n_first_updates.resize(1000,2); inherited::forget(); count_fin = 0; roc_values.resize(0); }
OptionList & PLearn::LiftStatsCollector::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
OptionMap & PLearn::LiftStatsCollector::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
RemoteMethodMap & PLearn::LiftStatsCollector::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::VecStatsCollector.
Definition at line 101 of file LiftStatsCollector.cc.
double PLearn::LiftStatsCollector::getStat | ( | const string & | statspec | ) | [virtual] |
In addition to the regular VecStatsCollector statistics, we understand specific lift statistics (see the .cc).
Reimplemented from PLearn::VecStatsCollector.
Definition at line 378 of file LiftStatsCollector.cc.
References computeAUC(), computeLift(), computeLiftMax(), PLearn::VecStatsCollector::getStat(), PLearn::openString(), PLearn::PStream::plearn_ascii, and PLearn::PStream::smartReadUntilNext().
{ PStream str = openString(statspec,PStream::plearn_ascii); string parsed; str.smartReadUntilNext("(",parsed); if (parsed == "LIFT") { return computeLift(); } else if (parsed == "LIFT_MAX") { return computeLiftMax(); } else if (parsed == "AUC") { return computeAUC(); } else return inherited::getStat(statspec); }
void PLearn::LiftStatsCollector::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 399 of file LiftStatsCollector.cc.
References PLearn::deepCopyField(), PLearn::VecStatsCollector::makeDeepCopyFromShallowCopy(), n_first_updates, roc_fractions, and roc_values.
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(roc_values, copies); deepCopyField(n_first_updates, copies); deepCopyField(roc_fractions, copies); }
Overridden because it is not supported in this VecStatsCollector.
Reimplemented from PLearn::VecStatsCollector.
Definition at line 410 of file LiftStatsCollector.cc.
References PLERROR.
{ // Not supported. PLERROR("In LiftStatsCollector::remove_observation - This method is not implemented"); }
updates the statistics when seeing x The weight applies to all elements of x
Reimplemented from PLearn::VecStatsCollector.
Definition at line 418 of file LiftStatsCollector.cc.
References count_fin, PLearn::FABS(), is_finalized, PLearn::is_missing(), PLearn::TMat< T >::length(), MAX, n_first_updates, nsamples, nstored, output_column_index, PLERROR, PLWARNING, PLearn::TMat< T >::resize(), roc_file, sign_trick, target_column, and PLearn::VecStatsCollector::update().
{ if (count_fin > 0) { // Depending on whether we compute additional lifts, this may be fatal or not. string msg = "In LiftStatsCollector::update - Called update after finalize (see help of LiftStatsCollector)"; if (roc_file != "") PLERROR(msg.c_str()); else PLWARNING(msg.c_str()); } if (nstored == n_first_updates.length()) { n_first_updates.resize(MAX(1000,10*n_first_updates.length()), 2); } real output_val = x[output_column_index]; if (is_missing(output_val)) { // Missing value: we just discard it. is_finalized = false; inherited::update(x,w); return; } real target = -1; switch(sign_trick) { case 0: // Normal behavior. n_first_updates(nstored, 0) = output_val; target = x[target_column]; break; case 1: // Sign trick. n_first_updates(nstored, 0) = FABS(output_val); if (output_val <= 0) { x[output_column_index] = -output_val; target = 0; } else { target = 1; // cout << "Positive example : " << x << " (output_val = " << output_val << ")" << endl; } break; default: PLERROR("Wrong value for sign_trick in LiftStatsCollector"); break; } n_first_updates(nstored, 1) = target; if (target != 0 && target != 1) { PLERROR("In LiftStatsCollector::update - Target must be 0 or 1 !"); } nsamples++; nstored++; is_finalized = false; inherited::update(x,w); }
Reimplemented from PLearn::VecStatsCollector.
Definition at line 159 of file LiftStatsCollector.h.
int PLearn::LiftStatsCollector::count_fin [protected] |
Definition at line 63 of file LiftStatsCollector.h.
Referenced by declareOptions(), finalize(), forget(), and update().
bool PLearn::LiftStatsCollector::is_finalized [protected] |
Set to true after each call to finalize().
Definition at line 73 of file LiftStatsCollector.h.
Referenced by computeAUC(), computeLift(), computeLiftMax(), finalize(), forget(), and update().
Definition at line 97 of file LiftStatsCollector.h.
Referenced by declareOptions(), and finalize().
Definition at line 98 of file LiftStatsCollector.h.
Referenced by computeLift(), declareOptions(), and finalize().
Mat PLearn::LiftStatsCollector::n_first_updates [protected] |
Matrix storing the output and target of the samples with highest output, as well as all the other data retrieved since the last call to finalize.
Definition at line 70 of file LiftStatsCollector.h.
Referenced by computeLift(), computeLiftMax(), finalize(), forget(), makeDeepCopyFromShallowCopy(), and update().
int PLearn::LiftStatsCollector::n_samples_to_keep [protected] |
Number of examples to keep (nsamples * lift_fraction).
Definition at line 86 of file LiftStatsCollector.h.
Referenced by computeLift(), computeLiftMax(), and finalize().
int PLearn::LiftStatsCollector::npos [protected] |
Number of positive examples that are not retained in the ones with the highest output (that is to say the ones in n_first_updates).
Definition at line 83 of file LiftStatsCollector.h.
Referenced by computeLift(), computeLiftMax(), finalize(), and forget().
int PLearn::LiftStatsCollector::nsamples [protected] |
Number of samples seen.
Definition at line 79 of file LiftStatsCollector.h.
Referenced by computeLift(), computeLiftMax(), finalize(), forget(), and update().
int PLearn::LiftStatsCollector::nstored [protected] |
Number of examples stored in the n_first_updates matrix.
Definition at line 76 of file LiftStatsCollector.h.
Referenced by computeLift(), finalize(), forget(), and update().
Definition at line 99 of file LiftStatsCollector.h.
Referenced by computeAUC(), computeLift(), and declareOptions().
Definition at line 100 of file LiftStatsCollector.h.
Referenced by build_(), computeLift(), and declareOptions().
int PLearn::LiftStatsCollector::output_column_index [protected] |
Index of the output column.
Definition at line 89 of file LiftStatsCollector.h.
Definition at line 101 of file LiftStatsCollector.h.
Referenced by declareOptions(), finalize(), and update().
Definition at line 102 of file LiftStatsCollector.h.
Referenced by computeAUC(), declareOptions(), finalize(), and makeDeepCopyFromShallowCopy().
Vec PLearn::LiftStatsCollector::roc_values [protected] |
Definition at line 64 of file LiftStatsCollector.h.
Referenced by computeAUC(), declareOptions(), finalize(), forget(), and makeDeepCopyFromShallowCopy().
Definition at line 103 of file LiftStatsCollector.h.
Referenced by computeLift(), declareOptions(), and update().
Definition at line 104 of file LiftStatsCollector.h.
Referenced by computeLift(), declareOptions(), and update().
Definition at line 105 of file LiftStatsCollector.h.
Referenced by computeLift(), declareOptions(), and finalize().