PLearn 0.1
|
#include <RepeatSplitter.h>
Public Member Functions | |
RepeatSplitter () | |
virtual void | setDataSet (VMat the_dataset) |
Set the dataset on which the splits are to be based. | |
virtual void | build () |
Post-constructor. | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual RepeatSplitter * | deepCopy (CopiesMap &copies) const |
virtual int | nsplits () const |
Returns the number of available different "splits". | |
virtual int | nSetsPerSplit () const |
Returns the number of sets per split. | |
virtual TVec< VMat > | getSplit (int i=0) |
Returns split number i. | |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
bool | do_not_shuffle_first |
real | force_proportion |
int | n |
int32_t | seed |
int | shuffle |
PP< Splitter > | to_repeat |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Protected Attributes | |
TMat< int > | indices |
A matrix where the i-th row is the vector of the indices of the i-th shuffled matrix (used only when shuffle = 1). | |
int | last_n |
Used to know if we need to reshuffle the matrix. | |
Private Types | |
typedef Splitter | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Definition at line 49 of file RepeatSplitter.h.
typedef Splitter PLearn::RepeatSplitter::inherited [private] |
Reimplemented from PLearn::Splitter.
Definition at line 51 of file RepeatSplitter.h.
PLearn::RepeatSplitter::RepeatSplitter | ( | ) |
Definition at line 51 of file RepeatSplitter.cc.
: last_n(-1), do_not_shuffle_first(0), force_proportion(-1), n(1), seed(-1), shuffle(0) { }
string PLearn::RepeatSplitter::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
OptionList & PLearn::RepeatSplitter::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
RemoteMethodMap & PLearn::RepeatSplitter::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
Object * PLearn::RepeatSplitter::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
StaticInitializer RepeatSplitter::_static_initializer_ & PLearn::RepeatSplitter::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
void PLearn::RepeatSplitter::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::Object.
Definition at line 101 of file RepeatSplitter.cc.
References PLearn::Object::build(), and build_().
Referenced by setDataSet().
{ inherited::build(); build_(); }
void PLearn::RepeatSplitter::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::Object.
Definition at line 110 of file RepeatSplitter.cc.
References PLearn::abs(), PLearn::Splitter::dataset, do_not_shuffle_first, PLearn::fast_exact_is_equal(), PLearn::StatsCollector::finalize(), force_proportion, PLearn::StatsCollector::getCounts(), i, indices, j, last_n, PLearn::TMat< T >::length(), PLearn::VMat::length(), m, PLearn::manual_seed(), n, nsplits(), PLERROR, PLWARNING, seed, shuffle, PLearn::shuffleElements(), PLearn::uniform_sample(), and PLearn::StatsCollector::update().
Referenced by build().
{ if (shuffle && dataset) { // Prepare the shuffled indices. if (seed >= 0) manual_seed(seed); else PLearn::seed(); int n_splits = nsplits(); indices = TMat<int>(n_splits, dataset.length()); TVec<int> shuffled; for (int i = 0; i < n_splits; i++) { shuffled = TVec<int>(0, dataset.length()-1, 1); // Don't shuffle if (i == 0) and do_not_shuffle_first is set to 1. if (!do_not_shuffle_first || i > 0) { shuffleElements(shuffled); if (force_proportion >= 1) { // We need to ensure the proportions of target values are respected. // First compute the target stats. StatsCollector tsc(2000); if (dataset->targetsize() != 1) { PLERROR("In RepeatSplitter::build_ - 'force_proportion' is only implemented for a 1-dimensional target"); } real t; for (int j = 0; j < dataset->length(); j++) { t = dataset->get(j, dataset->inputsize()); // We get the target. tsc.update(t); } tsc.finalize(); // Make sure the target is binary. int count = (int) tsc.getCounts()->size() - 1; if (count != 2) { PLERROR("In RepeatSplitter::build_ - 'force_proportion' is only implemented for a binary target"); } // Ensure the proportion of the targets respect the constraints. int index = 0; for (map<real,StatsCollectorCounts>::iterator it = tsc.getCounts()->begin(); index < count; index++) { t = it->first; real prop_t = real(it->second.n) / real(dataset->length()); // Find the step to use to check the proportion is ok. // We want a step such that each 'step' examples, there // should be at least two with this target, but less // than 'step - 10'. For instance, for a proportion of // 0.1, 'step' would be 20, and for a proportion of // 0.95, it would be 200. We also want the // approximation made when rounding to be negligible. int step = 20; bool ok = false; while (!ok) { int n = int(step * prop_t + 0.5); if (n >= 2 && n <= step - 10 && abs(step * prop_t - real(n)) / real(step) < 0.01) { ok = true; } else { // We try a higher step. step *= 2; } } int expected_count = int(step * prop_t + 0.5); // cout << "step = " << step << ", expected_count = " << expected_count << endl; // Now verify the proportion. ok = false; int tc = dataset->inputsize(); // The target column. while (!ok) { ok = true; // First pass: ensure there is enough. int first_pass_step = int(step * force_proportion + 0.5); int k,l; for (k = 0; k < shuffled.length(); k += first_pass_step) { int count_target = 0; for (l = k; l < k + first_pass_step && l < shuffled.length(); l++) { if (fast_exact_is_equal( dataset->get(shuffled[l], tc), t)) count_target++; } if (l - k == first_pass_step && count_target < expected_count) { // Not enough, need to add more. ok = false; // cout << "At l = " << l << ", need to add " << expected_count - count_target << " samples" << endl; for (int m = 0; m < expected_count - count_target; m++) { bool can_swap = false; int to_swap = -1; // Find a sample to swap in the current window. while (!can_swap) { to_swap = int(uniform_sample() * first_pass_step); if (!fast_exact_is_equal(dataset->get(shuffled[k + to_swap], tc), t)) { can_swap = true; } } to_swap += k; // Find a sample to swap in the next samples. int next = k + first_pass_step - 1; can_swap = false; while (!can_swap) { next++; if (next >= shuffled.length()) { next = 0; } if (fast_exact_is_equal(dataset->get(shuffled[next], tc), t)) { can_swap = true; } } // And swap baby! int tmp = shuffled[next]; shuffled[next] = shuffled[to_swap]; shuffled[to_swap] = tmp; } } } // Second pass: ensure there aren't too many. int second_pass_step = int(step / force_proportion + 0.5); for (k = 0; k < shuffled.length(); k += second_pass_step) { int count_target = 0; for (l = k; l < k + second_pass_step && l < shuffled.length(); l++) { if (fast_exact_is_equal(dataset->get(shuffled[l], tc), count_target)) { count_target++; } } if (l - k == second_pass_step && count_target > expected_count) { // Too many, need to remove some. ok = false; PLWARNING("In RepeatSplitter::build_ - The code reached hasn't been tested yet"); // cout << "At l = " << l << ", need to remove " << - expected_count + count_target << " samples" << endl; for (int m = 0; m < - expected_count + count_target; m++) { bool can_swap = false; int to_swap = k - 1; // Find a sample to swap in the current window. while (!can_swap) { to_swap++; if (fast_exact_is_equal(dataset->get(shuffled[to_swap], tc), t)) { can_swap = true; } } // Find a sample to swap in the next samples. int next = k + first_pass_step - 1; can_swap = false; while (!can_swap) { next++; if (next >= shuffled.length()) { next = 0; } if (!fast_exact_is_equal(dataset->get(shuffled[next], tc), t)) { can_swap = true; } } // And swap baby! int tmp = shuffled[next]; shuffled[next] = shuffled[to_swap]; shuffled[to_swap] = tmp; } } } } it++; } } } indices(i) << shuffled; } } else { indices = TMat<int>(); } last_n = -1; }
string PLearn::RepeatSplitter::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
void PLearn::RepeatSplitter::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::Splitter.
Definition at line 70 of file RepeatSplitter.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::Splitter::declareOptions(), do_not_shuffle_first, force_proportion, n, seed, shuffle, and to_repeat.
{ declareOption(ol, "do_not_shuffle_first", &RepeatSplitter::do_not_shuffle_first, OptionBase::buildoption, "If set to 1, then the dataset won't be shuffled the first time we do the splitting.\n" "It only makes sense to use this option if 'shuffle' is set to 1."); declareOption(ol, "force_proportion", &RepeatSplitter::force_proportion, OptionBase::buildoption, "If a target value appears at least once every x samples, will ensure that after\n" "shuffling it appears at least once every (x * 'force_proportion') samples, and not\n" "more than once every (x / 'force_proportion') samples. Will be ignored if < 1.\n" "Note that this currently only works for a binary target! (and hasn't been 100% tested)."); declareOption(ol, "n", &RepeatSplitter::n, OptionBase::buildoption, "How many times we want to repeat."); declareOption(ol, "seed", &RepeatSplitter::seed, OptionBase::buildoption, "Initializes the random number generator (only if shuffle is set to 1).\n" "If set to -1, the initialization will depend on the clock."); declareOption(ol, "shuffle", &RepeatSplitter::shuffle, OptionBase::buildoption, "If set to 1, the dataset will be shuffled differently at each repetition."); declareOption(ol, "to_repeat", &RepeatSplitter::to_repeat, OptionBase::buildoption, "The splitter we want to repeat."); inherited::declareOptions(ol); }
static const PPath& PLearn::RepeatSplitter::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::Splitter.
Definition at line 111 of file RepeatSplitter.h.
RepeatSplitter * PLearn::RepeatSplitter::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
OptionList & PLearn::RepeatSplitter::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
OptionMap & PLearn::RepeatSplitter::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
RemoteMethodMap & PLearn::RepeatSplitter::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
Returns split number i.
Implements PLearn::Splitter.
Definition at line 298 of file RepeatSplitter.cc.
References PLearn::Splitter::dataset, indices, last_n, m, nsplits(), PLERROR, shuffle, and to_repeat.
{ int n_splits = this->nsplits(); if (k >= n_splits) { PLERROR("In RepeatSplitter::getSplit: split asked is too high"); } int child_splits = to_repeat->nsplits(); int real_k = k % child_splits; if (shuffle && dataset) { int shuffle_indice = k / child_splits; if (shuffle_indice != last_n) { // We have to reshuffle the dataset, according to indices. VMat m = new SelectRowsVMatrix(dataset, indices(shuffle_indice), false, false); to_repeat->setDataSet(m); last_n = shuffle_indice; } } return to_repeat->getSplit(real_k); }
void PLearn::RepeatSplitter::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::Splitter.
Definition at line 281 of file RepeatSplitter.cc.
References PLearn::deepCopyField(), PLearn::Splitter::makeDeepCopyFromShallowCopy(), and to_repeat.
{ inherited::makeDeepCopyFromShallowCopy(copies); // ### Call deepCopyField on all "pointer-like" fields // ### that you wish to be deepCopied rather than // ### shallow-copied. // ### ex: // deepCopyField(trainvec, copies); deepCopyField(to_repeat, copies); }
int PLearn::RepeatSplitter::nSetsPerSplit | ( | ) | const [virtual] |
Returns the number of sets per split.
Implements PLearn::Splitter.
Definition at line 322 of file RepeatSplitter.cc.
References to_repeat.
{ return to_repeat->nSetsPerSplit(); }
int PLearn::RepeatSplitter::nsplits | ( | ) | const [virtual] |
Returns the number of available different "splits".
Implements PLearn::Splitter.
Definition at line 330 of file RepeatSplitter.cc.
Referenced by build_(), and getSplit().
void PLearn::RepeatSplitter::setDataSet | ( | VMat | the_dataset | ) | [virtual] |
Set the dataset on which the splits are to be based.
Reimplemented from PLearn::Splitter.
Definition at line 338 of file RepeatSplitter.cc.
References build(), PLearn::Splitter::setDataSet(), and to_repeat.
{ inherited::setDataSet(the_dataset); to_repeat->setDataSet(the_dataset); build(); // necessary to recompute the indices. }
Reimplemented from PLearn::Splitter.
Definition at line 111 of file RepeatSplitter.h.
Definition at line 72 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
TMat<int> PLearn::RepeatSplitter::indices [protected] |
A matrix where the i-th row is the vector of the indices of the i-th shuffled matrix (used only when shuffle = 1).
Definition at line 61 of file RepeatSplitter.h.
Referenced by build_(), and getSplit().
int PLearn::RepeatSplitter::last_n [protected] |
Used to know if we need to reshuffle the matrix.
Definition at line 64 of file RepeatSplitter.h.
Referenced by build_(), and getSplit().
Definition at line 74 of file RepeatSplitter.h.
Referenced by build_(), declareOptions(), and nsplits().
int32_t PLearn::RepeatSplitter::seed |
Definition at line 75 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
Definition at line 76 of file RepeatSplitter.h.
Referenced by build_(), declareOptions(), and getSplit().
Definition at line 77 of file RepeatSplitter.h.
Referenced by declareOptions(), getSplit(), makeDeepCopyFromShallowCopy(), nSetsPerSplit(), nsplits(), and setDataSet().