|
PLearn 0.1
|
#include <RepeatSplitter.h>


Public Member Functions | |
| RepeatSplitter () | |
| virtual void | setDataSet (VMat the_dataset) |
| Set the dataset on which the splits are to be based. | |
| virtual void | build () |
| Post-constructor. | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual RepeatSplitter * | deepCopy (CopiesMap &copies) const |
| virtual int | nsplits () const |
| Returns the number of available different "splits". | |
| virtual int | nSetsPerSplit () const |
| Returns the number of sets per split. | |
| virtual TVec< VMat > | getSplit (int i=0) |
| Returns split number i. | |
Static Public Member Functions | |
| static string | _classname_ () |
| Declares name and deepCopy methods. | |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Public Attributes | |
| bool | do_not_shuffle_first |
| real | force_proportion |
| int | n |
| int32_t | seed |
| int | shuffle |
| PP< Splitter > | to_repeat |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declares this class' options. | |
Protected Attributes | |
| TMat< int > | indices |
| A matrix where the i-th row is the vector of the indices of the i-th shuffled matrix (used only when shuffle = 1). | |
| int | last_n |
| Used to know if we need to reshuffle the matrix. | |
Private Types | |
| typedef Splitter | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
Definition at line 49 of file RepeatSplitter.h.
typedef Splitter PLearn::RepeatSplitter::inherited [private] |
Reimplemented from PLearn::Splitter.
Definition at line 51 of file RepeatSplitter.h.
| PLearn::RepeatSplitter::RepeatSplitter | ( | ) |
Definition at line 51 of file RepeatSplitter.cc.
:
last_n(-1),
do_not_shuffle_first(0),
force_proportion(-1),
n(1),
seed(-1),
shuffle(0)
{
}
| string PLearn::RepeatSplitter::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
| OptionList & PLearn::RepeatSplitter::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
| RemoteMethodMap & PLearn::RepeatSplitter::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
| Object * PLearn::RepeatSplitter::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
| StaticInitializer RepeatSplitter::_static_initializer_ & PLearn::RepeatSplitter::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
| void PLearn::RepeatSplitter::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::Object.
Definition at line 101 of file RepeatSplitter.cc.
References PLearn::Object::build(), and build_().
Referenced by setDataSet().
{
inherited::build();
build_();
}


| void PLearn::RepeatSplitter::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::Object.
Definition at line 110 of file RepeatSplitter.cc.
References PLearn::abs(), PLearn::Splitter::dataset, do_not_shuffle_first, PLearn::fast_exact_is_equal(), PLearn::StatsCollector::finalize(), force_proportion, PLearn::StatsCollector::getCounts(), i, indices, j, last_n, PLearn::TMat< T >::length(), PLearn::VMat::length(), m, PLearn::manual_seed(), n, nsplits(), PLERROR, PLWARNING, seed, shuffle, PLearn::shuffleElements(), PLearn::uniform_sample(), and PLearn::StatsCollector::update().
Referenced by build().
{
if (shuffle && dataset) {
// Prepare the shuffled indices.
if (seed >= 0)
manual_seed(seed);
else
PLearn::seed();
int n_splits = nsplits();
indices = TMat<int>(n_splits, dataset.length());
TVec<int> shuffled;
for (int i = 0; i < n_splits; i++) {
shuffled = TVec<int>(0, dataset.length()-1, 1);
// Don't shuffle if (i == 0) and do_not_shuffle_first is set to 1.
if (!do_not_shuffle_first || i > 0) {
shuffleElements(shuffled);
if (force_proportion >= 1) {
// We need to ensure the proportions of target values are respected.
// First compute the target stats.
StatsCollector tsc(2000);
if (dataset->targetsize() != 1) {
PLERROR("In RepeatSplitter::build_ - 'force_proportion' is only implemented for a 1-dimensional target");
}
real t;
for (int j = 0; j < dataset->length(); j++) {
t = dataset->get(j, dataset->inputsize()); // We get the target.
tsc.update(t);
}
tsc.finalize();
// Make sure the target is binary.
int count = (int) tsc.getCounts()->size() - 1;
if (count != 2) {
PLERROR("In RepeatSplitter::build_ - 'force_proportion' is only implemented for a binary target");
}
// Ensure the proportion of the targets respect the constraints.
int index = 0;
for (map<real,StatsCollectorCounts>::iterator it =
tsc.getCounts()->begin(); index < count; index++)
{
t = it->first;
real prop_t = real(it->second.n) /
real(dataset->length());
// Find the step to use to check the proportion is ok.
// We want a step such that each 'step' examples, there
// should be at least two with this target, but less
// than 'step - 10'. For instance, for a proportion of
// 0.1, 'step' would be 20, and for a proportion of
// 0.95, it would be 200. We also want the
// approximation made when rounding to be negligible.
int step = 20;
bool ok = false;
while (!ok) {
int n = int(step * prop_t + 0.5);
if (n >= 2 && n <= step - 10
&& abs(step * prop_t - real(n)) / real(step) < 0.01) {
ok = true;
} else {
// We try a higher step.
step *= 2;
}
}
int expected_count = int(step * prop_t + 0.5);
// cout << "step = " << step << ", expected_count = " << expected_count << endl;
// Now verify the proportion.
ok = false;
int tc = dataset->inputsize(); // The target column.
while (!ok) {
ok = true;
// First pass: ensure there is enough.
int first_pass_step = int(step * force_proportion + 0.5);
int k,l;
for (k = 0; k < shuffled.length(); k += first_pass_step) {
int count_target = 0;
for (l = k; l < k + first_pass_step && l < shuffled.length(); l++) {
if (fast_exact_is_equal(
dataset->get(shuffled[l], tc), t))
count_target++;
}
if (l - k == first_pass_step && count_target < expected_count) {
// Not enough, need to add more.
ok = false;
// cout << "At l = " << l << ", need to add " << expected_count - count_target << " samples" << endl;
for (int m = 0; m < expected_count - count_target; m++) {
bool can_swap = false;
int to_swap = -1;
// Find a sample to swap in the current window.
while (!can_swap) {
to_swap = int(uniform_sample() * first_pass_step);
if (!fast_exact_is_equal(dataset->get(shuffled[k + to_swap], tc), t)) {
can_swap = true;
}
}
to_swap += k;
// Find a sample to swap in the next samples.
int next = k + first_pass_step - 1;
can_swap = false;
while (!can_swap) {
next++;
if (next >= shuffled.length()) {
next = 0;
}
if (fast_exact_is_equal(dataset->get(shuffled[next], tc), t)) {
can_swap = true;
}
}
// And swap baby!
int tmp = shuffled[next];
shuffled[next] = shuffled[to_swap];
shuffled[to_swap] = tmp;
}
}
}
// Second pass: ensure there aren't too many.
int second_pass_step = int(step / force_proportion + 0.5);
for (k = 0; k < shuffled.length(); k += second_pass_step) {
int count_target = 0;
for (l = k; l < k + second_pass_step && l < shuffled.length(); l++) {
if (fast_exact_is_equal(dataset->get(shuffled[l], tc), count_target)) {
count_target++;
}
}
if (l - k == second_pass_step && count_target > expected_count) {
// Too many, need to remove some.
ok = false;
PLWARNING("In RepeatSplitter::build_ - The code reached hasn't been tested yet");
// cout << "At l = " << l << ", need to remove " << - expected_count + count_target << " samples" << endl;
for (int m = 0; m < - expected_count + count_target; m++) {
bool can_swap = false;
int to_swap = k - 1;
// Find a sample to swap in the current window.
while (!can_swap) {
to_swap++;
if (fast_exact_is_equal(dataset->get(shuffled[to_swap], tc), t)) {
can_swap = true;
}
}
// Find a sample to swap in the next samples.
int next = k + first_pass_step - 1;
can_swap = false;
while (!can_swap) {
next++;
if (next >= shuffled.length()) {
next = 0;
}
if (!fast_exact_is_equal(dataset->get(shuffled[next], tc), t)) {
can_swap = true;
}
}
// And swap baby!
int tmp = shuffled[next];
shuffled[next] = shuffled[to_swap];
shuffled[to_swap] = tmp;
}
}
}
}
it++;
}
}
}
indices(i) << shuffled;
}
} else {
indices = TMat<int>();
}
last_n = -1;
}


| string PLearn::RepeatSplitter::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
| void PLearn::RepeatSplitter::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::Splitter.
Definition at line 70 of file RepeatSplitter.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::Splitter::declareOptions(), do_not_shuffle_first, force_proportion, n, seed, shuffle, and to_repeat.
{
declareOption(ol, "do_not_shuffle_first", &RepeatSplitter::do_not_shuffle_first, OptionBase::buildoption,
"If set to 1, then the dataset won't be shuffled the first time we do the splitting.\n"
"It only makes sense to use this option if 'shuffle' is set to 1.");
declareOption(ol, "force_proportion", &RepeatSplitter::force_proportion, OptionBase::buildoption,
"If a target value appears at least once every x samples, will ensure that after\n"
"shuffling it appears at least once every (x * 'force_proportion') samples, and not\n"
"more than once every (x / 'force_proportion') samples. Will be ignored if < 1.\n"
"Note that this currently only works for a binary target! (and hasn't been 100% tested).");
declareOption(ol, "n", &RepeatSplitter::n, OptionBase::buildoption,
"How many times we want to repeat.");
declareOption(ol, "seed", &RepeatSplitter::seed, OptionBase::buildoption,
"Initializes the random number generator (only if shuffle is set to 1).\n"
"If set to -1, the initialization will depend on the clock.");
declareOption(ol, "shuffle", &RepeatSplitter::shuffle, OptionBase::buildoption,
"If set to 1, the dataset will be shuffled differently at each repetition.");
declareOption(ol, "to_repeat", &RepeatSplitter::to_repeat, OptionBase::buildoption,
"The splitter we want to repeat.");
inherited::declareOptions(ol);
}

| static const PPath& PLearn::RepeatSplitter::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::Splitter.
Definition at line 111 of file RepeatSplitter.h.
| RepeatSplitter * PLearn::RepeatSplitter::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::Splitter.
Definition at line 65 of file RepeatSplitter.cc.
| OptionList & PLearn::RepeatSplitter::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
| OptionMap & PLearn::RepeatSplitter::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
| RemoteMethodMap & PLearn::RepeatSplitter::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 65 of file RepeatSplitter.cc.
Returns split number i.
Implements PLearn::Splitter.
Definition at line 298 of file RepeatSplitter.cc.
References PLearn::Splitter::dataset, indices, last_n, m, nsplits(), PLERROR, shuffle, and to_repeat.
{
int n_splits = this->nsplits();
if (k >= n_splits) {
PLERROR("In RepeatSplitter::getSplit: split asked is too high");
}
int child_splits = to_repeat->nsplits();
int real_k = k % child_splits;
if (shuffle && dataset) {
int shuffle_indice = k / child_splits;
if (shuffle_indice != last_n) {
// We have to reshuffle the dataset, according to indices.
VMat m = new SelectRowsVMatrix(dataset, indices(shuffle_indice),
false, false);
to_repeat->setDataSet(m);
last_n = shuffle_indice;
}
}
return to_repeat->getSplit(real_k);
}

| void PLearn::RepeatSplitter::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::Splitter.
Definition at line 281 of file RepeatSplitter.cc.
References PLearn::deepCopyField(), PLearn::Splitter::makeDeepCopyFromShallowCopy(), and to_repeat.
{
inherited::makeDeepCopyFromShallowCopy(copies);
// ### Call deepCopyField on all "pointer-like" fields
// ### that you wish to be deepCopied rather than
// ### shallow-copied.
// ### ex:
// deepCopyField(trainvec, copies);
deepCopyField(to_repeat, copies);
}

| int PLearn::RepeatSplitter::nSetsPerSplit | ( | ) | const [virtual] |
Returns the number of sets per split.
Implements PLearn::Splitter.
Definition at line 322 of file RepeatSplitter.cc.
References to_repeat.
{
return to_repeat->nSetsPerSplit();
}
| int PLearn::RepeatSplitter::nsplits | ( | ) | const [virtual] |
Returns the number of available different "splits".
Implements PLearn::Splitter.
Definition at line 330 of file RepeatSplitter.cc.
Referenced by build_(), and getSplit().

| void PLearn::RepeatSplitter::setDataSet | ( | VMat | the_dataset | ) | [virtual] |
Set the dataset on which the splits are to be based.
Reimplemented from PLearn::Splitter.
Definition at line 338 of file RepeatSplitter.cc.
References build(), PLearn::Splitter::setDataSet(), and to_repeat.
{
inherited::setDataSet(the_dataset);
to_repeat->setDataSet(the_dataset);
build(); // necessary to recompute the indices.
}

Reimplemented from PLearn::Splitter.
Definition at line 111 of file RepeatSplitter.h.
Definition at line 72 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
Definition at line 73 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
TMat<int> PLearn::RepeatSplitter::indices [protected] |
A matrix where the i-th row is the vector of the indices of the i-th shuffled matrix (used only when shuffle = 1).
Definition at line 61 of file RepeatSplitter.h.
Referenced by build_(), and getSplit().
int PLearn::RepeatSplitter::last_n [protected] |
Used to know if we need to reshuffle the matrix.
Definition at line 64 of file RepeatSplitter.h.
Referenced by build_(), and getSplit().
Definition at line 74 of file RepeatSplitter.h.
Referenced by build_(), declareOptions(), and nsplits().
| int32_t PLearn::RepeatSplitter::seed |
Definition at line 75 of file RepeatSplitter.h.
Referenced by build_(), and declareOptions().
Definition at line 76 of file RepeatSplitter.h.
Referenced by build_(), declareOptions(), and getSplit().
Definition at line 77 of file RepeatSplitter.h.
Referenced by declareOptions(), getSplit(), makeDeepCopyFromShallowCopy(), nSetsPerSplit(), nsplits(), and setDataSet().
1.7.4