|
PLearn 0.1
|
#include <SentencesBlocks.h>


Public Member Functions | |
| SentencesBlocks (int n_blocks, VMat d, Vec separator) | |
| SentencesBlocks. | |
Build a vector of distributions which are SubRowDistributions of a given distribution, made up of consecutive blocks of approximately the same type, but where each block is a sequence of items terminating by a special "separator" value.
Definition at line 60 of file SentencesBlocks.h.
Definition at line 48 of file SentencesBlocks.cc.
References b, d, i, PLearn::VMat::length(), PLERROR, PLearn::VMat::subMatRows(), and PLearn::VMat::width().
: TVec<VMat>(n_blocks)
{
if (n_blocks==1)
{
(*this)[0]=d;
return;
}
int total_size = d->length();
if (total_size < n_blocks * 2)
PLERROR("SentencesBlocks: can't have blocks of size < 2 in average");
Vec v(d->width());
int b=0;
int previous_previous_block=0,previous_beginning_of_block = 0, previous_beginning_of_sentence=0;
int next_target = (int)(total_size / (real)n_blocks);
for (int i=0;i<total_size && b<n_blocks-1;i++)
{
d->getRow(i,v);
if (v==separator)
{
if (i>=next_target)
{
int cut=0;
if (i-next_target < next_target-previous_beginning_of_sentence ||
previous_beginning_of_sentence < previous_beginning_of_block)
cut=i+1;
else
{
cut=previous_beginning_of_sentence;
previous_beginning_of_sentence = i+1;
}
(*this)[b++] = d.subMatRows(previous_beginning_of_block,
cut-previous_beginning_of_block);
previous_previous_block = previous_beginning_of_block;
previous_beginning_of_block=cut;
if (b<n_blocks)
{
if (b>n_blocks-3)
next_target = (int)((total_size - cut) / (real)(n_blocks-b));
else
next_target = (int)(total_size * (real)(b+1.0) / n_blocks);
}
}
else
previous_beginning_of_sentence=i+1;
}
}
if (b==n_blocks-1)
(*this)[b++] = d.subMatRows(previous_beginning_of_block,
total_size-previous_beginning_of_block);
if (b<n_blocks-1) // we have to backtrack, split previous block in two
{
if (b<n_blocks-2)
PLERROR("SentencesBlocks: blocks are too small!");
if (previous_beginning_of_sentence<previous_beginning_of_block)
PLERROR("SentencesBlocks: Blocks are too small!");
int cut = previous_beginning_of_sentence;
(*this)[b++] = d.subMatRows(previous_beginning_of_block,
cut-previous_beginning_of_block);
previous_beginning_of_block=cut;
(*this)[b++] = d.subMatRows(previous_beginning_of_block,
total_size-previous_beginning_of_block);
}
}

1.7.4