PLearn 0.1
|
#include <SentencesBlocks.h>
Public Member Functions | |
SentencesBlocks (int n_blocks, VMat d, Vec separator) | |
SentencesBlocks. |
Build a vector of distributions which are SubRowDistributions of a given distribution, made up of consecutive blocks of approximately the same type, but where each block is a sequence of items terminating by a special "separator" value.
Definition at line 60 of file SentencesBlocks.h.
Definition at line 48 of file SentencesBlocks.cc.
References b, d, i, PLearn::VMat::length(), PLERROR, PLearn::VMat::subMatRows(), and PLearn::VMat::width().
: TVec<VMat>(n_blocks) { if (n_blocks==1) { (*this)[0]=d; return; } int total_size = d->length(); if (total_size < n_blocks * 2) PLERROR("SentencesBlocks: can't have blocks of size < 2 in average"); Vec v(d->width()); int b=0; int previous_previous_block=0,previous_beginning_of_block = 0, previous_beginning_of_sentence=0; int next_target = (int)(total_size / (real)n_blocks); for (int i=0;i<total_size && b<n_blocks-1;i++) { d->getRow(i,v); if (v==separator) { if (i>=next_target) { int cut=0; if (i-next_target < next_target-previous_beginning_of_sentence || previous_beginning_of_sentence < previous_beginning_of_block) cut=i+1; else { cut=previous_beginning_of_sentence; previous_beginning_of_sentence = i+1; } (*this)[b++] = d.subMatRows(previous_beginning_of_block, cut-previous_beginning_of_block); previous_previous_block = previous_beginning_of_block; previous_beginning_of_block=cut; if (b<n_blocks) { if (b>n_blocks-3) next_target = (int)((total_size - cut) / (real)(n_blocks-b)); else next_target = (int)(total_size * (real)(b+1.0) / n_blocks); } } else previous_beginning_of_sentence=i+1; } } if (b==n_blocks-1) (*this)[b++] = d.subMatRows(previous_beginning_of_block, total_size-previous_beginning_of_block); if (b<n_blocks-1) // we have to backtrack, split previous block in two { if (b<n_blocks-2) PLERROR("SentencesBlocks: blocks are too small!"); if (previous_beginning_of_sentence<previous_beginning_of_block) PLERROR("SentencesBlocks: Blocks are too small!"); int cut = previous_beginning_of_sentence; (*this)[b++] = d.subMatRows(previous_beginning_of_block, cut-previous_beginning_of_block); previous_beginning_of_block=cut; (*this)[b++] = d.subMatRows(previous_beginning_of_block, total_size-previous_beginning_of_block); } }