PLearn 0.1
|
This class handles a sequence of words/sense tag/POS triplets to present it as target words and their context. More...
#include <TextSenseSequenceVMatrix.h>
Public Types | |
typedef RowBufferedVMatrix | inherited |
Public Member Functions | |
TextSenseSequenceVMatrix () | |
Default constructor. After setting all options individually, build() should be called. | |
TextSenseSequenceVMatrix (VMat that_dvm, int that_window_size, TVec< int > that_res_pos=TVec< int >(0), bool that_rand_syn=false, WordNetOntology *that_wno=NULL) | |
int | getRestrictedRow (int i, Vec v) const |
This restricts the extraction of the context to the words that don't have their POS in res_pos and returns the position of the next non-overlapping context. | |
virtual void | build () |
Simply calls inherited::build() then build_(). | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
void | setOntology (WordNetOntology *that_wno) |
Sets the ontology. | |
void | setWindowSize (int that_window_size) |
Sets the number of context words. | |
void | setWordSequence (VMat that_dvm) |
Sets the VMatrix of word/sense_tag/POS sequence. | |
void | setRandomGeneration (bool that_rand_syn) |
Sets the activation/desactivation of the random generation of contexts and target words. | |
void | setRestrictedPOS (TVec< int > that_res_pos) |
Sets the vector of forbidden POS for the context words. | |
void | setSentenceBoundary (int b) |
Sets the sentence boundary symbol. | |
void | setUndefinedPOSId (int pos_id) |
Sets the undefined pos id. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual TextSenseSequenceVMatrix * | deepCopy (CopiesMap &copies) const |
Static Public Member Functions | |
static string | _classname_ () |
Declares name and deepCopy methods. | |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Protected Member Functions | |
virtual void | getNewRow (int i, const Vec &v) const |
This is the only method requiring implementation. | |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares this class' options. | |
Protected Attributes | |
VMat | dvm |
The VMatrix containing the sequence of words or lemmas, with their POS and WordNet (optional) tags. | |
int | window_size |
The number of context words. | |
bool | is_supervised_data |
Indication that at less some of the words or lemmas are semantically disambiguated. | |
TVec< int > | res_pos |
The vector containing the forbidden POS of the words given in the context of a target word. | |
bool | rand_syn |
Indication that examples can be randomly generated using random synonym replacements. | |
TVec< TVec< pair< int, real > > > | word_given_sense_priors |
Probability of a word given it has some sense. | |
WordNetOntology * | wno |
Ontology of the sense tagging. | |
int | my_current_row_index |
Index of the current row. | |
Vec | my_current_row |
Elements of the current row. | |
bool | keep_in_sentence |
Indication that the context must not spread over another sentence. | |
int | sentence_boundary |
Sentence boundary symbol. | |
bool | undefined_pos_set |
Indication that the undefined pos id is defined. | |
int | undefined_pos |
Undefined pos id. | |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
void | permute (Vec v) const |
This permutes randomly the words (target and context) with one of their corresponding synonym. | |
void | apply_boundary (const Vec &v) const |
This applies the sentence boundary. |
This class handles a sequence of words/sense tag/POS triplets to present it as target words and their context.
Definition at line 17 of file TextSenseSequenceVMatrix.h.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 128 of file TextSenseSequenceVMatrix.h.
PLearn::TextSenseSequenceVMatrix::TextSenseSequenceVMatrix | ( | ) |
Default constructor. After setting all options individually, build() should be called.
Definition at line 9 of file TextSenseSequenceVMatrix.cc.
:inherited(),window_size(0), is_supervised_data(true), res_pos(TVec<int>(0)), rand_syn(false), wno(NULL), keep_in_sentence(false), undefined_pos_set(false) /* ### Initialise all fields to their default value */ { // ### You may or may not want to call build_() to finish building the object //build_(); }
PLearn::TextSenseSequenceVMatrix::TextSenseSequenceVMatrix | ( | VMat | that_dvm, |
int | that_window_size, | ||
TVec< int > | that_res_pos = TVec<int>(0) , |
||
bool | that_rand_syn = false , |
||
WordNetOntology * | that_wno = NULL |
||
) | [inline] |
that_dvm | the sequence of words/lemmas |
that_window_size | the number of context words/lemmas |
that_res_pos | the forbidden POS for the context words |
that_rand_syn | indication that the user allow the random generation of contexts and target words using synonyms |
that_wno | the ontology used as a sense inventory |
Definition at line 67 of file TextSenseSequenceVMatrix.h.
:inherited(that_dvm->length(), 3*(that_window_size+1)),dvm(that_dvm),window_size(that_window_size), is_supervised_data(dvm->width()==3), res_pos(that_res_pos), rand_syn(that_rand_syn), wno(that_wno), my_current_row_index(-3*(that_window_size+1)), my_current_row(3*(that_window_size+1)), keep_in_sentence(false), undefined_pos_set(false) /* ### Initialise all fields to their default value */ { build_(); }
string PLearn::TextSenseSequenceVMatrix::_classname_ | ( | ) | [static] |
Declares name and deepCopy methods.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
OptionList & PLearn::TextSenseSequenceVMatrix::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
RemoteMethodMap & PLearn::TextSenseSequenceVMatrix::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
Object * PLearn::TextSenseSequenceVMatrix::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
StaticInitializer TextSenseSequenceVMatrix::_static_initializer_ & PLearn::TextSenseSequenceVMatrix::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
void PLearn::TextSenseSequenceVMatrix::apply_boundary | ( | const Vec & | v | ) | const [private] |
This applies the sentence boundary.
Definition at line 344 of file TextSenseSequenceVMatrix.cc.
References i, sentence_boundary, undefined_pos, undefined_pos_set, UNDEFINED_SS_ID, UNDEFINED_TYPE, and window_size.
Referenced by getNewRow(), and getRestrictedRow().
{ // Looking for left boundary bool found_boundary = false; for(int i=window_size/2-1; i>=0; i--) { if(v[3*i] == sentence_boundary) found_boundary = true; if(found_boundary) { v[3*i] = 0; v[3*i+1] = UNDEFINED_SS_ID; if(undefined_pos_set) v[3*i+2] = undefined_pos; else v[3*i+2] = UNDEFINED_TYPE; } } // Looking for right boundary found_boundary = false; for(int i=window_size/2; i<window_size; i--) { if(v[3*i] == sentence_boundary) found_boundary = true; if(found_boundary) { v[3*i] = 0; v[3*i+1] = UNDEFINED_SS_ID; if(undefined_pos_set) v[3*i+2] = undefined_pos; else v[3*i+2] = UNDEFINED_TYPE; } } }
void PLearn::TextSenseSequenceVMatrix::build | ( | ) | [virtual] |
Simply calls inherited::build() then build_().
Reimplemented from PLearn::VMatrix.
Definition at line 527 of file TextSenseSequenceVMatrix.cc.
References PLearn::VMatrix::build(), and build_().
{ inherited::build(); build_(); }
void PLearn::TextSenseSequenceVMatrix::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::VMatrix.
Definition at line 454 of file TextSenseSequenceVMatrix.cc.
References PLearn::Set::begin(), dvm, PLearn::Set::end(), PLearn::VMatrix::fieldinfos, PLearn::TVec< T >::first(), PLearn::WordNetOntology::getSenseSize(), PLearn::WordNetOntology::getWordsForSense(), i, PLearn::PP< T >::isNull(), j, PLearn::VMat::length(), PLearn::VMatrix::length_, PLERROR, PLWARNING, rand_syn, PLearn::TVec< T >::resize(), PLearn::Set::size(), PLearn::TVec< T >::size(), PLearn::sum(), PLearn::VMat::width(), PLearn::VMatrix::width_, window_size, wno, and word_given_sense_priors.
Referenced by build().
{ if(window_size%2 != 0) PLERROR("In TextSenseSequenceVMatrix: window_size must be even number"); if(window_size < 0) PLERROR("In TextSenseSequenceVMatrix: window_size must be non negative"); if(dvm->width() != 2 && dvm->width() != 3) PLERROR("In TextSenseSequenceVMatrix: VMat that_dvm should have width equal to 2 or 3"); width_ = 3*(window_size+1); length_ = dvm->length(); fieldinfos.resize(width_); //To do: Field Infos ? //oov_tag_id = wno->getWordId(OOV_TAG); if(dvm->width() == 2 && rand_syn) PLWARNING("In TextSenseSequenceVMatrix: cannot use permutation of same-sense words with unsupervised data"); if(dvm.isNull()) PLERROR("In TextSenseSequenceVMatrix: dvm (data of the matrix) is not defined"); if(dvm->width() == 3 && rand_syn) { if(wno == NULL) PLERROR("In TextSenseSequence: there is no WordNetOntology defined"); word_given_sense_priors.resize(wno->getSenseSize()); for(int i=0; i<word_given_sense_priors.size(); i++) { Set words_for_sense = wno->getWordsForSense(i); int n_words_for_sense = words_for_sense.size(); word_given_sense_priors[i]->resize(n_words_for_sense, 1); int j=0; for(SetIterator sit = words_for_sense.begin(); sit != words_for_sense.end(); sit++,j++) { word_given_sense_priors[i][j].first = *sit; word_given_sense_priors[i][j].second = 1; } } if(dvm.isNull()) PLERROR("In TextSenseSequenceVMatrix: dvm (data of the matrix) is not defined"); Vec triplet(3); for(int i=0; i<dvm.length(); i++) { dvm->getRow(i, triplet); int sense = (int)triplet[1]; int word = (int )triplet[0]; if(sense >= 0 && word >= 0) { int size = word_given_sense_priors[sense].size(); for(int j=0; j<size;j++) if(word == word_given_sense_priors[sense][j].first) { word_given_sense_priors[sense][j].second += word_given_sense_priors[sense][j].second == 1 ? 1 : 2; break; } } } //Normalization for(int i=0; i<word_given_sense_priors.size(); i++) { real sum = 0; for(int j=0; j<word_given_sense_priors[i].size(); j++) sum += word_given_sense_priors[i][j].second; if(sum != 0) for(int j=0; j<word_given_sense_priors[i].size(); j++) word_given_sense_priors[i][j].second /= sum; } } }
string PLearn::TextSenseSequenceVMatrix::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
void PLearn::TextSenseSequenceVMatrix::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares this class' options.
Reimplemented from PLearn::VMatrix.
Definition at line 440 of file TextSenseSequenceVMatrix.cc.
References PLearn::OptionBase::buildoption, PLearn::declareOption(), PLearn::VMatrix::declareOptions(), dvm, is_supervised_data, keep_in_sentence, rand_syn, res_pos, sentence_boundary, undefined_pos, undefined_pos_set, and window_size.
{ declareOption(ol, "window_size", &TextSenseSequenceVMatrix::window_size, OptionBase::buildoption,"Size of the context window"); declareOption(ol, "is_supervised_data", &TextSenseSequenceVMatrix::is_supervised_data, OptionBase::buildoption,"Data of VMatrix is supervised"); declareOption(ol, "res_pos", &TextSenseSequenceVMatrix::res_pos, OptionBase::buildoption,"TVec<int> containing the POSs of the words which should not be included in the target word context"); declareOption(ol, "dvm", &TextSenseSequenceVMatrix::dvm, OptionBase::buildoption,"VMatrix that contains the triplets word/sense/POS of a corpus"); declareOption(ol, "rand_syn", &TextSenseSequenceVMatrix::rand_syn, OptionBase::buildoption,"Use same-sense random permutation of words"); declareOption(ol, "keep_in_sentence", &TextSenseSequenceVMatrix::keep_in_sentence, OptionBase::buildoption,"Indication that the context must not spread over another sentence"); declareOption(ol, "sentence_boundary", &TextSenseSequenceVMatrix::sentence_boundary, OptionBase::buildoption,"Sentence boundary symbol"); declareOption(ol, "undefined_pos_set", &TextSenseSequenceVMatrix::undefined_pos_set, OptionBase::buildoption,"Indication that the undefined pos id is defined"); declareOption(ol, "undefined_pos", &TextSenseSequenceVMatrix::undefined_pos, OptionBase::buildoption,"Undefined pos id"); inherited::declareOptions(ol); }
static const PPath& PLearn::TextSenseSequenceVMatrix::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 130 of file TextSenseSequenceVMatrix.h.
TextSenseSequenceVMatrix * PLearn::TextSenseSequenceVMatrix::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
void PLearn::TextSenseSequenceVMatrix::getNewRow | ( | int | i, |
const Vec & | v | ||
) | const [protected, virtual] |
This is the only method requiring implementation.
Implements PLearn::RowBufferedVMatrix.
Definition at line 24 of file TextSenseSequenceVMatrix.cc.
References apply_boundary(), dvm, getRestrictedRow(), i, is_supervised_data, j, keep_in_sentence, PLearn::TVec< T >::length(), PLearn::VMat::length(), my_current_row, my_current_row_index, permute(), PLERROR, rand_syn, res_pos, PLearn::TVec< T >::size(), SYNSETTAG_ID, undefined_pos, undefined_pos_set, UNDEFINED_SS_ID, UNDEFINED_TYPE, PLearn::VMat::width(), and window_size.
{ if(res_pos.size() != 0) { getRestrictedRow(i,v); return; } if(i >= dvm->length() || i < 0) PLERROR("In TextSenseSequenceVMatrix: requesting %dth row of matrix of length %d", i, dvm->length()); if(v.length() != 3*(window_size+1)) PLERROR("In TextSenseSequenceVMatrix: getNewRow v.length() must be equal to VMat's width"); // Fetch context already in memory if(i == my_current_row_index) { for(int j=0; j<my_current_row.size(); j++) v[j] = my_current_row[j]; if(dvm->width() == 3 && rand_syn) permute(v); for(int j=0; j<my_current_row.size(); j++) my_current_row[j] = v[j]; return; } // Fetch context not found in memory int context_dist = -(window_size/2); int context_count = 0; while(context_dist <= window_size/2) { int context_dist_i = context_dist+i; int dist_my_current_row_index = (context_dist_i) - my_current_row_index; if(my_current_row_index != -1 && dist_my_current_row_index >= - window_size/2 && dist_my_current_row_index <= window_size/2) { int index = -1; if(dist_my_current_row_index == 0) index = window_size; if(dist_my_current_row_index < 0) index = window_size/2 + dist_my_current_row_index; if(dist_my_current_row_index > 0) index = window_size/2 + dist_my_current_row_index - 1; if(context_dist != 0) { v[3*context_count] = my_current_row[3*index]; v[3*context_count+1] = my_current_row[3*index+1]; v[3*context_count+2] = my_current_row[3*index+2]; context_count++; } else { v[3*window_size] = my_current_row[3*index]; v[3*window_size+1] = my_current_row[3*index+1]; v[3*window_size+2] = my_current_row[3*index+2]; } } else if(context_dist_i < 0 || context_dist_i >= dvm->length()) { v[3*context_count] = 0; //oov_tag_id : should'nt be handcoded; v[3*context_count+1] = UNDEFINED_SS_ID; v[3*context_count+2] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; context_count++; } else { if(context_dist == 0) { if(is_supervised_data) { Vec temp(3); dvm->getRow(i, temp); if(temp[0] == SYNSETTAG_ID) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = UNDEFINED_SS_ID; temp[2] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*window_size] = temp[0]; v[3*window_size+1] = temp[1]; v[3*window_size+2] = temp[2]; } else { Vec temp(2); dvm->getRow(i, temp); if(SYNSETTAG_ID == temp[0]) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*window_size] = temp[0]; v[3*window_size+1] = UNDEFINED_SS_ID; v[3*window_size+2] = temp[1]; } context_dist++; continue; } if(is_supervised_data) { Vec temp(3); dvm->getRow(context_dist_i, temp); if(temp[0] == SYNSETTAG_ID) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = UNDEFINED_SS_ID; temp[2] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*context_count] = temp[0]; v[3*context_count+1] = temp[1]; v[3*context_count+2] = temp[2]; } else { Vec temp(2); dvm->getRow(context_dist_i, temp); if(SYNSETTAG_ID == temp[0]) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*context_count] = temp[0]; v[3*context_count+1] = UNDEFINED_SS_ID; v[3*context_count+2] = temp[1]; } context_count++; } context_dist++; } if(context_count != window_size) PLERROR("What the hell!!!"); if(dvm->width() == 3 && rand_syn) permute(v); my_current_row_index = i; for(int j=0; j<my_current_row.size(); j++) my_current_row[j] = v[j]; if(keep_in_sentence) apply_boundary(v); }
OptionList & PLearn::TextSenseSequenceVMatrix::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
OptionMap & PLearn::TextSenseSequenceVMatrix::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
RemoteMethodMap & PLearn::TextSenseSequenceVMatrix::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 22 of file TextSenseSequenceVMatrix.cc.
This restricts the extraction of the context to the words that don't have their POS in res_pos and returns the position of the next non-overlapping context.
Definition at line 171 of file TextSenseSequenceVMatrix.cc.
References apply_boundary(), PLearn::TVec< T >::contains(), dvm, i, is_supervised_data, j, keep_in_sentence, PLearn::VMatrix::length(), PLearn::TVec< T >::length(), PLearn::VMat::length(), my_current_row, my_current_row_index, permute(), PLERROR, rand_syn, res_pos, PLearn::TVec< T >::size(), SYNSETTAG_ID, undefined_pos, undefined_pos_set, UNDEFINED_SS_ID, UNDEFINED_TYPE, PLearn::VMat::width(), and window_size.
Referenced by getNewRow().
{ if(i >= dvm->length() || i < 0) PLERROR("In TextSenseSequenceVMatrix: requesting %dth row of matrix of length %d", i, dvm.length()); if(v.length() != 3*(window_size+1)) PLERROR("In TextSenseSequenceVMatrix: getRestrictedRow v.length() must be equal to VMat's width"); // Initialization of context for(int j=0; j<window_size; j++) { v[3*j] = 0; //oov_tag_id : should'nt be handcoded; v[3*j+1] = UNDEFINED_SS_ID; v[3*j+2] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } // Fetch target word if(is_supervised_data) { Vec temp(3); dvm->getRow(i, temp); if(SYNSETTAG_ID == temp[0]) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = UNDEFINED_SS_ID; temp[2] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*window_size] = temp[0]; v[3*window_size+1] = temp[1]; v[3*window_size+2] = temp[2]; } else { Vec temp(2); dvm->getRow(i, temp); if(SYNSETTAG_ID == temp[0]) { temp[0] = 0; //oov_tag_id : should'nt be handcoded; temp[1] = undefined_pos_set ? undefined_pos : UNDEFINED_TYPE; } v[3*window_size] = temp[0]; v[3*window_size+1] = UNDEFINED_SS_ID; v[3*window_size+2] = temp[1]; } // Fetch words to the left int context_dist = -1; int context_found = 0; while(context_found != window_size/2 && context_dist+i >=0) { if(is_supervised_data) { Vec temp(3); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) break; if(!res_pos.contains((int)temp[2])) { context_found++; int index = window_size/2 - context_found; v[3*index] = temp[0]; v[3*index+1] = temp[1]; v[3*index+2] = temp[2]; } } else { Vec temp(2); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) break; if(!res_pos.contains((int)temp[1])) { context_found++; int index = window_size/2 - context_found; v[3*index] = temp[0]; v[3*index+1] = UNDEFINED_SS_ID;; v[3*index+2] = temp[1]; } } context_dist--; } // Fetch words to the right context_dist = 1; context_found = window_size/2; while(context_found != window_size && context_dist+i < dvm->length()) { if(is_supervised_data) { Vec temp(3); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) break; if(!res_pos.contains((int)temp[2])) { int index = context_found; context_found++; v[3*index] = temp[0]; v[3*index+1] = temp[1]; v[3*index+2] = temp[2]; } } else { Vec temp(2); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) break; if(!res_pos.contains((int)temp[1])) { int index = context_found; context_found++; v[3*index] = temp[0]; v[3*index+1] = UNDEFINED_SS_ID;; v[3*index+2] = temp[1]; } } context_dist++; } // Looking for next non-overlapping context context_found = 0; while(context_found != window_size/2+1 && context_dist+i < dvm->length()) { if(is_supervised_data) { Vec temp(3); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) { context_dist++; continue; } if(!res_pos.contains((int)temp[2])) context_found++; } else { Vec temp(2); dvm->getRow(context_dist+i, temp); if(temp[0] == SYNSETTAG_ID) { context_dist++; continue; } if(!res_pos.contains((int)temp[1])) context_found++; } context_dist++; } if(dvm->width() == 3 && rand_syn) permute(v); my_current_row_index = i; for(int j=0; j<my_current_row.size(); j++) my_current_row[j] = v[j]; if(keep_in_sentence) apply_boundary(v); return context_dist+i == dvm->length() ? context_dist+i : context_dist+i-1; }
void PLearn::TextSenseSequenceVMatrix::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 533 of file TextSenseSequenceVMatrix.cc.
References PLearn::deepCopyField(), dvm, PLearn::RowBufferedVMatrix::makeDeepCopyFromShallowCopy(), and res_pos.
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(dvm, copies); deepCopyField(res_pos,copies); // ### Remove this line when you have fully implemented this method. //PLERROR("TextSenseSequenceVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); }
void PLearn::TextSenseSequenceVMatrix::permute | ( | Vec | v | ) | const [private] |
This permutes randomly the words (target and context) with one of their corresponding synonym.
Definition at line 377 of file TextSenseSequenceVMatrix.cc.
References ADJ_TYPE, ADV_TYPE, PLearn::WordNetOntology::getSensesForWord(), PLearn::WordNetOntology::getWord(), PLearn::WordNetOntology::getWordId(), i, j, NOUN_TYPE, PLearn::TVec< T >::size(), PLearn::stemWord(), PLearn::sum(), PLearn::WordNetOntology::temp_word_to_adj_senses, PLearn::WordNetOntology::temp_word_to_adv_senses, PLearn::WordNetOntology::temp_word_to_noun_senses, PLearn::WordNetOntology::temp_word_to_verb_senses, UNDEFINED_TYPE, PLearn::uniform_sample(), VERB_TYPE, window_size, wno, and word_given_sense_priors.
Referenced by getNewRow(), and getRestrictedRow().
{ for(int i=0; i<window_size+1; i++) { int pos = (int)v[3*i+2]; if(pos == NOUN_TYPE || pos == VERB_TYPE || pos == ADJ_TYPE || pos == ADV_TYPE) { real rand = uniform_sample(); real sum = 0; int j=0; int sense = (int)v[3*i+1]; int word_id = (int)v[3*i]; if(sense >= 0 && word_id >= 0) { for(; j<word_given_sense_priors[sense].size(); j++) { if(rand < sum + word_given_sense_priors[sense][j].second) break; sum += word_given_sense_priors[sense][j].second; } string word = wno->getWord(word_given_sense_priors[sense][j].first); string stemmed_syn = stemWord(word, pos); int syn_word_id = wno->getWordId(stemmed_syn); if(syn_word_id != -1) { TVec<int> senses_of_target_word; switch (pos) { case NOUN_TYPE: senses_of_target_word = wno->temp_word_to_noun_senses[syn_word_id]; break; case VERB_TYPE: senses_of_target_word = wno->temp_word_to_verb_senses[syn_word_id]; break; case ADJ_TYPE: senses_of_target_word = wno->temp_word_to_adj_senses[syn_word_id]; break; case ADV_TYPE: senses_of_target_word = wno->temp_word_to_adv_senses[syn_word_id]; break; case UNDEFINED_TYPE: senses_of_target_word = wno->getSensesForWord(syn_word_id); break; default: //PLERROR("weird in train, target_pos = %d", target_pos); senses_of_target_word = wno->getSensesForWord(syn_word_id); } int k=0; while(k<senses_of_target_word.size()) { if(senses_of_target_word[k] == (int)v[3*i+1]) break; k++; } if(k != senses_of_target_word.size()) v[3*i] = syn_word_id; } } } } }
void PLearn::TextSenseSequenceVMatrix::setOntology | ( | WordNetOntology * | that_wno | ) | [inline] |
void PLearn::TextSenseSequenceVMatrix::setRandomGeneration | ( | bool | that_rand_syn | ) | [inline] |
Sets the activation/desactivation of the random generation of contexts and target words.
Definition at line 117 of file TextSenseSequenceVMatrix.h.
{rand_syn = that_rand_syn;}
Sets the vector of forbidden POS for the context words.
Definition at line 120 of file TextSenseSequenceVMatrix.h.
{res_pos = that_res_pos;}
void PLearn::TextSenseSequenceVMatrix::setSentenceBoundary | ( | int | b | ) | [inline] |
Sets the sentence boundary symbol.
Definition at line 123 of file TextSenseSequenceVMatrix.h.
References b.
{keep_in_sentence = true; sentence_boundary = b;}
void PLearn::TextSenseSequenceVMatrix::setUndefinedPOSId | ( | int | pos_id | ) | [inline] |
Sets the undefined pos id.
Definition at line 126 of file TextSenseSequenceVMatrix.h.
{undefined_pos_set = true; undefined_pos = pos_id;}
void PLearn::TextSenseSequenceVMatrix::setWindowSize | ( | int | that_window_size | ) | [inline] |
Sets the number of context words.
Definition at line 111 of file TextSenseSequenceVMatrix.h.
{window_size = that_window_size;}
void PLearn::TextSenseSequenceVMatrix::setWordSequence | ( | VMat | that_dvm | ) | [inline] |
Sets the VMatrix of word/sense_tag/POS sequence.
Definition at line 114 of file TextSenseSequenceVMatrix.h.
References PLearn::VMat::width().
{dvm = that_dvm; is_supervised_data = that_dvm->width()==3; }
Reimplemented from PLearn::RowBufferedVMatrix.
Definition at line 130 of file TextSenseSequenceVMatrix.h.
VMat PLearn::TextSenseSequenceVMatrix::dvm [protected] |
The VMatrix containing the sequence of words or lemmas, with their POS and WordNet (optional) tags.
Definition at line 25 of file TextSenseSequenceVMatrix.h.
Referenced by build_(), declareOptions(), getNewRow(), getRestrictedRow(), and makeDeepCopyFromShallowCopy().
Indication that at less some of the words or lemmas are semantically disambiguated.
Definition at line 29 of file TextSenseSequenceVMatrix.h.
Referenced by declareOptions(), getNewRow(), and getRestrictedRow().
Indication that the context must not spread over another sentence.
Definition at line 43 of file TextSenseSequenceVMatrix.h.
Referenced by declareOptions(), getNewRow(), and getRestrictedRow().
Vec PLearn::TextSenseSequenceVMatrix::my_current_row [mutable, protected] |
Elements of the current row.
Definition at line 41 of file TextSenseSequenceVMatrix.h.
Referenced by getNewRow(), and getRestrictedRow().
int PLearn::TextSenseSequenceVMatrix::my_current_row_index [mutable, protected] |
Index of the current row.
Definition at line 39 of file TextSenseSequenceVMatrix.h.
Referenced by getNewRow(), and getRestrictedRow().
bool PLearn::TextSenseSequenceVMatrix::rand_syn [protected] |
Indication that examples can be randomly generated using random synonym replacements.
Definition at line 33 of file TextSenseSequenceVMatrix.h.
Referenced by build_(), declareOptions(), getNewRow(), and getRestrictedRow().
TVec<int> PLearn::TextSenseSequenceVMatrix::res_pos [protected] |
The vector containing the forbidden POS of the words given in the context of a target word.
Definition at line 31 of file TextSenseSequenceVMatrix.h.
Referenced by declareOptions(), getNewRow(), getRestrictedRow(), and makeDeepCopyFromShallowCopy().
Sentence boundary symbol.
Definition at line 45 of file TextSenseSequenceVMatrix.h.
Referenced by apply_boundary(), and declareOptions().
int PLearn::TextSenseSequenceVMatrix::undefined_pos [protected] |
Undefined pos id.
Definition at line 49 of file TextSenseSequenceVMatrix.h.
Referenced by apply_boundary(), declareOptions(), getNewRow(), and getRestrictedRow().
Indication that the undefined pos id is defined.
Definition at line 47 of file TextSenseSequenceVMatrix.h.
Referenced by apply_boundary(), declareOptions(), getNewRow(), and getRestrictedRow().
int PLearn::TextSenseSequenceVMatrix::window_size [protected] |
The number of context words.
Definition at line 27 of file TextSenseSequenceVMatrix.h.
Referenced by apply_boundary(), build_(), declareOptions(), getNewRow(), getRestrictedRow(), and permute().
WordNetOntology* PLearn::TextSenseSequenceVMatrix::wno [protected] |
Ontology of the sense tagging.
Definition at line 37 of file TextSenseSequenceVMatrix.h.
TVec<TVec<pair<int, real> > > PLearn::TextSenseSequenceVMatrix::word_given_sense_priors [protected] |
Probability of a word given it has some sense.
Definition at line 35 of file TextSenseSequenceVMatrix.h.