PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // LemmatizeVMatrix.cc 00004 // 00005 // Copyright (C) 2005 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $ 00037 ******************************************************* */ 00038 00039 // Authors: Hugo Larochelle 00040 00044 #include "LemmatizeVMatrix.h" 00045 #include <plearn/base/stringutils.h> 00046 00047 namespace PLearn { 00048 using namespace std; 00049 00050 PLEARN_IMPLEMENT_OBJECT( 00051 LemmatizeVMatrix, 00052 "Takes a VMatrix with a word and a POS field and adds a field consisting of the lemma form of the word", 00053 "This field becomes the last input field. First, customized\n" 00054 "mapping tables (with different priorities) are used, \n" 00055 "and finally, if the previous step was unsuccesful, \n" 00056 "WordNet is used to obtain a lemma.\n" 00057 ); 00058 00060 // LemmatizeVMatrix // 00062 LemmatizeVMatrix::LemmatizeVMatrix() 00063 : word_field(-1), pos_field(-1) 00064 { 00065 lemma_dict = new Dictionary(); 00066 00067 if(morphinit<0) 00068 PLERROR("In WordNetSenseDictionary(): could not open WordNet exception list files"); 00069 // if(wninit()<0) 00070 // PLERROR("In WordNetSenseDictionary(): could not open WordNet database files"); 00071 } 00072 00073 00074 string LemmatizeVMatrix::getLemma(int row) const 00075 { 00076 string pos; 00077 string word; 00078 00079 word = lowerstring(source->getString(row,word_field)); 00080 pos = source->getString(row,pos_field); 00081 00082 // Verify if POS tag is compatible for word, using WordNet 00083 // wninit() most be called for that... 00084 /* 00085 TVec<string> ret_noun(0); 00086 TVec<string> ret_verb(0); 00087 TVec<string> ret_adj(0); 00088 TVec<string> ret_adv(0); 00089 TVec<string> stems; 00090 if(strstr(pos.c_str(),"NN")) 00091 { 00092 ret_noun.append(extractSenses(word,NOUN,"sense_key")); 00093 stems = stemsOfWord(word,NOUN); 00094 for(int i=0; i<stems.length(); i++) 00095 if(word != stems[i]) 00096 ret_noun.append(extractSenses(stems[i],NOUN,"sense_key")); 00097 } 00098 00099 if(strstr(pos.c_str(),"VB")) 00100 { 00101 ret_verb.append(extractSenses(word,VERB,"sense_key")); 00102 stems = stemsOfWord(word,VERB); 00103 for(int i=0; i<stems.length(); i++) 00104 if(word != stems[i]) 00105 ret_verb.append(extractSenses(stems[i],VERB,"sense_key")); 00106 } 00107 00108 if(strstr(pos.c_str(),"JJ")) 00109 { 00110 ret_adj.append(extractSenses(word,ADJ,"sense_key")); 00111 stems = stemsOfWord(word,ADJ); 00112 for(int i=0; i<stems.length(); i++) 00113 if(word != stems[i]) 00114 ret_adj.append(extractSenses(stems[i],ADJ,"sense_key")); 00115 } 00116 00117 if(strstr(pos.c_str(),"RB")) 00118 { 00119 ret_adv.append(extractSenses(word,ADV,"sense_key")); 00120 stems = stemsOfWord(word,ADV); 00121 for(int i=0; i<stems.length(); i++) 00122 if(word != stems[i]) 00123 ret_adv.append(extractSenses(stems[i],ADV,"sense_key")); 00124 } 00125 00126 if(!(ret_noun.length() == 0 && ret_verb.length() == 0 && ret_adj.length() == 0 && ret_adv.length() == 0)) 00127 { 00128 if(strstr(pos.c_str(),"NN") && ret_noun.length() == 0) 00129 PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be a noun", word.c_str()); 00130 if(strstr(pos.c_str(),"VB") && ret_verb.length() == 0) 00131 PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be a verb", word.c_str()); 00132 if(strstr(pos.c_str(),"JJ") && ret_adj.length() == 0) 00133 PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be an adjective", word.c_str()); 00134 if(strstr(pos.c_str(),"RB") && ret_adv.length() == 0) 00135 PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be an adverb", word.c_str()); 00136 } 00137 */ 00138 00139 // Use word_pos_to_lemma_table 00140 for(int i=0; i<word_pos_to_lemma_table.length(); i++) 00141 if(word_pos_to_lemma_table(i,0) == word && word_pos_to_lemma_table(i,1) == pos) 00142 return word_pos_to_lemma_table(i,2); 00143 00144 // Use word_to_lemma_table 00145 for(int i=0; i<word_to_lemma_table.length(); i++) 00146 if(word_to_lemma_table(i,0) == word) 00147 return word_to_lemma_table(i,1); 00148 00149 // Use pos_to_lemma_table 00150 for(int i=0; i<pos_to_lemma_table.length(); i++) 00151 if(pos_to_lemma_table(i,0) == pos) 00152 return pos_to_lemma_table(i,1); 00153 00154 if(strstr(pos.c_str(),"NN")) return stemWord(word,NOUN); 00155 else if(strstr(pos.c_str(),"VB")) return stemWord(word,VERB); 00156 else if(strstr(pos.c_str(),"JJ")) return stemWord(word,ADJ); 00157 else if(strstr(pos.c_str(),"RB")) return stemWord(word,ADV); 00158 00159 return word; 00160 } 00161 00163 // declareOptions // 00165 void LemmatizeVMatrix::declareOptions(OptionList& ol) 00166 { 00167 declareOption(ol, "lemma_dict", &LemmatizeVMatrix::lemma_dict, OptionBase::learntoption, 00168 "Dictionary for the lemma field\n"); 00169 declareOption(ol, "word_field", &LemmatizeVMatrix::word_field, OptionBase::buildoption, 00170 "Index (position) of word field\n"); 00171 declareOption(ol, "pos_field", &LemmatizeVMatrix::pos_field, OptionBase::buildoption, 00172 "Index (position) of POS field\n"); 00173 declareOption(ol, "word_pos_to_lemma_table", &LemmatizeVMatrix::word_pos_to_lemma_table, OptionBase::buildoption, 00174 "Customized table that uses the word and POS tag to obtain a lemma\n" 00175 "It must have exactly three columns [word POS lemma]. It has the first priority.\n"); 00176 declareOption(ol, "word_to_lemma_table", &LemmatizeVMatrix::word_to_lemma_table, OptionBase::buildoption, 00177 "Customized table that uses the word to obtain a lemma\n" 00178 "It must have exactly two columns [word lemma]. It has the second priority.\n"); 00179 declareOption(ol, "pos_to_lemma_table", &LemmatizeVMatrix::pos_to_lemma_table, OptionBase::buildoption, 00180 "Customized table that uses the POS tag to obtain a lemma\n" 00181 "It must have exactly two columns [POS lemma] . It has the third priority.\n"); 00182 00183 // Now call the parent class' declareOptions 00184 inherited::declareOptions(ol); 00185 } 00186 00188 // build // 00190 void LemmatizeVMatrix::build() 00191 { 00192 inherited::build(); 00193 build_(); 00194 } 00195 00197 // build_ // 00199 void LemmatizeVMatrix::build_() 00200 { 00201 if(source) 00202 { 00203 if(word_field < 0 || word_field >= source->inputsize()) 00204 PLERROR("In LemmatizeVMatrix::build_(): word_field(%d) is not in input part of source VMatrix", word_field); 00205 if(pos_field < 0 || pos_field >= source->inputsize()) 00206 PLERROR("In LemmatizeVMatrix::build_(): pos_field(%d) is not in input part of source VMatrix", pos_field); 00207 inputsize_ = source->inputsize() + 1; 00208 targetsize_ = source->targetsize(); 00209 weightsize_ = source->weightsize(); 00210 width_ = inputsize_+targetsize_+weightsize_; 00211 length_ = source->length(); 00212 updateMtime(source); 00213 if(word_pos_to_lemma_table.length() != 0 && word_pos_to_lemma_table.width() != 3) 00214 PLERROR("In LemmatizeVMatrix::build_(): word_pos_to_lemma_table doesn't have three columns"); 00215 if(word_to_lemma_table.length() != 0 && word_to_lemma_table.width() != 2) 00216 PLERROR("In LemmatizeVMatrix::build_(): word_to_lemma_table doesn't have two columns"); 00217 if(pos_to_lemma_table.length() != 0 && pos_to_lemma_table.width() != 2) 00218 PLERROR("In LemmatizeVMatrix::build_(): pos_to_lemma_table doesn't have two columns"); 00219 00220 string lemma; 00221 for(int i=0; i<length_;i++) 00222 { 00223 lemma = getLemma(i); 00224 lemma_dict->getId(lemma); 00225 } 00226 lemma_dict->update_mode = NO_UPDATE; 00227 src_row.resize(source->width()); 00228 } 00229 } 00230 00232 // getNewRow // 00234 void LemmatizeVMatrix::getNewRow(int i, const Vec& v) const 00235 { 00236 source->getRow(i,src_row); 00237 v.subVec(0,inputsize_-1) << src_row.subVec(0,inputsize_-1); 00238 v[inputsize_-1] = lemma_dict->getId(getLemma(i)); 00239 v.subVec(inputsize_,targetsize_+weightsize_) << src_row.subVec(inputsize_-1,targetsize_+weightsize_); 00240 } 00241 00243 real LemmatizeVMatrix::getStringVal(int col, const string & str) const 00244 { 00245 int ret; 00246 if(col == inputsize_-1) 00247 { 00248 ret = lemma_dict->getId(str); 00249 if(ret == -1) return MISSING_VALUE; 00250 else return ret; 00251 } 00252 return source->getStringVal(col - (col < inputsize_-1?0:1),str); 00253 } 00254 00255 string LemmatizeVMatrix::getValString(int col, real val) const 00256 { 00257 if(is_missing(val))return tostring(val); 00258 if(col == inputsize_-1) 00259 return lemma_dict->getSymbol((int)val); 00260 return source->getValString(col - (col < inputsize_-1?0:1),val); 00261 } 00262 00263 PP<Dictionary> LemmatizeVMatrix::getDictionary(int col) const 00264 { 00265 if(col == inputsize_-1) 00266 return lemma_dict; 00267 return source->getDictionary(col - (col < inputsize_-1?0:1)); 00268 } 00269 00270 00271 void LemmatizeVMatrix::getValues(int row, int col, Vec& values) const 00272 { 00273 if(row < 0 || row >= length_) PLERROR("In LemmatizeVMatrix::getValues() : invalid row %d, length()=%d", row, length_); 00274 if(col == inputsize_-1) 00275 { 00276 TVec<string> options(0); 00277 lemma_dict->getValues(options,values); 00278 } 00279 else 00280 source->getValues(row,col - (col < inputsize_-1?0:1),values); 00281 } 00282 00283 void LemmatizeVMatrix::getValues(const Vec& input, int col, Vec& values) const 00284 { 00285 if(col == inputsize_-1) 00286 { 00287 TVec<string> options(0); 00288 lemma_dict->getValues(options,values); 00289 } 00290 else 00291 source->getValues(input,col - (col < inputsize_-1?0:1),values); 00292 } 00293 00294 00295 00297 // makeDeepCopyFromShallowCopy // 00299 void LemmatizeVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00300 { 00301 inherited::makeDeepCopyFromShallowCopy(copies); 00302 deepCopyField(pos_to_lemma_table, copies); 00303 deepCopyField(word_to_lemma_table, copies); 00304 deepCopyField(word_pos_to_lemma_table, copies); 00305 deepCopyField(lemma_dict, copies); 00306 deepCopyField(src_row, copies); 00307 00308 //PLERROR("LemmatizeVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00309 } 00310 00311 } // end of namespace PLearn 00312 00313 00314 /* 00315 Local Variables: 00316 mode:c++ 00317 c-basic-offset:4 00318 c-file-style:"stroustrup" 00319 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00320 indent-tabs-mode:nil 00321 fill-column:79 00322 End: 00323 */ 00324 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :