PLearn 0.1
LemmatizeVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // LemmatizeVMatrix.cc
00004 //
00005 // Copyright (C) 2005 Hugo Larochelle
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036    * $Id: .pyskeleton_header 544 2003-09-01 00:05:31Z plearner $
00037    ******************************************************* */
00038 
00039 // Authors: Hugo Larochelle
00040 
00044 #include "LemmatizeVMatrix.h"
00045 #include <plearn/base/stringutils.h>
00046 
00047 namespace PLearn {
00048 using namespace std;
00049 
00050 PLEARN_IMPLEMENT_OBJECT(
00051     LemmatizeVMatrix,
00052     "Takes a VMatrix with a word and a POS field and adds a field consisting of the lemma form of the word",
00053     "This field becomes the last input field. First, customized\n"
00054     "mapping tables (with different priorities) are used, \n"
00055     "and finally, if the previous step was unsuccesful, \n"
00056     "WordNet is used to obtain a lemma.\n"
00057     );
00058 
00060 // LemmatizeVMatrix //
00062 LemmatizeVMatrix::LemmatizeVMatrix()
00063     : word_field(-1), pos_field(-1)
00064 {
00065     lemma_dict = new Dictionary();
00066 
00067     if(morphinit<0)
00068         PLERROR("In WordNetSenseDictionary(): could not open WordNet exception list files");
00069 //    if(wninit()<0)
00070 //        PLERROR("In WordNetSenseDictionary(): could not open WordNet database files");
00071 }
00072 
00073 
00074 string LemmatizeVMatrix::getLemma(int row) const
00075 {
00076     string pos;
00077     string word;
00078 
00079     word = lowerstring(source->getString(row,word_field));
00080     pos = source->getString(row,pos_field);
00081 
00082     // Verify if POS tag is compatible for word, using WordNet
00083     // wninit() most be called for that...
00084    /*
00085     TVec<string> ret_noun(0);
00086     TVec<string> ret_verb(0);
00087     TVec<string> ret_adj(0);
00088     TVec<string> ret_adv(0);
00089     TVec<string> stems;
00090     if(strstr(pos.c_str(),"NN"))
00091     {
00092         ret_noun.append(extractSenses(word,NOUN,"sense_key"));
00093         stems = stemsOfWord(word,NOUN);
00094         for(int i=0; i<stems.length(); i++)
00095             if(word != stems[i])
00096                 ret_noun.append(extractSenses(stems[i],NOUN,"sense_key"));
00097     }
00098 
00099     if(strstr(pos.c_str(),"VB"))
00100     {
00101         ret_verb.append(extractSenses(word,VERB,"sense_key"));
00102         stems = stemsOfWord(word,VERB);
00103         for(int i=0; i<stems.length(); i++)
00104             if(word != stems[i])
00105                 ret_verb.append(extractSenses(stems[i],VERB,"sense_key"));
00106     }
00107 
00108     if(strstr(pos.c_str(),"JJ"))
00109     {
00110         ret_adj.append(extractSenses(word,ADJ,"sense_key"));
00111         stems = stemsOfWord(word,ADJ);
00112         for(int i=0; i<stems.length(); i++)
00113             if(word != stems[i])
00114                 ret_adj.append(extractSenses(stems[i],ADJ,"sense_key"));
00115     }
00116 
00117     if(strstr(pos.c_str(),"RB"))
00118     {
00119         ret_adv.append(extractSenses(word,ADV,"sense_key"));
00120         stems = stemsOfWord(word,ADV);
00121         for(int i=0; i<stems.length(); i++)
00122             if(word != stems[i])
00123                 ret_adv.append(extractSenses(stems[i],ADV,"sense_key"));
00124     }
00125 
00126     if(!(ret_noun.length() == 0 && ret_verb.length() == 0 && ret_adj.length() == 0 && ret_adv.length() == 0))
00127     {
00128         if(strstr(pos.c_str(),"NN") && ret_noun.length() == 0)
00129             PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be a noun", word.c_str());
00130         if(strstr(pos.c_str(),"VB") && ret_verb.length() == 0)
00131             PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be a verb", word.c_str());
00132         if(strstr(pos.c_str(),"JJ") && ret_adj.length() == 0)
00133             PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be an adjective", word.c_str());
00134         if(strstr(pos.c_str(),"RB") && ret_adv.length() == 0)
00135             PLWARNING("In LemmatizeVMatrix::getLemma(): word %s cannot be an adverb", word.c_str());
00136     }
00137     */
00138 
00139     // Use word_pos_to_lemma_table
00140     for(int i=0; i<word_pos_to_lemma_table.length(); i++)
00141         if(word_pos_to_lemma_table(i,0) == word && word_pos_to_lemma_table(i,1) == pos)
00142             return word_pos_to_lemma_table(i,2);
00143 
00144     // Use word_to_lemma_table
00145     for(int i=0; i<word_to_lemma_table.length(); i++)
00146         if(word_to_lemma_table(i,0) == word)
00147             return word_to_lemma_table(i,1);
00148 
00149     // Use pos_to_lemma_table
00150     for(int i=0; i<pos_to_lemma_table.length(); i++)
00151         if(pos_to_lemma_table(i,0) == pos)
00152             return pos_to_lemma_table(i,1);
00153 
00154     if(strstr(pos.c_str(),"NN")) return stemWord(word,NOUN);
00155     else if(strstr(pos.c_str(),"VB")) return stemWord(word,VERB);
00156     else if(strstr(pos.c_str(),"JJ")) return stemWord(word,ADJ);
00157     else if(strstr(pos.c_str(),"RB")) return stemWord(word,ADV);
00158 
00159     return word;
00160 }
00161 
00163 // declareOptions //
00165 void LemmatizeVMatrix::declareOptions(OptionList& ol)
00166 {
00167     declareOption(ol, "lemma_dict", &LemmatizeVMatrix::lemma_dict, OptionBase::learntoption,
00168                   "Dictionary for the lemma field\n");
00169     declareOption(ol, "word_field", &LemmatizeVMatrix::word_field, OptionBase::buildoption,
00170                   "Index (position) of word field\n");
00171     declareOption(ol, "pos_field", &LemmatizeVMatrix::pos_field, OptionBase::buildoption,
00172                   "Index (position) of POS field\n");
00173     declareOption(ol, "word_pos_to_lemma_table", &LemmatizeVMatrix::word_pos_to_lemma_table, OptionBase::buildoption,
00174                   "Customized table that uses the word and POS tag to obtain a lemma\n"
00175                   "It must have exactly three columns [word POS lemma]. It has the first priority.\n");
00176     declareOption(ol, "word_to_lemma_table", &LemmatizeVMatrix::word_to_lemma_table, OptionBase::buildoption,
00177                   "Customized table that uses the word to obtain a lemma\n"
00178                   "It must have exactly two columns [word lemma]. It has the second priority.\n");
00179     declareOption(ol, "pos_to_lemma_table", &LemmatizeVMatrix::pos_to_lemma_table, OptionBase::buildoption,
00180                   "Customized table that uses the POS tag to obtain a lemma\n"
00181                   "It must have exactly two columns [POS lemma] . It has the third priority.\n");
00182 
00183     // Now call the parent class' declareOptions
00184     inherited::declareOptions(ol);
00185 }
00186 
00188 // build //
00190 void LemmatizeVMatrix::build()
00191 {
00192     inherited::build();
00193     build_();
00194 }
00195 
00197 // build_ //
00199 void LemmatizeVMatrix::build_()
00200 {
00201     if(source)
00202     {
00203         if(word_field < 0 || word_field >= source->inputsize())
00204             PLERROR("In LemmatizeVMatrix::build_(): word_field(%d) is not in input part of source VMatrix", word_field);
00205         if(pos_field < 0 || pos_field >= source->inputsize())
00206             PLERROR("In LemmatizeVMatrix::build_(): pos_field(%d) is not in input part of source VMatrix", pos_field);
00207         inputsize_ = source->inputsize() + 1;
00208         targetsize_ = source->targetsize();
00209         weightsize_ = source->weightsize();
00210         width_ = inputsize_+targetsize_+weightsize_;
00211         length_ = source->length();
00212         updateMtime(source);
00213         if(word_pos_to_lemma_table.length() != 0 && word_pos_to_lemma_table.width() != 3)
00214            PLERROR("In LemmatizeVMatrix::build_(): word_pos_to_lemma_table doesn't have three columns");
00215         if(word_to_lemma_table.length() != 0 && word_to_lemma_table.width() != 2)
00216            PLERROR("In LemmatizeVMatrix::build_(): word_to_lemma_table doesn't have two columns");
00217         if(pos_to_lemma_table.length() != 0 && pos_to_lemma_table.width() != 2)
00218            PLERROR("In LemmatizeVMatrix::build_(): pos_to_lemma_table doesn't have two columns");
00219 
00220         string lemma;
00221         for(int i=0; i<length_;i++)
00222         {
00223             lemma = getLemma(i);
00224             lemma_dict->getId(lemma);
00225         }
00226         lemma_dict->update_mode = NO_UPDATE;
00227         src_row.resize(source->width());
00228     }
00229 }
00230 
00232 // getNewRow //
00234 void LemmatizeVMatrix::getNewRow(int i, const Vec& v) const
00235 {
00236     source->getRow(i,src_row);
00237     v.subVec(0,inputsize_-1) << src_row.subVec(0,inputsize_-1);
00238     v[inputsize_-1] = lemma_dict->getId(getLemma(i));
00239     v.subVec(inputsize_,targetsize_+weightsize_) << src_row.subVec(inputsize_-1,targetsize_+weightsize_);
00240 }
00241 
00243 real LemmatizeVMatrix::getStringVal(int col, const string & str) const
00244 {
00245     int ret;
00246     if(col == inputsize_-1)
00247     {
00248         ret = lemma_dict->getId(str);
00249         if(ret == -1) return MISSING_VALUE;
00250         else return ret;
00251     }
00252     return source->getStringVal(col - (col < inputsize_-1?0:1),str);
00253 }
00254 
00255 string LemmatizeVMatrix::getValString(int col, real val) const
00256 {
00257     if(is_missing(val))return tostring(val);
00258     if(col == inputsize_-1)
00259         return lemma_dict->getSymbol((int)val);
00260     return source->getValString(col - (col < inputsize_-1?0:1),val);
00261 }
00262 
00263 PP<Dictionary>  LemmatizeVMatrix::getDictionary(int col) const
00264 {
00265     if(col == inputsize_-1)
00266         return lemma_dict;
00267     return source->getDictionary(col - (col < inputsize_-1?0:1));
00268 }
00269 
00270 
00271 void LemmatizeVMatrix::getValues(int row, int col, Vec& values) const
00272 {
00273     if(row < 0 || row >= length_) PLERROR("In LemmatizeVMatrix::getValues() : invalid row %d, length()=%d", row, length_);
00274     if(col == inputsize_-1)
00275     {
00276         TVec<string> options(0);
00277         lemma_dict->getValues(options,values);
00278     }
00279     else
00280         source->getValues(row,col - (col < inputsize_-1?0:1),values);
00281 }
00282 
00283 void LemmatizeVMatrix::getValues(const Vec& input, int col, Vec& values) const
00284 {
00285     if(col == inputsize_-1)
00286     {
00287         TVec<string> options(0);
00288         lemma_dict->getValues(options,values);
00289     }
00290     else
00291         source->getValues(input,col - (col < inputsize_-1?0:1),values);
00292 }
00293 
00294 
00295 
00297 // makeDeepCopyFromShallowCopy //
00299 void LemmatizeVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00300 {
00301     inherited::makeDeepCopyFromShallowCopy(copies);
00302     deepCopyField(pos_to_lemma_table, copies);
00303     deepCopyField(word_to_lemma_table, copies);
00304     deepCopyField(word_pos_to_lemma_table, copies);
00305     deepCopyField(lemma_dict, copies);
00306     deepCopyField(src_row, copies);
00307 
00308     //PLERROR("LemmatizeVMatrix::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!");
00309 }
00310 
00311 } // end of namespace PLearn
00312 
00313 
00314 /*
00315   Local Variables:
00316   mode:c++
00317   c-basic-offset:4
00318   c-file-style:"stroustrup"
00319   c-file-offsets:((innamespace . 0)(inline-open . 0))
00320   indent-tabs-mode:nil
00321   fill-column:79
00322   End:
00323 */
00324 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines