PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // Dictionary.cc 00004 // 00005 // Copyright (C) 2004 Hugo Larochelle, Christopher Kermorvant 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: Dictionary.cc 6392 2006-11-07 18:32:40Z larocheh $ 00037 ******************************************************* */ 00038 00039 // Authors: Hugo Larochelle, Christopher Kermorvant 00040 00044 #include "Dictionary.h" 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 Dictionary::Dictionary() 00050 : 00051 update_mode(UPDATE), 00052 dont_insert_oov_symbol(false), 00053 oov_symbol("<oov>") 00054 { 00055 } 00056 00057 PLEARN_IMPLEMENT_OBJECT(Dictionary, 00058 "Mapping string->int and int->string", 00059 "A dictionary is a mapping between a string and an index or ID (int).\n" 00060 "Depending on the update mode, the dictionay can include an unknown \n" 00061 "word when asking for its ID with getId();\n" 00062 "if update_mode == UPDATE, add the word and return its ID\n" 00063 "if update_mode == NO_UPDATE, return OOV symbol's ID (see below)\n" 00064 "\n" 00065 "Also, when asking for the symbol associated to an ID, no update is \n" 00066 "possible. If the ID has not been assigned by the dictionary, \n" 00067 "then the returned symbol is \"\".\n" 00068 "\n" 00069 "A Dictionary object is instantiated empty, and then symbols can \n" 00070 "be added in the dictionary by using getId() (with update_mode == UPDATE).\n" 00071 "By default, IDs are assigned by starting with the ID 0 and incrementing\n" 00072 "the assigned IDs as symbols are added in the dictionary. \n" 00073 "\n" 00074 "Also, a \"out-of-vocabulary\" (OOV) symbol is assigned an ID which \n" 00075 "is equal to the number of symbols (other than the OOV symbol) \n" 00076 "that are in the dictionary at the current state. By default, \n" 00077 "this symbol corresponds to the string \"<oov>\". \n" 00078 "\n" 00079 "The OOV symbol's ID is returned by the getId(symbol) function if\n" 00080 "\"symbol\" is not in the dictionary. Also by default, this OOV token is\n" 00081 "considered as part of the dictionary, but this can be changed\n" 00082 "using the dont_insert_oov_symbol option. If dont_insert_oov_token\n" 00083 "is true, then the size(), isIn() and getValues() functions\n" 00084 "do not consider the oov symbol as part of the dictionary. Note that\n" 00085 "getId() will still return the oov symbol's ID for unknown symbols,\n" 00086 "and getSymbol() will still return the OOV symbol for its ID.\n" 00087 ); 00088 00089 void Dictionary::declareOptions(OptionList& ol) 00090 { 00091 declareOption(ol, "update_mode", &Dictionary::update_mode, 00092 OptionBase::buildoption, 00093 "update_mode : 0(no_update)/1(update). Default is update"); 00094 declareOption(ol, "oov_symbol", &Dictionary::oov_symbol, 00095 OptionBase::buildoption, 00096 "String symbol for \"out-of-vocabulary\" token."); 00097 declareOption(ol, "string_to_int", &Dictionary::string_to_int, 00098 OptionBase::learntoption, "string to int mapping"); 00099 declareOption(ol, "int_to_string", &Dictionary::int_to_string, 00100 OptionBase::learntoption, "int to string mapping"); 00101 // For backward compatibility... 00102 declareOption(ol, "oov_not_in_possible_values", 00103 &Dictionary::dont_insert_oov_symbol, 00104 OptionBase::buildoption, 00105 "DEPRECATED, use dont_insert_oov_symbol instead: Indication that\n" 00106 "the OOV symbol should not be considered as part of the dictionary"); 00107 declareOption(ol, "dont_insert_oov_symbol", 00108 &Dictionary::dont_insert_oov_symbol, 00109 OptionBase::buildoption, 00110 "Indication that the OOV symbol should not be considered\n" 00111 "as part of the dictionary"); 00112 00113 inherited::declareOptions(ol); 00114 } 00115 00116 void Dictionary::build_(){ 00117 } 00118 00119 // ### Nothing to add here, simply calls build_ 00120 void Dictionary::build() 00121 { 00122 inherited::build(); 00123 build_(); 00124 } 00125 00126 void Dictionary::setUpdateMode(int up_mode) 00127 { 00128 if(up_mode != UPDATE && up_mode != NO_UPDATE) 00129 PLERROR("In Dictionary::setUpdateMode(): incompatible update mode %d", up_mode); 00130 update_mode =up_mode; 00131 } 00132 00133 int Dictionary::getId(string symbol, TVec<string> options) 00134 { 00135 // Gives the id of a symbol in the dictionary 00136 // If the symbol is not in the dictionary, 00137 // returns index of OOV symbol if update_mode = NO_UPDATE, 00138 // insert the new word otherwise and return its index 00139 00140 if(symbol == oov_symbol) return string_to_int.size(); 00141 00142 int index; 00143 if(update_mode== UPDATE) 00144 { 00145 if(string_to_int.find(symbol) == string_to_int.end()){ 00146 // word not found, add it 00147 index=int(string_to_int.size()); 00148 string_to_int[symbol] = index; 00149 int_to_string[index] = symbol; 00150 } 00151 return string_to_int[symbol]; 00152 } 00153 else 00154 { 00155 // No update mode 00156 if(string_to_int.find(symbol) == string_to_int.end()){ 00157 // word not found, return OOV symbol's ID 00158 return string_to_int.size(); 00159 }else{ 00160 return string_to_int[symbol]; 00161 } 00162 } 00163 } 00164 00165 string Dictionary::getSymbol(int id, TVec<string>options)const 00166 { 00167 if(int_to_string.find(id) == int_to_string.end()) 00168 { 00169 if(id == ((int)string_to_int.size())) 00170 return oov_symbol; 00171 else 00172 return ""; 00173 } 00174 else 00175 return int_to_string.find(id)->second; 00176 } 00177 00178 int Dictionary::size(TVec<string> options){ 00179 if(dont_insert_oov_symbol) 00180 return int(string_to_int.size()); 00181 else 00182 return int(string_to_int.size())+1; 00183 } 00184 00185 void Dictionary::getValues(TVec<string> options, Vec& values) 00186 { 00187 values.resize(size()); 00188 int i=0; 00189 for(map<int,string>::iterator it = int_to_string.begin(); it != int_to_string.end(); it++) 00190 values[i++] = it->first; 00191 if(!dont_insert_oov_symbol) 00192 values[i] = i; // OOV symbol's ID 00193 } 00194 00195 bool Dictionary::isIn(string symbol, TVec<string> options){ 00196 if(symbol == oov_symbol) return !dont_insert_oov_symbol; 00197 int last_update_mode = update_mode; 00198 update_mode = NO_UPDATE; 00199 int id = getId(symbol,options); 00200 update_mode = last_update_mode; 00201 return id != ((int)string_to_int.size()); 00202 } 00203 00204 bool Dictionary::isIn(int id, TVec<string> options) 00205 { 00206 return id < size() && id >= 0; 00207 } 00208 00209 void Dictionary::clear() 00210 { 00211 string_to_int.clear(); 00212 int_to_string.clear(); 00213 } 00214 00215 void Dictionary::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00216 { 00217 inherited::makeDeepCopyFromShallowCopy(copies); 00218 deepCopyField(string_to_int, copies); 00219 deepCopyField(int_to_string, copies); 00220 } 00221 00222 } // end of namespace PLearn 00223 00224 00225 /* 00226 Local Variables: 00227 mode:c++ 00228 c-basic-offset:4 00229 c-file-style:"stroustrup" 00230 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00231 indent-tabs-mode:nil 00232 fill-column:79 00233 End: 00234 */ 00235 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :