PLearn 0.1
Dictionary.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // Dictionary.cc
00004 //
00005 // Copyright (C) 2004 Hugo Larochelle, Christopher Kermorvant
00006 // 
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 // 
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 // 
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 // 
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 // 
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 // 
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************      
00036  * $Id: Dictionary.cc 6392 2006-11-07 18:32:40Z larocheh $ 
00037  ******************************************************* */
00038 
00039 // Authors: Hugo Larochelle, Christopher Kermorvant
00040 
00044 #include "Dictionary.h"
00045 
00046 namespace PLearn {
00047 using namespace std;
00048   
00049 Dictionary::Dictionary()
00050     :
00051     update_mode(UPDATE), 
00052     dont_insert_oov_symbol(false),
00053     oov_symbol("<oov>")
00054 {
00055 }
00056 
00057 PLEARN_IMPLEMENT_OBJECT(Dictionary,
00058                         "Mapping string->int and int->string",
00059   "A dictionary is a mapping between a string and an index or ID (int).\n"
00060   "Depending on the update mode, the dictionay can include an unknown \n"
00061   "word when asking for its ID with getId();\n"
00062   "if update_mode == UPDATE, add the word and return its ID\n"
00063   "if update_mode == NO_UPDATE, return OOV symbol's ID (see below)\n"
00064   "\n"
00065   "Also, when asking for the symbol associated to an ID, no update is \n"
00066   "possible. If the ID has not been assigned by the dictionary, \n"
00067   "then the returned symbol is \"\".\n"
00068   "\n"
00069   "A Dictionary object is instantiated empty, and then symbols can \n"
00070   "be added in the dictionary by using getId() (with update_mode == UPDATE).\n"
00071   "By default, IDs are assigned by starting with the ID 0 and incrementing\n"
00072   "the assigned IDs as symbols are added in the dictionary. \n"
00073   "\n"
00074   "Also, a \"out-of-vocabulary\" (OOV) symbol is assigned an ID which \n"
00075   "is equal to the number of symbols (other than the OOV symbol) \n"
00076   "that are in the dictionary at the current state. By default, \n"
00077   "this symbol corresponds to the string \"<oov>\". \n"
00078   "\n"
00079   "The OOV symbol's ID is returned by the getId(symbol) function if\n"
00080   "\"symbol\" is not in the dictionary. Also by default, this OOV token is\n" 
00081   "considered as part of the dictionary, but this can be changed\n"
00082   "using the dont_insert_oov_symbol option. If dont_insert_oov_token\n"
00083   "is true, then the size(), isIn() and getValues() functions\n"
00084   "do not consider the oov symbol as part of the dictionary. Note that\n"
00085   "getId() will still return the oov symbol's ID for unknown symbols,\n"
00086   "and getSymbol() will still return the OOV symbol for its ID.\n"
00087 );
00088 
00089 void Dictionary::declareOptions(OptionList& ol)
00090 {
00091     declareOption(ol, "update_mode", &Dictionary::update_mode, 
00092                   OptionBase::buildoption, 
00093                   "update_mode : 0(no_update)/1(update). Default is update");
00094     declareOption(ol, "oov_symbol", &Dictionary::oov_symbol, 
00095                   OptionBase::buildoption, 
00096                   "String symbol for \"out-of-vocabulary\" token.");
00097     declareOption(ol, "string_to_int", &Dictionary::string_to_int, 
00098                   OptionBase::learntoption, "string to int mapping");
00099     declareOption(ol, "int_to_string", &Dictionary::int_to_string, 
00100                   OptionBase::learntoption, "int to string mapping");
00101     // For backward compatibility...
00102     declareOption(ol, "oov_not_in_possible_values", 
00103                   &Dictionary::dont_insert_oov_symbol, 
00104                   OptionBase::buildoption,
00105                   "DEPRECATED, use dont_insert_oov_symbol instead: Indication that\n"
00106                   "the OOV symbol should not be considered as part of the dictionary"); 
00107     declareOption(ol, "dont_insert_oov_symbol", 
00108                   &Dictionary::dont_insert_oov_symbol, 
00109                   OptionBase::buildoption, 
00110                   "Indication that the OOV symbol should not be considered\n"
00111                   "as part of the dictionary"); 
00112 
00113     inherited::declareOptions(ol);
00114 }
00115 
00116 void Dictionary::build_(){
00117 }
00118 
00119 // ### Nothing to add here, simply calls build_
00120 void Dictionary::build()
00121 {
00122     inherited::build();
00123     build_();
00124 }
00125 
00126 void Dictionary::setUpdateMode(int up_mode)
00127 {
00128     if(up_mode != UPDATE && up_mode != NO_UPDATE)
00129         PLERROR("In Dictionary::setUpdateMode(): incompatible update mode %d", up_mode);
00130     update_mode =up_mode;
00131 }
00132 
00133 int Dictionary::getId(string symbol, TVec<string> options)
00134 {
00135     // Gives the id of a symbol in the dictionary
00136     // If the symbol is not in the dictionary, 
00137     // returns index of OOV symbol if update_mode = NO_UPDATE,
00138     // insert the new word otherwise and return its index
00139 
00140     if(symbol == oov_symbol) return string_to_int.size();
00141 
00142     int index;
00143     if(update_mode== UPDATE)
00144     {
00145         if(string_to_int.find(symbol) == string_to_int.end()){
00146             // word not found, add it
00147             index=int(string_to_int.size());
00148             string_to_int[symbol] = index;
00149             int_to_string[index] = symbol;
00150         }
00151         return string_to_int[symbol];
00152     }
00153     else
00154     {
00155         // No update mode
00156         if(string_to_int.find(symbol) == string_to_int.end()){
00157             // word not found, return OOV symbol's ID
00158             return string_to_int.size();
00159         }else{
00160             return string_to_int[symbol];
00161         }
00162     }
00163 }
00164 
00165 string Dictionary::getSymbol(int id, TVec<string>options)const
00166 {
00167     if(int_to_string.find(id) == int_to_string.end())
00168     {
00169         if(id == ((int)string_to_int.size()))
00170             return oov_symbol;
00171         else
00172             return "";
00173     }
00174     else
00175         return int_to_string.find(id)->second;
00176 }
00177 
00178 int Dictionary::size(TVec<string> options){
00179     if(dont_insert_oov_symbol)
00180         return int(string_to_int.size());
00181     else
00182         return int(string_to_int.size())+1;
00183 }
00184 
00185 void Dictionary::getValues(TVec<string> options, Vec& values)
00186 { 
00187     values.resize(size());
00188     int i=0;
00189     for(map<int,string>::iterator it = int_to_string.begin(); it != int_to_string.end(); it++)
00190         values[i++] = it->first;
00191     if(!dont_insert_oov_symbol)
00192         values[i] = i; // OOV symbol's ID
00193 }
00194 
00195 bool Dictionary::isIn(string symbol, TVec<string> options){
00196     if(symbol == oov_symbol) return !dont_insert_oov_symbol;
00197     int last_update_mode = update_mode;
00198     update_mode = NO_UPDATE;
00199     int id = getId(symbol,options);
00200     update_mode = last_update_mode;
00201     return id != ((int)string_to_int.size());
00202 }
00203 
00204 bool Dictionary::isIn(int id, TVec<string> options) 
00205 { 
00206     return id < size() && id >= 0;
00207 }
00208 
00209 void Dictionary::clear()
00210 {
00211     string_to_int.clear(); 
00212     int_to_string.clear();
00213 }
00214 
00215 void Dictionary::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00216 {
00217     inherited::makeDeepCopyFromShallowCopy(copies);
00218     deepCopyField(string_to_int, copies);
00219     deepCopyField(int_to_string, copies);
00220 }
00221 
00222 } // end of namespace PLearn
00223 
00224 
00225 /*
00226   Local Variables:
00227   mode:c++
00228   c-basic-offset:4
00229   c-file-style:"stroustrup"
00230   c-file-offsets:((innamespace . 0)(inline-open . 0))
00231   indent-tabs-mode:nil
00232   fill-column:79
00233   End:
00234 */
00235 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines