PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // HashMapFeatureSet.cc 00004 // 00005 // Copyright (C) 2006 Hugo Larochelle 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Hugo Larochelle 00036 00040 #include "HashMapFeatureSet.h" 00041 #include <plearn/vmat/VMatrix.h> 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 PLEARN_IMPLEMENT_ABSTRACT_OBJECT( 00047 HashMapFeatureSet, 00048 "Base class for feature sets that maintains an explicit mapping between index and string form features", 00049 "This class facilitates the conception of FeatureSet objects.\n" 00050 "Classes that inherits from this class only need to define the\n" 00051 "getNewFeaturesString() function, which is called when adding new features\n" 00052 "or when asking for the features of a token (in index or string format).\n" 00053 "This function should return the list of features in string format\n" 00054 "for a given token. It is assumed that no feature is present more than once\n" 00055 "in the output of that function, for any input token.\n" 00056 "This base class maintains two hash_maps between the index and string forms\n" 00057 "of the features. These mappings contain all the necessary\n" 00058 "information to define the different functions of a FeatureSet object.\n" 00059 ); 00060 00061 HashMapFeatureSet::HashMapFeatureSet() 00062 : report_progress(false) 00063 {} 00064 00065 // ### Nothing to add here, simply calls build_ 00066 void HashMapFeatureSet::build() 00067 { 00068 inherited::build(); 00069 build_(); 00070 } 00071 00072 void HashMapFeatureSet::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00073 { 00074 inherited::makeDeepCopyFromShallowCopy(copies); 00075 deepCopyField(index_to_string_feats, copies); 00076 deepCopyField(string_to_index_feats, copies); 00077 00078 //PLERROR("HashMapFeatureSet::makeDeepCopyFromShallowCopy not fully (correctly) implemented yet!"); 00079 } 00080 00081 void HashMapFeatureSet::declareOptions(OptionList& ol) 00082 { 00083 declareOption(ol, "report_progress", &HashMapFeatureSet::report_progress, 00084 OptionBase::buildoption, 00085 "Indication that progress should be reported in a ProgressBar\n" 00086 "when filling the feature set.\n"); 00087 declareOption(ol, "index_to_string_feats", &HashMapFeatureSet::index_to_string_feats, 00088 OptionBase::learntoption, 00089 "Index feature to string feature mapping"); 00090 declareOption(ol, "string_to_index_feats", &HashMapFeatureSet::string_to_index_feats, 00091 OptionBase::learntoption, 00092 "String feature to index feature mapping"); 00093 00094 // Now call the parent class' declareOptions 00095 inherited::declareOptions(ol); 00096 } 00097 00098 void HashMapFeatureSet::build_() 00099 {} 00100 00101 void HashMapFeatureSet::getFeatures(string token, TVec<int>& feats) 00102 { 00103 int index; 00104 string str; 00105 getNewFeaturesString(token,f_str); 00106 feats.resize(0); 00107 00108 for(int t=0; t<f_str.length(); t++) 00109 { 00110 str = f_str[t]; 00111 if(string_to_index_feats.find(str) != string_to_index_feats.end()) 00112 { 00113 index = string_to_index_feats[str]; 00114 feats.push_back(index); 00115 } 00116 } 00117 } 00118 00119 string HashMapFeatureSet::getStringFeature(int index) 00120 { 00121 #ifdef BOUNDCHECK 00122 if(index_to_string_feats.find(index) == index_to_string_feats.end()) 00123 PLERROR("In HashMapFeatureSet::getStringFeature(): index %d is an invalid feature index", index); 00124 #endif 00125 return index_to_string_feats[index]; 00126 00127 } 00128 00129 int HashMapFeatureSet::getIndexFeature(string str) 00130 { 00131 if(string_to_index_feats.find(str) == string_to_index_feats.end()) 00132 PLERROR("In HashMapFeatureSet::getIndexFeature(): string %s is an invalid feature", str.c_str()); 00133 00134 return string_to_index_feats[str]; 00135 } 00136 00137 int HashMapFeatureSet::size() 00138 { 00139 return int(index_to_string_feats.size()); 00140 } 00141 00142 void HashMapFeatureSet::addFeatures(string token) 00143 { 00144 string str; 00145 int index; 00146 00147 getNewFeaturesString(token,f_str); 00148 for(int t=0; t<f_str.length(); t++) 00149 { 00150 str = f_str[t]; 00151 if(string_to_index_feats.find(str) == string_to_index_feats.end()){ 00152 index = int(string_to_index_feats.size()); 00153 string_to_index_feats[str] = index; 00154 index_to_string_feats[index] = str; 00155 } 00156 } 00157 00158 } 00159 00160 void HashMapFeatureSet::addFeatures(VMat tokens, int min_freq) 00161 { 00162 string str, token; 00163 int index; 00164 map<string,int> frequencies; 00165 map<string,TVec<string> > token_features; 00166 map<string,bool> token_feat_inserted; 00167 Vec row(tokens->width()); 00168 TVec<string> f_str_temp; 00169 00170 PP<ProgressBar> pb; 00171 int id=0; 00172 00173 if(min_freq > 1) 00174 { 00175 if(report_progress) 00176 pb = new ProgressBar("Counting frequencies for the features of " + tostring(tokens->length()*tokens->width()) + " tokens", tokens->length()*tokens->width()); 00177 00178 for(int i=0; i<tokens->length(); i++) 00179 { 00180 tokens->getRow(i,row); 00181 for(int j=0; j<tokens->width(); j++) 00182 { 00183 token = tokens->getValString(j,row[j]); 00184 if(token_features.find(token) == token_features.end()) 00185 { 00186 getNewFeaturesString(token,f_str); 00187 token_features[token] = f_str.copy(); 00188 } 00189 else 00190 { 00191 f_str_temp = token_features[token]; 00192 f_str.resize(f_str_temp.length()); 00193 f_str << f_str_temp; 00194 } 00195 00196 for(int t=0; t<f_str.length(); t++) 00197 { 00198 str = f_str[t]; 00199 if(frequencies.find(str) == frequencies.end()) 00200 frequencies[str] = 1; 00201 else 00202 frequencies[str]++; 00203 } 00204 00205 if(report_progress) pb->update(++id); 00206 } 00207 } 00208 token_features.clear(); 00209 } 00210 00211 id=0; 00212 if(report_progress) 00213 pb = new ProgressBar("Filling FeatureSet with the features of " + tostring(tokens->length()*tokens->width()) + " tokens", tokens->length()*tokens->width()); 00214 00215 00216 for(int i=0; i<tokens->length(); i++) 00217 { 00218 tokens->getRow(i,row); 00219 for(int j=0; j<tokens->width(); j++) 00220 { 00221 token = tokens->getValString(j,row[j]); 00222 if(token_feat_inserted.find(token) == token_feat_inserted.end()) 00223 { 00224 getNewFeaturesString(token,f_str); 00225 for(int t=0; t<f_str.length(); t++) 00226 { 00227 str = f_str[t]; 00228 if(string_to_index_feats.find(str) == string_to_index_feats.end() 00229 && (min_freq <=1 || frequencies[str] >= min_freq)){ 00230 index = int(string_to_index_feats.size()); 00231 string_to_index_feats[str] = index; 00232 index_to_string_feats[index] = str; 00233 } 00234 } 00235 token_feat_inserted[token] = true; 00236 } 00237 if(report_progress) pb->update(++id); 00238 } 00239 } 00240 } 00241 00242 void HashMapFeatureSet::clear() 00243 { 00244 index_to_string_feats.clear(); 00245 string_to_index_feats.clear(); 00246 } 00247 00248 } // end of namespace PLearn 00249 00250 00251 /* 00252 Local Variables: 00253 mode:c++ 00254 c-basic-offset:4 00255 c-file-style:"stroustrup" 00256 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00257 indent-tabs-mode:nil 00258 fill-column:79 00259 End: 00260 */ 00261 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :