PLearn 0.1
|
#include <plearn/base/general.h>
#include <plearn_learners/language/WordNet/WordNetOntology.h>
#include <plearn/io/TypesNumeriques.h>
#include <plearn/base/stringutils.h>
Go to the source code of this file.
Functions | |
set< string > | extractWordSet (string file) |
bool | isPunctuation (string &word) |
int | main (int argc, char **argv) |
Variables | |
set< string > | proper_nouns |
set< string > | stop_words |
WordNetOntology | wn |
set<string> extractWordSet | ( | string | file | ) |
Definition at line 14 of file full_filter.cc.
References in, and PLearn::pgetline().
Referenced by main().
{ set<string> words; ifstream in(file.c_str()); while (!in.eof()) { string line = pgetline(in); if (line == "") continue; words.insert(line); } in.close(); return words; }
bool isPunctuation | ( | string & | word | ) |
Definition at line 28 of file full_filter.cc.
References i, and PLearn::isAlpha().
Referenced by main().
{ for (unsigned int i = 0; i < word.size(); i++) { if (isAlpha(word[i])) return false; } return true; }
Definition at line 38 of file full_filter.cc.
References PLearn::endl(), extractWordSet(), PLearn::WordNetOntology::isInWordNet(), isPunctuation(), PLearn::looksNumeric(), PLearn::lowerstring(), NUMERIC_TAG, PLERROR, PROPER_NOUN_TAG, proper_nouns, PUNCTUATION_TAG, and wn.
{ if (argc != 3) PLERROR("usage : full_filter proper_nouns stop_words < in.text > filtered_out.text"); proper_nouns = extractWordSet(argv[1]); //stop_words = extractWordSet(argv[2]); string word; while (true) { cin >> word; if (!cin) break; word = lowerstring(word); /*if (stop_words.find(word) != stop_words.end()) cout << STOP_TAG << endl; else*/ if (looksNumeric(word.c_str())) cout << NUMERIC_TAG << endl; else if (isPunctuation(word)) cout << PUNCTUATION_TAG << endl; else if ((proper_nouns.find(word) != proper_nouns.end()) && !wn.isInWordNet(word)) cout << PROPER_NOUN_TAG << endl; else cout << word << endl; } }
set<string> proper_nouns |
Definition at line 10 of file full_filter.cc.
Referenced by main().
set<string> stop_words |
Definition at line 11 of file full_filter.cc.
Referenced by main().
Definition at line 12 of file full_filter.cc.
Referenced by main().