|
PLearn 0.1
|
#include <plearn/base/general.h>#include <plearn_learners/language/WordNet/WordNetOntology.h>#include <plearn/io/TypesNumeriques.h>#include <plearn/base/stringutils.h>
Go to the source code of this file.
Functions | |
| set< string > | extractWordSet (string file) |
| bool | isPunctuation (string &word) |
| int | main (int argc, char **argv) |
Variables | |
| set< string > | proper_nouns |
| set< string > | stop_words |
| WordNetOntology | wn |
| set<string> extractWordSet | ( | string | file | ) |
Definition at line 14 of file full_filter.cc.
References in, and PLearn::pgetline().
Referenced by main().
{
set<string> words;
ifstream in(file.c_str());
while (!in.eof())
{
string line = pgetline(in);
if (line == "") continue;
words.insert(line);
}
in.close();
return words;
}


| bool isPunctuation | ( | string & | word | ) |
Definition at line 28 of file full_filter.cc.
References i, and PLearn::isAlpha().
Referenced by main().
{
for (unsigned int i = 0; i < word.size(); i++)
{
if (isAlpha(word[i]))
return false;
}
return true;
}


Definition at line 38 of file full_filter.cc.
References PLearn::endl(), extractWordSet(), PLearn::WordNetOntology::isInWordNet(), isPunctuation(), PLearn::looksNumeric(), PLearn::lowerstring(), NUMERIC_TAG, PLERROR, PROPER_NOUN_TAG, proper_nouns, PUNCTUATION_TAG, and wn.
{
if (argc != 3)
PLERROR("usage : full_filter proper_nouns stop_words < in.text > filtered_out.text");
proper_nouns = extractWordSet(argv[1]);
//stop_words = extractWordSet(argv[2]);
string word;
while (true)
{
cin >> word;
if (!cin) break;
word = lowerstring(word);
/*if (stop_words.find(word) != stop_words.end())
cout << STOP_TAG << endl;
else*/
if (looksNumeric(word.c_str()))
cout << NUMERIC_TAG << endl;
else if (isPunctuation(word))
cout << PUNCTUATION_TAG << endl;
else if ((proper_nouns.find(word) != proper_nouns.end()) && !wn.isInWordNet(word))
cout << PROPER_NOUN_TAG << endl;
else
cout << word << endl;
}
}

| set<string> proper_nouns |
Definition at line 10 of file full_filter.cc.
Referenced by main().
| set<string> stop_words |
Definition at line 11 of file full_filter.cc.
Referenced by main().
Definition at line 12 of file full_filter.cc.
Referenced by main().
1.7.4