PLearn 0.1
Functions | Variables
full_filter.cc File Reference
#include <plearn/base/general.h>
#include <plearn_learners/language/WordNet/WordNetOntology.h>
#include <plearn/io/TypesNumeriques.h>
#include <plearn/base/stringutils.h>
Include dependency graph for full_filter.cc:

Go to the source code of this file.

Functions

set< string > extractWordSet (string file)
bool isPunctuation (string &word)
int main (int argc, char **argv)

Variables

set< string > proper_nouns
set< string > stop_words
WordNetOntology wn

Function Documentation

set<string> extractWordSet ( string  file)

Definition at line 14 of file full_filter.cc.

References in, and PLearn::pgetline().

Referenced by main().

{
    set<string> words;
    ifstream in(file.c_str());
    while (!in.eof())
    {
        string line = pgetline(in);
        if (line == "") continue;
        words.insert(line);
    }
    in.close();
    return words;
}

Here is the call graph for this function:

Here is the caller graph for this function:

bool isPunctuation ( string &  word)

Definition at line 28 of file full_filter.cc.

References i, and PLearn::isAlpha().

Referenced by main().

{
    for (unsigned int i = 0; i < word.size(); i++)
    {
        if (isAlpha(word[i])) 
            return false;
    }
    return true;
}

Here is the call graph for this function:

Here is the caller graph for this function:

int main ( int  argc,
char **  argv 
)

Definition at line 38 of file full_filter.cc.

References PLearn::endl(), extractWordSet(), PLearn::WordNetOntology::isInWordNet(), isPunctuation(), PLearn::looksNumeric(), PLearn::lowerstring(), NUMERIC_TAG, PLERROR, PROPER_NOUN_TAG, proper_nouns, PUNCTUATION_TAG, and wn.

{
    if (argc != 3)
        PLERROR("usage : full_filter proper_nouns stop_words < in.text > filtered_out.text");
    proper_nouns = extractWordSet(argv[1]);
    //stop_words = extractWordSet(argv[2]);
    string word;
    while (true)
    {
        cin >> word;
        if (!cin) break;
        word = lowerstring(word);
        /*if (stop_words.find(word) != stop_words.end())
          cout << STOP_TAG << endl;
          else*/
        if (looksNumeric(word.c_str()))
            cout << NUMERIC_TAG << endl;
        else if (isPunctuation(word))
            cout << PUNCTUATION_TAG << endl;
        else if ((proper_nouns.find(word) != proper_nouns.end()) && !wn.isInWordNet(word))
            cout << PROPER_NOUN_TAG << endl;
        else
            cout << word << endl;
    }
}

Here is the call graph for this function:


Variable Documentation

set<string> proper_nouns

Definition at line 10 of file full_filter.cc.

Referenced by main().

set<string> stop_words

Definition at line 11 of file full_filter.cc.

Referenced by main().

Definition at line 12 of file full_filter.cc.

Referenced by main().

 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines