PLearn 0.1
Public Member Functions | Public Attributes
PLearn::SDBWithStats Class Reference

#include <SDBWithStats.h>

Inheritance diagram for PLearn::SDBWithStats:
Inheritance graph
[legend]
Collaboration diagram for PLearn::SDBWithStats:
Collaboration graph
[legend]

List of all members.

Public Member Functions

int nfields ()
string fieldname (int i)
 SDBWithStats (string basename, string path=".", AccessType access=readwrite, bool verbose=true)
void forgetStats ()
void computeStats (unsigned int nrows)
void computeStats ()
bool hasStats ()
 returns true if the stats files exist
void saveStats ()
void loadStats ()
FieldStatgetStat (int i)
const FieldStatgetStat (int i) const
FieldStatgetStat (const string &fieldname)
const FieldStatgetStat (const string &fieldname) const

Public Attributes

vector< FieldStatfieldstat

Detailed Description

Definition at line 106 of file SDBWithStats.h.


Constructor & Destructor Documentation

PLearn::SDBWithStats::SDBWithStats ( string  basename,
string  path = ".",
AccessType  access = readwrite,
bool  verbose = true 
)

Definition at line 98 of file SDBWithStats.cc.

References fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), hasStats(), loadStats(), and PLearn::SimpleDB< KeyType, QueryResult >::size().

    :SDB(basename,path,access,verbose)
{ 
    fieldstat.resize(getSchema().size());
    if(hasStats())
        loadStats();
}

Here is the call graph for this function:


Member Function Documentation

void PLearn::SDBWithStats::computeStats ( unsigned int  nrows)

Definition at line 112 of file SDBWithStats.cc.

References PLearn::Row::begin(), PLearn::CharacterType, PLearn::DateType, PLearn::DoubleType, PLearn::endl(), fieldstat, PLearn::FloatType, forgetStats(), PLearn::RowIterator::getFieldType(), PLearn::SimpleDB< KeyType, QueryResult >::getInRow(), PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), i, PLearn::IntType, PLearn::RowIterator::isMissing(), j, nfields(), PLERROR, PLearn::ShortType, PLearn::SignedCharType, PLearn::StringType, PLearn::todouble(), and PLearn::tostring().

Referenced by PLearn::SDBVMatrix::SDBVMatrix().

{
    forgetStats();
    const Schema& sc = getSchema();
    Row row(&sc);

    for(SDB::RowNumber i=0; i<nrows;i++) 
    {
        getInRow(i, row);
        Row::const_iterator it = row.begin();
        if (nrows>100000 && i%100000==0) 
            cout << "SDBWithStats::computeStats processing row " << i << " of " << nrows << endl;

        for (int j=0; j<nfields(); ++j, ++it) 
        {
            if(it.isMissing())
                fieldstat[j].updateMissing();
            else
            {
                switch(it.getFieldType())
                {
                case StringType: 
                case CharacterType:
                    fieldstat[j].updateString(tostring(it));
                    break;
                case SignedCharType:
                case ShortType:
                case IntType:
                case FloatType:
                case DoubleType:
                case DateType:
                    fieldstat[j].updateNumber(todouble(it));
                    break;
                default: 
                    PLERROR("Unknown field type");
                }
            }
        }
    }    
    cout << "boucle terminee" << endl;
    for (int j=0; j<nfields(); ++j)
        fieldstat[j].finalize();
    cout << "fini computestats" << endl;
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PLearn::SDBWithStats::computeStats ( ) [inline]

Definition at line 119 of file SDBWithStats.h.

References computeStats().

Referenced by computeStats().

Here is the call graph for this function:

Here is the caller graph for this function:

string PLearn::SDBWithStats::fieldname ( int  i) [inline]

Definition at line 111 of file SDBWithStats.h.

References i.

Referenced by loadStats(), and saveStats().

{ return getSchema()[i].name; }

Here is the caller graph for this function:

void PLearn::SDBWithStats::forgetStats ( )

Definition at line 106 of file SDBWithStats.cc.

References PLearn::FieldStat::clear(), getStat(), j, and PLearn::SimpleDB< KeyType, QueryResult >::width().

Referenced by computeStats(), and loadStats().

{
    for(int j=0; j<width(); j++)
        getStat(j).clear();
}

Here is the call graph for this function:

Here is the caller graph for this function:

FieldStat & PLearn::SDBWithStats::getStat ( const string &  fieldname)

Definition at line 268 of file SDBWithStats.cc.

References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.

{ 
    int pos = indexOfField(fieldname);
    if(pos<0)
        PLERROR("No field named %s",fieldname.c_str());
    return getStat(pos); 
}

Here is the call graph for this function:

FieldStat & PLearn::SDBWithStats::getStat ( int  i)

Definition at line 254 of file SDBWithStats.cc.

References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().

Referenced by forgetStats(), PLearn::SDBVMSource::getFieldStat(), and getStat().

{
    if(i<0 || i>=width())
        PLERROR("Out of bounds");
    return fieldstat[i]; 
}

Here is the call graph for this function:

Here is the caller graph for this function:

const FieldStat & PLearn::SDBWithStats::getStat ( int  i) const

Definition at line 261 of file SDBWithStats.cc.

References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().

{
    if(i<0 || i>=width())
        PLERROR("Out of bounds");
    return fieldstat[i]; 
}

Here is the call graph for this function:

const FieldStat & PLearn::SDBWithStats::getStat ( const string &  fieldname) const

Definition at line 276 of file SDBWithStats.cc.

References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.

{ 
    int pos = indexOfField(fieldname);
    if(pos<0)
        PLERROR("No field named %s",fieldname.c_str());
    return getStat(pos); 
}

Here is the call graph for this function:

bool PLearn::SDBWithStats::hasStats ( )

returns true if the stats files exist

Definition at line 157 of file SDBWithStats.cc.

References PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), and PLearn::isfile().

Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().

{
    string numstatsfile = getPath()+getName()+".stats"; 
    string symstatsfile = getPath()+getName()+".symbols";
    return isfile(numstatsfile) && isfile(symstatsfile);
}

Here is the call graph for this function:

Here is the caller graph for this function:

void PLearn::SDBWithStats::loadStats ( )

Definition at line 195 of file SDBWithStats.cc.

References fieldname(), fieldstat, forgetStats(), PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max_, PLearn::FieldStat::mean_, PLearn::FieldStat::min_, PLearn::FieldStat::missing_, PLearn::SimpleDB< KeyType, QueryResult >::name, PLearn::FieldStat::nonmissing_, PLERROR, PLearn::FieldStat::stddev_, PLearn::FieldStat::symbolcount, PLearn::FieldStat::symbolid, PLearn::tofloat(), and PLearn::toint().

Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().

{
    forgetStats();
    string numstatsfile = getPath()+getName()+".stats"; 
    string symstatsfile = getPath()+getName()+".symbols";
    ifstream numstats(numstatsfile.c_str());
    if(!numstats)
        PLERROR("could not open file %s for reading",numstatsfile.c_str());

    ifstream symstats(symstatsfile.c_str());
    if(!symstats)
        PLERROR("could not open file %s for reading",symstatsfile.c_str());
      
    for(unsigned int j=0; j<fieldstat.size(); j++)
    {
        string str_nonmissing_,str_missing_,str_mean_,str_stddev_,str_min_,str_max_;
        FieldStat& s = fieldstat[j];
        string name;
        numstats >> name;

        //cout<<"field : "<<j<<" name:"<<name<<" fieldname(j):"<<fieldname(j)<<endl;
        if(name!=fieldname(j))
            PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,numstatsfile.c_str(),j,fieldname(j).c_str()); 

        // **** we use strings as intermediate type to handle nans (julien)
        numstats >> str_nonmissing_ >> str_missing_ >> str_mean_ >> str_stddev_ >> str_min_ >> str_max_;
        
        if(str_mean_=="nan")str_mean_="";
        if(str_stddev_=="nan")str_stddev_="";
        if(str_min_=="nan")str_min_="";        
        if(str_max_=="nan")str_max_="";
        
        s.nonmissing_ = toint(str_nonmissing_);
        s.missing_ = toint(str_missing_);
        s.mean_ = tofloat(str_mean_);
        s.stddev_ = tofloat(str_stddev_);
        s.min_ = tofloat(str_min_);
        s.max_ = tofloat(str_max_);

        //cout <<" "<< s.nonmissing_ <<" "<< s.missing_ <<" "<< s.mean_ <<" "<< s.stddev_ <<" "<< s.min_ <<" "<< s.max_<<endl;
                
        symstats >> name;
        if(name!=fieldname(j))
            PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,symstatsfile.c_str(),j,fieldname(j).c_str()); 

        int nsymbols;
        symstats >> nsymbols;
        string sym;
        int symcount;
        for(int k=0; k<nsymbols; k++)
        {
            symstats >> sym >> symcount;
            s.symbolcount[sym] = symcount;
            s.symbolid[sym]=k;
        }
    }
    
}

Here is the call graph for this function:

Here is the caller graph for this function:

int PLearn::SDBWithStats::nfields ( ) [inline]

Definition at line 110 of file SDBWithStats.h.

Referenced by computeStats().

{ return (int)getSchema().size(); }

Here is the caller graph for this function:

void PLearn::SDBWithStats::saveStats ( )

Definition at line 164 of file SDBWithStats.cc.

References PLearn::endl(), fieldname(), fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max(), PLearn::FieldStat::mean(), PLearn::FieldStat::min(), PLearn::FieldStat::missing(), PLearn::FieldStat::nonmissing(), PLearn::FieldStat::nsymbols(), PLERROR, PLearn::FieldStat::stddev(), and PLearn::FieldStat::symbolcount.

Referenced by PLearn::SDBVMatrix::SDBVMatrix().

{
    string numstatsfile = getPath()+getName()+".stats"; 
    string symstatsfile = getPath()+getName()+".symbols";
    
    ofstream numstats(numstatsfile.c_str());
    if(!numstats)
        PLERROR("could not open file %s for writing",numstatsfile.c_str());

    ofstream symstats(symstatsfile.c_str());
    if(!symstats)
        PLERROR("could not open file %s for writing",symstatsfile.c_str());
      
    numstats.precision(8);
    symstats.precision(8);

    for(unsigned int j=0; j<fieldstat.size(); j++)
    {
        FieldStat& s = fieldstat[j];
        numstats << fieldname(j) << ' ' << s.nonmissing() << ' ' << s.missing() << ' ' 
                 << s.mean() << ' ' << s.stddev() << ' ' << s.min() << ' ' << s.max() << endl;

        symstats << fieldname(j) << ' ';
        symstats << s.nsymbols() << "     ";
        map<string,int>::iterator it;
        for(it = s.symbolcount.begin(); it!= s.symbolcount.end(); ++it)
            symstats << it->first << ' ' << it->second << "   ";
        symstats << endl;
    }
}

Here is the call graph for this function:

Here is the caller graph for this function:


Member Data Documentation

Definition at line 109 of file SDBWithStats.h.

Referenced by computeStats(), getStat(), loadStats(), saveStats(), and SDBWithStats().


The documentation for this class was generated from the following files:
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines