PLearn 0.1
|
#include <SDBWithStats.h>
Public Member Functions | |
int | nfields () |
string | fieldname (int i) |
SDBWithStats (string basename, string path=".", AccessType access=readwrite, bool verbose=true) | |
void | forgetStats () |
void | computeStats (unsigned int nrows) |
void | computeStats () |
bool | hasStats () |
returns true if the stats files exist | |
void | saveStats () |
void | loadStats () |
FieldStat & | getStat (int i) |
const FieldStat & | getStat (int i) const |
FieldStat & | getStat (const string &fieldname) |
const FieldStat & | getStat (const string &fieldname) const |
Public Attributes | |
vector< FieldStat > | fieldstat |
Definition at line 106 of file SDBWithStats.h.
PLearn::SDBWithStats::SDBWithStats | ( | string | basename, |
string | path = "." , |
||
AccessType | access = readwrite , |
||
bool | verbose = true |
||
) |
Definition at line 98 of file SDBWithStats.cc.
References fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), hasStats(), loadStats(), and PLearn::SimpleDB< KeyType, QueryResult >::size().
:SDB(basename,path,access,verbose) { fieldstat.resize(getSchema().size()); if(hasStats()) loadStats(); }
void PLearn::SDBWithStats::computeStats | ( | unsigned int | nrows | ) |
Definition at line 112 of file SDBWithStats.cc.
References PLearn::Row::begin(), PLearn::CharacterType, PLearn::DateType, PLearn::DoubleType, PLearn::endl(), fieldstat, PLearn::FloatType, forgetStats(), PLearn::RowIterator::getFieldType(), PLearn::SimpleDB< KeyType, QueryResult >::getInRow(), PLearn::SimpleDB< KeyType, QueryResult >::getSchema(), i, PLearn::IntType, PLearn::RowIterator::isMissing(), j, nfields(), PLERROR, PLearn::ShortType, PLearn::SignedCharType, PLearn::StringType, PLearn::todouble(), and PLearn::tostring().
Referenced by PLearn::SDBVMatrix::SDBVMatrix().
{ forgetStats(); const Schema& sc = getSchema(); Row row(&sc); for(SDB::RowNumber i=0; i<nrows;i++) { getInRow(i, row); Row::const_iterator it = row.begin(); if (nrows>100000 && i%100000==0) cout << "SDBWithStats::computeStats processing row " << i << " of " << nrows << endl; for (int j=0; j<nfields(); ++j, ++it) { if(it.isMissing()) fieldstat[j].updateMissing(); else { switch(it.getFieldType()) { case StringType: case CharacterType: fieldstat[j].updateString(tostring(it)); break; case SignedCharType: case ShortType: case IntType: case FloatType: case DoubleType: case DateType: fieldstat[j].updateNumber(todouble(it)); break; default: PLERROR("Unknown field type"); } } } } cout << "boucle terminee" << endl; for (int j=0; j<nfields(); ++j) fieldstat[j].finalize(); cout << "fini computestats" << endl; }
void PLearn::SDBWithStats::computeStats | ( | ) | [inline] |
Definition at line 119 of file SDBWithStats.h.
References computeStats().
Referenced by computeStats().
{ computeStats(size()); }
string PLearn::SDBWithStats::fieldname | ( | int | i | ) | [inline] |
Definition at line 111 of file SDBWithStats.h.
References i.
Referenced by loadStats(), and saveStats().
void PLearn::SDBWithStats::forgetStats | ( | ) |
Definition at line 106 of file SDBWithStats.cc.
References PLearn::FieldStat::clear(), getStat(), j, and PLearn::SimpleDB< KeyType, QueryResult >::width().
Referenced by computeStats(), and loadStats().
FieldStat & PLearn::SDBWithStats::getStat | ( | const string & | fieldname | ) |
Definition at line 268 of file SDBWithStats.cc.
References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.
{ int pos = indexOfField(fieldname); if(pos<0) PLERROR("No field named %s",fieldname.c_str()); return getStat(pos); }
Definition at line 254 of file SDBWithStats.cc.
References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().
Referenced by forgetStats(), PLearn::SDBVMSource::getFieldStat(), and getStat().
Definition at line 261 of file SDBWithStats.cc.
References fieldstat, i, PLERROR, and PLearn::SimpleDB< KeyType, QueryResult >::width().
const FieldStat & PLearn::SDBWithStats::getStat | ( | const string & | fieldname | ) | const |
Definition at line 276 of file SDBWithStats.cc.
References getStat(), PLearn::SimpleDB< KeyType, QueryResult >::indexOfField(), and PLERROR.
{ int pos = indexOfField(fieldname); if(pos<0) PLERROR("No field named %s",fieldname.c_str()); return getStat(pos); }
bool PLearn::SDBWithStats::hasStats | ( | ) |
returns true if the stats files exist
Definition at line 157 of file SDBWithStats.cc.
References PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), and PLearn::isfile().
Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().
{ string numstatsfile = getPath()+getName()+".stats"; string symstatsfile = getPath()+getName()+".symbols"; return isfile(numstatsfile) && isfile(symstatsfile); }
void PLearn::SDBWithStats::loadStats | ( | ) |
Definition at line 195 of file SDBWithStats.cc.
References fieldname(), fieldstat, forgetStats(), PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max_, PLearn::FieldStat::mean_, PLearn::FieldStat::min_, PLearn::FieldStat::missing_, PLearn::SimpleDB< KeyType, QueryResult >::name, PLearn::FieldStat::nonmissing_, PLERROR, PLearn::FieldStat::stddev_, PLearn::FieldStat::symbolcount, PLearn::FieldStat::symbolid, PLearn::tofloat(), and PLearn::toint().
Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and SDBWithStats().
{ forgetStats(); string numstatsfile = getPath()+getName()+".stats"; string symstatsfile = getPath()+getName()+".symbols"; ifstream numstats(numstatsfile.c_str()); if(!numstats) PLERROR("could not open file %s for reading",numstatsfile.c_str()); ifstream symstats(symstatsfile.c_str()); if(!symstats) PLERROR("could not open file %s for reading",symstatsfile.c_str()); for(unsigned int j=0; j<fieldstat.size(); j++) { string str_nonmissing_,str_missing_,str_mean_,str_stddev_,str_min_,str_max_; FieldStat& s = fieldstat[j]; string name; numstats >> name; //cout<<"field : "<<j<<" name:"<<name<<" fieldname(j):"<<fieldname(j)<<endl; if(name!=fieldname(j)) PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,numstatsfile.c_str(),j,fieldname(j).c_str()); // **** we use strings as intermediate type to handle nans (julien) numstats >> str_nonmissing_ >> str_missing_ >> str_mean_ >> str_stddev_ >> str_min_ >> str_max_; if(str_mean_=="nan")str_mean_=""; if(str_stddev_=="nan")str_stddev_=""; if(str_min_=="nan")str_min_=""; if(str_max_=="nan")str_max_=""; s.nonmissing_ = toint(str_nonmissing_); s.missing_ = toint(str_missing_); s.mean_ = tofloat(str_mean_); s.stddev_ = tofloat(str_stddev_); s.min_ = tofloat(str_min_); s.max_ = tofloat(str_max_); //cout <<" "<< s.nonmissing_ <<" "<< s.missing_ <<" "<< s.mean_ <<" "<< s.stddev_ <<" "<< s.min_ <<" "<< s.max_<<endl; symstats >> name; if(name!=fieldname(j)) PLERROR("Row number %d of file %s does not correpond to field number %d: %s",j,symstatsfile.c_str(),j,fieldname(j).c_str()); int nsymbols; symstats >> nsymbols; string sym; int symcount; for(int k=0; k<nsymbols; k++) { symstats >> sym >> symcount; s.symbolcount[sym] = symcount; s.symbolid[sym]=k; } } }
int PLearn::SDBWithStats::nfields | ( | ) | [inline] |
Definition at line 110 of file SDBWithStats.h.
Referenced by computeStats().
{ return (int)getSchema().size(); }
void PLearn::SDBWithStats::saveStats | ( | ) |
Definition at line 164 of file SDBWithStats.cc.
References PLearn::endl(), fieldname(), fieldstat, PLearn::SimpleDB< KeyType, QueryResult >::getName(), PLearn::SimpleDB< KeyType, QueryResult >::getPath(), j, PLearn::FieldStat::max(), PLearn::FieldStat::mean(), PLearn::FieldStat::min(), PLearn::FieldStat::missing(), PLearn::FieldStat::nonmissing(), PLearn::FieldStat::nsymbols(), PLERROR, PLearn::FieldStat::stddev(), and PLearn::FieldStat::symbolcount.
Referenced by PLearn::SDBVMatrix::SDBVMatrix().
{ string numstatsfile = getPath()+getName()+".stats"; string symstatsfile = getPath()+getName()+".symbols"; ofstream numstats(numstatsfile.c_str()); if(!numstats) PLERROR("could not open file %s for writing",numstatsfile.c_str()); ofstream symstats(symstatsfile.c_str()); if(!symstats) PLERROR("could not open file %s for writing",symstatsfile.c_str()); numstats.precision(8); symstats.precision(8); for(unsigned int j=0; j<fieldstat.size(); j++) { FieldStat& s = fieldstat[j]; numstats << fieldname(j) << ' ' << s.nonmissing() << ' ' << s.missing() << ' ' << s.mean() << ' ' << s.stddev() << ' ' << s.min() << ' ' << s.max() << endl; symstats << fieldname(j) << ' '; symstats << s.nsymbols() << " "; map<string,int>::iterator it; for(it = s.symbolcount.begin(); it!= s.symbolcount.end(); ++it) symstats << it->first << ' ' << it->second << " "; symstats << endl; } }
Definition at line 109 of file SDBWithStats.h.
Referenced by computeStats(), getStat(), loadStats(), saveStats(), and SDBWithStats().