PLearn 0.1
|
#include <SimpleDB.h>
Public Types | |
enum | { InvalidRow = ULONG_MAX } |
enum | AccessType { readwrite = 0, readonly = 1 } |
Whether the user is granted read/write or read-only access. More... | |
typedef unsigned long | RowNumber |
Friend functions and classes Template constraints function. | |
typedef unsigned long | Offset |
A physical offset_into the database. | |
typedef QueryResult | QueryResult_t |
The result of making a query. | |
typedef SimpleDBIndexKey< KeyType > | IndexKey |
An index is simply a hash table from IndexKey to QueryResult. | |
typedef Hash< IndexKey, QueryResult > | Index |
typedef PP< Index > | PIndex |
typedef vector< const unsigned char * > | vuc |
Use linear search to find multiple "lookfor". | |
Public Member Functions | |
SimpleDB (string rootname, string path=".", AccessType=readwrite, bool verbose=true) | |
virtual | ~SimpleDB () |
string | getName () const |
string | getPath () const |
void | setSchema (const Schema &s) |
const Schema & | getSchema () const |
void | saveSchema () |
void | loadSchema () |
bool | findColumn (string name, int &position, int &start, int &precision) const |
int | indexOfField (const string &fieldname) const |
returns the index of the given field inside the Schema, -1 if not found | |
Row | getRow (RowNumber) const |
Row & | getInRow (RowNumber, Row &) const |
RowNumber | size () const |
1 beyond maximum row number | |
int | length () const |
int | width () const |
void | addRow (const Row &) |
add at end of database | |
void | setRow (const Row &, RowNumber) |
set a particular row | |
void | truncateFromRow (RowNumber n) |
erase all rows from row n (included) until end | |
bool | indexColumn (string columnName, string secondColumn=string("")) |
void | clearIndex (string columnName) |
Clear an index from memory to free up space. | |
QueryResult | findEqual (const unsigned char *lookfor, string columnName, string secondColumn=string("")) |
const QueryResult & | findEqualIndexed (const unsigned char *lookfor, string columnName, string secondColumn=string("")) |
Always use the index to find "lookfor". | |
QueryResult | findEqualLinear (const unsigned char *lookfor, string columnName, string secondColumn=string("")) |
Use linear search to find "lookfor". | |
QueryResult | findEqualLinear (const vuc &lookfor, string columnName, string secondColumn=string("")) |
double | tableSizeMultiplier () const |
Access the table size multiplier (see description below) | |
void | tableSizeMultiplier (double x) |
Static Public Attributes | |
static const Offset | AbsoluteFileLimit = 512ul * 1024ul * 1024ul - 1 |
Maximum size that a single database file can hold; this is currently set to 512 MB. | |
static QueryResult | EmptyResult |
A null query. | |
Private Member Functions | |
void | computeSize () |
(should provide union, intersection, and such operators...) | |
void | memoryToDisk (Row &) const |
convert a row from machine-dependent endianness to disk-standard little-endian format. | |
void | diskToMemory (Row &) const |
convert a row from disk-standard little-endian format to machine-dependent endianness | |
int | seekToRow (RowNumber) const |
int | seekToEnd () const |
seek to end of database, return FD to the correct file | |
void | openAllFiles () const |
open all existing segments in database | |
void | closeAllFiles () const |
close all open file descriptors | |
int | lastSegment () const |
return the index (NOT FD!) of the last open segment | |
string | getSegmentPath (int i) const |
return full path of segment i (zero-based) | |
SimpleDB (const SimpleDB &) | |
For now, disable copy construction and assignment. | |
void | operator= (const SimpleDB &) |
Private Attributes | |
string | name |
database base name | |
string | path |
database root path | |
AccessType | access_type |
readwrite or readonly | |
int | access_mask |
Unix constants for access_type. | |
RowNumber | size_ |
cached number of rows | |
Schema | schema |
database schema | |
int | row_size |
length of a row in bytes | |
RowNumber | max_records_file |
maximum number of full rows in a single file | |
vector< int > | allfd |
File descriptors of open segments; -1 for not open. | |
double | table_size_multiplier |
vector< PIndex > | indexes |
There's one possible index per column in the database. | |
bool | verbose |
print debugging info to cerr |
Simple Database
This class permits the representation of a simple quasi-flat-file database in an efficient binary format, and enables indexing string columns to obtain the rows that match a given string.
Definition at line 500 of file SimpleDB.h.
typedef Hash<IndexKey,QueryResult> PLearn::SimpleDB< KeyType, QueryResult >::Index |
Definition at line 534 of file SimpleDB.h.
typedef SimpleDBIndexKey<KeyType> PLearn::SimpleDB< KeyType, QueryResult >::IndexKey |
An index is simply a hash table from IndexKey to QueryResult.
Definition at line 533 of file SimpleDB.h.
typedef unsigned long PLearn::SimpleDB< KeyType, QueryResult >::Offset |
A physical offset_into the database.
Definition at line 516 of file SimpleDB.h.
typedef PP<Index> PLearn::SimpleDB< KeyType, QueryResult >::PIndex |
Definition at line 535 of file SimpleDB.h.
typedef QueryResult PLearn::SimpleDB< KeyType, QueryResult >::QueryResult_t |
typedef unsigned long PLearn::SimpleDB< KeyType, QueryResult >::RowNumber |
Friend functions and classes Template constraints function.
--- Public Type Definitions A row number in the database. Rows are numbered starting with 0. InvalidRow is a constant denoting an invalid row number.
Definition at line 510 of file SimpleDB.h.
typedef vector<const unsigned char*> PLearn::SimpleDB< KeyType, QueryResult >::vuc |
Use linear search to find multiple "lookfor".
Definition at line 634 of file SimpleDB.h.
anonymous enum |
enum PLearn::SimpleDB::AccessType |
Whether the user is granted read/write or read-only access.
Definition at line 519 of file SimpleDB.h.
PLearn::SimpleDB< KT, QR >::SimpleDB | ( | string | rootname, |
string | path = "." , |
||
AccessType | the_access_type = readwrite , |
||
bool | verbose = true |
||
) |
--- Constructors, etc.
Definition at line 1209 of file SimpleDB.h.
References PLearn::SimpleDB< KeyType, QueryResult >::access_mask, PLearn::SimpleDB< KeyType, QueryResult >::access_type, PLearn::SimpleDB< KeyType, QueryResult >::computeSize(), PLearn::SimpleDB< KeyType, QueryResult >::loadSchema(), PLearn::SimpleDB< KeyType, QueryResult >::openAllFiles(), PLearn::SimpleDB< KeyType, QueryResult >::path, PLearn::SimpleDB< KeyType, QueryResult >::readonly, PLearn::SimpleDB< KeyType, QueryResult >::readwrite, and slash.
: name(rootname), path(the_path), access_type(the_access_type), access_mask(0), schema(), row_size(), allfd(), table_size_multiplier(1.8), indexes(), verbose(the_verbose) { if (path != "") path += slash; switch (access_type) { case readwrite: access_mask = O_RDWR; break; case readonly: access_mask = O_RDONLY; break; } loadSchema(); openAllFiles(); computeSize(); }
PLearn::SimpleDB< KT, QR >::~SimpleDB | ( | ) | [virtual] |
Upon destroying the database, save the current schema
Definition at line 1233 of file SimpleDB.h.
{ closeAllFiles(); saveSchema(); }
PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB | ( | const SimpleDB< KeyType, QueryResult > & | ) | [private] |
For now, disable copy construction and assignment.
void PLearn::SimpleDB< KT, QR >::addRow | ( | const Row & | row | ) |
add at end of database
--- Functions dealing with database modifications
Handle writing error
Preserve database integrity by truncating from the point where we should have started writing
< increment length of db
Definition at line 1357 of file SimpleDB.h.
References PLERROR, PLWARNING, PLearn::Row::raw(), PLearn::Row::sanitize(), PLearn::Row::size(), and PLearn::write().
{ if(row_size != row.size()) PLERROR("In addRow row_size != row.size() (%d != %d)", row_size, row.size()); row.sanitize(); int fd = seekToEnd(); off_t curpos = lseek(fd, 0L, SEEK_CUR); #ifdef LITTLEENDIAN //ssize_t writtensize = ::write(fd, row.raw(), row_size); int writtensize = ::write(fd, row.raw(), row_size); #endif #ifdef BIGENDIAN Row newrow(row); memoryToDisk(newrow); int writtensize = ::write(fd, newrow.raw(), row_size); #endif if (writtensize == -1) { #if defined(_MINGW_) || defined(WIN32) PLWARNING("could not truncate database file, end may be corrupted!"); #else ftruncate(fd, curpos); #endif PLERROR("Could not write to database: %s", strerror(errno)); } else size_++; }
void PLearn::SimpleDB< KT, QR >::clearIndex | ( | string | columnName | ) |
Clear an index from memory to free up space.
Definition at line 1778 of file SimpleDB.h.
References n.
{ int n, start_pos, column_precision=0; if (findColumn(column_name, n, start_pos, column_precision)) indexes[n] = 0; }
void PLearn::SimpleDB< KT, QR >::closeAllFiles | ( | ) | const [private] |
close all open file descriptors
Definition at line 1650 of file SimpleDB.h.
void PLearn::SimpleDB< KT, QR >::computeSize | ( | ) | [private] |
(should provide union, intersection, and such operators...)
computes the size_ field (called by constructor)
! cerr << "computing size" << endl; if(row_size<=0) //!< schema not yet set size_ = 0; else { int last = lastSegment(); int fd = seekToEnd(); assert (fd != -1 && last >= 0); off_t pos = lseek(fd, 0ul, SEEK_CUR); PLASSERT(row_size > 0 && pos % row_size == 0); size_ = pos / row_size + last * max_records_file; }
Definition at line 1493 of file SimpleDB.h.
References i.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
{ if(row_size<=0) size_ = 0; else { size_ = 0; int i=0; int bytesinfile = file_size(getSegmentPath(i++)); while(bytesinfile>0) { size_ += bytesinfile/row_size; bytesinfile = file_size(getSegmentPath(i++)); } } }
void PLearn::SimpleDB< KT, QR >::diskToMemory | ( | Row & | row | ) | const [private] |
convert a row from disk-standard little-endian format to machine-dependent endianness
Definition at line 1553 of file SimpleDB.h.
{ memoryToDisk(row); }
bool PLearn::SimpleDB< KeyType, QueryResult >::findColumn | ( | string | name, |
int & | position, | ||
int & | start, | ||
int & | precision | ||
) | const [inline] |
Find a column by name, return true if found, and return the position, total bytes before in the row, and number of bytes in the column
Definition at line 568 of file SimpleDB.h.
QR PLearn::SimpleDB< KT, QR >::findEqual | ( | const unsigned char * | lookfor, |
string | columnName, | ||
string | secondColumn = string("") |
||
) |
Find all rows in the database having the sequence of bytes contained in lookfor in the column columnName. (The number of bytes to match is determined by the precision of the field type of the specified column). If the column has been indexed before, use the index; otherwise, use linear search through the database.
Definition at line 1786 of file SimpleDB.h.
References n.
{ int n, start_pos, column_precision; if (!findColumn(column_name, n, start_pos, column_precision)) return EmptyResult; if (indexes[n]) return findEqualIndexed(lookfor, column_name, second_column); else return findEqualLinear(lookfor, column_name, second_column); }
const QR & PLearn::SimpleDB< KT, QR >::findEqualIndexed | ( | const unsigned char * | lookfor, |
string | columnName, | ||
string | secondColumn = string("") |
||
) |
Always use the index to find "lookfor".
Definition at line 1800 of file SimpleDB.h.
References PLearn::Hash_UNUSED_TAG, n, and PLERROR.
{ bool has_second_column = (second_column.size() > 0); int n, start_pos, column_precision =0, n2, start_pos2, column_precision2=0; if (!findColumn(column_name, n, start_pos, column_precision)) return EmptyResult; if (has_second_column && !findColumn(second_column, n2, start_pos2, column_precision2)) return EmptyResult; if (!indexes[n]) PLERROR("SimpleDB::indexColumn must be done before performing " "indexed searches on column %s", column_name.c_str()); Index& index = *indexes[n]; IndexKey key(lookfor, column_precision+column_precision2); unsigned int addr = index.hashAddress(key); if (addr == Hash_UNUSED_TAG) return EmptyResult; else return *index[addr]; }
QR PLearn::SimpleDB< KT, QR >::findEqualLinear | ( | const unsigned char * | lookfor, |
string | columnName, | ||
string | secondColumn = string("") |
||
) |
Use linear search to find "lookfor".
Definition at line 1828 of file SimpleDB.h.
{ vuc lf(1); lf[0] = lookfor; return findEqualLinear(lf, column_name, second_column); }
QR PLearn::SimpleDB< KT, QR >::findEqualLinear | ( | const vuc & | lookfor, |
string | columnName, | ||
string | secondColumn = string("") |
||
) |
Make a vector of keys from all strings to look for
Indeed search linearly...
(the -1 below assumes that the first column type is a string;--- should enforce this in the future)
Look among all keys to lookfor
Definition at line 1839 of file SimpleDB.h.
References PLearn::SimpleDBIndexKey< KeyType >::begin(), std::copy(), PLearn::endl(), i, n, and PLearn::Row::raw().
{ bool has_second_column = (second_column.size() > 0); int n, start_pos, column_precision =0, n2, start_pos2, column_precision2=0; if (!findColumn(column_name, n, start_pos, column_precision)) return EmptyResult; if (has_second_column && !findColumn(second_column, n2, start_pos2, column_precision2)) return EmptyResult; QR qr; vector<IndexKey> key_lookfor(lookfor.size()); vuc::const_iterator lookit, lookbeg = lookfor.begin(), lookend = lookfor.end(); typename vector<IndexKey>::iterator keyit, keybeg = key_lookfor.begin(), keyend = key_lookfor.end(); size_t len = column_precision+column_precision2; for (lookit=lookbeg, keyit=keybeg ; lookit != lookend ; ++lookit, ++keyit) { keyit->resize(len); copy(*lookit, *lookit+len, keyit->begin()); } IndexKey key_dbrow(column_precision+column_precision2); typename IndexKey::iterator keybegin = key_dbrow.begin(); RowNumber maxrows = size(); Row currow(&schema); unsigned char* begin1 = currow.raw() + start_pos; unsigned char* end1 = begin1 + column_precision; unsigned char* begin2 = currow.raw() + start_pos2; unsigned char* end2 = begin2 + column_precision2; for (RowNumber i=0; i<maxrows; ++i) { if (verbose && i % 1000000 == 0) { cerr << "Searching row " << i << endl; } getInRow(i, currow); copy(begin1, end1, keybegin); if (has_second_column) copy(begin2, end2, keybegin+column_precision-1); for (keyit = keybeg ; keyit != keyend ; ++keyit) if (*keyit == key_dbrow) { qr.push_back(i); if (verbose) cerr << "Found string \"" << *keyit << "\" at row " << i << endl; } } return qr; }
Row & PLearn::SimpleDB< KT, QR >::getInRow | ( | RowNumber | n, |
Row & | row | ||
) | const |
Definition at line 1463 of file SimpleDB.h.
References PLERROR, PLearn::Row::raw(), PLearn::read(), and PLearn::Row::size().
Referenced by PLearn::SDBWithStats::computeStats(), PLearn::AutoSDBVMatrix::getNewRow(), PLearn::SDBVMatrix::getRow(), PLearn::halfShuffleRows(), and PLearn::randomShuffleRows().
{ #ifdef __INTEL_COMPILER #pragma warning(disable:186) // Get rid of compiler warning. #endif if(n<0 || n>=size()) #ifdef __INTEL_COMPILER #pragma warning(default:186) #endif PLERROR("Out of Bounds in SimpleDB::getInRow"); if(row_size != row.size()) PLERROR("In getInRow row_size!=row_size()"); int fd = seekToRow(n); int size_read = ::read(fd, row.raw(), row_size); if (size_read == -1) PLERROR("Could not read from database: %s",strerror(errno)); diskToMemory(row); return row; }
string PLearn::SimpleDB< KeyType, QueryResult >::getName | ( | ) | const [inline] |
--- Functions dealing with database name and location
Definition at line 546 of file SimpleDB.h.
Referenced by PLearn::SDBWithStats::hasStats(), PLearn::SDBWithStats::loadStats(), and PLearn::SDBWithStats::saveStats().
{ return name; }
string PLearn::SimpleDB< KeyType, QueryResult >::getPath | ( | ) | const [inline] |
Definition at line 549 of file SimpleDB.h.
Referenced by PLearn::SDBWithStats::hasStats(), PLearn::SDBWithStats::loadStats(), and PLearn::SDBWithStats::saveStats().
{ return path; }
Row PLearn::SimpleDB< KT, QR >::getRow | ( | RowNumber | n | ) | const |
--- Functions dealing with simple database queries
Definition at line 1485 of file SimpleDB.h.
const Schema& PLearn::SimpleDB< KeyType, QueryResult >::getSchema | ( | ) | const [inline] |
Definition at line 557 of file SimpleDB.h.
Referenced by PLearn::AutoSDBVMatrix::AutoSDBVMatrix(), PLearn::SDBWithStats::computeStats(), PLearn::AutoSDBVMatrix::getMappings(), PLearn::halfShuffleRows(), PLearn::randomShuffleRows(), PLearn::SDBVMatrix::SDBVMatrix(), and PLearn::SDBWithStats::SDBWithStats().
{ return schema; }
string PLearn::SimpleDB< KT, QR >::getSegmentPath | ( | int | i | ) | const [private] |
return full path of segment i (zero-based)
Definition at line 1669 of file SimpleDB.h.
References PLERROR.
bool PLearn::SimpleDB< KT, QR >::indexColumn | ( | string | columnName, |
string | secondColumn = string("") |
||
) |
--- Functions dealing with indexing and more complex queries
Index according to the field having a given name. Return TRUE if indexing has been successful. At the moment, indexes are strictly kept in memory; they are not saved to disk. Optionally, the contents of a second column can be concatenated for indexing purposes.
Add all records to the index. Create one if necessary. Make initial size a constant times the number of records in the DB, as a heuristic.
If current key already exists, just add current row number; otherwise create new query result (the -1 below assumes that the first column type is a string;--- should enforce this in the future).
Avoid creating a one-element QueryResult and then copy it over. We add an EMPTY key/value pair to the hash, table, then access the value object (through the normal code to add a new row into an existing key, then set the row i into it.
Normal pathway: add row i to existing address
Definition at line 1687 of file SimpleDB.h.
References PLearn::Hash< KeyType, DataType >::add(), PLearn::SimpleDBIndexKey< KeyType >::begin(), std::copy(), PLearn::Hash< KeyType, DataType >::diagnostics(), PLearn::endl(), PLearn::Hash< KeyType, DataType >::flush(), PLearn::Hash_UNUSED_TAG, PLearn::Hash< KeyType, DataType >::hashAddress(), i, n, PLWARNING, PLearn::Row::raw(), and PLearn::Hash< KeyType, DataType >::resize().
{ bool has_second_column = (second_column.size() > 0); int n, start_pos, column_precision =0, n2, start_pos2, column_precision2=0; if (!findColumn(column_name, n, start_pos, column_precision)) return false; if (has_second_column && !findColumn(second_column, n2, start_pos2, column_precision2)) return false; RowNumber maxrows = size(); RowNumber tablesize = RowNumber(table_size_multiplier*maxrows); if (maxrows <= 0 || tablesize <= 0) { PLWARNING("SimpleDB::indexColumn: cannot index a database of " "zero size."); return false; } if (!indexes[n]) { indexes[n] = new Index(tablesize, true); indexes[n]->initializeTable((unsigned int)tablesize); } Index& index = *indexes[n]; index.resize(tablesize); index.flush(); Row currow(&schema); IndexKey key(column_precision + column_precision2); typename IndexKey::iterator keybegin = key.begin(); unsigned char* begin1 = currow.raw() + start_pos; unsigned char* end1 = begin1 + column_precision; unsigned char* begin2 = currow.raw() + start_pos2; unsigned char* end2 = begin2 + column_precision2; for(RowNumber i=0; i<maxrows; ++i) { if (verbose && i % 1000000 == 0) { unsigned numclusters, maxcluster; index.diagnostics(numclusters, maxcluster); cerr << "indexing row " << i << "\t num. clusters=" << numclusters << "\t max. cluster size=" << maxcluster << endl; } getInRow(i,currow); copy(begin1, end1, keybegin); if (has_second_column) copy(begin2, end2, keybegin+column_precision-1); unsigned int addr = index.hashAddress(key); if (addr == Hash_UNUSED_TAG) { bool needresize = !index.add(key,QueryResult_t()); if (needresize) { cerr << "Hash table unexpectedly full; exiting..." << endl; exit(1); } addr = index.hashAddress(key); } QueryResult_t* qr = index[addr]; try { qr->push_back(i); } catch (logic_error& e) { cerr << "Exception caught during indexing: " << typeid(e).name() << endl << "Containing: " << e.what() << endl; throw; } } return true; }
int PLearn::SimpleDB< KeyType, QueryResult >::indexOfField | ( | const string & | fieldname | ) | const [inline] |
returns the index of the given field inside the Schema, -1 if not found
Definition at line 575 of file SimpleDB.h.
Referenced by PLearn::SDBWithStats::getStat().
{ return schema(fieldname).field_index(); }
int PLearn::SimpleDB< KT, QR >::lastSegment | ( | ) | const [inline, private] |
return the index (NOT FD!) of the last open segment
Definition at line 1662 of file SimpleDB.h.
{ return (int)allfd.size() - 1; }
int PLearn::SimpleDB< KeyType, QueryResult >::length | ( | ) | const [inline] |
Definition at line 584 of file SimpleDB.h.
Referenced by PLearn::AutoSDBVMatrix::AutoSDBVMatrix(), PLearn::halfShuffleRows(), and PLearn::randomShuffleRows().
void PLearn::SimpleDB< KT, QR >::loadSchema | ( | ) |
If a schema does not exist, an empty schema is set into the current database!
Definition at line 1315 of file SimpleDB.h.
References PLearn::CharacterType, PLearn::DateType, PLearn::DoubleType, PLearn::endl(), PLearn::FloatType, PLearn::IntType, PLearn::lowerstring(), PLearn::ShortType, PLearn::SignedCharType, and PLearn::StringType.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
{ string fullpath = path + name + ".ssc"; ifstream sf(fullpath.c_str()); Schema schema; while (sf) { string name,type; sf >> name >> type; if (name.size() == 0 || type.size() == 0) break; type = lowerstring(type); if (type == "string") { int length; sf >> length; schema.push_back(Field(name,StringType,length)); } else if (type == "character") schema.push_back(Field(name,CharacterType)); else if (type == "signedchar") schema.push_back(Field(name,SignedCharType)); else if (type == "short") schema.push_back(Field(name,ShortType)); else if (type == "int") schema.push_back(Field(name,IntType)); else if (type == "float") schema.push_back(Field(name,FloatType)); else if (type == "double") schema.push_back(Field(name,DoubleType)); else if (type == "date") schema.push_back(Field(name,DateType)); else { cerr << "Unexpected input type \"" << type << "\" in schema file " << fullpath << endl; exit(1); } } setSchema(schema); }
void PLearn::SimpleDB< KT, QR >::memoryToDisk | ( | Row & | row | ) | const [private] |
convert a row from machine-dependent endianness to disk-standard little-endian format.
Definition at line 1527 of file SimpleDB.h.
References PLearn::RowIterator::asDate(), PLearn::RowIterator::asDouble(), PLearn::RowIterator::asFloat(), PLearn::RowIterator::asInt(), PLearn::RowIterator::asShort(), PLearn::Row::begin(), PLearn::Row::end(), PLearn::reverse_double(), PLearn::reverse_float(), PLearn::reverse_int(), PLearn::reverse_short(), and x.
{ #ifdef LITTLEENDIAN #endif #ifdef BIGENDIAN Row newr(row); Row::iterator it = newr.begin(), end = newr.end(); for(; it != end; ++it) { if (short* x = it.asShort()) reverse_short(x,1); if (int* x = it.asInt()) reverse_int(x,1); if (float* x = it.asFloat()) reverse_float(x,1); if (double* x = it.asDouble()) reverse_double(x,1); if (PDate* x = it.asDate()) { reverse_short(&(x->year),1); } } #endif }
void PLearn::SimpleDB< KT, QR >::openAllFiles | ( | ) | const [private] |
open all existing segments in database
Since we don't know in advance how many files are in the database, we start by opening segment zero (which always exists), and then successively try to open more segments
< safeguard
Definition at line 1620 of file SimpleDB.h.
References PLERROR.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
{ closeAllFiles(); int fd; fd = open(getSegmentPath(0).c_str(), access_mask | O_CREAT, 0777); if (fd == -1) PLERROR("Could not open main database segment %s: %s", getSegmentPath(0).c_str(), strerror(errno)); allfd.push_back(fd); int index = 1; for ( ; ; ++index ) { fd = open(getSegmentPath(index).c_str(), access_mask); if (fd == -1) break; else allfd.push_back(fd); } }
void PLearn::SimpleDB< KeyType, QueryResult >::operator= | ( | const SimpleDB< KeyType, QueryResult > & | ) | [private] |
void PLearn::SimpleDB< KT, QR >::saveSchema | ( | ) |
Definition at line 1264 of file SimpleDB.h.
References PLearn::CharacterType, PLearn::DateType, PLearn::DoubleType, PLearn::endl(), PLearn::FloatType, PLearn::IntType, PLERROR, PLearn::ShortType, PLearn::SignedCharType, PLearn::StringType, and PLearn::Unknown.
{ if (access_type==readwrite) { string fullpath = path + name + ".ssc"; ofstream sf(fullpath.c_str(), ios::out); Schema::iterator it = schema.begin(), end = schema.end(); for (; it != end; ++it) { sf << it->name << " "; switch (it->field_type) { case Unknown: break; case StringType: sf << "string " << it->precision << endl; break; case CharacterType: sf << "character" << endl; break; case SignedCharType: sf << "signedchar" << endl; break; case ShortType: sf << "short" << endl; break; case IntType: sf << "int" << endl; break; case FloatType: sf << "float" << endl; break; case DoubleType: sf << "double" << endl; break; case DateType: sf << "date" << endl; break; default: PLERROR("Unknown field type in database: %d", it->field_type); } } } }
int PLearn::SimpleDB< KT, QR >::seekToEnd | ( | ) | const [private] |
seek to end of database, return FD to the correct file
cout << "seekToEnd in db " << getName() << endl;
There is a slight subtlety here: if the last segment is full, we have to seek to the beginning of the next segment
Definition at line 1595 of file SimpleDB.h.
References PLERROR.
{ if (max_records_file == 0) PLERROR("Attempting to seekToEnd without schema set"); int last = lastSegment(); int fd = allfd[last]; if (fd == -1) PLERROR("Problem accessing database segment %d at path %s", last, getSegmentPath(last).c_str()); off_t pos = lseek(fd, 0ul, SEEK_END); if (Offset(pos) / Offset(row_size) >= max_records_file) fd = seekToRow((last+1)*max_records_file); return fd; }
int PLearn::SimpleDB< KT, QR >::seekToRow | ( | RowNumber | i | ) | const [private] |
given a row number, seek to the beginning of the row of the correct underlying physical file (segment), and return a file descriptor to the file; open a new file if necessary
cout << "seekToRow " << i << " in db " << getSegmentPath(0) << endl;
verify that segment is indeed open
Definition at line 1562 of file SimpleDB.h.
{ if (max_records_file == 0) PLERROR("Attempting to seekToRow without schema set"); int segmentNumber = int(i / max_records_file); Offset rowInSegment = Offset(i % max_records_file); if (segmentNumber > lastSegment()) { for (int i = lastSegment()+1; i <= segmentNumber; ++i) { int fd = open(getSegmentPath(i).c_str(), access_mask | O_CREAT, 0777); if (fd == -1) PLERROR("Could not open database segment %d at path %s: %s", i, getSegmentPath(i).c_str(), strerror(errno)); allfd.push_back(fd); } } if (allfd[segmentNumber] == -1) PLERROR("Problem accessing database segment %d at path %s", segmentNumber, getSegmentPath(segmentNumber).c_str()); if (lseek(allfd[segmentNumber], rowInSegment * Offset(row_size), SEEK_SET)<0) PLERROR("problem in lseek: %s",strerror(errno)); return allfd[segmentNumber]; }
void PLearn::SimpleDB< KT, QR >::setRow | ( | const Row & | row, |
RowNumber | n | ||
) |
set a particular row
Handle writing error
Definition at line 1392 of file SimpleDB.h.
References PLERROR, PLearn::Row::raw(), PLearn::Row::sanitize(), PLearn::Row::size(), and PLearn::write().
Referenced by PLearn::halfShuffleRows(), and PLearn::randomShuffleRows().
{ #ifdef __INTEL_COMPILER #pragma warning(disable:186) // Get rid of compiler warning. #endif if(n<0 || n>=size()) #ifdef __INTEL_COMPILER #pragma warning(default:186) #endif PLERROR("Out of bounds in SimpleDB::setRow"); if(row_size != row.size()) PLERROR("In setRow row_size != row.size() (%d != %d)", row_size, row.size()); row.sanitize(); int fd = seekToRow(n); #ifdef LITTLEENDIAN int writtensize = ::write(fd, row.raw(), row_size); #endif #ifdef BIGENDIAN Row newrow(row); memoryToDisk(newrow); int writtensize = ::write(fd, newrow.raw(), row_size); #endif if (writtensize == -1) PLERROR("Could not write to database: %s",strerror(errno)); }
void PLearn::SimpleDB< KT, QR >::setSchema | ( | const Schema & | s | ) |
--- Functions dealing with schema representation.
Compute the maximum size of a single file, taking into account the row size.
cout << "In setSchema for db " << getName() << " row_size=" << row_size << " max_records_file=" << max_records_file << endl;
Reopen files with new schema in effect
Definition at line 1241 of file SimpleDB.h.
References PLearn::Row::size().
{ schema = s; Row row(&s); row_size = row.size(); indexes.resize(s.size()); if (row_size > 0) max_records_file = RowNumber(AbsoluteFileLimit / row_size); else max_records_file = 0; closeAllFiles(); openAllFiles(); computeSize(); }
RowNumber PLearn::SimpleDB< KeyType, QueryResult >::size | ( | ) | const [inline] |
1 beyond maximum row number
Definition at line 582 of file SimpleDB.h.
Referenced by PLearn::SDBVMatrix::SDBVMatrix(), and PLearn::SDBWithStats::SDBWithStats().
void PLearn::SimpleDB< KeyType, QueryResult >::tableSizeMultiplier | ( | double | x | ) | [inline] |
double PLearn::SimpleDB< KeyType, QueryResult >::tableSizeMultiplier | ( | ) | const [inline] |
Access the table size multiplier (see description below)
Definition at line 640 of file SimpleDB.h.
{ return table_size_multiplier; }
void PLearn::SimpleDB< KT, QR >::truncateFromRow | ( | RowNumber | n | ) |
erase all rows from row n (included) until end
We must perform the following: seek to the proper row, truncate the current file, unlink all remaining files until end, close everything, re-open everything.
Definition at line 1423 of file SimpleDB.h.
References PLearn::find(), PLERROR, PLWARNING, and PLearn::tostring().
{ int curfd = seekToRow(n); off_t curpos = lseek(curfd, 0L, SEEK_CUR); if (ftruncate(curfd, curpos) == -1) { PLERROR((string("Could not truncate database at row ") + tostring(n) + ": " + strerror(errno)).c_str()); } vector<int>::iterator found = find(allfd.begin(), allfd.end(), curfd); int fromfd = found-allfd.begin() + 1; int last = lastSegment(); closeAllFiles(); bool allok = true; for ( ; fromfd <= last; ++fromfd) { string path = getSegmentPath(fromfd); if(unlink(path.c_str()) == -1) { PLWARNING((string("Could not unlink database segment ") + path + ": " + strerror(errno)).c_str()); allok = false; } } if (allok) { openAllFiles(); computeSize(); } else PLERROR("Error during truncation"); }
int PLearn::SimpleDB< KeyType, QueryResult >::width | ( | ) | const [inline] |
Definition at line 587 of file SimpleDB.h.
Referenced by PLearn::AutoSDBVMatrix::AutoSDBVMatrix(), PLearn::SDBWithStats::forgetStats(), and PLearn::SDBWithStats::getStat().
const Offset PLearn::SimpleDB< KeyType, QueryResult >::AbsoluteFileLimit = 512ul * 1024ul * 1024ul - 1 [static] |
Maximum size that a single database file can hold; this is currently set to 512 MB.
Definition at line 526 of file SimpleDB.h.
int PLearn::SimpleDB< KeyType, QueryResult >::access_mask [private] |
Unix constants for access_type.
Definition at line 687 of file SimpleDB.h.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
AccessType PLearn::SimpleDB< KeyType, QueryResult >::access_type [private] |
readwrite or readonly
Definition at line 686 of file SimpleDB.h.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
vector<int> PLearn::SimpleDB< KeyType, QueryResult >::allfd [mutable, private] |
File descriptors of open segments; -1 for not open.
Definition at line 697 of file SimpleDB.h.
QR PLearn::SimpleDB< KT, QR >::EmptyResult [static] |
A null query.
Definition at line 530 of file SimpleDB.h.
vector<PIndex> PLearn::SimpleDB< KeyType, QueryResult >::indexes [private] |
There's one possible index per column in the database.
An index is made only when indexColumn is called for a particular column.
Definition at line 710 of file SimpleDB.h.
RowNumber PLearn::SimpleDB< KeyType, QueryResult >::max_records_file [private] |
maximum number of full rows in a single file
Definition at line 694 of file SimpleDB.h.
string PLearn::SimpleDB< KeyType, QueryResult >::name [private] |
database base name
-- Physical characteristics of database
Definition at line 684 of file SimpleDB.h.
Referenced by PLearn::SDBWithStats::loadStats().
string PLearn::SimpleDB< KeyType, QueryResult >::path [private] |
database root path
Definition at line 685 of file SimpleDB.h.
Referenced by PLearn::SimpleDB< KeyType, QueryResult >::SimpleDB().
int PLearn::SimpleDB< KeyType, QueryResult >::row_size [private] |
length of a row in bytes
Definition at line 693 of file SimpleDB.h.
Schema PLearn::SimpleDB< KeyType, QueryResult >::schema [private] |
RowNumber PLearn::SimpleDB< KeyType, QueryResult >::size_ [private] |
cached number of rows
Definition at line 688 of file SimpleDB.h.
double PLearn::SimpleDB< KeyType, QueryResult >::table_size_multiplier [private] |
-- Indexing-related
The multiplier factor for converting the number of database entries into the size of the hash table for indexing. By default, 1.8, but should be considerably less if there are many repeated keys.
Definition at line 706 of file SimpleDB.h.
bool PLearn::SimpleDB< KeyType, QueryResult >::verbose [private] |