PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // SimpleDB.cc: Simple Database Representation (implementation) 00004 // 00005 // Copyright (C) 2000 Nicolas Chapados 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00037 #ifndef SIMPLEDB_H 00038 #define SIMPLEDB_H 00039 00040 00042 #include <limits.h> 00043 #include <string> 00044 #include <vector> 00045 #include <iostream> 00046 #include <stdexcept> 00047 #include <typeinfo> 00048 00049 #include <assert.h> 00050 00051 #ifndef WIN32 00052 #include <unistd.h> 00053 #else 00054 #include <io.h> 00055 #endif 00056 00057 #include <fcntl.h> 00058 #include <errno.h> 00059 #include <cstdlib> 00060 00061 00063 #include <plearn/base/stringutils.h> 00064 #include <plearn/base/PDate.h> 00065 #include <plearn/math/Hash.h> 00066 #include <plearn/base/TinyVector.h> 00067 00068 #ifdef WIN32 00069 // norman: potentially dangerous if there is a function called with the same name in this 00070 // file. Beware! 00071 #define open _open 00072 #define close _close 00073 #define lseek _lseek 00074 #define read _read 00075 #define write _write 00076 #define unlink _unlink 00077 #endif 00078 00079 namespace PLearn { 00080 using namespace std; 00081 00083 class Row; 00084 00085 00086 //############################## CLASS FIELD ########################### 00094 // StringType length of string 00095 // (including null terminator) 00096 // CharacterType 1 == unsigned char 00097 // SignedCharType 1 == signed char 00098 // ShortType 2 == signed short 00099 // IntType 4 == signed int 00100 // FloatType 4 == float 00101 // DoubleType 8 == double 00102 // DateType 4 == PDate 00103 enum FieldType { 00104 Unknown = 0, 00105 StringType, 00106 CharacterType, 00107 SignedCharType, 00108 ShortType, 00109 IntType, 00110 FloatType, 00111 DoubleType, 00112 DateType 00113 }; 00114 00115 struct Field { 00116 Field() : name(), field_type(Unknown), precision() {} 00117 Field(string name, FieldType t, int p = 0) 00118 : name(name), field_type(t) 00119 { 00122 switch(field_type) { 00123 case Unknown: precision=0; break; 00124 case StringType: precision=p; break; 00125 case CharacterType: precision=1; break; 00126 case SignedCharType: precision=1; break; 00127 case ShortType: precision=2; break; 00128 case IntType: precision=4; break; 00129 case FloatType: precision=4; break; 00130 case DoubleType: precision=8; break; 00131 case DateType: precision=4; break; 00132 default: 00133 PLERROR("Unknown field type %d with name %s", 00134 int(field_type), name.c_str()); 00135 } 00136 if (sizeof(PDate) != 4) 00137 PLERROR("A PLearn PDate must have sizeof equal to 4"); 00138 } 00139 00140 bool operator==(const Field& x) const { 00141 return name==x.name && 00142 field_type == x.field_type && 00143 precision == x.precision; 00144 } 00145 00146 string name; 00147 FieldType field_type; 00148 int precision; 00149 }; 00150 00152 extern const char MissingString; 00153 extern const unsigned char MissingCharacter; 00154 extern const signed char MissingSignedChar; 00155 extern const short MissingShort; 00156 extern const int MissingInt; 00157 extern const float MissingFloat; 00158 extern const double MissingDouble; 00159 extern const PDate MissingDate; 00160 00161 00162 //############################# CLASS FIELDPTR ######################### 00175 class FieldPtr { 00176 friend class Row; 00177 friend class Schema; 00178 00179 public: 00181 FieldPtr() : field_index_(-1), offset_(-1) {} 00182 00184 00186 int field_index() const { 00187 return field_index_; 00188 } 00189 00190 ptrdiff_t offset() const { 00191 return offset_; 00192 } 00193 00195 operator bool() const { 00196 return field_index_ >= 0; 00197 } 00198 00200 bool operator!() const { 00201 return field_index_ == -1; 00202 } 00203 00205 bool operator==(const FieldPtr& x) const { 00206 return field_index_ == x.field_index_ && 00207 offset_ == x.offset_; 00208 } 00209 00210 bool operator!=(const FieldPtr& x) const { 00211 return !(*this == x); 00212 } 00213 00214 private: 00216 FieldPtr(int fi, ptrdiff_t o) : field_index_(fi), offset_(o) {} 00217 00218 private: 00219 int field_index_; 00220 ptrdiff_t offset_; 00221 }; 00222 00223 00224 //############################ CLASS FIELDVALUE ########################## 00231 class FieldValue 00232 { 00233 friend class FieldRowRef; 00234 00235 public: 00237 FieldValue(); 00238 FieldValue(const FieldValue& fv); 00239 ~FieldValue(); 00240 00242 explicit FieldValue(const char*); 00243 explicit FieldValue(unsigned char); 00244 explicit FieldValue(signed char); 00245 explicit FieldValue(short); 00246 explicit FieldValue(int); 00247 explicit FieldValue(float); 00248 explicit FieldValue(double); 00249 explicit FieldValue(const PDate&); 00250 00252 bool isMissing() const; 00253 void setMissing(); 00254 00256 string toString() const; 00257 double toDouble() const; 00258 PDate toDate() const; 00259 00260 operator double() const { return toDouble(); } 00261 operator float() const { return float(toDouble()); } 00262 operator int() const { return int(toDouble()); } 00263 operator string() const { return toString(); } 00264 operator PDate() const { return toDate(); } 00265 00267 00269 FieldValue& operator=(FieldValue); 00270 00272 bool operator==(const FieldValue&) const; 00273 bool operator< (const FieldValue&) const; 00274 00275 bool operator!=(const FieldValue& rhs) const { 00276 return !(*this == rhs); 00277 } 00278 00279 bool operator<=(const FieldValue& rhs) const { 00280 return (*this == rhs) || (*this < rhs); 00281 } 00282 00283 bool operator> (const FieldValue& rhs) const { 00284 return !(*this <= rhs); 00285 } 00286 00287 bool operator>=(const FieldValue& rhs) const { 00288 return !(*this < rhs); 00289 } 00290 00292 FieldValue operator+(const FieldValue&) const; 00293 FieldValue operator-(const FieldValue&) const; 00294 FieldValue operator*(const FieldValue&) const; 00295 FieldValue operator/(const FieldValue&) const; 00296 00298 void swap(FieldValue&); 00299 00301 bool isString() const { 00302 return field_type_ == StringType; 00303 } 00304 00305 bool isIntegral() const { 00306 return 00307 field_type_ == CharacterType || field_type_ == SignedCharType || 00308 field_type_ == ShortType || field_type_ == IntType; 00309 } 00310 00311 bool isFloating() const { 00312 return field_type_ == FloatType || field_type_ == DoubleType; 00313 } 00314 00315 bool isDate() const { 00316 return field_type_ == DateType; 00317 } 00318 00319 private: 00320 FieldType field_type_; 00321 int precision_; 00322 struct DateVal_t { 00323 short year; 00324 unsigned char month, day; 00325 }; 00326 union { 00327 char* string_val_; 00328 long long_val_; 00329 double double_val_; 00330 struct DateVal_t date_val_; 00331 }; 00332 }; 00333 00334 ostream& operator<<(ostream&, const FieldValue&); 00335 00336 00337 00338 //############################## CLASS SCHEMA ############################ 00352 class Schema : public vector<Field> 00353 { 00354 typedef vector<Field> inherited; 00355 typedef Field T; 00356 00357 public: 00359 Schema() : inherited() {} 00360 Schema(size_type n, const T& value) : inherited(n, value) {} 00361 Schema(int n, const T& value) : inherited(n, value) {} 00362 Schema(long n, const T& value) : inherited(n, value) {} 00363 explicit Schema(size_type n) : inherited(n) {} 00364 Schema(const Schema& x) : inherited(x) {} 00365 00366 #ifdef __STL_MEMBER_TEMPLATES 00367 template <class InputIterator> 00368 Schema(InputIterator first, InputIterator last) 00369 : inherited(first, last) {} 00370 #else 00371 Schema(const_iterator first, const_iterator last) 00372 : inherited(first, last) {} 00373 #endif 00375 00376 00377 00378 00383 bool findColumn(const string& name, int& position, int& start, 00384 int& precision) const; 00385 00387 FieldPtr findColumn(int position) const; 00388 00390 FieldPtr findColumn(const string& name) const; 00391 00396 FieldPtr operator()(int position) const { 00397 return findColumn(position); 00398 } 00399 00400 FieldPtr operator()(const string& name) const { 00401 return findColumn(name); 00402 } 00403 }; 00404 00405 00406 00407 //######################### CLASS SIMPLEDBINDEXKEY ####################### 00414 template <class KeyType> 00415 class SimpleDBIndexKey { 00416 public: 00427 typedef KeyType ByteArr; 00428 typedef typename ByteArr::iterator iterator; 00429 00430 SimpleDBIndexKey() {} 00431 00432 explicit SimpleDBIndexKey(size_t len) 00433 : raw(len, '\0') {} 00434 00435 SimpleDBIndexKey(const unsigned char* the_raw, size_t len) 00436 : raw(len, '\0') 00437 { 00438 copy(the_raw, the_raw+len, begin()); 00439 } 00440 00441 SimpleDBIndexKey(const ByteArr& the_raw) 00442 : raw(the_raw) {} 00443 00454 operator char*() const { 00455 return (char*)(&raw[0]); 00456 } 00457 00458 size_t byteLength() const { 00459 return raw.size(); 00460 } 00461 00463 bool operator==(const SimpleDBIndexKey& other) const { 00464 return raw == other.raw; 00465 } 00466 00467 bool operator!=(const SimpleDBIndexKey& other) const { 00468 return raw != other.raw; 00469 } 00470 00471 void resize(size_t len) { 00472 raw.resize(len); 00473 } 00474 00475 typename ByteArr::iterator begin() { 00476 return raw.begin(); 00477 } 00478 00479 typename ByteArr::iterator end(){ 00480 return raw.end(); 00481 } 00482 00483 private: 00484 ByteArr raw; 00485 }; 00486 00487 00488 00489 //############################# CLASS SIMPLEDB ######################### 00498 template <class KeyType = TinyVector<unsigned char, 8>, 00499 class QueryResult = TinyVector<unsigned int, 4> > 00500 class SimpleDB 00501 { 00504 00505 public: 00507 00510 typedef unsigned long RowNumber; 00511 enum { 00512 InvalidRow = ULONG_MAX 00513 }; 00514 00516 typedef unsigned long Offset; 00517 00519 enum AccessType { 00520 readwrite = 0, 00521 readonly = 1 00522 }; 00523 00526 static const Offset AbsoluteFileLimit = 512ul * 1024ul * 1024ul - 1; 00527 00529 typedef QueryResult QueryResult_t; 00530 static QueryResult EmptyResult; 00531 00533 typedef SimpleDBIndexKey<KeyType> IndexKey; 00534 typedef Hash<IndexKey,QueryResult> Index; 00535 typedef PP<Index> PIndex; 00536 00537 public: 00538 00540 SimpleDB(string rootname, string path=".", AccessType = readwrite, 00541 bool verbose=true); 00542 virtual ~SimpleDB(); 00543 00544 00546 string getName() const { 00547 return name; 00548 } 00549 string getPath() const { 00550 return path; 00551 } 00552 00553 00555 void setSchema(const Schema& s); 00556 00557 const Schema& getSchema() const { 00558 return schema; 00559 } 00560 00561 void saveSchema(); 00562 void loadSchema(); 00563 00568 bool findColumn(string name, int& position, int& start, 00569 int& precision) const { 00570 return schema.findColumn(name,position,start,precision); 00571 } 00572 00575 int indexOfField(const string& fieldname) const { 00576 return schema(fieldname).field_index(); 00577 } 00578 00580 Row getRow(RowNumber) const; 00581 Row& getInRow(RowNumber, Row&) const; 00582 RowNumber size() const { return size_; } 00583 00584 int length() const { 00585 return int(size()); 00586 } 00587 int width() const { 00588 return int(schema.size()); 00589 } 00590 00592 void addRow(const Row&); 00593 void setRow(const Row&, RowNumber); 00594 void truncateFromRow(RowNumber n); 00595 00596 00597 00599 00606 bool indexColumn(string columnName, 00607 string secondColumn = string("")); 00608 00610 void clearIndex(string columnName); 00611 00619 QueryResult findEqual(const unsigned char* lookfor, 00620 string columnName, 00621 string secondColumn = string("")); 00622 00624 const QueryResult& findEqualIndexed(const unsigned char* lookfor, 00625 string columnName, 00626 string secondColumn = string("")); 00627 00629 QueryResult findEqualLinear(const unsigned char* lookfor, 00630 string columnName, 00631 string secondColumn = string("")); 00632 00634 typedef vector<const unsigned char*> vuc; 00635 QueryResult findEqualLinear(const vuc& lookfor, 00636 string columnName, 00637 string secondColumn = string("")); 00638 00640 double tableSizeMultiplier() const { 00641 return table_size_multiplier; 00642 } 00643 void tableSizeMultiplier(double x) { 00644 table_size_multiplier = x; 00645 } 00646 00648 00649 private: 00651 void computeSize(); 00652 00655 void memoryToDisk(Row&) const; 00656 00659 void diskToMemory(Row&) const; 00660 00665 int seekToRow(RowNumber) const; 00666 00668 int seekToEnd() const; 00669 00671 void openAllFiles() const; 00672 00674 void closeAllFiles() const; 00675 00677 inline int lastSegment() const; 00678 00680 string getSegmentPath(int i) const; 00681 00682 private: 00684 string name; 00685 string path; 00686 AccessType access_type; 00687 int access_mask; 00688 RowNumber size_; 00689 00690 00692 Schema schema; 00693 int row_size; 00694 RowNumber max_records_file; 00695 00696 00697 mutable vector<int> allfd; 00698 00699 00701 00706 double table_size_multiplier; 00707 00710 vector<PIndex> indexes; 00711 00713 bool verbose; 00714 00715 private: 00717 SimpleDB(const SimpleDB&); 00718 void operator=(const SimpleDB&); 00719 }; 00720 00721 00722 //########################### CLASS ROWITERATOR ########################## 00733 // *myit1 = *myit2; 00734 00735 class FieldRowRef; 00736 00737 class RowIterator { 00738 public: 00740 RowIterator() : curfield(0), curptr(0), schema(0) { } 00741 RowIterator(const RowIterator& x) { 00742 curfield = x.curfield; 00743 curptr = x.curptr; 00744 schema = x.schema; 00745 } 00746 bool operator==(const RowIterator& x) { 00747 return 00748 curfield==x.curfield && 00749 curptr ==x.curptr && 00750 schema ==x.schema; 00751 } 00752 bool operator!=(const RowIterator& x) { 00753 return !((*this) == x); 00754 } 00755 00757 RowIterator(int curf, unsigned char* curp, const Schema* sc) 00758 : curfield(curf), curptr(curp), schema(sc) 00759 { } 00760 00762 00764 RowIterator& operator=(const RowIterator& x) 00765 { 00766 if (&x != this) 00767 { 00768 curfield = x.curfield; 00769 curptr = x.curptr; 00770 schema = x.schema; 00771 } 00772 return *this; 00773 } 00774 00776 void copyFrom(const RowIterator& it) 00777 { 00778 #ifdef BOUNDCHECK 00779 if(it.precision()!=precision() || it.getFieldType()!=getFieldType()) 00780 PLERROR("In Row::iterator::copyFrom Source and destination fields not of same type or precision"); 00781 #endif 00782 copy(it.raw(),it.raw()+it.precision(),raw()); 00783 } 00784 00786 inline FieldRowRef operator*() const; 00787 00789 00795 RowIterator& operator++() { 00796 if (schema && curptr && curfield < schema->size()) { 00797 curptr += (*schema)[curfield].precision; 00798 ++curfield; 00799 } 00800 return *this; 00801 } 00802 00804 RowIterator operator++(int) { 00805 RowIterator x = *this; 00806 ++(*this); 00807 return x; 00808 } 00809 00812 RowIterator operator[](int i) { 00813 PLASSERT(i >= 0); 00814 RowIterator it = *this; 00815 while (i--) 00816 ++it; 00817 return it; 00818 } 00819 00820 FieldType getFieldType() const 00821 { return (*schema)[curfield].field_type; } 00822 00825 00826 bool isString() const { 00827 return schema && curptr && curfield < schema->size() && 00828 (*schema)[curfield].field_type == StringType; 00829 } 00830 00831 bool isCharacter() const { 00832 return schema && curptr && curfield < schema->size() && 00833 (*schema)[curfield].field_type == CharacterType; 00834 } 00835 00836 bool isSignedChar() const { 00837 return schema && curptr && curfield < schema->size() && 00838 (*schema)[curfield].field_type == SignedCharType; 00839 } 00840 00841 bool isShort() const { 00842 return schema && curptr && curfield < schema->size() && 00843 (*schema)[curfield].field_type == ShortType; 00844 } 00845 00846 bool isInt() const { 00847 return schema && curptr && curfield < schema->size() && 00848 (*schema)[curfield].field_type == IntType; 00849 } 00850 00851 bool isFloat() const { 00852 return schema && curptr && curfield < schema->size() && 00853 (*schema)[curfield].field_type == FloatType; 00854 } 00855 00856 bool isDouble() const { 00857 return schema && curptr && curfield < schema->size() && 00858 (*schema)[curfield].field_type == DoubleType; 00859 } 00860 00861 bool isDate() const { 00862 return schema && curptr && curfield < schema->size() && 00863 (*schema)[curfield].field_type == DateType; 00864 } 00865 00866 00872 char* asString() { 00873 bool iss = isString(); 00874 if (iss) 00875 return reinterpret_cast<char*>(curptr); 00876 else 00877 return 0; 00878 //return isString()? reinterpret_cast<char*>(curptr) : 0; 00879 } 00880 00881 unsigned char* asCharacter() { 00882 return isCharacter()? 00883 reinterpret_cast<unsigned char*>(curptr) : 0; 00884 } 00885 00886 signed char* asSignedChar() { 00887 return isSignedChar()? 00888 reinterpret_cast<signed char*>(curptr) : 0; 00889 } 00890 00891 short* asShort() { 00892 return isShort()? reinterpret_cast<short*>(curptr) : 0; 00893 } 00894 00895 int* asInt() { 00896 return isInt()? reinterpret_cast<int*>(curptr) : 0; 00897 } 00898 00899 float* asFloat() { 00900 return isFloat()? reinterpret_cast<float*>(curptr) : 0; 00901 } 00902 00903 double* asDouble() { 00904 return isDouble()? reinterpret_cast<double*>(curptr) : 0; 00905 } 00906 00907 PDate* asDate() { 00908 return isDate()? reinterpret_cast<PDate*>(curptr) : 0; 00909 } 00910 00911 00918 const char* asString() const { 00919 return isString()? reinterpret_cast<const char*>(curptr) : 0; 00920 } 00921 00922 const unsigned char* asCharacter() const { 00923 return isCharacter()? 00924 reinterpret_cast<const unsigned char*>(curptr) : 0; 00925 } 00926 00927 const signed char* asSignedChar() const { 00928 return isSignedChar()? 00929 reinterpret_cast<const signed char*>(curptr) : 0; 00930 } 00931 00932 const short* asShort() const { 00933 return isShort()? reinterpret_cast<const short*>(curptr) : 0; 00934 } 00935 00936 const int* asInt() const { 00937 return isInt()? reinterpret_cast<const int*>(curptr) : 0; 00938 } 00939 00940 const float* asFloat() const { 00941 return isFloat()? reinterpret_cast<const float*>(curptr) : 0; 00942 } 00943 00944 const double* asDouble() const { 00945 return isDouble()? reinterpret_cast<const double*>(curptr) : 0; 00946 } 00947 00948 const PDate* asDate() const { 00949 return isDate()? reinterpret_cast<const PDate*>(curptr) : 0; 00950 } 00951 00952 00955 double toDouble() const; 00956 string toString() const; 00957 00958 00960 bool isMissing() const; 00961 void setMissing(); 00962 00963 00966 string name() const { 00967 return (schema && curfield < schema->size())? 00968 (*schema)[curfield].name : string(""); 00969 } 00970 00973 int precision() const { 00974 return (schema && curfield < schema->size())? 00975 (*schema)[curfield].precision : -1; 00976 } 00977 00980 int char_width() const; 00981 00983 unsigned char* raw() { 00984 return curptr; 00985 } 00986 00987 const unsigned char* raw() const { 00988 return curptr; 00989 } 00990 00995 FieldPtr makePtr() const { 00996 return schema->findColumn(curfield); 00997 } 00998 00999 private: 01000 unsigned curfield; 01001 unsigned char* curptr; 01002 const Schema* schema; 01003 }; 01004 01005 01006 //########################### CLASS FIELDROWREF ########################## 01013 class FieldRowRef 01014 { 01015 public: 01016 FieldRowRef(const RowIterator& it) 01017 : it_(it) {} 01018 01020 operator FieldValue() const; 01021 FieldRowRef& operator=(const FieldValue&); 01022 FieldRowRef& operator=(const FieldRowRef rhs) { 01023 return operator=(FieldValue(rhs)); 01024 } 01025 01027 inline RowIterator operator&() const; 01028 01029 private: 01030 RowIterator it_; 01031 01032 }; 01033 01034 01035 //############################### CLASS ROW ############################ 01042 class Row { 01043 01044 public: 01046 typedef RowIterator iterator; 01047 typedef iterator const_iterator; 01048 typedef long size_type; 01049 01050 public: 01055 Row() : rawrow(), schema(0) { } 01056 Row(const Row& r) : rawrow(r.rawrow), schema(r.schema) { } 01057 01059 Row(const Schema* s); 01060 01063 Row(const vector<unsigned char>& raw, const Schema* s) 01064 : rawrow(raw), schema(s) { } 01065 01066 01068 01069 01071 iterator begin() { 01072 return iterator(0, raw(), schema); 01073 } 01074 01075 iterator end() { 01076 if (schema) 01077 return iterator(schema->size(), raw()+rawrow.size(), 01078 schema); 01079 else 01080 return iterator(0,0,0); 01081 } 01082 01084 size_type size() const { 01085 return (size_type)rawrow.size(); 01086 } 01087 01088 size_type max_size() const { 01089 return (size_type)rawrow.size(); 01090 } 01091 01092 bool empty() const { 01093 return (schema && schema->empty()) || !schema; 01094 } 01095 01097 const unsigned char* raw() const { 01098 if (rawrow.size()) 01099 return &rawrow[0]; 01100 else 01101 return 0; 01102 } 01103 01104 unsigned char* raw() { 01105 if (rawrow.size()) 01106 return &rawrow[0]; 01107 else 01108 return 0; 01109 } 01110 01111 const Schema* getSchema() const { 01112 return schema; 01113 } 01114 01122 iterator operator[](int fieldNumber); 01123 iterator operator[](string fieldName); 01124 01130 iterator bind(const FieldPtr& p) const { 01131 if (!p) 01132 PLERROR("Trying to dereference a null FieldPtr"); 01133 return iterator(p.field_index_, 01134 const_cast<unsigned char*>(raw()) + p.offset_, 01135 schema); 01136 } 01137 01142 void sanitize() const; 01143 01144 private: 01145 vector<unsigned char> rawrow; 01146 const Schema* schema; 01147 }; 01148 01149 01150 //##### Row-related global functions #################################### 01151 01153 double todouble(const Row::iterator& it); 01154 string tostring(const Row::iterator& it); 01155 01158 ostream& operator<<(ostream& o, const Row::iterator& field); 01159 01161 ostream& operator<<(ostream&, const Row& row); 01162 01164 void printFieldName(ostream& o, const Row::iterator& field); 01165 01167 void printFieldNames(ostream& o, const Row& row); 01168 01169 01170 //##### Miscellaneous Declarations ######################################## 01171 01173 typedef SimpleDB<> SDB; 01174 01177 void randomShuffleRows(SDB& sdb); 01178 01181 void halfShuffleRows(SDB& sdb); 01182 01183 01184 01185 //##### Non-Template Inline Functions ##################################### 01186 01187 FieldRowRef RowIterator::operator*() const 01188 { 01189 return FieldRowRef(*this); 01190 } 01191 01192 RowIterator FieldRowRef::operator&() const 01193 { 01194 return it_; 01195 } 01196 01197 01198 01199 //##### Implementation of Templates ####################################### 01200 01201 template <class KT, class QR> 01202 QR SimpleDB<KT,QR>::EmptyResult; 01203 01204 01205 01206 //##### SimpleDB-related functions ######################################## 01207 01208 template <class KT, class QR> 01209 SimpleDB<KT,QR>::SimpleDB(string rootname, string the_path, 01210 AccessType the_access_type, bool the_verbose) 01211 : name(rootname), path(the_path), access_type(the_access_type), 01212 access_mask(0), schema(), row_size(), allfd(), 01213 table_size_multiplier(1.8), indexes(), verbose(the_verbose) 01214 { 01215 if (path != "") 01216 path += slash; 01217 01218 switch (access_type) { 01219 case readwrite: 01220 access_mask = O_RDWR; 01221 break; 01222 case readonly: 01223 access_mask = O_RDONLY; 01224 break; 01225 } 01226 01227 loadSchema(); 01228 openAllFiles(); 01229 computeSize(); 01230 } 01231 01232 template <class KT, class QR> 01233 SimpleDB<KT,QR>::~SimpleDB() 01234 { 01236 closeAllFiles(); 01237 saveSchema(); 01238 } 01239 01240 template <class KT, class QR> 01241 void SimpleDB<KT,QR>::setSchema(const Schema& s) 01242 { 01243 schema = s; 01244 Row row(&s); 01245 row_size = row.size(); 01246 indexes.resize(s.size()); 01247 01250 if (row_size > 0) 01251 max_records_file = RowNumber(AbsoluteFileLimit / row_size); 01252 else 01253 max_records_file = 0; 01254 01256 01258 closeAllFiles(); 01259 openAllFiles(); 01260 computeSize(); 01261 } 01262 01263 template <class KT, class QR> 01264 void SimpleDB<KT,QR>::saveSchema() 01265 { 01266 if (access_type==readwrite) { 01267 string fullpath = path + name + ".ssc"; 01268 ofstream sf(fullpath.c_str(), ios::out); 01269 Schema::iterator it = schema.begin(), end = schema.end(); 01270 for (; it != end; ++it) { 01271 sf << it->name << " "; 01272 switch (it->field_type) { 01273 case Unknown: 01274 break; 01275 case StringType: 01276 sf << "string " << it->precision << endl; 01277 break; 01278 01279 case CharacterType: 01280 sf << "character" << endl; 01281 break; 01282 01283 case SignedCharType: 01284 sf << "signedchar" << endl; 01285 break; 01286 01287 case ShortType: 01288 sf << "short" << endl; 01289 break; 01290 01291 case IntType: 01292 sf << "int" << endl; 01293 break; 01294 01295 case FloatType: 01296 sf << "float" << endl; 01297 break; 01298 01299 case DoubleType: 01300 sf << "double" << endl; 01301 break; 01302 01303 case DateType: 01304 sf << "date" << endl; 01305 break; 01306 01307 default: 01308 PLERROR("Unknown field type in database: %d", it->field_type); 01309 } 01310 } 01311 } 01312 } 01313 01314 template <class KT, class QR> 01315 void SimpleDB<KT,QR>::loadSchema() 01316 { 01319 string fullpath = path + name + ".ssc"; 01320 ifstream sf(fullpath.c_str()); 01321 Schema schema; 01322 while (sf) { 01323 string name,type; 01324 sf >> name >> type; 01325 if (name.size() == 0 || type.size() == 0) 01326 break; 01327 type = lowerstring(type); 01328 if (type == "string") { 01329 int length; 01330 sf >> length; 01331 schema.push_back(Field(name,StringType,length)); 01332 } 01333 else if (type == "character") 01334 schema.push_back(Field(name,CharacterType)); 01335 else if (type == "signedchar") 01336 schema.push_back(Field(name,SignedCharType)); 01337 else if (type == "short") 01338 schema.push_back(Field(name,ShortType)); 01339 else if (type == "int") 01340 schema.push_back(Field(name,IntType)); 01341 else if (type == "float") 01342 schema.push_back(Field(name,FloatType)); 01343 else if (type == "double") 01344 schema.push_back(Field(name,DoubleType)); 01345 else if (type == "date") 01346 schema.push_back(Field(name,DateType)); 01347 else { 01348 cerr << "Unexpected input type \"" << type 01349 << "\" in schema file " << fullpath << endl; 01350 exit(1); 01351 } 01352 } 01353 setSchema(schema); 01354 } 01355 01356 template <class KT, class QR> 01357 void SimpleDB<KT,QR>::addRow(const Row& row) 01358 { 01359 if(row_size != row.size()) 01360 PLERROR("In addRow row_size != row.size() (%d != %d)", row_size, row.size()); 01361 row.sanitize(); 01362 int fd = seekToEnd(); 01363 off_t curpos = lseek(fd, 0L, SEEK_CUR); 01364 01365 #ifdef LITTLEENDIAN 01366 01367 //ssize_t writtensize = ::write(fd, row.raw(), row_size); 01368 int writtensize = ::write(fd, row.raw(), row_size); 01369 #endif 01370 #ifdef BIGENDIAN 01371 Row newrow(row); 01372 memoryToDisk(newrow); 01373 int writtensize = ::write(fd, newrow.raw(), row_size); 01374 #endif 01375 01377 if (writtensize == -1) { 01380 #if defined(_MINGW_) || defined(WIN32) 01381 PLWARNING("could not truncate database file, end may be corrupted!"); 01382 #else 01383 ftruncate(fd, curpos); 01384 #endif 01385 PLERROR("Could not write to database: %s", strerror(errno)); 01386 } 01387 else 01388 size_++; 01389 } 01390 01391 template <class KT, class QR> 01392 void SimpleDB<KT,QR>::setRow(const Row& row, RowNumber n) 01393 { 01394 #ifdef __INTEL_COMPILER 01395 #pragma warning(disable:186) // Get rid of compiler warning. 01396 #endif 01397 if(n<0 || n>=size()) 01398 #ifdef __INTEL_COMPILER 01399 #pragma warning(default:186) 01400 #endif 01401 PLERROR("Out of bounds in SimpleDB::setRow"); 01402 if(row_size != row.size()) 01403 PLERROR("In setRow row_size != row.size() (%d != %d)", row_size, row.size()); 01404 row.sanitize(); 01405 int fd = seekToRow(n); 01406 01407 #ifdef LITTLEENDIAN 01408 int writtensize = ::write(fd, row.raw(), row_size); 01409 #endif 01410 #ifdef BIGENDIAN 01411 Row newrow(row); 01412 memoryToDisk(newrow); 01413 int writtensize = ::write(fd, newrow.raw(), row_size); 01414 #endif 01415 01417 if (writtensize == -1) 01418 PLERROR("Could not write to database: %s",strerror(errno)); 01419 } 01420 01421 01422 template <class KT, class QR> 01423 void SimpleDB<KT,QR>::truncateFromRow(RowNumber n) 01424 { 01430 int curfd = seekToRow(n); 01431 off_t curpos = lseek(curfd, 0L, SEEK_CUR); 01432 if (ftruncate(curfd, curpos) == -1) { 01433 PLERROR((string("Could not truncate database at row ") + 01434 tostring(n) + ": " + strerror(errno)).c_str()); 01435 } 01436 01437 vector<int>::iterator found = find(allfd.begin(), allfd.end(), curfd); 01438 int fromfd = found-allfd.begin() + 1; 01439 int last = lastSegment(); 01440 01441 closeAllFiles(); 01442 bool allok = true; 01443 01444 for ( ; fromfd <= last; ++fromfd) { 01445 string path = getSegmentPath(fromfd); 01446 if(unlink(path.c_str()) == -1) { 01447 PLWARNING((string("Could not unlink database segment ") + path + 01448 ": " + strerror(errno)).c_str()); 01449 allok = false; 01450 } 01451 } 01452 01453 if (allok) { 01454 openAllFiles(); 01455 computeSize(); 01456 } 01457 else 01458 PLERROR("Error during truncation"); 01459 } 01460 01461 01462 template <class KT, class QR> 01463 Row& SimpleDB<KT,QR>::getInRow(RowNumber n, Row& row) const 01464 { 01465 #ifdef __INTEL_COMPILER 01466 #pragma warning(disable:186) // Get rid of compiler warning. 01467 #endif 01468 if(n<0 || n>=size()) 01469 #ifdef __INTEL_COMPILER 01470 #pragma warning(default:186) 01471 #endif 01472 PLERROR("Out of Bounds in SimpleDB::getInRow"); 01473 if(row_size != row.size()) 01474 PLERROR("In getInRow row_size!=row_size()"); 01475 int fd = seekToRow(n); 01476 01477 int size_read = ::read(fd, row.raw(), row_size); 01478 if (size_read == -1) 01479 PLERROR("Could not read from database: %s",strerror(errno)); 01480 diskToMemory(row); 01481 return row; 01482 } 01483 01484 template <class KT, class QR> 01485 Row SimpleDB<KT,QR>::getRow(RowNumber n) const 01486 { 01487 Row row(&schema); 01488 getInRow(n, row); 01489 return row; 01490 } 01491 01492 template <class KT, class QR> 01493 void SimpleDB<KT,QR>::computeSize() 01494 { 01495 if(row_size<=0) 01496 size_ = 0; 01497 else 01498 { 01499 size_ = 0; 01500 int i=0; 01501 int bytesinfile = file_size(getSegmentPath(i++)); 01502 while(bytesinfile>0) 01503 { 01504 size_ += bytesinfile/row_size; 01505 bytesinfile = file_size(getSegmentPath(i++)); 01506 } 01507 } 01508 01511 01524 } 01525 01526 template <class KT, class QR> 01527 void SimpleDB<KT,QR>::memoryToDisk(Row& row) const 01528 { 01529 #ifdef LITTLEENDIAN 01530 01531 #endif 01532 #ifdef BIGENDIAN 01533 Row newr(row); 01534 Row::iterator it = newr.begin(), end = newr.end(); 01535 for(; it != end; ++it) { 01537 if (short* x = it.asShort()) 01538 reverse_short(x,1); 01539 if (int* x = it.asInt()) 01540 reverse_int(x,1); 01541 if (float* x = it.asFloat()) 01542 reverse_float(x,1); 01543 if (double* x = it.asDouble()) 01544 reverse_double(x,1); 01545 if (PDate* x = it.asDate()) { 01546 reverse_short(&(x->year),1); 01547 } 01548 } 01549 #endif 01550 } 01551 01552 template <class KT, class QR> 01553 void SimpleDB<KT,QR>::diskToMemory(Row& row) const 01554 { 01555 memoryToDisk(row); 01556 } 01557 01558 01559 //##### Physical Splitting Among Multiple Files ######################### 01560 01561 template <class KT, class QR> 01562 int SimpleDB<KT,QR>::seekToRow(RowNumber i) const 01563 { 01565 01566 if (max_records_file == 0) 01567 PLERROR("Attempting to seekToRow without schema set"); 01568 01569 int segmentNumber = int(i / max_records_file); 01570 Offset rowInSegment = Offset(i % max_records_file); 01571 01573 if (segmentNumber > lastSegment()) { 01574 for (int i = lastSegment()+1; i <= segmentNumber; ++i) { 01575 int fd = open(getSegmentPath(i).c_str(), 01576 access_mask | O_CREAT, 0777); 01577 if (fd == -1) 01578 PLERROR("Could not open database segment %d at path %s: %s", 01579 i, getSegmentPath(i).c_str(), strerror(errno)); 01580 allfd.push_back(fd); 01581 } 01582 } 01583 if (allfd[segmentNumber] == -1) 01584 PLERROR("Problem accessing database segment %d at path %s", 01585 segmentNumber, getSegmentPath(segmentNumber).c_str()); 01586 01587 if (lseek(allfd[segmentNumber], 01588 rowInSegment * Offset(row_size), SEEK_SET)<0) 01589 PLERROR("problem in lseek: %s",strerror(errno)); 01590 return allfd[segmentNumber]; 01591 } 01592 01593 01594 template <class KT, class QR> 01595 int SimpleDB<KT,QR>::seekToEnd() const 01596 { 01598 01599 if (max_records_file == 0) 01600 PLERROR("Attempting to seekToEnd without schema set"); 01601 01602 int last = lastSegment(); 01603 int fd = allfd[last]; 01604 if (fd == -1) 01605 PLERROR("Problem accessing database segment %d at path %s", 01606 last, getSegmentPath(last).c_str()); 01607 01608 off_t pos = lseek(fd, 0ul, SEEK_END); 01609 01612 if (Offset(pos) / Offset(row_size) >= max_records_file) 01613 fd = seekToRow((last+1)*max_records_file); 01614 01615 return fd; 01616 } 01617 01618 01619 template <class KT, class QR> 01620 void SimpleDB<KT,QR>::openAllFiles() const 01621 { 01627 closeAllFiles(); 01628 01629 int fd; 01630 fd = open(getSegmentPath(0).c_str(), 01631 access_mask | O_CREAT, 0777); 01632 if (fd == -1) 01633 PLERROR("Could not open main database segment %s: %s", 01634 getSegmentPath(0).c_str(), strerror(errno)); 01635 allfd.push_back(fd); 01636 01637 int index = 1; 01638 for ( ; ; ++index ) { 01639 fd = open(getSegmentPath(index).c_str(), 01640 access_mask); 01641 if (fd == -1) 01642 break; 01643 else 01644 allfd.push_back(fd); 01645 } 01646 } 01647 01648 01649 template <class KT, class QR> 01650 void SimpleDB<KT,QR>::closeAllFiles() const 01651 { 01652 vector<int>::iterator it=allfd.begin(), end=allfd.end(); 01653 for (; it != end; ++it) 01654 if (*it != -1) { 01655 close(*it); 01656 } 01657 allfd.clear(); 01658 } 01659 01660 01661 template <class KT, class QR> 01662 inline int SimpleDB<KT,QR>::lastSegment() const 01663 { 01664 return (int)allfd.size() - 1; 01665 } 01666 01667 01668 template <class KT, class QR> 01669 string SimpleDB<KT,QR>::getSegmentPath(int i) const 01670 { 01671 string fullpath = path + name; 01672 if (i >= 1 && i <= 26) { 01673 string postfix(1, char('a'+i-1)); 01674 fullpath += string("_") + postfix; 01675 } 01676 else if (i > 26) 01677 PLERROR("Too many segments in the database."); 01678 if(fullpath.find(".sdb")==string::npos) 01679 fullpath += ".sdb"; 01680 return fullpath; 01681 } 01682 01683 01684 //##### Indexing- and Query-Related Functions ########################### 01685 01686 template <class KT, class QR> 01687 bool SimpleDB<KT,QR>::indexColumn(string column_name, 01688 string second_column) 01689 { 01690 bool has_second_column = (second_column.size() > 0); 01691 int n, start_pos, column_precision =0, 01692 n2, start_pos2, column_precision2=0; 01693 if (!findColumn(column_name, n, start_pos, column_precision)) 01694 return false; 01695 if (has_second_column && 01696 !findColumn(second_column, n2, start_pos2, column_precision2)) 01697 return false; 01698 01703 RowNumber maxrows = size(); 01704 RowNumber tablesize = RowNumber(table_size_multiplier*maxrows); 01705 if (maxrows <= 0 || tablesize <= 0) { 01706 PLWARNING("SimpleDB::indexColumn: cannot index a database of " 01707 "zero size."); 01708 return false; 01709 } 01710 if (!indexes[n]) { 01711 indexes[n] = new Index(tablesize, true); 01712 indexes[n]->initializeTable((unsigned int)tablesize); 01713 } 01714 Index& index = *indexes[n]; 01715 index.resize(tablesize); 01716 index.flush(); 01717 01718 Row currow(&schema); 01719 IndexKey key(column_precision + column_precision2); 01720 typename IndexKey::iterator keybegin = key.begin(); 01721 unsigned char* begin1 = currow.raw() + start_pos; 01722 unsigned char* end1 = begin1 + column_precision; 01723 unsigned char* begin2 = currow.raw() + start_pos2; 01724 unsigned char* end2 = begin2 + column_precision2; 01725 01726 for(RowNumber i=0; i<maxrows; ++i) { 01727 if (verbose && i % 1000000 == 0) { 01728 unsigned numclusters, maxcluster; 01729 index.diagnostics(numclusters, maxcluster); 01730 cerr << "indexing row " << i 01731 << "\t num. clusters=" << numclusters 01732 << "\t max. cluster size=" << maxcluster 01733 << endl; 01734 } 01735 01736 getInRow(i,currow); 01737 01743 copy(begin1, end1, keybegin); 01744 if (has_second_column) 01745 copy(begin2, end2, keybegin+column_precision-1); 01746 unsigned int addr = index.hashAddress(key); 01747 01748 if (addr == Hash_UNUSED_TAG) { 01754 bool needresize = !index.add(key,QueryResult_t()); 01755 if (needresize) { 01756 cerr << "Hash table unexpectedly full; exiting..." << endl; 01757 exit(1); 01758 } 01759 addr = index.hashAddress(key); 01760 } 01761 01763 QueryResult_t* qr = index[addr]; 01764 try { 01765 qr->push_back(i); 01766 } 01767 catch (logic_error& e) { 01768 cerr << "Exception caught during indexing: " 01769 << typeid(e).name() << endl 01770 << "Containing: " << e.what() << endl; 01771 throw; 01772 } 01773 } 01774 return true; 01775 } 01776 01777 template <class KT, class QR> 01778 void SimpleDB<KT,QR>::clearIndex(string column_name) 01779 { 01780 int n, start_pos, column_precision=0; 01781 if (findColumn(column_name, n, start_pos, column_precision)) 01782 indexes[n] = 0; 01783 } 01784 01785 template <class KT, class QR> 01786 QR SimpleDB<KT,QR>::findEqual(const unsigned char* lookfor, 01787 string column_name, string second_column) 01788 { 01789 int n, start_pos, column_precision; 01790 if (!findColumn(column_name, n, start_pos, column_precision)) 01791 return EmptyResult; 01792 if (indexes[n]) 01793 return findEqualIndexed(lookfor, column_name, second_column); 01794 else 01795 return findEqualLinear(lookfor, column_name, second_column); 01796 } 01797 01798 01799 template <class KT, class QR> 01800 const QR& SimpleDB<KT,QR>::findEqualIndexed(const unsigned char* lookfor, 01801 string column_name, 01802 string second_column) 01803 { 01804 bool has_second_column = (second_column.size() > 0); 01805 int n, start_pos, column_precision =0, 01806 n2, start_pos2, column_precision2=0; 01807 if (!findColumn(column_name, n, start_pos, column_precision)) 01808 return EmptyResult; 01809 if (has_second_column && 01810 !findColumn(second_column, n2, start_pos2, column_precision2)) 01811 return EmptyResult; 01812 01813 if (!indexes[n]) 01814 PLERROR("SimpleDB::indexColumn must be done before performing " 01815 "indexed searches on column %s", column_name.c_str()); 01816 01817 Index& index = *indexes[n]; 01818 IndexKey key(lookfor, column_precision+column_precision2); 01819 unsigned int addr = index.hashAddress(key); 01820 if (addr == Hash_UNUSED_TAG) 01821 return EmptyResult; 01822 else 01823 return *index[addr]; 01824 } 01825 01826 01827 template <class KT, class QR> 01828 QR SimpleDB<KT,QR>::findEqualLinear(const unsigned char* lookfor, 01829 string column_name, 01830 string second_column) 01831 { 01832 vuc lf(1); 01833 lf[0] = lookfor; 01834 return findEqualLinear(lf, column_name, second_column); 01835 } 01836 01837 01838 template <class KT, class QR> 01839 QR SimpleDB<KT,QR>::findEqualLinear( 01840 const vuc& lookfor, 01841 string column_name, string second_column) 01842 { 01843 bool has_second_column = (second_column.size() > 0); 01844 int n, start_pos, column_precision =0, 01845 n2, start_pos2, column_precision2=0; 01846 if (!findColumn(column_name, n, start_pos, column_precision)) 01847 return EmptyResult; 01848 if (has_second_column && 01849 !findColumn(second_column, n2, start_pos2, column_precision2)) 01850 return EmptyResult; 01851 01852 QR qr; 01853 01855 vector<IndexKey> key_lookfor(lookfor.size()); 01856 vuc::const_iterator 01857 lookit, lookbeg = lookfor.begin(), lookend = lookfor.end(); 01858 typename vector<IndexKey>::iterator 01859 keyit, keybeg = key_lookfor.begin(), keyend = key_lookfor.end(); 01860 size_t len = column_precision+column_precision2; 01861 01862 for (lookit=lookbeg, keyit=keybeg ; lookit != lookend ; 01863 ++lookit, ++keyit) { 01864 keyit->resize(len); 01865 copy(*lookit, *lookit+len, keyit->begin()); 01866 } 01867 01868 IndexKey key_dbrow(column_precision+column_precision2); 01869 typename IndexKey::iterator keybegin = key_dbrow.begin(); 01870 01871 RowNumber maxrows = size(); 01872 Row currow(&schema); 01873 unsigned char* begin1 = currow.raw() + start_pos; 01874 unsigned char* end1 = begin1 + column_precision; 01875 unsigned char* begin2 = currow.raw() + start_pos2; 01876 unsigned char* end2 = begin2 + column_precision2; 01877 01879 for (RowNumber i=0; i<maxrows; ++i) { 01880 if (verbose && i % 1000000 == 0) { 01881 cerr << "Searching row " << i << endl; 01882 } 01883 01884 getInRow(i, currow); 01885 01886 copy(begin1, end1, keybegin); 01887 if (has_second_column) 01890 copy(begin2, end2, keybegin+column_precision-1); 01891 01893 for (keyit = keybeg ; keyit != keyend ; ++keyit) 01894 if (*keyit == key_dbrow) { 01895 qr.push_back(i); 01896 if (verbose) 01897 cerr << "Found string \"" << *keyit 01898 << "\" at row " << i << endl; 01899 } 01900 } 01901 01902 return qr; 01903 } 01904 01905 #ifdef WIN32 01906 #undef open 01907 #undef close 01908 #undef lseek 01909 #undef read 01910 #undef write 01911 #undef unlink 01912 #endif 01913 01914 } // end of namespace PLearn 01915 01916 #endif 01917 01918 01919 /* 01920 Local Variables: 01921 mode:c++ 01922 c-basic-offset:4 01923 c-file-style:"stroustrup" 01924 c-file-offsets:((innamespace . 0)(inline-open . 0)) 01925 indent-tabs-mode:nil 01926 fill-column:79 01927 End: 01928 */ 01929 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :