PLearn 0.1
|
#include <Storage.h>
Inherits PLearn::PPointable.
Public Types | |
typedef T | value_type |
typedef int | size_type |
typedef T * | iterator |
typedef const T * | const_iterator |
Public Member Functions | |
Storage (const Storage &other) | |
Storage (long the_length, T *dataptr) | |
int | length () const |
int | size () const |
iterator | begin () const |
iterator | end () const |
void | mem_alloc (int len) |
Storage (long the_length=0) | |
data is initially filled with zeros | |
Storage (const char *filename, bool readonly) | |
void | pointTo (int the_length, T *dataptr) |
~Storage () | |
void | resize (long lnewlength) |
void | resizeMat (int new_length, int new_width, int extrarows, int extracols, int new_offset, int old_mod, int old_length, int old_width, int old_offset) |
Storage< T > * | deepCopy (CopiesMap &copies) const |
Deep copying. | |
T & | operator[] (int idx) const |
void | push_back (const T &x) |
Public Attributes | |
int | length_ |
T * | data |
bool | dont_delete_data |
if true, the destructor won't delete[] data, because it will assume it is somebody else's responsibility | |
tFileHandle | fd |
The descriptor for the memory-mapped file (-1 if there is no memory mapping) |
typedef const T* PLearn::Storage< T >::const_iterator |
typedef T* PLearn::Storage< T >::iterator |
typedef int PLearn::Storage< T >::size_type |
typedef T PLearn::Storage< T >::value_type |
PLearn::Storage< T >::Storage | ( | const Storage< T > & | other | ) | [inline] |
Definition at line 84 of file Storage.h.
:length_(other.length()), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) { try { data = new T[length()]; if(!data) PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length()); //memcpy(data,other.data,length()*sizeof(T)); copy(other.data, other.data+length(), data); } catch(...) { PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length()); } }
PLearn::Storage< T >::Storage | ( | long | the_length, |
T * | dataptr | ||
) | [inline] |
Definition at line 101 of file Storage.h.
:length_(int(the_length)), data(dataptr), dont_delete_data(true), fd(STORAGE_UNUSED_HANDLE) { //we do the check outside a BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(the_length>std::numeric_limits<int>::max()) PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " "is possible (limited to 2e31, int)",the_length); }
PLearn::Storage< T >::Storage | ( | long | the_length = 0 | ) | [inline] |
data is initially filled with zeros
Definition at line 138 of file Storage.h.
:length_((int)the_length), data(0), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) { //we do the check outside the BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(the_length>std::numeric_limits<int>::max()) PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " "is possible (limited to 2e31, int)",the_length); int l = length(); #ifdef BOUNDCHECK if(l<0) PLERROR("new Storage called with a length() < 0: length = %d", l); #endif if (l>0) { mem_alloc(l); } }
PLearn::Storage< T >::Storage | ( | const char * | filename, |
bool | readonly | ||
) | [inline] |
Constructor for memory-mapped file The file is supposed to exist and have the correct size length() of the storage will be set to the size of the file divided by sizeof(T)
< read-write
Definition at line 164 of file Storage.h.
{ void* addr; off_t filesize; if(readonly) { #if !(!defined(_MSC_VER) && !defined(_MINGW_)) PLERROR("Not implemented for MinGW"); #else addr = (T*)MemoryMap(filename, fd, true, filesize); #endif } else { #if !(!defined(_MSC_VER) && !defined(_MINGW_)) PLERROR("Not implemtned for MinGW"); #else addr = (T*)MemoryMap(filename, fd, false, filesize); #endif } if(addr==0) { perror("Error when calling mmap: "); PLERROR("In Storage: Memory-mapping failed"); } data = (T*) addr; length_ = filesize/sizeof(T); dont_delete_data = false; }
PLearn::Storage< T >::~Storage | ( | ) | [inline] |
Definition at line 220 of file Storage.h.
{ if (data && !dont_delete_data) { if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file { #if !(!defined(_MSC_VER) && !defined(_MINGW_)) PLERROR("Not implemented for MinGW"); #else memoryUnmap((void *)data,fd,length()*sizeof(T)); #endif } else delete[] data; } }
iterator PLearn::Storage< T >::begin | ( | ) | const [inline] |
Storage<T>* PLearn::Storage< T >::deepCopy | ( | CopiesMap & | copies | ) | const [inline] |
Deep copying.
< a copy already exists, so return it
Otherwise call the copy constructor to obtain a copy
Put the copy in the map
return the completed deep_copy
iterator PLearn::Storage< T >::end | ( | ) | const [inline] |
int PLearn::Storage< T >::length | ( | ) | const [inline] |
void PLearn::Storage< T >::mem_alloc | ( | int | len | ) | [inline] |
T& PLearn::Storage< T >::operator[] | ( | int | idx | ) | const [inline] |
void PLearn::Storage< T >::pointTo | ( | int | the_length, |
T * | dataptr | ||
) | [inline] |
< allocated elsewhere
Definition at line 198 of file Storage.h.
{ if (data && !dont_delete_data) { if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file { #if !(!defined(_MSC_VER) && !defined(_MINGW_)) PLERROR("Not implemented for MinGW"); #else memoryUnmap((void *)data,fd,length()*sizeof(T)); #endif } else delete[] data; } length_=the_length; data=dataptr; fd=STORAGE_UNUSED_HANDLE; dont_delete_data=true; }
void PLearn::Storage< T >::push_back | ( | const T & | x | ) | [inline] |
void PLearn::Storage< T >::resize | ( | long | lnewlength | ) | [inline] |
Grow or shrink data memory If newlength==length() this call does nothing If newlength<=0 it outputs an PLERROR(i.e. cannot shrink memory to 0) Otherwise this call ALWAYS: -> allocates a new block of exactly the given size -> copies all the possible the data of the old block to the new one -> fills the remaining of the new block (if any) with 0.0 -> frees the old block It is the job of the CALLER (Mat and Vec) to have an appropriate policy to minimize the number of calls to Storage::resize
< we are using a memory-mapped file
< growing
< newlength<length() (shrinking)
Definition at line 249 of file Storage.h.
Referenced by PLearn::CompactVMatrix::append(), PLearn::CompactVMatrix::CompactVMatrix(), and PLearn::TVec< PP< RegressionTreeNode > >::resize().
{ //we do the check outside a BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(lnewlength>std::numeric_limits<int>::max() || lnewlength<std::numeric_limits<int>::min()) PLERROR("In Storage(%ld) - we ask to create a bigger/smaller" " Storage than is possible with an int", lnewlength); #ifdef BOUNDCHECK if(lnewlength<0) PLERROR("Storage::resize(%ld) called with a length() <0", lnewlength); #endif int newlength=(int)lnewlength; if (newlength==length()) return; #if defined(_MINGW_) || defined(WIN32) else if(fd>0 || dont_delete_data) #else else if(fd>=0 || dont_delete_data) #endif PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere"); else if (newlength==0) { if (data) delete[] data; data = 0; length_ = 0; } else if (newlength > length()) { #ifdef DEBUG_PLEARN_STORAGE_RESIZE int mem_before = getProcessDataMemory(); int length_before = length(); #endif try { T* newdata = new T[newlength]; if(!newdata) PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resize, trying to allocate %d elements",newlength); if(data) { // memcpy(newdata,data,length()*sizeof(T)); copy(data,data+length(),newdata); delete[] data; } // memset(&newdata[length()],0,(newlength-length())*sizeof(T)); clear_n(newdata+length(),newlength-length()); length_ = newlength; data = newdata; } catch(...) { PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newlength); } #ifdef DEBUG_PLEARN_STORAGE_RESIZE int mem_after = getProcessDataMemory(); if (mem_after - mem_before > 256*1024) cerr << "Storage::resize: for storage at " << hex << this << dec << " fromsize=" << length_before << " tosize=" << newlength << " : memusage " << (mem_before/1024) << " kB ==> " << (mem_after/1024) << " kB" << endl; if (mem_after - mem_before > 10000*1024) PLWARNING("Storage::resize: memory usage increased by more than 10000 kB"); #endif } else { try { T* newdata = new T[newlength]; if(!newdata) PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length()); if(data) { //memcpy(newdata,data,newlength*sizeof(T)); copy(data,data+newlength,newdata); delete[] data; } length_ = newlength; data = newdata; } catch(...) { PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length()); } } }
void PLearn::Storage< T >::resizeMat | ( | int | new_length, |
int | new_width, | ||
int | extrarows, | ||
int | extracols, | ||
int | new_offset, | ||
int | old_mod, | ||
int | old_length, | ||
int | old_width, | ||
int | old_offset | ||
) | [inline] |
< we are using a memory-mapped file
Definition at line 343 of file Storage.h.
{ long ls = new_length*new_width; long lextrabytes = (new_length+extrarows)*(new_width+extracols) - ls; long lnewsize = new_offset+ls+lextrabytes; //we do the check outside a BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(lnewsize>std::numeric_limits<int>::max()) PLERROR("In Storage.resizeMat - we ask to create a bigger Storage " " %ld then is possible (limited to 2e31, int)",lnewsize); int newsize=(int)lnewsize; int new_mod = new_width+extracols; #ifdef BOUNDCHECK if(newsize<0) PLERROR("Storage::resize called with a length() <0"); #endif if (newsize==length()) return; #if defined(_MINGW_) || defined(WIN32) else if(fd>0 || dont_delete_data) #else else if(fd>=0 || dont_delete_data) #endif PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere"); else if (newsize==0) { if (data) delete[] data; data = 0; length_ = 0; } else { #ifdef DEBUG_PLEARN_STORAGE_RESIZE int mem_before = getProcessDataMemory(); int length_before = length(); #endif try { T* newdata = new T[newsize]; if(!newdata) PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resizeMat, trying to allocate %d elements",newsize); if(data) { // perform a 'structured' copy that keeps all the old values T* oldp = data+old_offset; T* newp = newdata+new_offset; int w = min(old_width,new_width); int l = min(old_length,new_length); if (new_offset!=0) { if (new_offset!=old_offset) PLERROR("Storage::resizeMat: when new_offset!=0 it should equal old_offset"); copy(data,data+new_offset,newdata); } for (int row=0;row<l;row++, oldp+=old_mod, newp+=new_mod) { copy(oldp,oldp+w,newp); if(new_width>old_width) clear_n(newp+old_width,new_width+extracols-old_width); } if (new_length>old_length) clear_n(newp,(new_length+extrarows-old_length)*new_mod); delete[] data; } length_ = newsize; data = newdata; } catch(...) { PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newsize); } #ifdef DEBUG_PLEARN_STORAGE_RESIZE int mem_after = getProcessDataMemory(); if (mem_after - mem_before > 256*1024) cerr << "Storage::resize: for storage at " << hex << this << dec << " fromsize=" << length_before << " tosize=" << newsize << " : memusage " << (mem_before/1024) << " kB ==> " << (mem_after/1024) << " kB" << endl; if (mem_after - mem_before > 10000*1024) PLWARNING("Storage::resize: memory usage increased by more than 10000 kB"); #endif } }
int PLearn::Storage< T >::size | ( | ) | const [inline] |
T* PLearn::Storage< T >::data |
Definition at line 80 of file Storage.h.
Referenced by PLearn::CompactVMatrix::append(), PLearn::Storage< PP< RegressionTreeNode > >::deepCopy(), PLearn::CompactVMatrix::dot(), PLearn::CompactVMatrix::encodeAndPutRow(), PLearn::CompactVMatrix::getNewRow(), PLearn::CompactVMatrix::perturb(), PLearn::CompactVMatrix::putSubRow(), and PLearn::Storage< PP< RegressionTreeNode > >::Storage().
bool PLearn::Storage< T >::dont_delete_data |
tFileHandle PLearn::Storage< T >::fd |
int PLearn::Storage< T >::length_ |