|
PLearn 0.1
|
#include <Storage.h>
Inherits PLearn::PPointable.

Public Types | |
| typedef T | value_type |
| typedef int | size_type |
| typedef T * | iterator |
| typedef const T * | const_iterator |
Public Member Functions | |
| Storage (const Storage &other) | |
| Storage (long the_length, T *dataptr) | |
| int | length () const |
| int | size () const |
| iterator | begin () const |
| iterator | end () const |
| void | mem_alloc (int len) |
| Storage (long the_length=0) | |
| data is initially filled with zeros | |
| Storage (const char *filename, bool readonly) | |
| void | pointTo (int the_length, T *dataptr) |
| ~Storage () | |
| void | resize (long lnewlength) |
| void | resizeMat (int new_length, int new_width, int extrarows, int extracols, int new_offset, int old_mod, int old_length, int old_width, int old_offset) |
| Storage< T > * | deepCopy (CopiesMap &copies) const |
| Deep copying. | |
| T & | operator[] (int idx) const |
| void | push_back (const T &x) |
Public Attributes | |
| int | length_ |
| T * | data |
| bool | dont_delete_data |
| if true, the destructor won't delete[] data, because it will assume it is somebody else's responsibility | |
| tFileHandle | fd |
| The descriptor for the memory-mapped file (-1 if there is no memory mapping) | |
| typedef const T* PLearn::Storage< T >::const_iterator |
| typedef T* PLearn::Storage< T >::iterator |
| typedef int PLearn::Storage< T >::size_type |
| typedef T PLearn::Storage< T >::value_type |
| PLearn::Storage< T >::Storage | ( | const Storage< T > & | other | ) | [inline] |
Definition at line 84 of file Storage.h.
:length_(other.length()), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) { try { data = new T[length()]; if(!data) PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length()); //memcpy(data,other.data,length()*sizeof(T)); copy(other.data, other.data+length(), data); } catch(...) { PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length()); } }
| PLearn::Storage< T >::Storage | ( | long | the_length, |
| T * | dataptr | ||
| ) | [inline] |
Definition at line 101 of file Storage.h.
:length_(int(the_length)), data(dataptr), dont_delete_data(true), fd(STORAGE_UNUSED_HANDLE) { //we do the check outside a BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(the_length>std::numeric_limits<int>::max()) PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " "is possible (limited to 2e31, int)",the_length); }
| PLearn::Storage< T >::Storage | ( | long | the_length = 0 | ) | [inline] |
data is initially filled with zeros
Definition at line 138 of file Storage.h.
:length_((int)the_length), data(0), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) { //we do the check outside the BOUNDCHECK as we normaly do our test with //small dataset. Also, this is not a performance bottleneck and is a //fraction of the time of the malloc. if(the_length>std::numeric_limits<int>::max()) PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " "is possible (limited to 2e31, int)",the_length); int l = length(); #ifdef BOUNDCHECK if(l<0) PLERROR("new Storage called with a length() < 0: length = %d", l); #endif if (l>0) { mem_alloc(l); } }
| PLearn::Storage< T >::Storage | ( | const char * | filename, |
| bool | readonly | ||
| ) | [inline] |
Constructor for memory-mapped file The file is supposed to exist and have the correct size length() of the storage will be set to the size of the file divided by sizeof(T)
< read-write
Definition at line 164 of file Storage.h.
{
void* addr;
off_t filesize;
if(readonly)
{
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
PLERROR("Not implemented for MinGW");
#else
addr = (T*)MemoryMap(filename, fd, true, filesize);
#endif
}
else
{
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
PLERROR("Not implemtned for MinGW");
#else
addr = (T*)MemoryMap(filename, fd, false, filesize);
#endif
}
if(addr==0)
{
perror("Error when calling mmap: ");
PLERROR("In Storage: Memory-mapping failed");
}
data = (T*) addr;
length_ = filesize/sizeof(T);
dont_delete_data = false;
}
| PLearn::Storage< T >::~Storage | ( | ) | [inline] |
Definition at line 220 of file Storage.h.
{
if (data && !dont_delete_data)
{
if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file
{
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
PLERROR("Not implemented for MinGW");
#else
memoryUnmap((void *)data,fd,length()*sizeof(T));
#endif
}
else
delete[] data;
}
}
| iterator PLearn::Storage< T >::begin | ( | ) | const [inline] |
| Storage<T>* PLearn::Storage< T >::deepCopy | ( | CopiesMap & | copies | ) | const [inline] |
Deep copying.
< a copy already exists, so return it
Otherwise call the copy constructor to obtain a copy
Put the copy in the map
return the completed deep_copy
| iterator PLearn::Storage< T >::end | ( | ) | const [inline] |
| int PLearn::Storage< T >::length | ( | ) | const [inline] |
| void PLearn::Storage< T >::mem_alloc | ( | int | len | ) | [inline] |
| T& PLearn::Storage< T >::operator[] | ( | int | idx | ) | const [inline] |
| void PLearn::Storage< T >::pointTo | ( | int | the_length, |
| T * | dataptr | ||
| ) | [inline] |
< allocated elsewhere
Definition at line 198 of file Storage.h.
{
if (data && !dont_delete_data)
{
if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file
{
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
PLERROR("Not implemented for MinGW");
#else
memoryUnmap((void *)data,fd,length()*sizeof(T));
#endif
}
else
delete[] data;
}
length_=the_length;
data=dataptr;
fd=STORAGE_UNUSED_HANDLE;
dont_delete_data=true;
}
| void PLearn::Storage< T >::push_back | ( | const T & | x | ) | [inline] |
| void PLearn::Storage< T >::resize | ( | long | lnewlength | ) | [inline] |
Grow or shrink data memory If newlength==length() this call does nothing If newlength<=0 it outputs an PLERROR(i.e. cannot shrink memory to 0) Otherwise this call ALWAYS: -> allocates a new block of exactly the given size -> copies all the possible the data of the old block to the new one -> fills the remaining of the new block (if any) with 0.0 -> frees the old block It is the job of the CALLER (Mat and Vec) to have an appropriate policy to minimize the number of calls to Storage::resize
< we are using a memory-mapped file
< growing
< newlength<length() (shrinking)
Definition at line 249 of file Storage.h.
Referenced by PLearn::CompactVMatrix::append(), PLearn::CompactVMatrix::CompactVMatrix(), and PLearn::TVec< PP< RegressionTreeNode > >::resize().
{
//we do the check outside a BOUNDCHECK as we normaly do our test with
//small dataset. Also, this is not a performance bottleneck and is a
//fraction of the time of the malloc.
if(lnewlength>std::numeric_limits<int>::max() || lnewlength<std::numeric_limits<int>::min())
PLERROR("In Storage(%ld) - we ask to create a bigger/smaller"
" Storage than is possible with an int",
lnewlength);
#ifdef BOUNDCHECK
if(lnewlength<0)
PLERROR("Storage::resize(%ld) called with a length() <0",
lnewlength);
#endif
int newlength=(int)lnewlength;
if (newlength==length())
return;
#if defined(_MINGW_) || defined(WIN32)
else if(fd>0 || dont_delete_data)
#else
else if(fd>=0 || dont_delete_data)
#endif
PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
else if (newlength==0)
{
if (data) delete[] data;
data = 0;
length_ = 0;
}
else if (newlength > length())
{
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
int mem_before = getProcessDataMemory();
int length_before = length();
#endif
try
{
T* newdata = new T[newlength];
if(!newdata)
PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resize, trying to allocate %d elements",newlength);
if(data)
{
// memcpy(newdata,data,length()*sizeof(T));
copy(data,data+length(),newdata);
delete[] data;
}
// memset(&newdata[length()],0,(newlength-length())*sizeof(T));
clear_n(newdata+length(),newlength-length());
length_ = newlength;
data = newdata;
}
catch(...)
{
PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newlength);
}
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
int mem_after = getProcessDataMemory();
if (mem_after - mem_before > 256*1024)
cerr << "Storage::resize: for storage at "
<< hex << this << dec
<< " fromsize=" << length_before << " tosize=" << newlength
<< " : memusage " << (mem_before/1024) << " kB ==> "
<< (mem_after/1024) << " kB" << endl;
if (mem_after - mem_before > 10000*1024)
PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
#endif
}
else
{
try
{
T* newdata = new T[newlength];
if(!newdata)
PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length());
if(data)
{
//memcpy(newdata,data,newlength*sizeof(T));
copy(data,data+newlength,newdata);
delete[] data;
}
length_ = newlength;
data = newdata;
}
catch(...)
{
PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length());
}
}
}

| void PLearn::Storage< T >::resizeMat | ( | int | new_length, |
| int | new_width, | ||
| int | extrarows, | ||
| int | extracols, | ||
| int | new_offset, | ||
| int | old_mod, | ||
| int | old_length, | ||
| int | old_width, | ||
| int | old_offset | ||
| ) | [inline] |
< we are using a memory-mapped file
Definition at line 343 of file Storage.h.
{
long ls = new_length*new_width;
long lextrabytes = (new_length+extrarows)*(new_width+extracols) - ls;
long lnewsize = new_offset+ls+lextrabytes;
//we do the check outside a BOUNDCHECK as we normaly do our test with
//small dataset. Also, this is not a performance bottleneck and is a
//fraction of the time of the malloc.
if(lnewsize>std::numeric_limits<int>::max())
PLERROR("In Storage.resizeMat - we ask to create a bigger Storage "
" %ld then is possible (limited to 2e31, int)",lnewsize);
int newsize=(int)lnewsize;
int new_mod = new_width+extracols;
#ifdef BOUNDCHECK
if(newsize<0)
PLERROR("Storage::resize called with a length() <0");
#endif
if (newsize==length())
return;
#if defined(_MINGW_) || defined(WIN32)
else if(fd>0 || dont_delete_data)
#else
else if(fd>=0 || dont_delete_data)
#endif
PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
else if (newsize==0)
{
if (data) delete[] data;
data = 0;
length_ = 0;
}
else
{
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
int mem_before = getProcessDataMemory();
int length_before = length();
#endif
try
{
T* newdata = new T[newsize];
if(!newdata)
PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resizeMat, trying to allocate %d elements",newsize);
if(data)
{
// perform a 'structured' copy that keeps all the old values
T* oldp = data+old_offset;
T* newp = newdata+new_offset;
int w = min(old_width,new_width);
int l = min(old_length,new_length);
if (new_offset!=0)
{
if (new_offset!=old_offset)
PLERROR("Storage::resizeMat: when new_offset!=0 it should equal old_offset");
copy(data,data+new_offset,newdata);
}
for (int row=0;row<l;row++, oldp+=old_mod, newp+=new_mod)
{
copy(oldp,oldp+w,newp);
if(new_width>old_width)
clear_n(newp+old_width,new_width+extracols-old_width);
}
if (new_length>old_length)
clear_n(newp,(new_length+extrarows-old_length)*new_mod);
delete[] data;
}
length_ = newsize;
data = newdata;
}
catch(...)
{
PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newsize);
}
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
int mem_after = getProcessDataMemory();
if (mem_after - mem_before > 256*1024)
cerr << "Storage::resize: for storage at "
<< hex << this << dec
<< " fromsize=" << length_before << " tosize=" << newsize
<< " : memusage " << (mem_before/1024) << " kB ==> "
<< (mem_after/1024) << " kB" << endl;
if (mem_after - mem_before > 10000*1024)
PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
#endif
}
}
| int PLearn::Storage< T >::size | ( | ) | const [inline] |
| T* PLearn::Storage< T >::data |
Definition at line 80 of file Storage.h.
Referenced by PLearn::CompactVMatrix::append(), PLearn::Storage< PP< RegressionTreeNode > >::deepCopy(), PLearn::CompactVMatrix::dot(), PLearn::CompactVMatrix::encodeAndPutRow(), PLearn::CompactVMatrix::getNewRow(), PLearn::CompactVMatrix::perturb(), PLearn::CompactVMatrix::putSubRow(), and PLearn::Storage< PP< RegressionTreeNode > >::Storage().
| bool PLearn::Storage< T >::dont_delete_data |
| tFileHandle PLearn::Storage< T >::fd |
| int PLearn::Storage< T >::length_ |
1.7.4