PLearn 0.1
Public Types | Public Member Functions | Public Attributes
PLearn::Storage< T > Class Template Reference

#include <Storage.h>

Inherits PLearn::PPointable.

Collaboration diagram for PLearn::Storage< T >:
Collaboration graph
[legend]

List of all members.

Public Types

typedef T value_type
typedef int size_type
typedef T * iterator
typedef const T * const_iterator

Public Member Functions

 Storage (const Storage &other)
 Storage (long the_length, T *dataptr)
int length () const
int size () const
iterator begin () const
iterator end () const
void mem_alloc (int len)
 Storage (long the_length=0)
 data is initially filled with zeros
 Storage (const char *filename, bool readonly)
void pointTo (int the_length, T *dataptr)
 ~Storage ()
void resize (long lnewlength)
void resizeMat (int new_length, int new_width, int extrarows, int extracols, int new_offset, int old_mod, int old_length, int old_width, int old_offset)
Storage< T > * deepCopy (CopiesMap &copies) const
 Deep copying.
T & operator[] (int idx) const
void push_back (const T &x)

Public Attributes

int length_
T * data
bool dont_delete_data
 if true, the destructor won't delete[] data, because it will assume it is somebody else's responsibility
tFileHandle fd
 The descriptor for the memory-mapped file (-1 if there is no memory mapping)

Detailed Description

template<class T>
class PLearn::Storage< T >

Definition at line 70 of file Storage.h.


Member Typedef Documentation

template<class T>
typedef const T* PLearn::Storage< T >::const_iterator

Definition at line 76 of file Storage.h.

template<class T>
typedef T* PLearn::Storage< T >::iterator

Definition at line 75 of file Storage.h.

template<class T>
typedef int PLearn::Storage< T >::size_type

Definition at line 74 of file Storage.h.

template<class T>
typedef T PLearn::Storage< T >::value_type

Definition at line 73 of file Storage.h.


Constructor & Destructor Documentation

template<class T>
PLearn::Storage< T >::Storage ( const Storage< T > &  other) [inline]

Definition at line 84 of file Storage.h.

        :length_(other.length()), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE)
    {
        try 
        {
            data = new T[length()];
            if(!data)
                PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length());
            //memcpy(data,other.data,length()*sizeof(T));
            copy(other.data, other.data+length(), data);
        }
        catch(...)
        {
            PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length());
        }
    }
template<class T>
PLearn::Storage< T >::Storage ( long  the_length,
T *  dataptr 
) [inline]

Definition at line 101 of file Storage.h.

        :length_(int(the_length)), data(dataptr), 
         dont_delete_data(true), fd(STORAGE_UNUSED_HANDLE)
    {
        //we do the check outside a BOUNDCHECK as we normaly do our test with
        //small dataset. Also, this is not a performance bottleneck and is a
        //fraction of the time of the malloc.
        if(the_length>std::numeric_limits<int>::max())
            PLERROR("In Storage(%ld) - we ask to create a bigger Storage than "
                    "is possible (limited to 2e31, int)",the_length);

    }
template<class T>
PLearn::Storage< T >::Storage ( long  the_length = 0) [inline]

data is initially filled with zeros

Definition at line 138 of file Storage.h.

        :length_((int)the_length), data(0), 
         dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE)
    {
        //we do the check outside the BOUNDCHECK as we normaly do our test with
        //small dataset. Also, this is not a performance bottleneck and is a
        //fraction of the time of the malloc.
        if(the_length>std::numeric_limits<int>::max())
            PLERROR("In Storage(%ld) - we ask to create a bigger Storage than "
                    "is possible (limited to 2e31, int)",the_length);

        int l = length();
#ifdef BOUNDCHECK
        if(l<0)
            PLERROR("new Storage called with a length() < 0: length = %d", l);
#endif
        if (l>0) 
        {
            mem_alloc(l);
        }
    }
template<class T>
PLearn::Storage< T >::Storage ( const char *  filename,
bool  readonly 
) [inline]

Constructor for memory-mapped file The file is supposed to exist and have the correct size length() of the storage will be set to the size of the file divided by sizeof(T)

< read-write

Definition at line 164 of file Storage.h.

    {
        void* addr;
        off_t filesize;
        if(readonly)
        {
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
            PLERROR("Not implemented for MinGW");
#else
            addr = (T*)MemoryMap(filename, fd, true, filesize);
#endif
        }
        else 
        {
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
            PLERROR("Not implemtned for MinGW");
#else
            addr = (T*)MemoryMap(filename, fd, false, filesize);
#endif
        }

        if(addr==0)
        {
            perror("Error when calling mmap: ");
            PLERROR("In Storage: Memory-mapping failed");
        }

        data = (T*) addr;
        length_ = filesize/sizeof(T);
        dont_delete_data = false;
    }
template<class T>
PLearn::Storage< T >::~Storage ( ) [inline]

Definition at line 220 of file Storage.h.

    {
        if (data && !dont_delete_data) 
        {
            if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!<  we are using a memory-mapped file
            { 
#if !(!defined(_MSC_VER) && !defined(_MINGW_))
                PLERROR("Not implemented for MinGW");
#else
                memoryUnmap((void *)data,fd,length()*sizeof(T));
#endif
            }
            else
                delete[] data;
        }
    }

Member Function Documentation

template<class T>
iterator PLearn::Storage< T >::begin ( ) const [inline]

Definition at line 120 of file Storage.h.

    { return data; }
template<class T>
Storage<T>* PLearn::Storage< T >::deepCopy ( CopiesMap copies) const [inline]

Deep copying.

< a copy already exists, so return it

Otherwise call the copy constructor to obtain a copy

Put the copy in the map

return the completed deep_copy

Definition at line 433 of file Storage.h.

    {
        CopiesMap::iterator it = copies.find(this);
        if(it!=copies.end())  
            return (Storage<T>*) it->second;
      
        Storage<T>* deep_copy = new Storage<T>(*this);
        for (int i = 0; i < size(); i++) {
            deepCopyField(deep_copy->data[i], copies);
        }
        //if (usage() > 1)
        copies[this] = deep_copy;
        return deep_copy;
    }
template<class T>
iterator PLearn::Storage< T >::end ( ) const [inline]

Definition at line 123 of file Storage.h.

    { return data+length_; }
template<class T>
int PLearn::Storage< T >::length ( ) const [inline]

Definition at line 114 of file Storage.h.

    { return length_; }
template<class T>
void PLearn::Storage< T >::mem_alloc ( int  len) [inline]

Definition at line 128 of file Storage.h.

    {
        data = new T[len];
        if(!data)
            PLERROR("OUT OF MEMORY (new returned NULL) in constructor of storage, trying to allocate %d elements",length());
        clear_n(data,len); // clear the zone
    }
template<class T>
T& PLearn::Storage< T >::operator[] ( int  idx) const [inline]

Definition at line 451 of file Storage.h.

{return data[idx];}
template<class T>
void PLearn::Storage< T >::pointTo ( int  the_length,
T *  dataptr 
) [inline]

< allocated elsewhere

Definition at line 198 of file Storage.h.

    {
        if (data && !dont_delete_data) 
        {
            if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!<  we are using a memory-mapped file
            { 
#if !(!defined(_MSC_VER) && !defined(_MINGW_))

                PLERROR("Not implemented for MinGW");
#else
                memoryUnmap((void *)data,fd,length()*sizeof(T));
#endif
            }
            else
                delete[] data;
        }
        length_=the_length;
        data=dataptr;
        fd=STORAGE_UNUSED_HANDLE;
        dont_delete_data=true; 
    }
template<class T>
void PLearn::Storage< T >::push_back ( const T &  x) [inline]

Definition at line 453 of file Storage.h.

    {
        int n = size();
        resize(n+1);
        data[n] = x;
    }
template<class T>
void PLearn::Storage< T >::resize ( long  lnewlength) [inline]

Grow or shrink data memory If newlength==length() this call does nothing If newlength<=0 it outputs an PLERROR(i.e. cannot shrink memory to 0) Otherwise this call ALWAYS: -> allocates a new block of exactly the given size -> copies all the possible the data of the old block to the new one -> fills the remaining of the new block (if any) with 0.0 -> frees the old block It is the job of the CALLER (Mat and Vec) to have an appropriate policy to minimize the number of calls to Storage::resize

< we are using a memory-mapped file

< growing

< newlength<length() (shrinking)

Definition at line 249 of file Storage.h.

Referenced by PLearn::CompactVMatrix::append(), PLearn::CompactVMatrix::CompactVMatrix(), and PLearn::TVec< PP< RegressionTreeNode > >::resize().

    {
        //we do the check outside a BOUNDCHECK as we normaly do our test with
        //small dataset. Also, this is not a performance bottleneck and is a
        //fraction of the time of the malloc.
        if(lnewlength>std::numeric_limits<int>::max() || lnewlength<std::numeric_limits<int>::min())
            PLERROR("In Storage(%ld) - we ask to create a bigger/smaller"
                    " Storage than is possible with an int",
                    lnewlength);
#ifdef BOUNDCHECK
        if(lnewlength<0)
            PLERROR("Storage::resize(%ld) called with a length() <0",
                    lnewlength);
#endif

        int newlength=(int)lnewlength;

        if (newlength==length())
            return;
#if defined(_MINGW_) || defined(WIN32)
        else if(fd>0 || dont_delete_data) 
#else
            else if(fd>=0 || dont_delete_data) 
#endif
                PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
        else if (newlength==0)
        {
            if (data) delete[] data;
            data = 0;
            length_ = 0;
        }
        else if (newlength > length()) 
        {
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
            int mem_before = getProcessDataMemory();
            int length_before = length();
#endif
            try 
            {
                T* newdata = new T[newlength];
                if(!newdata)
                    PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resize, trying to allocate %d elements",newlength);
                if(data)
                {
                    // memcpy(newdata,data,length()*sizeof(T));
                    copy(data,data+length(),newdata);
                    delete[] data;
                }
                // memset(&newdata[length()],0,(newlength-length())*sizeof(T));
                clear_n(newdata+length(),newlength-length());
                length_ = newlength;
                data = newdata;
            }
            catch(...)
            {
                PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newlength);
            }
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
            int mem_after = getProcessDataMemory();
            if (mem_after - mem_before > 256*1024)
                cerr << "Storage::resize: for storage at "
                     << hex << this << dec
                     << " fromsize=" << length_before << " tosize=" << newlength
                     << " : memusage " << (mem_before/1024) << " kB  ==>  "
                     << (mem_after/1024) << " kB" << endl;
            if (mem_after - mem_before > 10000*1024)
                PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
#endif
        }
        else 
        {
            try 
            { 
                T* newdata = new T[newlength];
                if(!newdata)
                    PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length());

                if(data)
                {
                    //memcpy(newdata,data,newlength*sizeof(T));
                    copy(data,data+newlength,newdata);
                    delete[] data;
                }
                length_ = newlength;
                data = newdata;          
            }
            catch(...)
            {
                PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length());
            }
        }
    }

Here is the caller graph for this function:

template<class T>
void PLearn::Storage< T >::resizeMat ( int  new_length,
int  new_width,
int  extrarows,
int  extracols,
int  new_offset,
int  old_mod,
int  old_length,
int  old_width,
int  old_offset 
) [inline]

< we are using a memory-mapped file

Definition at line 343 of file Storage.h.

    {
        long ls = new_length*new_width;
        long lextrabytes = (new_length+extrarows)*(new_width+extracols) - ls;
        long lnewsize = new_offset+ls+lextrabytes;

        //we do the check outside a BOUNDCHECK as we normaly do our test with
        //small dataset. Also, this is not a performance bottleneck and is a
        //fraction of the time of the malloc.
        if(lnewsize>std::numeric_limits<int>::max())
            PLERROR("In Storage.resizeMat - we ask to create a bigger Storage "
                    " %ld then is possible (limited to 2e31, int)",lnewsize);
        int newsize=(int)lnewsize;
        int new_mod = new_width+extracols;

#ifdef BOUNDCHECK
        if(newsize<0)
            PLERROR("Storage::resize called with a length() <0");
#endif
        if (newsize==length())
            return;
#if defined(_MINGW_) || defined(WIN32)
        else if(fd>0 || dont_delete_data) 
#else
            else if(fd>=0 || dont_delete_data) 
#endif
                PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
        else if (newsize==0)
        {
            if (data) delete[] data;
            data = 0;
            length_ = 0;
        }
        else
        {
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
            int mem_before = getProcessDataMemory();
            int length_before = length();
#endif
            try 
            {
                T* newdata = new T[newsize];
                if(!newdata)
                    PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resizeMat, trying to allocate %d elements",newsize);
                if(data)
                {
                    // perform a 'structured' copy that keeps all the old values
                    T* oldp = data+old_offset;
                    T* newp = newdata+new_offset;
                    int w = min(old_width,new_width);
                    int l = min(old_length,new_length);
                    if (new_offset!=0)
                    { 
                        if (new_offset!=old_offset)
                            PLERROR("Storage::resizeMat: when new_offset!=0 it should equal old_offset");
                        copy(data,data+new_offset,newdata);
                    }
                    for (int row=0;row<l;row++, oldp+=old_mod, newp+=new_mod)
                    {
                        copy(oldp,oldp+w,newp);
                        if(new_width>old_width)
                            clear_n(newp+old_width,new_width+extracols-old_width);
                    }
                    if (new_length>old_length)
                        clear_n(newp,(new_length+extrarows-old_length)*new_mod);

                    delete[] data;
                }
                length_ = newsize;
                data = newdata;
            }
            catch(...)
            {
                PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newsize);
            }
#ifdef DEBUG_PLEARN_STORAGE_RESIZE
            int mem_after = getProcessDataMemory();
            if (mem_after - mem_before > 256*1024)
                cerr << "Storage::resize: for storage at "
                     << hex << this << dec
                     << " fromsize=" << length_before << " tosize=" << newsize
                     << " : memusage " << (mem_before/1024) << " kB  ==>  "
                     << (mem_after/1024) << " kB" << endl;
            if (mem_after - mem_before > 10000*1024)
                PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
#endif
        }
    }
template<class T>
int PLearn::Storage< T >::size ( ) const [inline]

Definition at line 117 of file Storage.h.

    { return length_; }

Member Data Documentation

template<class T>
T* PLearn::Storage< T >::data
template<class T>
bool PLearn::Storage< T >::dont_delete_data

if true, the destructor won't delete[] data, because it will assume it is somebody else's responsibility

Definition at line 81 of file Storage.h.

template<class T>
tFileHandle PLearn::Storage< T >::fd

The descriptor for the memory-mapped file (-1 if there is no memory mapping)

Definition at line 82 of file Storage.h.

template<class T>
int PLearn::Storage< T >::length_

Definition at line 79 of file Storage.h.


The documentation for this class was generated from the following file:
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines