PLearn 0.1
Storage.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037  
00038 
00039 /* *******************************************************      
00040  * $Id: Storage.h 10219 2009-05-27 20:38:46Z tihocan $
00041  * AUTHORS: Pascal Vincent & Yoshua Bengio
00042  * This file is part of the PLearn library.
00043  ******************************************************* */
00044 
00045 
00048 #ifndef STORAGE_INC
00049 #define STORAGE_INC
00050 
00051 #include <map>
00052 #include "general.h"
00053 #include <plearn/sys/MemoryMap.h>
00054 #include "PP.h"
00055 #include <plearn/io/PStream.h>
00056 #include <limits>
00057 
00060 #ifdef DEBUG_PLEARN_STORAGE_RESIZE
00061 #  include <plearn/sys/procinfo.h>
00062 #endif
00063 //#include "Object.h"
00064 
00065 namespace PLearn {
00066 using namespace std;
00067 
00068     
00069 template <class T>
00070 class Storage: public PPointable
00071 {
00072 public: // A few STL-like typedefs
00073     typedef T value_type;
00074     typedef int size_type;
00075     typedef T* iterator;
00076     typedef const T* const_iterator;
00077 
00078 public:
00079     int length_;
00080     T* data;
00081     bool dont_delete_data; 
00082     tFileHandle fd; 
00083 
00084     inline Storage(const Storage& other)
00085         :length_(other.length()), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE)
00086     {
00087         try 
00088         {
00089             data = new T[length()];
00090             if(!data)
00091                 PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length());
00092             //memcpy(data,other.data,length()*sizeof(T));
00093             copy(other.data, other.data+length(), data);
00094         }
00095         catch(...)
00096         {
00097             PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length());
00098         }
00099     }
00100 
00101     inline Storage(long the_length, T* dataptr)
00102         :length_(int(the_length)), data(dataptr), 
00103          dont_delete_data(true), fd(STORAGE_UNUSED_HANDLE)
00104     {
00105         //we do the check outside a BOUNDCHECK as we normaly do our test with
00106         //small dataset. Also, this is not a performance bottleneck and is a
00107         //fraction of the time of the malloc.
00108         if(the_length>std::numeric_limits<int>::max())
00109             PLERROR("In Storage(%ld) - we ask to create a bigger Storage than "
00110                     "is possible (limited to 2e31, int)",the_length);
00111 
00112     }
00113 
00114     inline int length() const
00115     { return length_; }
00116 
00117     inline int size() const
00118     { return length_; }
00119 
00120     inline iterator begin() const
00121     { return data; }
00122 
00123     inline iterator end() const
00124     { return data+length_; }
00125 
00126     // If you're wondering why I did this stupid (and useless) function, ask GCC 3.0 that does not accept
00127     // to allocate a template into the constructor!!!! (and GCC 3.0 is the standard version on SGI)
00128     void mem_alloc(int len)
00129     {
00130         data = new T[len];
00131         if(!data)
00132             PLERROR("OUT OF MEMORY (new returned NULL) in constructor of storage, trying to allocate %d elements",length());
00133         clear_n(data,len); // clear the zone
00134     }
00135         
00136 
00138     Storage(long the_length=0)
00139         :length_((int)the_length), data(0), 
00140          dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE)
00141     {
00142         //we do the check outside the BOUNDCHECK as we normaly do our test with
00143         //small dataset. Also, this is not a performance bottleneck and is a
00144         //fraction of the time of the malloc.
00145         if(the_length>std::numeric_limits<int>::max())
00146             PLERROR("In Storage(%ld) - we ask to create a bigger Storage than "
00147                     "is possible (limited to 2e31, int)",the_length);
00148 
00149         int l = length();
00150 #ifdef BOUNDCHECK
00151         if(l<0)
00152             PLERROR("new Storage called with a length() < 0: length = %d", l);
00153 #endif
00154         if (l>0) 
00155         {
00156             mem_alloc(l);
00157         }
00158     }
00159 
00164     inline Storage(const char* filename, bool readonly)
00165     {
00166         void* addr;
00167         off_t filesize;
00168         if(readonly)
00169         {
00170 #if !(!defined(_MSC_VER) && !defined(_MINGW_))
00171             PLERROR("Not implemented for MinGW");
00173 #else
00174             addr = (T*)MemoryMap(filename, fd, true, filesize);
00175 #endif
00176         }
00177         else 
00178         {
00179 #if !(!defined(_MSC_VER) && !defined(_MINGW_))
00180             PLERROR("Not implemtned for MinGW");
00182 #else
00183             addr = (T*)MemoryMap(filename, fd, false, filesize);
00184 #endif
00185         }
00186 
00187         if(addr==0)
00188         {
00189             perror("Error when calling mmap: ");
00190             PLERROR("In Storage: Memory-mapping failed");
00191         }
00192 
00193         data = (T*) addr;
00194         length_ = filesize/sizeof(T);
00195         dont_delete_data = false;
00196     }
00197 
00198     inline void pointTo(int the_length, T* dataptr)
00199     {
00200         if (data && !dont_delete_data) 
00201         {
00202             if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!<  we are using a memory-mapped file
00203             { 
00204 #if !(!defined(_MSC_VER) && !defined(_MINGW_))
00205 
00206                 PLERROR("Not implemented for MinGW");
00207 #else
00208                 memoryUnmap((void *)data,fd,length()*sizeof(T));
00209 #endif
00210             }
00211             else
00212                 delete[] data;
00213         }
00214         length_=the_length;
00215         data=dataptr;
00216         fd=STORAGE_UNUSED_HANDLE;
00217         dont_delete_data=true; 
00218     }
00219 
00220     inline ~Storage()
00221     {
00222         if (data && !dont_delete_data) 
00223         {
00224             if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!<  we are using a memory-mapped file
00225             { 
00226 #if !(!defined(_MSC_VER) && !defined(_MINGW_))
00227                 PLERROR("Not implemented for MinGW");
00229 #else
00230                 memoryUnmap((void *)data,fd,length()*sizeof(T));
00231 #endif
00232             }
00233             else
00234                 delete[] data;
00235         }
00236     }
00237       
00249     inline void resize(long lnewlength)
00250     {
00251         //we do the check outside a BOUNDCHECK as we normaly do our test with
00252         //small dataset. Also, this is not a performance bottleneck and is a
00253         //fraction of the time of the malloc.
00254         if(lnewlength>std::numeric_limits<int>::max() || lnewlength<std::numeric_limits<int>::min())
00255             PLERROR("In Storage(%ld) - we ask to create a bigger/smaller"
00256                     " Storage than is possible with an int",
00257                     lnewlength);
00258 #ifdef BOUNDCHECK
00259         if(lnewlength<0)
00260             PLERROR("Storage::resize(%ld) called with a length() <0",
00261                     lnewlength);
00262 #endif
00263 
00264         int newlength=(int)lnewlength;
00265 
00266         if (newlength==length())
00267             return;
00268 #if defined(_MINGW_) || defined(WIN32)
00269         else if(fd>0 || dont_delete_data) 
00270 #else
00271             else if(fd>=0 || dont_delete_data) 
00272 #endif
00273                 PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
00274         else if (newlength==0)
00275         {
00276             if (data) delete[] data;
00277             data = 0;
00278             length_ = 0;
00279         }
00280         else if (newlength > length()) 
00281         {
00282 #ifdef DEBUG_PLEARN_STORAGE_RESIZE
00283             int mem_before = getProcessDataMemory();
00284             int length_before = length();
00285 #endif
00286             try 
00287             {
00288                 T* newdata = new T[newlength];
00289                 if(!newdata)
00290                     PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resize, trying to allocate %d elements",newlength);
00291                 if(data)
00292                 {
00293                     // memcpy(newdata,data,length()*sizeof(T));
00294                     copy(data,data+length(),newdata);
00295                     delete[] data;
00296                 }
00297                 // memset(&newdata[length()],0,(newlength-length())*sizeof(T));
00298                 clear_n(newdata+length(),newlength-length());
00299                 length_ = newlength;
00300                 data = newdata;
00301             }
00302             catch(...)
00303             {
00304                 PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newlength);
00305             }
00306 #ifdef DEBUG_PLEARN_STORAGE_RESIZE
00307             int mem_after = getProcessDataMemory();
00308             if (mem_after - mem_before > 256*1024)
00309                 cerr << "Storage::resize: for storage at "
00310                      << hex << this << dec
00311                      << " fromsize=" << length_before << " tosize=" << newlength
00312                      << " : memusage " << (mem_before/1024) << " kB  ==>  "
00313                      << (mem_after/1024) << " kB" << endl;
00314             if (mem_after - mem_before > 10000*1024)
00315                 PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
00316 #endif
00317         }
00318         else 
00319         {
00320             try 
00321             { 
00322                 T* newdata = new T[newlength];
00323                 if(!newdata)
00324                     PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length());
00325 
00326                 if(data)
00327                 {
00328                     //memcpy(newdata,data,newlength*sizeof(T));
00329                     copy(data,data+newlength,newdata);
00330                     delete[] data;
00331                 }
00332                 length_ = newlength;
00333                 data = newdata;          
00334             }
00335             catch(...)
00336             {
00337                 PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length());
00338             }
00339         }
00340     }
00341 
00342 
00343     inline void resizeMat(int new_length, int new_width, int extrarows, int extracols, int new_offset, int old_mod, int old_length, int old_width, int old_offset)
00344     {
00345         long ls = new_length*new_width;
00346         long lextrabytes = (new_length+extrarows)*(new_width+extracols) - ls;
00347         long lnewsize = new_offset+ls+lextrabytes;
00348 
00349         //we do the check outside a BOUNDCHECK as we normaly do our test with
00350         //small dataset. Also, this is not a performance bottleneck and is a
00351         //fraction of the time of the malloc.
00352         if(lnewsize>std::numeric_limits<int>::max())
00353             PLERROR("In Storage.resizeMat - we ask to create a bigger Storage "
00354                     " %ld then is possible (limited to 2e31, int)",lnewsize);
00355         int newsize=(int)lnewsize;
00356         int new_mod = new_width+extracols;
00357 
00358 #ifdef BOUNDCHECK
00359         if(newsize<0)
00360             PLERROR("Storage::resize called with a length() <0");
00361 #endif
00362         if (newsize==length())
00363             return;
00364 #if defined(_MINGW_) || defined(WIN32)
00365         else if(fd>0 || dont_delete_data) 
00366 #else
00367             else if(fd>=0 || dont_delete_data) 
00368 #endif
00369                 PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere");
00370         else if (newsize==0)
00371         {
00372             if (data) delete[] data;
00373             data = 0;
00374             length_ = 0;
00375         }
00376         else
00377         {
00378 #ifdef DEBUG_PLEARN_STORAGE_RESIZE
00379             int mem_before = getProcessDataMemory();
00380             int length_before = length();
00381 #endif
00382             try 
00383             {
00384                 T* newdata = new T[newsize];
00385                 if(!newdata)
00386                     PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resizeMat, trying to allocate %d elements",newsize);
00387                 if(data)
00388                 {
00389                     // perform a 'structured' copy that keeps all the old values
00390                     T* oldp = data+old_offset;
00391                     T* newp = newdata+new_offset;
00392                     int w = min(old_width,new_width);
00393                     int l = min(old_length,new_length);
00394                     if (new_offset!=0)
00395                     { 
00396                         if (new_offset!=old_offset)
00397                             PLERROR("Storage::resizeMat: when new_offset!=0 it should equal old_offset");
00398                         copy(data,data+new_offset,newdata);
00399                     }
00400                     for (int row=0;row<l;row++, oldp+=old_mod, newp+=new_mod)
00401                     {
00402                         copy(oldp,oldp+w,newp);
00403                         if(new_width>old_width)
00404                             clear_n(newp+old_width,new_width+extracols-old_width);
00405                     }
00406                     if (new_length>old_length)
00407                         clear_n(newp,(new_length+extrarows-old_length)*new_mod);
00408 
00409                     delete[] data;
00410                 }
00411                 length_ = newsize;
00412                 data = newdata;
00413             }
00414             catch(...)
00415             {
00416                 PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newsize);
00417             }
00418 #ifdef DEBUG_PLEARN_STORAGE_RESIZE
00419             int mem_after = getProcessDataMemory();
00420             if (mem_after - mem_before > 256*1024)
00421                 cerr << "Storage::resize: for storage at "
00422                      << hex << this << dec
00423                      << " fromsize=" << length_before << " tosize=" << newsize
00424                      << " : memusage " << (mem_before/1024) << " kB  ==>  "
00425                      << (mem_after/1024) << " kB" << endl;
00426             if (mem_after - mem_before > 10000*1024)
00427                 PLWARNING("Storage::resize: memory usage increased by more than 10000 kB");
00428 #endif
00429         }
00430     }
00431 
00433     Storage<T>* deepCopy(CopiesMap& copies) const
00434     {
00435         CopiesMap::iterator it = copies.find(this);
00436         if(it!=copies.end())  
00437             return (Storage<T>*) it->second;
00438       
00440         Storage<T>* deep_copy = new Storage<T>(*this);
00441         for (int i = 0; i < size(); i++) {
00442             deepCopyField(deep_copy->data[i], copies);
00443         }
00445         //if (usage() > 1)
00446         copies[this] = deep_copy;
00448         return deep_copy;
00449     }
00450 
00451     T& operator[](int idx) const {return data[idx];}
00452 
00453     inline void push_back(const T& x)
00454     {
00455         int n = size();
00456         resize(n+1);
00457         data[n] = x;
00458     }
00459 
00460 };
00461 
00462 
00463 template<class T>
00464 PStream& operator<<(PStream& out, const Storage<T>& seq)
00465 {
00466     writeSequence(out, seq);
00467     return out;
00468 }
00469 
00470 template<class T>
00471 PStream& operator>>(PStream& in, Storage<T>& seq)
00472 {
00473     readSequence(in, seq);
00474     return in;
00475 }
00476 
00477 
00478 
00479 } // end of namespace PLearn
00480 
00481 #endif
00482 
00483 
00484 /*
00485   Local Variables:
00486   mode:c++
00487   c-basic-offset:4
00488   c-file-style:"stroustrup"
00489   c-file-offsets:((innamespace . 0)(inline-open . 0))
00490   indent-tabs-mode:nil
00491   fill-column:79
00492   End:
00493 */
00494 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines