PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // 00007 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 00038 00039 /* ******************************************************* 00040 * $Id: Storage.h 10219 2009-05-27 20:38:46Z tihocan $ 00041 * AUTHORS: Pascal Vincent & Yoshua Bengio 00042 * This file is part of the PLearn library. 00043 ******************************************************* */ 00044 00045 00048 #ifndef STORAGE_INC 00049 #define STORAGE_INC 00050 00051 #include <map> 00052 #include "general.h" 00053 #include <plearn/sys/MemoryMap.h> 00054 #include "PP.h" 00055 #include <plearn/io/PStream.h> 00056 #include <limits> 00057 00060 #ifdef DEBUG_PLEARN_STORAGE_RESIZE 00061 # include <plearn/sys/procinfo.h> 00062 #endif 00063 //#include "Object.h" 00064 00065 namespace PLearn { 00066 using namespace std; 00067 00068 00069 template <class T> 00070 class Storage: public PPointable 00071 { 00072 public: // A few STL-like typedefs 00073 typedef T value_type; 00074 typedef int size_type; 00075 typedef T* iterator; 00076 typedef const T* const_iterator; 00077 00078 public: 00079 int length_; 00080 T* data; 00081 bool dont_delete_data; 00082 tFileHandle fd; 00083 00084 inline Storage(const Storage& other) 00085 :length_(other.length()), dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) 00086 { 00087 try 00088 { 00089 data = new T[length()]; 00090 if(!data) 00091 PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length()); 00092 //memcpy(data,other.data,length()*sizeof(T)); 00093 copy(other.data, other.data+length(), data); 00094 } 00095 catch(...) 00096 { 00097 PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length()); 00098 } 00099 } 00100 00101 inline Storage(long the_length, T* dataptr) 00102 :length_(int(the_length)), data(dataptr), 00103 dont_delete_data(true), fd(STORAGE_UNUSED_HANDLE) 00104 { 00105 //we do the check outside a BOUNDCHECK as we normaly do our test with 00106 //small dataset. Also, this is not a performance bottleneck and is a 00107 //fraction of the time of the malloc. 00108 if(the_length>std::numeric_limits<int>::max()) 00109 PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " 00110 "is possible (limited to 2e31, int)",the_length); 00111 00112 } 00113 00114 inline int length() const 00115 { return length_; } 00116 00117 inline int size() const 00118 { return length_; } 00119 00120 inline iterator begin() const 00121 { return data; } 00122 00123 inline iterator end() const 00124 { return data+length_; } 00125 00126 // If you're wondering why I did this stupid (and useless) function, ask GCC 3.0 that does not accept 00127 // to allocate a template into the constructor!!!! (and GCC 3.0 is the standard version on SGI) 00128 void mem_alloc(int len) 00129 { 00130 data = new T[len]; 00131 if(!data) 00132 PLERROR("OUT OF MEMORY (new returned NULL) in constructor of storage, trying to allocate %d elements",length()); 00133 clear_n(data,len); // clear the zone 00134 } 00135 00136 00138 Storage(long the_length=0) 00139 :length_((int)the_length), data(0), 00140 dont_delete_data(false), fd((tFileHandle)STORAGE_UNUSED_HANDLE) 00141 { 00142 //we do the check outside the BOUNDCHECK as we normaly do our test with 00143 //small dataset. Also, this is not a performance bottleneck and is a 00144 //fraction of the time of the malloc. 00145 if(the_length>std::numeric_limits<int>::max()) 00146 PLERROR("In Storage(%ld) - we ask to create a bigger Storage than " 00147 "is possible (limited to 2e31, int)",the_length); 00148 00149 int l = length(); 00150 #ifdef BOUNDCHECK 00151 if(l<0) 00152 PLERROR("new Storage called with a length() < 0: length = %d", l); 00153 #endif 00154 if (l>0) 00155 { 00156 mem_alloc(l); 00157 } 00158 } 00159 00164 inline Storage(const char* filename, bool readonly) 00165 { 00166 void* addr; 00167 off_t filesize; 00168 if(readonly) 00169 { 00170 #if !(!defined(_MSC_VER) && !defined(_MINGW_)) 00171 PLERROR("Not implemented for MinGW"); 00173 #else 00174 addr = (T*)MemoryMap(filename, fd, true, filesize); 00175 #endif 00176 } 00177 else 00178 { 00179 #if !(!defined(_MSC_VER) && !defined(_MINGW_)) 00180 PLERROR("Not implemtned for MinGW"); 00182 #else 00183 addr = (T*)MemoryMap(filename, fd, false, filesize); 00184 #endif 00185 } 00186 00187 if(addr==0) 00188 { 00189 perror("Error when calling mmap: "); 00190 PLERROR("In Storage: Memory-mapping failed"); 00191 } 00192 00193 data = (T*) addr; 00194 length_ = filesize/sizeof(T); 00195 dont_delete_data = false; 00196 } 00197 00198 inline void pointTo(int the_length, T* dataptr) 00199 { 00200 if (data && !dont_delete_data) 00201 { 00202 if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file 00203 { 00204 #if !(!defined(_MSC_VER) && !defined(_MINGW_)) 00205 00206 PLERROR("Not implemented for MinGW"); 00207 #else 00208 memoryUnmap((void *)data,fd,length()*sizeof(T)); 00209 #endif 00210 } 00211 else 00212 delete[] data; 00213 } 00214 length_=the_length; 00215 data=dataptr; 00216 fd=STORAGE_UNUSED_HANDLE; 00217 dont_delete_data=true; 00218 } 00219 00220 inline ~Storage() 00221 { 00222 if (data && !dont_delete_data) 00223 { 00224 if (fd!=STORAGE_UNUSED_HANDLE)//(fd>=0) //!< we are using a memory-mapped file 00225 { 00226 #if !(!defined(_MSC_VER) && !defined(_MINGW_)) 00227 PLERROR("Not implemented for MinGW"); 00229 #else 00230 memoryUnmap((void *)data,fd,length()*sizeof(T)); 00231 #endif 00232 } 00233 else 00234 delete[] data; 00235 } 00236 } 00237 00249 inline void resize(long lnewlength) 00250 { 00251 //we do the check outside a BOUNDCHECK as we normaly do our test with 00252 //small dataset. Also, this is not a performance bottleneck and is a 00253 //fraction of the time of the malloc. 00254 if(lnewlength>std::numeric_limits<int>::max() || lnewlength<std::numeric_limits<int>::min()) 00255 PLERROR("In Storage(%ld) - we ask to create a bigger/smaller" 00256 " Storage than is possible with an int", 00257 lnewlength); 00258 #ifdef BOUNDCHECK 00259 if(lnewlength<0) 00260 PLERROR("Storage::resize(%ld) called with a length() <0", 00261 lnewlength); 00262 #endif 00263 00264 int newlength=(int)lnewlength; 00265 00266 if (newlength==length()) 00267 return; 00268 #if defined(_MINGW_) || defined(WIN32) 00269 else if(fd>0 || dont_delete_data) 00270 #else 00271 else if(fd>=0 || dont_delete_data) 00272 #endif 00273 PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere"); 00274 else if (newlength==0) 00275 { 00276 if (data) delete[] data; 00277 data = 0; 00278 length_ = 0; 00279 } 00280 else if (newlength > length()) 00281 { 00282 #ifdef DEBUG_PLEARN_STORAGE_RESIZE 00283 int mem_before = getProcessDataMemory(); 00284 int length_before = length(); 00285 #endif 00286 try 00287 { 00288 T* newdata = new T[newlength]; 00289 if(!newdata) 00290 PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resize, trying to allocate %d elements",newlength); 00291 if(data) 00292 { 00293 // memcpy(newdata,data,length()*sizeof(T)); 00294 copy(data,data+length(),newdata); 00295 delete[] data; 00296 } 00297 // memset(&newdata[length()],0,(newlength-length())*sizeof(T)); 00298 clear_n(newdata+length(),newlength-length()); 00299 length_ = newlength; 00300 data = newdata; 00301 } 00302 catch(...) 00303 { 00304 PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newlength); 00305 } 00306 #ifdef DEBUG_PLEARN_STORAGE_RESIZE 00307 int mem_after = getProcessDataMemory(); 00308 if (mem_after - mem_before > 256*1024) 00309 cerr << "Storage::resize: for storage at " 00310 << hex << this << dec 00311 << " fromsize=" << length_before << " tosize=" << newlength 00312 << " : memusage " << (mem_before/1024) << " kB ==> " 00313 << (mem_after/1024) << " kB" << endl; 00314 if (mem_after - mem_before > 10000*1024) 00315 PLWARNING("Storage::resize: memory usage increased by more than 10000 kB"); 00316 #endif 00317 } 00318 else 00319 { 00320 try 00321 { 00322 T* newdata = new T[newlength]; 00323 if(!newdata) 00324 PLERROR("OUT OF MEMORY (new returned NULL) in copy constructor of storage, trying to allocate %d elements",length()); 00325 00326 if(data) 00327 { 00328 //memcpy(newdata,data,newlength*sizeof(T)); 00329 copy(data,data+newlength,newdata); 00330 delete[] data; 00331 } 00332 length_ = newlength; 00333 data = newdata; 00334 } 00335 catch(...) 00336 { 00337 PLERROR("OUT OF MEMORY in copy constructor of storage, trying to allocate %d elements",length()); 00338 } 00339 } 00340 } 00341 00342 00343 inline void resizeMat(int new_length, int new_width, int extrarows, int extracols, int new_offset, int old_mod, int old_length, int old_width, int old_offset) 00344 { 00345 long ls = new_length*new_width; 00346 long lextrabytes = (new_length+extrarows)*(new_width+extracols) - ls; 00347 long lnewsize = new_offset+ls+lextrabytes; 00348 00349 //we do the check outside a BOUNDCHECK as we normaly do our test with 00350 //small dataset. Also, this is not a performance bottleneck and is a 00351 //fraction of the time of the malloc. 00352 if(lnewsize>std::numeric_limits<int>::max()) 00353 PLERROR("In Storage.resizeMat - we ask to create a bigger Storage " 00354 " %ld then is possible (limited to 2e31, int)",lnewsize); 00355 int newsize=(int)lnewsize; 00356 int new_mod = new_width+extracols; 00357 00358 #ifdef BOUNDCHECK 00359 if(newsize<0) 00360 PLERROR("Storage::resize called with a length() <0"); 00361 #endif 00362 if (newsize==length()) 00363 return; 00364 #if defined(_MINGW_) || defined(WIN32) 00365 else if(fd>0 || dont_delete_data) 00366 #else 00367 else if(fd>=0 || dont_delete_data) 00368 #endif 00369 PLERROR("In Storage::resize cannot change size of memory-mapped data or of data allocated elsewhere"); 00370 else if (newsize==0) 00371 { 00372 if (data) delete[] data; 00373 data = 0; 00374 length_ = 0; 00375 } 00376 else 00377 { 00378 #ifdef DEBUG_PLEARN_STORAGE_RESIZE 00379 int mem_before = getProcessDataMemory(); 00380 int length_before = length(); 00381 #endif 00382 try 00383 { 00384 T* newdata = new T[newsize]; 00385 if(!newdata) 00386 PLERROR("OUT OF MEMORY (new returned NULL) in Storage::resizeMat, trying to allocate %d elements",newsize); 00387 if(data) 00388 { 00389 // perform a 'structured' copy that keeps all the old values 00390 T* oldp = data+old_offset; 00391 T* newp = newdata+new_offset; 00392 int w = min(old_width,new_width); 00393 int l = min(old_length,new_length); 00394 if (new_offset!=0) 00395 { 00396 if (new_offset!=old_offset) 00397 PLERROR("Storage::resizeMat: when new_offset!=0 it should equal old_offset"); 00398 copy(data,data+new_offset,newdata); 00399 } 00400 for (int row=0;row<l;row++, oldp+=old_mod, newp+=new_mod) 00401 { 00402 copy(oldp,oldp+w,newp); 00403 if(new_width>old_width) 00404 clear_n(newp+old_width,new_width+extracols-old_width); 00405 } 00406 if (new_length>old_length) 00407 clear_n(newp,(new_length+extrarows-old_length)*new_mod); 00408 00409 delete[] data; 00410 } 00411 length_ = newsize; 00412 data = newdata; 00413 } 00414 catch(...) 00415 { 00416 PLERROR("OUT OF MEMORY in Storage::resize, trying to allocate %d elements",newsize); 00417 } 00418 #ifdef DEBUG_PLEARN_STORAGE_RESIZE 00419 int mem_after = getProcessDataMemory(); 00420 if (mem_after - mem_before > 256*1024) 00421 cerr << "Storage::resize: for storage at " 00422 << hex << this << dec 00423 << " fromsize=" << length_before << " tosize=" << newsize 00424 << " : memusage " << (mem_before/1024) << " kB ==> " 00425 << (mem_after/1024) << " kB" << endl; 00426 if (mem_after - mem_before > 10000*1024) 00427 PLWARNING("Storage::resize: memory usage increased by more than 10000 kB"); 00428 #endif 00429 } 00430 } 00431 00433 Storage<T>* deepCopy(CopiesMap& copies) const 00434 { 00435 CopiesMap::iterator it = copies.find(this); 00436 if(it!=copies.end()) 00437 return (Storage<T>*) it->second; 00438 00440 Storage<T>* deep_copy = new Storage<T>(*this); 00441 for (int i = 0; i < size(); i++) { 00442 deepCopyField(deep_copy->data[i], copies); 00443 } 00445 //if (usage() > 1) 00446 copies[this] = deep_copy; 00448 return deep_copy; 00449 } 00450 00451 T& operator[](int idx) const {return data[idx];} 00452 00453 inline void push_back(const T& x) 00454 { 00455 int n = size(); 00456 resize(n+1); 00457 data[n] = x; 00458 } 00459 00460 }; 00461 00462 00463 template<class T> 00464 PStream& operator<<(PStream& out, const Storage<T>& seq) 00465 { 00466 writeSequence(out, seq); 00467 return out; 00468 } 00469 00470 template<class T> 00471 PStream& operator>>(PStream& in, Storage<T>& seq) 00472 { 00473 readSequence(in, seq); 00474 return in; 00475 } 00476 00477 00478 00479 } // end of namespace PLearn 00480 00481 #endif 00482 00483 00484 /* 00485 Local Variables: 00486 mode:c++ 00487 c-basic-offset:4 00488 c-file-style:"stroustrup" 00489 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00490 indent-tabs-mode:nil 00491 fill-column:79 00492 End: 00493 */ 00494 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :