PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // PLearn (A C++ Machine Learning Library) 00004 // Copyright (C) 1998 Pascal Vincent 00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal 00006 // 00007 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 00037 00038 00039 /* ******************************************************* 00040 * $Id$ 00041 * AUTHORS: Yoshua Bengio 00042 * This file is part of the PLearn library. 00043 ******************************************************* */ 00044 00045 00048 #ifndef Cache_INC 00049 #define Cache_INC 00050 00051 #include <plearn/base/BoundedMemoryCache.h> 00052 #include <plearn/io/fileutils.h> 00053 #include <plearn/io/openFile.h> 00054 namespace PLearn { 00055 using namespace std; 00056 00074 template <class KeyType, class ValueType> 00075 class Cache : public BoundedMemoryCache<KeyType,ValueType> 00076 { 00077 protected: 00078 public: 00079 bool single_file; // whether the filed entries are in a single large file or one file per element 00080 string files_directory; 00081 PStream::mode_t file_format; // possible values are: PStream::{raw,plearn}_{ascii,binary} 00082 00083 inline Cache(string dir = "cache", int max_memory_=0, bool singlefile=false) 00084 : BoundedMemoryCache<KeyType,ValueType>(max_memory_), single_file(singlefile), files_directory(dir), file_format(PStream::plearn_ascii) 00085 {} 00086 00090 inline const ValueType* operator()(const KeyType& key) const { 00091 ValueType* value_p = BoundedMemoryCache<KeyType,ValueType>::operator()(key); 00092 if (value_p) // we have it in memory 00093 return value_p; 00094 // else check if we have it on file 00095 value_p = loadValue(key); 00096 if (value_p) 00097 BoundedMemoryCache<KeyType,ValueType>::set(key,*value_p); 00098 return value_p; 00099 } 00100 inline ValueType* operator()(const KeyType& key) { 00101 ValueType* value_p = BoundedMemoryCache<KeyType,ValueType>::operator()(key); 00102 if (value_p) // we have it in memory 00103 return value_p; 00104 // else check if we have it on file 00105 value_p = loadValue(key); 00106 if (value_p) 00107 BoundedMemoryCache<KeyType,ValueType>::set(key,*value_p); 00108 return value_p; 00109 } 00110 00112 inline bool isCached(const KeyType& key) const { return BoundedMemoryCache<KeyType,ValueType>::isCached(key) || isOnFile(key); } 00113 inline bool isInMemory(const KeyType& key) const { return BoundedMemoryCache<KeyType,ValueType>::isCached(key); } 00114 inline string filename(const KeyType& key) const { return files_directory + "/" + tostring(key) + ".psave"; } 00115 inline bool isOnFile(const KeyType& key) const { return isfile(filename(key)); } 00116 // note that clear REMOVES EVERYTHING: stuff on disk and stuff in memory 00117 inline void clear() { 00118 BoundedMemoryCache<KeyType,ValueType>::clear(); 00119 if (files_directory!="") 00120 { 00121 vector<string> filenames = lsdir(files_directory); 00122 for (unsigned int i=0;i<filenames.size();i++) 00123 rm(files_directory + "/" + filenames[i]); 00124 } 00125 } 00126 // whereas destroying the cache or calling BoundedMemoryCache::clear() will 00127 // delete the stuff from memory WHILE SAVING IT TO DISK, EVERYTHING IS PRESERVED ON DISK. 00128 inline virtual ~Cache() { 00129 BoundedMemoryCache<KeyType,ValueType>::clear(); 00130 } 00131 00132 // makes sure everything in memory also resides on disk (brutally copy everything, which is simpler) 00133 inline virtual void synchronizeDisk() const { 00134 typename map<KeyType,pair<ValueType,DoublyLinkedListElement<KeyType>*> >::const_iterator it = BoundedMemoryCache<KeyType,ValueType>::elements.begin(); 00135 typename map<KeyType,pair<ValueType,DoublyLinkedListElement<KeyType>*> >::const_iterator end = BoundedMemoryCache<KeyType,ValueType>::elements.end(); 00136 for (;it!=end;++it) 00137 saveValue(it->first,it->second.first); 00138 } 00139 00140 protected: 00141 inline ValueType* loadValue(const KeyType& key) { 00142 if (single_file) 00143 { 00144 PLERROR("Cache: single_file mode not yet implemented"); 00145 } 00146 else 00147 { 00148 string fname = filename(key); 00149 if (!isfile(fname)) 00150 return 0; 00151 PStream filestream = openFile(fname, file_format, "r"); 00152 ValueType* v = new ValueType; 00153 filestream >> *v; 00154 return v; 00155 } 00156 return 0; 00157 } 00158 inline void saveValue(const KeyType& key, const ValueType& value) const { 00159 if (single_file) 00160 { 00161 PLERROR("Cache: single_file mode not yet implemented"); 00162 } 00163 else 00164 { 00165 PStream filestream = openFile(filename(key), file_format, "w"); 00166 filestream << value; 00167 } 00168 } 00169 00171 inline void removeExcess() 00172 { 00173 while (BoundedMemoryCache<KeyType,ValueType>::current_memory > BoundedMemoryCache<KeyType,ValueType>::max_memory) 00174 { 00175 KeyType& key = BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->last->entry; 00176 00177 // STORE THE ELEMENT TO BE DELETED ON DISK: 00178 pair<ValueType,DoublyLinkedListElement<KeyType>*> v = BoundedMemoryCache<KeyType,ValueType>::elements[key]; 00179 saveValue(key, v.first); 00180 00181 BoundedMemoryCache<KeyType,ValueType>::current_memory -= sizeInBytes(BoundedMemoryCache<KeyType,ValueType>::elements[key]); 00182 v.second=0; // this is just to help debugging, really, to help track wrong pointers 00183 BoundedMemoryCache<KeyType,ValueType>::elements.erase(key); 00184 BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->removeLast(); 00185 BoundedMemoryCache<KeyType,ValueType>::n_elements--; 00186 } 00187 00188 //int left_elements = BoundedMemoryCache<KeyType,ValueType>::n_elements; 00189 } 00190 00192 inline void removeAll() 00193 { 00194 while (BoundedMemoryCache<KeyType,ValueType>::n_elements) 00195 { 00196 KeyType& key = BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->last->entry; 00197 00198 // STORE THE ELEMENT TO BE DELETED ON DISK: 00199 pair<ValueType,DoublyLinkedListElement<KeyType>*> v = BoundedMemoryCache<KeyType,ValueType>::elements[key]; 00200 saveValue(key, v.first); 00201 00202 // BoundedMemoryCache<KeyType,ValueType>::current_memory -= sizeInBytes(BoundedMemoryCache<KeyType,ValueType>::elements[key]); 00203 v.second=0; // this is just to help debugging, really, to help track wrong pointers 00204 BoundedMemoryCache<KeyType,ValueType>::elements.erase(key); 00205 BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->removeLast(); 00206 BoundedMemoryCache<KeyType,ValueType>::n_elements--; 00207 } 00208 00209 BoundedMemoryCache<KeyType,ValueType>::current_memory = 0; 00210 //int left_elements = BoundedMemoryCache<KeyType,ValueType>::n_elements; 00211 } 00212 00213 protected: 00214 00215 }; 00216 00217 } // end of namespace PLearn 00218 00219 00220 #endif 00221 00222 00223 /* 00224 Local Variables: 00225 mode:c++ 00226 c-basic-offset:4 00227 c-file-style:"stroustrup" 00228 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00229 indent-tabs-mode:nil 00230 fill-column:79 00231 End: 00232 */ 00233 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :