PLearn 0.1
Cache.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999-2002 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037  
00038 
00039 /* *******************************************************      
00040  * $Id$
00041  * AUTHORS: Yoshua Bengio
00042  * This file is part of the PLearn library.
00043  ******************************************************* */
00044 
00045 
00048 #ifndef Cache_INC
00049 #define Cache_INC
00050 
00051 #include <plearn/base/BoundedMemoryCache.h>
00052 #include <plearn/io/fileutils.h>
00053 #include <plearn/io/openFile.h>
00054 namespace PLearn {
00055 using namespace std;
00056 
00074 template <class KeyType, class ValueType>
00075 class Cache : public BoundedMemoryCache<KeyType,ValueType>
00076 {
00077 protected:
00078 public:
00079     bool single_file; // whether the filed entries are in a single large file or one file per element
00080     string files_directory;
00081     PStream::mode_t file_format; // possible values are: PStream::{raw,plearn}_{ascii,binary}
00082 
00083     inline Cache(string dir = "cache", int max_memory_=0, bool singlefile=false) 
00084         : BoundedMemoryCache<KeyType,ValueType>(max_memory_), single_file(singlefile), files_directory(dir), file_format(PStream::plearn_ascii)
00085     {}
00086 
00090     inline const ValueType* operator()(const KeyType& key) const { 
00091         ValueType* value_p = BoundedMemoryCache<KeyType,ValueType>::operator()(key);
00092         if (value_p) // we have it in memory
00093             return value_p;
00094         // else check if we have it on file
00095         value_p = loadValue(key);
00096         if (value_p)
00097             BoundedMemoryCache<KeyType,ValueType>::set(key,*value_p);
00098         return value_p;
00099     }
00100     inline ValueType* operator()(const KeyType& key) { 
00101         ValueType* value_p = BoundedMemoryCache<KeyType,ValueType>::operator()(key);
00102         if (value_p) // we have it in memory
00103             return value_p;
00104         // else check if we have it on file
00105         value_p = loadValue(key);
00106         if (value_p)
00107             BoundedMemoryCache<KeyType,ValueType>::set(key,*value_p);
00108         return value_p;
00109     }
00110 
00112     inline bool isCached(const KeyType& key) const { return BoundedMemoryCache<KeyType,ValueType>::isCached(key) || isOnFile(key); }
00113     inline bool isInMemory(const KeyType& key) const { return BoundedMemoryCache<KeyType,ValueType>::isCached(key); }
00114     inline string filename(const KeyType& key) const { return files_directory + "/" + tostring(key) + ".psave"; }
00115     inline bool isOnFile(const KeyType& key) const { return isfile(filename(key)); }
00116     // note that clear REMOVES EVERYTHING: stuff on disk and stuff in memory
00117     inline void clear() {
00118         BoundedMemoryCache<KeyType,ValueType>::clear();
00119         if (files_directory!="")
00120         {
00121             vector<string> filenames = lsdir(files_directory);
00122             for (unsigned int i=0;i<filenames.size();i++)
00123                 rm(files_directory + "/" + filenames[i]);
00124         }
00125     }
00126     // whereas destroying the cache or calling BoundedMemoryCache::clear() will 
00127     // delete the stuff from memory WHILE SAVING IT TO DISK, EVERYTHING IS PRESERVED ON DISK.
00128     inline virtual ~Cache() { 
00129         BoundedMemoryCache<KeyType,ValueType>::clear(); 
00130     }
00131 
00132     // makes sure everything in memory also resides on disk (brutally copy everything, which is simpler)
00133     inline virtual void synchronizeDisk() const {
00134         typename map<KeyType,pair<ValueType,DoublyLinkedListElement<KeyType>*> >::const_iterator it = BoundedMemoryCache<KeyType,ValueType>::elements.begin();
00135         typename map<KeyType,pair<ValueType,DoublyLinkedListElement<KeyType>*> >::const_iterator end = BoundedMemoryCache<KeyType,ValueType>::elements.end();
00136         for (;it!=end;++it)
00137             saveValue(it->first,it->second.first);
00138     }
00139 
00140 protected:
00141     inline ValueType* loadValue(const KeyType& key) {
00142         if (single_file)
00143         {
00144             PLERROR("Cache: single_file mode not yet implemented");
00145         }
00146         else
00147         {
00148             string fname = filename(key);
00149             if (!isfile(fname))
00150                 return 0;
00151             PStream filestream = openFile(fname, file_format, "r");
00152             ValueType* v = new ValueType;
00153             filestream >> *v;
00154             return v;
00155         }
00156         return 0;
00157     }
00158     inline void saveValue(const KeyType& key, const ValueType& value) const {
00159         if (single_file)
00160         {
00161             PLERROR("Cache: single_file mode not yet implemented");
00162         }
00163         else
00164         {
00165             PStream filestream = openFile(filename(key), file_format, "w");
00166             filestream << value;
00167         }
00168     }
00169 
00171     inline void removeExcess()
00172     {
00173         while (BoundedMemoryCache<KeyType,ValueType>::current_memory > BoundedMemoryCache<KeyType,ValueType>::max_memory)
00174         {
00175             KeyType& key = BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->last->entry;
00176 
00177             // STORE THE ELEMENT TO BE DELETED ON DISK:
00178             pair<ValueType,DoublyLinkedListElement<KeyType>*> v = BoundedMemoryCache<KeyType,ValueType>::elements[key];
00179             saveValue(key, v.first);
00180 
00181             BoundedMemoryCache<KeyType,ValueType>::current_memory -= sizeInBytes(BoundedMemoryCache<KeyType,ValueType>::elements[key]);
00182             v.second=0;  // this is just to help debugging, really, to help track wrong pointers
00183             BoundedMemoryCache<KeyType,ValueType>::elements.erase(key);
00184             BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->removeLast();
00185             BoundedMemoryCache<KeyType,ValueType>::n_elements--;
00186         }
00187 
00188         //int left_elements = BoundedMemoryCache<KeyType,ValueType>::n_elements;
00189     }
00190 
00192     inline void removeAll()
00193     {
00194         while (BoundedMemoryCache<KeyType,ValueType>::n_elements)
00195         {
00196             KeyType& key = BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->last->entry;
00197 
00198             // STORE THE ELEMENT TO BE DELETED ON DISK:
00199             pair<ValueType,DoublyLinkedListElement<KeyType>*> v = BoundedMemoryCache<KeyType,ValueType>::elements[key];
00200             saveValue(key, v.first);
00201 
00202 //            BoundedMemoryCache<KeyType,ValueType>::current_memory -= sizeInBytes(BoundedMemoryCache<KeyType,ValueType>::elements[key]);
00203             v.second=0;  // this is just to help debugging, really, to help track wrong pointers
00204             BoundedMemoryCache<KeyType,ValueType>::elements.erase(key);
00205             BoundedMemoryCache<KeyType,ValueType>::doubly_linked_list->removeLast();
00206             BoundedMemoryCache<KeyType,ValueType>::n_elements--;
00207         }
00208 
00209         BoundedMemoryCache<KeyType,ValueType>::current_memory = 0;
00210         //int left_elements = BoundedMemoryCache<KeyType,ValueType>::n_elements;
00211     }
00212 
00213 protected:
00214 
00215 };
00216 
00217 } // end of namespace PLearn
00218 
00219 
00220 #endif
00221 
00222 
00223 /*
00224   Local Variables:
00225   mode:c++
00226   c-basic-offset:4
00227   c-file-style:"stroustrup"
00228   c-file-offsets:((innamespace . 0)(inline-open . 0))
00229   indent-tabs-mode:nil
00230   fill-column:79
00231   End:
00232 */
00233 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines