PLearn 0.1
TextStreamVMatrix.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // TextStreamVMatrix.cc
00004 //
00005 // Copyright (C) 2007 Vytenis Sakenas
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 // Authors: Vytenis Sakenas
00036 
00040 #include "TextStreamVMatrix.h"
00041 #include <fstream>
00042 
00043 namespace PLearn {
00044 using namespace std;
00045 
00046 
00047 PLEARN_IMPLEMENT_OBJECT(
00048     TextStreamVMatrix,
00049     "Character based sliding window data matrix from given text file.",
00050     "Given the text file, and symbols that must be parsed this creates a data"
00051         "matrix that every row contains concecutive number_of_symbols chars "
00052         "expressed in one-hot vector. For example:\n"
00053         "text file contents: abaab\n"
00054         "symbols:       ab\n"
00055         "number_of_symbols: 3\n"
00056         "result matrix will be:\n"
00057         "1 0 0 1 1 0 <-- aba\n"
00058         "0 1 1 0 1 0 <-- baa\n"
00059         "1 0 1 0 0 1 <-- aab\n"
00060         "Position of 1 in one-hot vector is defined by chars position in symbols string."
00061     );
00062 
00063 TextStreamVMatrix::TextStreamVMatrix()
00064 /* ### Initialize all fields to their default value */
00065 {
00066 }
00067 
00068 void TextStreamVMatrix::getNewRow(int i, const Vec& v) const
00069 {
00070     v.subVec(0, symbol_width*number_of_symbols) << data.subVec( i*symbol_width, symbol_width*number_of_symbols);
00071     v[symbol_width*number_of_symbols] = target[i];
00072 }
00073 
00074 void TextStreamVMatrix::declareOptions(OptionList& ol)
00075 {
00076      declareOption(ol, "data_file", &TextStreamVMatrix::data_file,
00077                    OptionBase::buildoption,
00078                    "File that contains data.");
00079      declareOption(ol, "number_of_symbols", &TextStreamVMatrix::number_of_symbols,
00080                    OptionBase::buildoption,
00081                    "Number of symbols to represent in one row (window size).");
00082      declareOption(ol, "symbols", &TextStreamVMatrix::symbols,
00083                    OptionBase::buildoption,
00084                    "Symbols that are represented. All symbols that are not in this string will be ignored. "
00085                         "Also the index of a symbol will show which bit will be set to one in the one-hot vector.");
00086 
00087     // Now call the parent class' declareOptions
00088     inherited::declareOptions(ol);
00089 }
00090 
00091 void TextStreamVMatrix::build_()
00092 {
00093         int ids[256];
00094 
00095         symbol_width = symbols.length();
00096 
00097         // Calculate id table
00098         memset(ids, -1, sizeof(ids));
00099         for (int i = 0; i < symbol_width; ++i)
00100                 ids[(unsigned int)symbols[i]] = i;
00101 
00102         updateMtime(data_file);
00103 
00104         int text_length = 0;
00105         string text;
00106 
00107         // Read text from file
00108         // TODO: optimize
00109         ifstream fin(data_file.c_str(), ios::in | ios::binary);
00110         unsigned char chr;
00111         
00112         fin.unsetf(ios::skipws);
00113 
00114         while (fin >> chr) {
00115                 if (ids[(unsigned int)chr] == -1) continue;
00116                 text += chr;
00117                 ++text_length;
00118         }
00119 
00120         fin.close();
00121 
00122         data.resize(symbol_width*text_length);
00123         data.fill(0);
00124 
00125         target.resize(text_length - number_of_symbols);
00126 
00127         //cout << text_length << endl;
00128         for (int i = 0; i < text_length; ++i) {
00129                 //cout << symbol_width*i << endl;
00130                 //cout << "id: " << ids[text[i]] << " " << i << endl;
00131                 data[ symbol_width*i + ids[(unsigned int)text[i]] ] = 1;
00132         }
00133 
00134         for (int i = 0; i < text_length-number_of_symbols; ++i)
00135                 target[i] = ids[(unsigned int)text[i + number_of_symbols]];
00136 
00137         length_ = text_length - number_of_symbols;
00138         width_ = symbol_width*number_of_symbols + 1;
00139         inputsize_ = symbol_width*number_of_symbols;
00140         targetsize_ = 1;
00141         weightsize_ = 0;
00142         extrasize_ = 0;
00143 }
00144 
00145 // ### Nothing to add here, simply calls build_
00146 void TextStreamVMatrix::build()
00147 {
00148     inherited::build();
00149     build_();
00150 }
00151 
00152 void TextStreamVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00153 {
00154     inherited::makeDeepCopyFromShallowCopy(copies);
00155 }
00156 
00157 } // end of namespace PLearn
00158 
00159 
00160 /*
00161   Local Variables:
00162   mode:c++
00163   c-basic-offset:4
00164   c-file-style:"stroustrup"
00165   c-file-offsets:((innamespace . 0)(inline-open . 0))
00166   indent-tabs-mode:nil
00167   fill-column:79
00168   End:
00169 */
00170 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines