PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // TextStreamVMatrix.cc 00004 // 00005 // Copyright (C) 2007 Vytenis Sakenas 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 // Authors: Vytenis Sakenas 00036 00040 #include "TextStreamVMatrix.h" 00041 #include <fstream> 00042 00043 namespace PLearn { 00044 using namespace std; 00045 00046 00047 PLEARN_IMPLEMENT_OBJECT( 00048 TextStreamVMatrix, 00049 "Character based sliding window data matrix from given text file.", 00050 "Given the text file, and symbols that must be parsed this creates a data" 00051 "matrix that every row contains concecutive number_of_symbols chars " 00052 "expressed in one-hot vector. For example:\n" 00053 "text file contents: abaab\n" 00054 "symbols: ab\n" 00055 "number_of_symbols: 3\n" 00056 "result matrix will be:\n" 00057 "1 0 0 1 1 0 <-- aba\n" 00058 "0 1 1 0 1 0 <-- baa\n" 00059 "1 0 1 0 0 1 <-- aab\n" 00060 "Position of 1 in one-hot vector is defined by chars position in symbols string." 00061 ); 00062 00063 TextStreamVMatrix::TextStreamVMatrix() 00064 /* ### Initialize all fields to their default value */ 00065 { 00066 } 00067 00068 void TextStreamVMatrix::getNewRow(int i, const Vec& v) const 00069 { 00070 v.subVec(0, symbol_width*number_of_symbols) << data.subVec( i*symbol_width, symbol_width*number_of_symbols); 00071 v[symbol_width*number_of_symbols] = target[i]; 00072 } 00073 00074 void TextStreamVMatrix::declareOptions(OptionList& ol) 00075 { 00076 declareOption(ol, "data_file", &TextStreamVMatrix::data_file, 00077 OptionBase::buildoption, 00078 "File that contains data."); 00079 declareOption(ol, "number_of_symbols", &TextStreamVMatrix::number_of_symbols, 00080 OptionBase::buildoption, 00081 "Number of symbols to represent in one row (window size)."); 00082 declareOption(ol, "symbols", &TextStreamVMatrix::symbols, 00083 OptionBase::buildoption, 00084 "Symbols that are represented. All symbols that are not in this string will be ignored. " 00085 "Also the index of a symbol will show which bit will be set to one in the one-hot vector."); 00086 00087 // Now call the parent class' declareOptions 00088 inherited::declareOptions(ol); 00089 } 00090 00091 void TextStreamVMatrix::build_() 00092 { 00093 int ids[256]; 00094 00095 symbol_width = symbols.length(); 00096 00097 // Calculate id table 00098 memset(ids, -1, sizeof(ids)); 00099 for (int i = 0; i < symbol_width; ++i) 00100 ids[(unsigned int)symbols[i]] = i; 00101 00102 updateMtime(data_file); 00103 00104 int text_length = 0; 00105 string text; 00106 00107 // Read text from file 00108 // TODO: optimize 00109 ifstream fin(data_file.c_str(), ios::in | ios::binary); 00110 unsigned char chr; 00111 00112 fin.unsetf(ios::skipws); 00113 00114 while (fin >> chr) { 00115 if (ids[(unsigned int)chr] == -1) continue; 00116 text += chr; 00117 ++text_length; 00118 } 00119 00120 fin.close(); 00121 00122 data.resize(symbol_width*text_length); 00123 data.fill(0); 00124 00125 target.resize(text_length - number_of_symbols); 00126 00127 //cout << text_length << endl; 00128 for (int i = 0; i < text_length; ++i) { 00129 //cout << symbol_width*i << endl; 00130 //cout << "id: " << ids[text[i]] << " " << i << endl; 00131 data[ symbol_width*i + ids[(unsigned int)text[i]] ] = 1; 00132 } 00133 00134 for (int i = 0; i < text_length-number_of_symbols; ++i) 00135 target[i] = ids[(unsigned int)text[i + number_of_symbols]]; 00136 00137 length_ = text_length - number_of_symbols; 00138 width_ = symbol_width*number_of_symbols + 1; 00139 inputsize_ = symbol_width*number_of_symbols; 00140 targetsize_ = 1; 00141 weightsize_ = 0; 00142 extrasize_ = 0; 00143 } 00144 00145 // ### Nothing to add here, simply calls build_ 00146 void TextStreamVMatrix::build() 00147 { 00148 inherited::build(); 00149 build_(); 00150 } 00151 00152 void TextStreamVMatrix::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00153 { 00154 inherited::makeDeepCopyFromShallowCopy(copies); 00155 } 00156 00157 } // end of namespace PLearn 00158 00159 00160 /* 00161 Local Variables: 00162 mode:c++ 00163 c-basic-offset:4 00164 c-file-style:"stroustrup" 00165 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00166 indent-tabs-mode:nil 00167 fill-column:79 00168 End: 00169 */ 00170 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :