PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // VMat_operations.cc 00004 // 00005 // Copyright (C) 2004 Pascal Vincent 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: VMat_operations.cc 7042 2007-05-09 23:44:20Z saintmlx $ 00037 ******************************************************* */ 00038 00039 // Authors: Pascal Vincent 00040 00044 #include "VMat.h" 00045 #include "VMat_operations.h" 00046 #include <plearn/math/random.h> 00047 #include <plearn/io/TmpFilenames.h> 00048 #include <plearn/io/IntVecFile.h> 00049 00050 namespace PLearn { 00051 using namespace std; 00052 00053 00054 VMat grep(VMat d, int col, Vec values, bool exclude) 00055 { 00056 Vec indices(d.length()); 00057 int nrows = 0; 00058 for(int i=0; i<d.length(); i++) 00059 { 00060 bool contains = values.contains(d(i,col)); 00061 if( (!exclude && contains) || (exclude && !contains) ) 00062 indices[nrows++] = i; 00063 } 00064 indices = indices.subVec(0,nrows); 00065 return d.rows(indices.copy()); 00066 } 00067 00069 map<real, int> countOccurencesInColumn(VMat m, int col) 00070 { 00071 map<real, int> counts; // result we will return 00072 map<real, int>::iterator found; 00073 int l = m.length(); 00074 for(int i=0; i<l; i++) 00075 { 00076 real val = m(i,col); 00077 if (is_missing(val)) 00078 // The 'nan' real value has to be dealt with separately. Here, we just 00079 // raise an error to keep it simple. 00080 PLERROR("In countOccurencesInColumn - Found a missing value, this case is currently not handled"); 00081 found = counts.find(val); 00082 if(found==counts.end()) 00083 counts[val] = 1; 00084 else 00085 found->second++; 00086 } 00087 return counts; 00088 } 00089 00091 map<real, TVec<int> > indicesOfOccurencesInColumn(VMat m, int col) 00092 { 00093 map< real, TVec<int> > indices; // result we will return 00094 map<real, int> counts = countOccurencesInColumn(m,col); 00095 map<real, int>::iterator it = counts.begin(); 00096 map<real, int>::iterator itend = counts.end(); 00097 for(; it!=itend; ++it) 00098 { 00099 indices[it->first].resize(it->second); // allocate the exact amount of memory 00100 indices[it->first].resize(0); // reset the size to 0 so we can do appends... 00101 } 00102 int l = m.length(); 00103 for(int i=0; i<l; i++) 00104 indices[m(i,col)].push_back(i); 00105 return indices; 00106 } 00107 00108 VMat grep(VMat d, int col, Vec values, const string& indexfile, bool exclude) 00109 { 00110 if(!isfile(indexfile)) 00111 { 00112 IntVecFile indices(indexfile,true); 00113 for(int i=0; i<d.length(); i++) 00114 { 00115 bool contains = values.contains(d(i,col)); 00116 if( (!exclude && contains) || (exclude && !contains) ) 00117 indices.append(i); 00118 } 00119 } 00120 return d.rows(indexfile); 00121 } 00122 00123 VMat filter(VMat d, const string& indexfile) 00124 { 00125 if(!isfile(indexfile) || filesize(indexfile)==0) 00126 { 00127 IntVecFile indices(indexfile,true); 00128 Vec v(d.width()); 00129 for(int i=0; i<d.length(); i++) 00130 { 00131 d->getRow(i,v); 00132 if(!v.hasMissing()) 00133 indices.append(i); 00134 } 00135 } 00136 return d.rows(indexfile); 00137 } 00138 00139 00140 VMat shuffle(VMat d) 00141 { 00142 Vec indices(0, d.length()-1, 1); // Range-vector 00143 shuffleElements(indices); 00144 return d.rows(indices); 00145 } 00146 00147 VMat bootstrap(VMat d, bool reorder, bool norepeat) 00148 { 00149 Vec indices; 00150 if (norepeat) 00151 { 00152 indices = Vec(0, d.length()-1, 1); // Range-vector 00153 shuffleElements(indices); 00154 indices = indices.subVec(0,int(0.667 * d.length())); 00155 if (reorder) 00156 sortElements(indices); 00157 return d.rows(indices); 00158 } 00159 else 00160 { 00161 indices.resize(d.length()); 00162 for (int i=0;i<d.length();i++) 00163 indices[i] = uniform_multinomial_sample(d.length()); 00164 } 00165 if (reorder) 00166 sortElements(indices); 00167 return d.rows(indices); 00168 } 00169 00170 00171 VMat rebalanceNClasses(VMat inputs, int nclasses, const string& filename) 00172 { 00173 if (!isfile(filename)) 00174 { 00175 IntVecFile indices(filename, true); 00176 Vec last = inputs.lastColumn()->toMat().toVecCopy(); 00177 const int len = last.length(); 00178 Vec capacity(nclasses); 00179 Array<Vec> index(nclasses); 00180 Array<Vec> index_used(nclasses); 00181 for (int i=0; i<nclasses; i++) index[i].resize(len); 00182 for (int i=0; i<nclasses; i++) index_used[i].resize(len); 00183 real** p_index; 00184 p_index = new real*[nclasses]; 00185 for (int i=0; i<nclasses; i++) p_index[i] = index[i].data(); 00186 for (int i=0; i<nclasses; i++) index_used[i].clear(); 00187 for (int i=0; i<len; i++) 00188 { 00189 int class_i = int(last[i]); 00190 *p_index[class_i]++ = i; 00191 capacity[class_i]++; 00192 } 00193 for (int i=0; i<nclasses; i++) index[i].resize(int(capacity[i])); 00194 for (int i=0; i<nclasses; i++) index_used[i].resize(int(capacity[i])); 00195 00196 Mat class_length(nclasses,2); 00197 for (int i=0; i<nclasses; i++) 00198 { 00199 class_length(i,0) = capacity[i]; 00200 class_length(i,1) = i; 00201 } 00202 sortRows(class_length); 00203 Vec remap = class_length.column(1).toVecCopy(); 00204 00205 vector<int> n(nclasses,0); 00206 int new_index = -1; 00207 for (int i=0; i<len; i++) 00208 { 00209 int c = i%nclasses; 00210 int c_map = int(remap[c]); 00211 if (c == 0) 00212 { 00213 if (fast_exact_is_equal(n[0], capacity[c_map])) n[0] = 0; 00214 new_index = int(index[c_map][n[0]++]); 00215 } 00216 else 00217 { 00218 if (fast_exact_is_equal(n[c], capacity[c_map])) 00219 { 00220 n[c] = 0; 00221 index_used[c_map].clear(); 00222 } 00223 bool index_found = false; 00224 int start_pos = binary_search(index[c_map], real(new_index)); 00225 for (int j=start_pos+1; j<capacity[c_map]; j++) 00226 { 00227 if (fast_exact_is_equal(index_used[c_map][j], 0)) 00228 { 00229 index_used[c_map][j] = 1; 00230 new_index = int(index[c_map][j]); 00231 index_found = true; 00232 n[c]++; 00233 break; 00234 } 00235 } 00236 if (!index_found) 00237 { 00238 for (int j=0; j<start_pos; j++) 00239 { 00240 if (fast_exact_is_equal(index_used[c_map][j], 0)) 00241 { 00242 index_used[c_map][j] = 1; 00243 new_index = int(index[c_map][j]); 00244 index_found = true; 00245 n[c]++; 00246 break; 00247 } 00248 } 00249 } 00250 if (!index_found) 00251 PLERROR("In rebalanceNClasses: something got wrong!"); 00252 } 00253 indices.put(i, new_index); 00254 } 00255 00256 delete[] p_index; 00257 } 00258 return inputs.rows(filename); 00259 } 00260 00261 void fullyRebalance2Classes(VMat inputs, const string& filename, bool save_indices) 00262 { 00263 if (!isfile(filename)) 00264 { 00265 int len = inputs.length(); 00266 00267 int n_zeros = 0; 00268 int n_ones = 0; 00269 Vec zeros(len); 00270 Vec ones(len); 00271 00272 Vec last = inputs.lastColumn()->toMat().toVecCopy(); 00273 for (int i=0; i<len;i++) 00274 { 00275 if (fast_exact_is_equal(last[i], 0)) 00276 zeros[n_zeros++] = i; 00277 else 00278 ones[n_ones++] = i; 00279 } 00280 zeros.resize(n_zeros); 00281 ones.resize(n_ones); 00282 00283 TmpFilenames tmpfile(1); 00284 string fname = save_indices ? filename : tmpfile.addFilename(); 00285 IntVecFile indices(fname, true); 00286 int max_symbols = MAX(n_zeros, n_ones); 00287 for (int i=0; i<max_symbols; i++) 00288 { 00289 indices.put(2*i, int(zeros[i%n_zeros])); 00290 indices.put(2*i+1, int(ones[i%n_ones])); 00291 } 00292 if (!save_indices) 00293 { 00294 VMat vm = inputs.rows(fname); 00295 vm.save(filename); 00296 } 00297 } 00298 } 00299 00300 VMat temporalThreshold(VMat distr, int threshold_date, bool is_before, 00301 int yyyymmdd_col) 00302 { 00303 Vec indices(distr->length()); 00304 int n_data = 0; 00305 for (int i=0; i<distr->length(); i++) 00306 { 00307 int reference_date = (int)distr(i, yyyymmdd_col); 00308 if (is_before ? reference_date<=threshold_date : reference_date>=threshold_date) 00309 indices[n_data++] = i; 00310 } 00311 indices.resize(n_data); 00312 00313 return distr.rows(indices); 00314 } 00315 00316 VMat temporalThreshold(VMat distr, int threshold_date, bool is_before, 00317 int yyyy_col, int mm_col, int dd_col) 00318 { 00319 Vec indices(distr->length()); 00320 int n_data = 0; 00321 for (int i=0; i<distr->length(); i++) 00322 { 00323 int reference_date = 10000*(int)distr(i, yyyy_col) + 100*(int)distr(i, mm_col) + (int)distr(i, dd_col); 00324 if (is_before ? reference_date<=threshold_date : reference_date>=threshold_date) 00325 indices[n_data++] = i; 00326 } 00327 indices.resize(n_data); 00328 00329 return distr.rows(indices); 00330 } 00331 00332 } // end of namespace PLearn 00333 00334 00335 /* 00336 Local Variables: 00337 mode:c++ 00338 c-basic-offset:4 00339 c-file-style:"stroustrup" 00340 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00341 indent-tabs-mode:nil 00342 fill-column:79 00343 End: 00344 */ 00345 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :