PLearn 0.1
VMat_operations.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // VMat_operations.cc
00004 //
00005 // Copyright (C) 2004 Pascal Vincent
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: VMat_operations.cc 7042 2007-05-09 23:44:20Z saintmlx $
00037  ******************************************************* */
00038 
00039 // Authors: Pascal Vincent
00040 
00044 #include "VMat.h"
00045 #include "VMat_operations.h"
00046 #include <plearn/math/random.h>
00047 #include <plearn/io/TmpFilenames.h>
00048 #include <plearn/io/IntVecFile.h>
00049 
00050 namespace PLearn {
00051 using namespace std;
00052 
00053 
00054 VMat grep(VMat d, int col, Vec values, bool exclude)
00055 {
00056     Vec indices(d.length());
00057     int nrows = 0;
00058     for(int i=0; i<d.length(); i++)
00059     {
00060         bool contains = values.contains(d(i,col));
00061         if( (!exclude && contains) || (exclude && !contains) )
00062             indices[nrows++] = i;
00063     }
00064     indices = indices.subVec(0,nrows);
00065     return d.rows(indices.copy());
00066 }
00067 
00069 map<real, int> countOccurencesInColumn(VMat m, int col)
00070 {
00071     map<real, int> counts; // result we will return
00072     map<real, int>::iterator found;
00073     int l = m.length();
00074     for(int i=0; i<l; i++)
00075     {
00076         real val = m(i,col);
00077         if (is_missing(val))
00078             // The 'nan' real value has to be dealt with separately. Here, we just
00079             // raise an error to keep it simple.
00080             PLERROR("In countOccurencesInColumn - Found a missing value, this case is currently not handled");
00081         found = counts.find(val);
00082         if(found==counts.end())
00083             counts[val] = 1;
00084         else
00085             found->second++;
00086     }
00087     return counts;
00088 }
00089 
00091 map<real, TVec<int> > indicesOfOccurencesInColumn(VMat m, int col)
00092 {
00093     map< real, TVec<int> > indices; // result we will return
00094     map<real, int> counts = countOccurencesInColumn(m,col);
00095     map<real, int>::iterator it = counts.begin();
00096     map<real, int>::iterator itend = counts.end();
00097     for(; it!=itend; ++it)
00098     {
00099         indices[it->first].resize(it->second); // allocate the exact amount of memory
00100         indices[it->first].resize(0); // reset the size to 0 so we can do appends...
00101     }
00102     int l = m.length();
00103     for(int i=0; i<l; i++)
00104         indices[m(i,col)].push_back(i);
00105     return indices;
00106 }
00107 
00108 VMat grep(VMat d, int col, Vec values, const string& indexfile, bool exclude)
00109 {
00110     if(!isfile(indexfile))
00111     {
00112         IntVecFile indices(indexfile,true);
00113         for(int i=0; i<d.length(); i++)
00114         {
00115             bool contains = values.contains(d(i,col));
00116             if( (!exclude && contains) || (exclude && !contains) )
00117                 indices.append(i);
00118         }
00119     }
00120     return d.rows(indexfile);
00121 }
00122 
00123 VMat filter(VMat d, const string& indexfile)
00124 {
00125     if(!isfile(indexfile) || filesize(indexfile)==0)
00126     {
00127         IntVecFile indices(indexfile,true);
00128         Vec v(d.width());
00129         for(int i=0; i<d.length(); i++)
00130         {
00131             d->getRow(i,v);
00132             if(!v.hasMissing())
00133                 indices.append(i);
00134         }
00135     }
00136     return d.rows(indexfile);
00137 }
00138 
00139 
00140 VMat shuffle(VMat d)
00141 {
00142     Vec indices(0, d.length()-1, 1); // Range-vector
00143     shuffleElements(indices);
00144     return d.rows(indices);
00145 }
00146 
00147 VMat bootstrap(VMat d, bool reorder, bool norepeat)
00148 {
00149     Vec indices;
00150     if (norepeat)
00151     {
00152         indices = Vec(0, d.length()-1, 1); // Range-vector
00153         shuffleElements(indices);
00154         indices = indices.subVec(0,int(0.667 * d.length()));
00155         if (reorder)
00156             sortElements(indices);
00157         return d.rows(indices);
00158     }
00159     else
00160     {
00161         indices.resize(d.length());
00162         for (int i=0;i<d.length();i++)
00163             indices[i] = uniform_multinomial_sample(d.length());
00164     }
00165     if (reorder)
00166         sortElements(indices);
00167     return d.rows(indices);
00168 }
00169 
00170 
00171 VMat rebalanceNClasses(VMat inputs, int nclasses, const string& filename)
00172 {
00173     if (!isfile(filename))
00174     {
00175         IntVecFile indices(filename, true);
00176         Vec last = inputs.lastColumn()->toMat().toVecCopy();
00177         const int len = last.length();
00178         Vec capacity(nclasses);
00179         Array<Vec> index(nclasses);
00180         Array<Vec> index_used(nclasses);
00181         for (int i=0; i<nclasses; i++) index[i].resize(len);
00182         for (int i=0; i<nclasses; i++) index_used[i].resize(len);
00183         real** p_index;
00184         p_index = new real*[nclasses];
00185         for (int i=0; i<nclasses; i++) p_index[i] = index[i].data();
00186         for (int i=0; i<nclasses; i++) index_used[i].clear();
00187         for (int i=0; i<len; i++)
00188         {
00189             int class_i = int(last[i]);
00190             *p_index[class_i]++ = i;
00191             capacity[class_i]++;
00192         }
00193         for (int i=0; i<nclasses; i++) index[i].resize(int(capacity[i]));
00194         for (int i=0; i<nclasses; i++) index_used[i].resize(int(capacity[i]));
00195 
00196         Mat class_length(nclasses,2);
00197         for (int i=0; i<nclasses; i++)
00198         {
00199             class_length(i,0) = capacity[i];
00200             class_length(i,1) = i;
00201         }
00202         sortRows(class_length);
00203         Vec remap = class_length.column(1).toVecCopy();
00204 
00205         vector<int> n(nclasses,0);
00206         int new_index = -1;
00207         for (int i=0; i<len; i++)
00208         {
00209             int c = i%nclasses;
00210             int c_map = int(remap[c]);
00211             if (c == 0)
00212             {
00213                 if (fast_exact_is_equal(n[0], capacity[c_map])) n[0] = 0;
00214                 new_index = int(index[c_map][n[0]++]);
00215             }
00216             else
00217             {
00218                 if (fast_exact_is_equal(n[c], capacity[c_map]))
00219                 {
00220                     n[c] = 0;
00221                     index_used[c_map].clear();
00222                 }
00223                 bool index_found = false;
00224                 int start_pos = binary_search(index[c_map], real(new_index));
00225                 for (int j=start_pos+1; j<capacity[c_map]; j++)
00226                 {
00227                     if (fast_exact_is_equal(index_used[c_map][j], 0))
00228                     {
00229                         index_used[c_map][j] = 1;
00230                         new_index = int(index[c_map][j]);
00231                         index_found = true;
00232                         n[c]++;
00233                         break;
00234                     }
00235                 }
00236                 if (!index_found)
00237                 {
00238                     for (int j=0; j<start_pos; j++)
00239                     {
00240                         if (fast_exact_is_equal(index_used[c_map][j], 0))
00241                         {
00242                             index_used[c_map][j] = 1;
00243                             new_index = int(index[c_map][j]);
00244                             index_found = true;
00245                             n[c]++;
00246                             break;
00247                         }
00248                     }
00249                 }
00250                 if (!index_found)
00251                     PLERROR("In rebalanceNClasses:  something got wrong!");
00252             }
00253             indices.put(i, new_index);
00254         }
00255 
00256         delete[] p_index;
00257     }
00258     return inputs.rows(filename);
00259 }
00260 
00261 void fullyRebalance2Classes(VMat inputs, const string& filename, bool save_indices)
00262 {
00263     if (!isfile(filename))
00264     {
00265         int len = inputs.length();
00266 
00267         int n_zeros = 0;
00268         int n_ones = 0;
00269         Vec zeros(len);
00270         Vec ones(len);
00271 
00272         Vec last = inputs.lastColumn()->toMat().toVecCopy();
00273         for (int i=0; i<len;i++)
00274         {
00275             if (fast_exact_is_equal(last[i], 0))
00276                 zeros[n_zeros++] = i;
00277             else
00278                 ones[n_ones++] = i;
00279         }
00280         zeros.resize(n_zeros);
00281         ones.resize(n_ones);
00282 
00283         TmpFilenames tmpfile(1);
00284         string fname = save_indices ? filename : tmpfile.addFilename();
00285         IntVecFile indices(fname, true);
00286         int max_symbols = MAX(n_zeros, n_ones);
00287         for (int i=0; i<max_symbols; i++)
00288         {
00289             indices.put(2*i, int(zeros[i%n_zeros]));
00290             indices.put(2*i+1, int(ones[i%n_ones]));
00291         }
00292         if (!save_indices)
00293         {
00294             VMat vm = inputs.rows(fname);
00295             vm.save(filename);
00296         }
00297     }
00298 }
00299 
00300 VMat temporalThreshold(VMat distr, int threshold_date, bool is_before,
00301                        int yyyymmdd_col)
00302 {
00303     Vec indices(distr->length());
00304     int n_data = 0;
00305     for (int i=0; i<distr->length(); i++)
00306     {
00307         int reference_date = (int)distr(i, yyyymmdd_col);
00308         if (is_before ? reference_date<=threshold_date : reference_date>=threshold_date)
00309             indices[n_data++] = i;
00310     }
00311     indices.resize(n_data);
00312 
00313     return distr.rows(indices);
00314 }
00315 
00316 VMat temporalThreshold(VMat distr, int threshold_date, bool is_before,
00317                        int yyyy_col, int mm_col, int dd_col)
00318 {
00319     Vec indices(distr->length());
00320     int n_data = 0;
00321     for (int i=0; i<distr->length(); i++)
00322     {
00323         int reference_date = 10000*(int)distr(i, yyyy_col) + 100*(int)distr(i, mm_col) + (int)distr(i, dd_col);
00324         if (is_before ? reference_date<=threshold_date : reference_date>=threshold_date)
00325             indices[n_data++] = i;
00326     }
00327     indices.resize(n_data);
00328 
00329     return distr.rows(indices);
00330 }
00331 
00332 } // end of namespace PLearn
00333 
00334 
00335 /*
00336   Local Variables:
00337   mode:c++
00338   c-basic-offset:4
00339   c-file-style:"stroustrup"
00340   c-file-offsets:((innamespace . 0)(inline-open . 0))
00341   indent-tabs-mode:nil
00342   fill-column:79
00343   End:
00344 */
00345 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines