PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // Splitter.cc 00004 // 00005 // Copyright (C) 2002 Pascal Vincent, Frederic Morin 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: Splitter.cc 8333 2007-12-04 22:17:32Z saintmlx $ 00037 ******************************************************* */ 00038 00040 #include "Splitter.h" 00041 #include "VMat.h" 00042 #include "ConcatRowsVMatrix.h" 00043 #include "ConcatColumnsVMatrix.h" 00044 #include <plearn/math/random.h> 00045 00046 namespace PLearn { 00047 using namespace std; 00048 00049 PLEARN_IMPLEMENT_ABSTRACT_OBJECT( 00050 Splitter, 00051 "Allows to split a VMat into one or several (sub-)VMats", 00052 "This class is an abstract base class for mechanisms allowing to \"split\" a\n" 00053 "dataset into one or several partitions (or \"splits\").\n" 00054 "\n" 00055 "Thus for instance a subclass can be used to implement k-fold splits (for\n" 00056 "k-fold cross validation), where each of the k splits returned by\n" 00057 "getSplit(i=0..k-1) would be an 2-element array containing the corresponding\n" 00058 "training-set and test-set.\n" 00059 "\n" 00060 "A splitter is an essential part of a PTester.\n" 00061 ); 00062 00063 void Splitter::declareOptions(OptionList& ol) 00064 { 00065 declareOption(ol,"dataset", &Splitter::dataset, 00066 OptionBase::learntoption | OptionBase::nosave 00067 | OptionBase::remotetransmit, 00068 "Dataset to split."); 00069 00070 inherited::declareOptions(ol); 00071 } 00072 00073 void Splitter::declareMethods(RemoteMethodMap& rmm) 00074 { 00075 // Insert a backpointer to remote methods; note that this 00076 // different than for declareOptions() 00077 rmm.inherited(inherited::_getRemoteMethodMap_()); 00078 00079 declareMethod( 00080 rmm, "setDataSet", &Splitter::setDataSet, 00081 (BodyDoc("Set this splitter's dataset\n"), 00082 ArgDoc ("the_dataset","The dataset to split"))); 00083 declareMethod( 00084 rmm, "nSetsPerSplit", &Splitter::nSetsPerSplit, 00085 (BodyDoc("Returns the number of sets per split\n"), 00086 RetDoc ("the numer of sets per split"))); 00087 declareMethod( 00088 rmm, "nsplits", &Splitter::nsplits, 00089 (BodyDoc(" Returns the number of available different 'splits'\n"), 00090 RetDoc (" the numer of available splits"))); 00091 declareMethod( 00092 rmm, "getSplit", &Splitter::getSplit, 00093 (BodyDoc("Get one of the splits\n"), 00094 ArgDoc ("i","The split to get"), 00095 RetDoc ("The ith split (vec. of N sets)"))); 00096 00097 00099 00100 } 00101 void Splitter::makeDeepCopyFromShallowCopy(CopiesMap& copies) 00102 { 00103 inherited::makeDeepCopyFromShallowCopy(copies); 00104 deepCopyField(dataset, copies); 00105 } 00106 00107 void Splitter::setDataSet(VMat the_dataset) 00108 { 00109 dataset = the_dataset; 00110 } 00111 00112 // Useful splitting functions 00113 00114 void split(VMat d, real test_fraction, VMat& train, VMat& test, int i, bool use_all) 00115 { 00116 int n = d.length(); 00117 real ftest = test_fraction>=1.0 ? test_fraction : test_fraction*real(n); 00118 int ntest = int(ftest); 00119 int ntrain_before_test = n - (i+1)*ntest; 00120 int ntrain_after_test = i*ntest; 00121 if (use_all) { 00122 // See how many splits there are. 00123 int nsplits = int(n / ftest + 0.5); 00124 // See how many examples will be left. 00125 int nleft = n - nsplits * ntest; 00126 // Deduce how many examples to add in each split. 00127 int ntest_more = nleft / nsplits; 00128 // And, finally, how many splits will have one more example so that they are 00129 // all taken somewhere. 00130 int nsplits_one_more = nleft % nsplits; 00131 // Now recompute ntest, ntrain_before_test and ntrain_after_test. 00132 ntest = ntest + ntest_more; 00133 if (i < nsplits_one_more) { 00134 ntest++; 00135 ntrain_before_test = n - (i+1) * ntest; 00136 } else { 00137 ntrain_before_test = 00138 n 00139 - (nsplits_one_more) * (ntest + 1) 00140 - (i - nsplits_one_more + 1) * ntest; 00141 } 00142 ntrain_after_test = n - ntest - ntrain_before_test; 00143 } 00144 00145 test = d.subMatRows(ntrain_before_test, ntest); 00146 if(ntrain_after_test == 0) 00147 train = d.subMatRows(0,ntrain_before_test); 00148 else if(ntrain_before_test==0) 00149 train = d.subMatRows(ntest, ntrain_after_test); 00150 else 00151 train = vconcat( d.subMatRows(0,ntrain_before_test), 00152 d.subMatRows(ntrain_before_test+ntest, ntrain_after_test) ); 00153 } 00154 00155 Vec randomSplit(VMat d, real test_fraction, VMat& train, VMat& test) 00156 { 00157 int ntest = int( test_fraction>=1.0 ?test_fraction :test_fraction*d.length() ); 00158 int ntrain = d.length()-ntest; 00159 Vec indices(0, d.length()-1, 1); // Range-vector 00160 shuffleElements(indices); 00161 train = d.rows(indices.subVec(0,ntrain)); 00162 test = d.rows(indices.subVec(ntrain,ntest)); 00163 return indices; 00164 } 00165 00166 void split(VMat d, real validation_fraction, real test_fraction, VMat& train, VMat& valid, VMat& test,bool do_shuffle) 00167 { 00168 int ntest = int( test_fraction>=1.0 ?test_fraction :test_fraction*d.length() ); 00169 int nvalid = int( validation_fraction>=1.0 ?validation_fraction :validation_fraction*d.length() ); 00170 int ntrain = d.length()-(ntest+nvalid); 00171 Vec indices(0, d.length()-1, 1); // Range-vector 00172 if (do_shuffle){ 00173 cout<<"shuffle !"<<endl; 00174 shuffleElements(indices); 00175 } 00176 train = d.rows(indices.subVec(0,ntrain)); 00177 valid = d.rows(indices.subVec(ntrain,nvalid)); 00178 test = d.rows(indices.subVec(ntrain+nvalid,ntest)); 00179 cout<<"n_train : "<<ntrain<<endl<<"n_valid : "<<nvalid<<endl<<"n_test : "<<(d.length()-ntrain+nvalid)<<endl; 00180 } 00181 00182 void randomSplit(VMat d, real validation_fraction, real test_fraction, VMat& train, VMat& valid, VMat& test) 00183 { 00184 split(d,validation_fraction,test_fraction,train,valid,test,true); 00185 } 00186 00187 } // end of namespace PLearn 00188 00189 00190 /* 00191 Local Variables: 00192 mode:c++ 00193 c-basic-offset:4 00194 c-file-style:"stroustrup" 00195 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00196 indent-tabs-mode:nil 00197 fill-column:79 00198 End: 00199 */ 00200 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :