PLearn 0.1
Splitter.cc
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // Splitter.cc
00004 //
00005 // Copyright (C) 2002 Pascal Vincent, Frederic Morin
00006 //
00007 // Redistribution and use in source and binary forms, with or without
00008 // modification, are permitted provided that the following conditions are met:
00009 //
00010 //  1. Redistributions of source code must retain the above copyright
00011 //     notice, this list of conditions and the following disclaimer.
00012 //
00013 //  2. Redistributions in binary form must reproduce the above copyright
00014 //     notice, this list of conditions and the following disclaimer in the
00015 //     documentation and/or other materials provided with the distribution.
00016 //
00017 //  3. The name of the authors may not be used to endorse or promote
00018 //     products derived from this software without specific prior written
00019 //     permission.
00020 //
00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00031 //
00032 // This file is part of the PLearn library. For more information on the PLearn
00033 // library, go to the PLearn Web site at www.plearn.org
00034 
00035 /* *******************************************************
00036  * $Id: Splitter.cc 8333 2007-12-04 22:17:32Z saintmlx $
00037  ******************************************************* */
00038 
00040 #include "Splitter.h"
00041 #include "VMat.h"
00042 #include "ConcatRowsVMatrix.h"
00043 #include "ConcatColumnsVMatrix.h"
00044 #include <plearn/math/random.h>
00045 
00046 namespace PLearn {
00047 using namespace std;
00048 
00049 PLEARN_IMPLEMENT_ABSTRACT_OBJECT(
00050     Splitter,
00051     "Allows to split a VMat into one or several (sub-)VMats",
00052     "This class is an abstract base class for mechanisms allowing to \"split\" a\n"
00053     "dataset into one or several partitions (or \"splits\").\n"
00054     "\n"
00055     "Thus for instance a subclass can be used to implement k-fold splits (for\n"
00056     "k-fold cross validation), where each of the k splits returned by\n"
00057     "getSplit(i=0..k-1) would be an 2-element array containing the corresponding\n"
00058     "training-set and test-set.\n"
00059     "\n"
00060     "A splitter is an essential part of a PTester.\n"
00061     );
00062 
00063 void Splitter::declareOptions(OptionList& ol)
00064 {
00065     declareOption(ol,"dataset", &Splitter::dataset,
00066                   OptionBase::learntoption | OptionBase::nosave 
00067                   | OptionBase::remotetransmit,
00068                   "Dataset to split.");
00069   
00070     inherited::declareOptions(ol);
00071 }
00072 
00073 void Splitter::declareMethods(RemoteMethodMap& rmm)
00074 {
00075     // Insert a backpointer to remote methods; note that this
00076     // different than for declareOptions()
00077     rmm.inherited(inherited::_getRemoteMethodMap_());
00078 
00079     declareMethod(
00080         rmm, "setDataSet", &Splitter::setDataSet,
00081         (BodyDoc("Set this splitter's dataset\n"),
00082          ArgDoc ("the_dataset","The dataset to split")));
00083     declareMethod(
00084         rmm, "nSetsPerSplit", &Splitter::nSetsPerSplit,
00085         (BodyDoc("Returns the number of sets per split\n"),
00086          RetDoc ("the numer of sets per split")));
00087     declareMethod(
00088         rmm, "nsplits", &Splitter::nsplits,
00089         (BodyDoc(" Returns the number of available different 'splits'\n"),
00090          RetDoc (" the numer of available splits")));
00091     declareMethod(
00092         rmm, "getSplit", &Splitter::getSplit,
00093         (BodyDoc("Get one of the splits\n"),
00094          ArgDoc ("i","The split to get"),
00095          RetDoc ("The ith split (vec. of N sets)")));
00096 
00097 
00099 
00100 }
00101 void Splitter::makeDeepCopyFromShallowCopy(CopiesMap& copies)
00102 {
00103     inherited::makeDeepCopyFromShallowCopy(copies);
00104     deepCopyField(dataset, copies);
00105 }
00106 
00107 void Splitter::setDataSet(VMat the_dataset)
00108 {
00109     dataset = the_dataset;
00110 }
00111 
00112 // Useful splitting functions
00113 
00114 void split(VMat d, real test_fraction, VMat& train, VMat& test, int i, bool use_all)
00115 {
00116     int n = d.length();
00117     real ftest = test_fraction>=1.0 ? test_fraction : test_fraction*real(n);
00118     int ntest = int(ftest);
00119     int ntrain_before_test = n - (i+1)*ntest;
00120     int ntrain_after_test = i*ntest;
00121     if (use_all) {
00122         // See how many splits there are.
00123         int nsplits = int(n / ftest + 0.5);
00124         // See how many examples will be left.
00125         int nleft = n - nsplits * ntest;
00126         // Deduce how many examples to add in each split.
00127         int ntest_more = nleft / nsplits;
00128         // And, finally, how many splits will have one more example so that they are
00129         // all taken somewhere.
00130         int nsplits_one_more = nleft % nsplits;
00131         // Now recompute ntest, ntrain_before_test and ntrain_after_test.
00132         ntest = ntest + ntest_more;
00133         if (i < nsplits_one_more) {
00134             ntest++;
00135             ntrain_before_test = n - (i+1) * ntest;
00136         } else {
00137             ntrain_before_test =
00138                 n
00139                 - (nsplits_one_more)          * (ntest + 1)
00140                 - (i - nsplits_one_more + 1)  * ntest;
00141         }
00142         ntrain_after_test = n - ntest - ntrain_before_test;
00143     }
00144 
00145     test = d.subMatRows(ntrain_before_test, ntest);
00146     if(ntrain_after_test == 0)
00147         train = d.subMatRows(0,ntrain_before_test);
00148     else if(ntrain_before_test==0)
00149         train = d.subMatRows(ntest, ntrain_after_test);
00150     else
00151         train = vconcat( d.subMatRows(0,ntrain_before_test),
00152                          d.subMatRows(ntrain_before_test+ntest, ntrain_after_test) );
00153 }
00154 
00155 Vec randomSplit(VMat d, real test_fraction, VMat& train, VMat& test)
00156 {
00157     int ntest = int( test_fraction>=1.0 ?test_fraction :test_fraction*d.length() );
00158     int ntrain = d.length()-ntest;
00159     Vec indices(0, d.length()-1, 1); // Range-vector
00160     shuffleElements(indices);
00161     train = d.rows(indices.subVec(0,ntrain));
00162     test = d.rows(indices.subVec(ntrain,ntest));
00163     return indices;
00164 }
00165 
00166 void split(VMat d, real validation_fraction, real test_fraction, VMat& train, VMat& valid, VMat& test,bool do_shuffle)
00167 {
00168     int ntest = int( test_fraction>=1.0 ?test_fraction :test_fraction*d.length() );
00169     int nvalid = int( validation_fraction>=1.0 ?validation_fraction :validation_fraction*d.length() );
00170     int ntrain = d.length()-(ntest+nvalid);
00171     Vec indices(0, d.length()-1, 1); // Range-vector
00172     if (do_shuffle){
00173         cout<<"shuffle !"<<endl;
00174         shuffleElements(indices);
00175     }
00176     train = d.rows(indices.subVec(0,ntrain));
00177     valid = d.rows(indices.subVec(ntrain,nvalid));
00178     test = d.rows(indices.subVec(ntrain+nvalid,ntest));
00179     cout<<"n_train : "<<ntrain<<endl<<"n_valid : "<<nvalid<<endl<<"n_test : "<<(d.length()-ntrain+nvalid)<<endl;
00180 }
00181 
00182 void randomSplit(VMat d, real validation_fraction, real test_fraction, VMat& train, VMat& valid, VMat& test)
00183 {
00184     split(d,validation_fraction,test_fraction,train,valid,test,true);
00185 }
00186 
00187 } // end of namespace PLearn
00188 
00189 
00190 /*
00191   Local Variables:
00192   mode:c++
00193   c-basic-offset:4
00194   c-file-style:"stroustrup"
00195   c-file-offsets:((innamespace . 0)(inline-open . 0))
00196   indent-tabs-mode:nil
00197   fill-column:79
00198   End:
00199 */
00200 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines