PLearn 0.1
databases.h
Go to the documentation of this file.
00001 // -*- C++ -*-
00002 
00003 // PLearn (A C++ Machine Learning Library)
00004 // Copyright (C) 1998 Pascal Vincent
00005 // Copyright (C) 1999,2000 Pascal Vincent, Yoshua Bengio and University of Montreal
00006 //
00007 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 
00037 
00038 /* *******************************************************      
00039  * $Id: databases.h 8231 2007-11-07 20:50:01Z nouiz $
00040  * AUTHORS: Pascal Vincent
00041  * This file is part of the PLearn library.
00042  ******************************************************* */
00043 
00044 
00047 #ifndef DATABASES_INC
00048 #define DATABASES_INC
00049 
00050 #include "UCISpecification.h"
00051 #include <plearn/vmat/VMat.h>
00052 //#include "NistDB.h"
00053 
00054 namespace PLearn {
00055 using namespace std;
00056 
00057 
00059 Mat input2dSet(const PPath& filename="data2d.amat");
00060 
00062 void normalizeDataSets(Mat& training_set, Mat& test_set);
00063 void normalizeDataSets(Mat& training_set, Mat& validation_set, Mat& test_set);
00064 void normalizeDataSets(VMat& training_set, VMat& validation_set, VMat& test_set);
00065 
00066 // split the data_set into training_set, valid_set and test_set
00067 void splitTrainValidTest(VMat &data_set,VMat &train_set,VMat &valid_set,
00068                          real valid_fraction,VMat &test_set, real test_fraction,
00069                          bool normalize=true);
00070 
00071 // reduce nb of inputs (usefull for rapid experimentation)
00072 VMat reduceInputSize(real fraction,VMat data);
00073 
00074 // reduce nb of examples 
00075 VMat reduceDataSetSize(real fraction,VMat data);
00076 
00078 void remapClassnums(VMat& data, real remap_minval_to, real remap_maxval_to);
00079 
00081 int loadBreastCancer(VMat& training_set, VMat& validation_set, VMat& test_set, int ntrain, int nvalid, bool uniq=true);
00082 int loadDiabetes(VMat& training_set, VMat& validation_set, VMat& test_set, int ntrain, int nvalid);
00083 int loadATT800(VMat& training_set, VMat& test_set);
00084 int loadLetters(VMat& training_set, VMat& validation_set, VMat& test_set, char* letters="ABCDEFGHIJKLMNOPQRSTUVWXYZ", real validation_fraction=.2, real test_fraction=.4,bool do_shuffle=true);
00085 int loadLetters(VMat& training_set, VMat& validation_set, VMat& test_set, int n_letters, real validation_fraction=.2, real test_fraction=.4,bool do_shuffle=true);
00086 
00087 void loadCorelDatamat(int classnum, Mat& train, Mat& valid, Mat& test);
00088 Mat smoothCorelHisto(Mat& data);
00089 void loadCorel(Mat& training_set, Mat& validation_set, Mat& test_set, int negative_class=2, int positive_class=3);
00090 void loadCallxx(int year, VMat& d);
00091 
00092 VMat loadBreastCancerWisconsin(bool normalize=true, bool uniq=true);
00093 VMat loadSonar();
00094 VMat loadIonosphere();
00095 VMat loadDiabetes(bool normalize=true);
00096 VMat loadPimaIndians(bool normalize=true);
00097 VMat loadLetters(const char* class0, const char* class1, bool normalize=true);
00098 VMat loadLetters(bool normalize=true);
00099 VMat loadLetters(int n_letters, bool do_shuffle);
00100 void loadUSPS(VMat& trainset, VMat& testset, bool use_smooth=true);
00101 VMat loadUSPS(bool use_smooth=true);
00102 VMat loadHousing(bool normalize=true);
00107 void loadUCI(VMat& trainset, VMat& testset, VMat& allset, string db_spec, string id, bool &normalize, const string& type);
00109 void loadUCISet(VMat& data, string file, PP<UCISpecification> uci_spec);
00111 void loadUCIAMat(VMat& data, string file, PP<UCISpecification> uci_spec);
00112 
00114 void loadUCISet(VMat& data, PP<UCISpecification> uci_spec);
00115 
00129 inline string loadClassificationDatasetHelp()
00130 {
00131     return "  Preprogrammed datasets are: \n"
00132         "    2d \n"
00133         "    letters \n"
00134         "    breast \n"
00135         "    usps \n"
00136         "    mnist \n"
00137         "    usps0 ... usps9 \n"
00138         "    nist0 ... usps9 \n"
00139         "    They can optionally be followed by :size in which case only the 'size' \n"
00140         "    first rows will be kept. \n";
00141 }
00142 
00143 inline string loadUCIDatasetsHelp()
00144 {
00145     return 
00146         "In order to access the UCI datasets the dataset name must start with UCI_. The possible\n"
00147         "dataset names are:\n"
00148         "    UCI_annealing\n"
00149         "    UCI_heart-disease_ID=va\n"
00150         "    UCI_heart-disease_ID=cleveland\n"
00151         "    UCI_heart-disease_ID=hungarian\n"
00152         "    UCI_heart-disease_ID=switzerland\n"
00153         "    UCI_housing\n"
00154         "    UCI_image\n"
00155         "    UCI_ionosphere\n"
00156         "    UCI_iris\n"
00157         "    UCI_iris_ID=bezdekIris\n"
00158         "    UCI_isolet_ID=1+2+3+4\n"
00159         "    UCI_isolet_ID=5\n"
00160         "    UCI_monks-problems_ID=monks-1\n"
00161         "    UCI_monks-problems_ID=monks-2\n"
00162         "    UCI_monks-problems_ID=monks-3\n"
00163         "    UCI_mushroom\n"
00164         "    UCI_musk_ID=clean1\n"
00165         "    UCI_musk_ID=clean2\n"
00166         "    UCI_page-blocks\n"
00167         "    UCI_pima-indians-diabetes\n"
00168         "    UCI_solar-flare_ID=data1\n"
00169         "    UCI_solar-flare_ID=data2\n"
00170         "    UCI_statlog_ID=german\n"
00171         "    UCI_statlog_ID=australian\n"
00172         "    UCI_statlog_ID=heart\n"
00173         "    UCI_statlog_ID=satimage\n"
00174         "    UCI_statlog_ID=segment\n"
00175         "    UCI_statlog_ID=vehicle\n"
00176         "    UCI_statlog_ID=shuttle\n"
00177         "    UCI_thyroid-disease_ID=allbp\n"
00178         "    UCI_thyroid-disease_ID=allhyper\n"
00179         "    UCI_thyroid-disease_ID=allhypo\n"
00180         "    UCI_thyroid-disease_ID=allrep\n"
00181         "    UCI_thyroid-disease_ID=ann\n"
00182         "    UCI_thyroid-disease_ID=dis\n"
00183         "    UCI_thyroid-disease_ID=sick\n"
00184         "    UCI_thyroid-disease_ID=hypothyroid\n"
00185         "    UCI_thyroid-disease_ID=new-thyroid\n"
00186         "    UCI_thyroid-disease_ID=sick-euthyroid\n"
00187         "    UCI_thyroid-disease_ID=thyroid0387\n"
00188         "    UCI_abalone\n"
00189         "    UCI_adult\n"
00190         "    UCI_covtype\n"
00191         "    UCI_internet_ads\n"
00192         "    UCI_nursery\n"
00193         "    UCI_pendigits\n"
00194         "    UCI_spambase\n"
00195         "    UCI_yeast\n" 
00196         "In order to access the UCI KDD datasets the dataset name must start with UCI_KDD_. The possible\n"
00197         "dataset names are:\n"
00198         "    UCI_KDD_corel_ID=ColorMoments\n"
00199         "    UCI_KDD_corel_ID=ColorHistogram\n"
00200         "    UCI_KDD_corel_ID=CoocTexture\n"
00201         "    UCI_KDD_corel_ID=LayoutHistogram\n"
00202         "    UCI_KDD_insurance-bench\n";
00203 }
00204 
00205 void loadClassificationDataset(const string& dbname, int& inputsize, int& nclasses, VMat& trainset, VMat& testset, bool normalizeinputs, VMat& allset);
00206 
00207 
00208 } // end of namespace PLearn
00209 
00210 
00211 #endif
00212 
00213 
00214 /*
00215   Local Variables:
00216   mode:c++
00217   c-basic-offset:4
00218   c-file-style:"stroustrup"
00219   c-file-offsets:((innamespace . 0)(inline-open . 0))
00220   indent-tabs-mode:nil
00221   fill-column:79
00222   End:
00223 */
00224 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines