PLearn 0.1
VMatCommand.cc
Go to the documentation of this file.
00001 
00002 // -*- C++ -*-
00003 
00004 // VMatCommand.cc
00005 //
00006 // Copyright (C) 2003  Pascal Vincent 
00007 // 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 /* *******************************************************      
00037  * $Id: VMatCommand.cc 10303 2009-10-06 20:33:55Z nouiz $ 
00038  ******************************************************* */
00039 
00041 #include <plearn/misc/vmatmain.h>
00042 #include "VMatCommand.h"
00043 #include <plearn/db/getDataSet.h>
00044 #include <plearn/base/lexical_cast.h>
00045 #include <plearn/vmat/FileVMatrix.h>
00046 #include <plearn/base/plerror.h>
00047 #include <plearn/io/fileutils.h>
00048 
00049 namespace PLearn {
00050 using namespace std;
00051 
00053 PLearnCommandRegistry VMatCommand::reg_(new VMatCommand);
00054 
00055 VMatCommand::VMatCommand():
00056     PLearnCommand(
00057         "vmat",
00058         "Examination and manipulation of vmat datasets",
00059         "Usage: vmat info <dataset>... \n"
00060         "       Will info about dataset (size, etc..)\n"
00061         "   or: vmat fields <dataset> [name_only] [transpose] \n"
00062         "       To list the fields with their names (if 'name_only' is specified, the indexes won't be displayed,\n"
00063         "       and if 'transpose' is also added, the fields will be listed on a single line)\n"
00064         "   or: vmat fieldinfo <dataset> <fieldname_or_num> [--bin]\n"
00065         "       To display statistics for that field \n"
00066         "   or: vmat bbox <dataset> [<extra_percent>] \n"
00067         "       To display the data bounding box (i.e., for each field, its min and max, possibly extended by +-extra_percent ex: 0.10 for +-10% of the data range )\n"
00068         "   or: vmat cat <dataset>... [--precision=N] [<optional_vpl_filtering_code>]\n"
00069         "       To display the dataset \n"
00070         "   or: vmat sascat <dataset.vmat> <dataset.txt>\n"
00071         "       To output in <dataset.txt> the dataset in SAS-like tab-separated format with field names on the first line\n"
00072         "   or: vmat view <dataset>...\n"
00073         "       Interactive display to browse on the data. \n"
00074         "       ( will work only if your executable includes commands/PLearnCommands/VMatViewCommand.h )\n"
00075         "   or: vmat stats <dataset> \n"
00076         "       Will display basic statistics for each field \n"
00077         "   or: vmat convert <source> <destination> [--cols=col1,col2,col3,...] [--mat_to_mem] [--save_vmat] [--force_float]\n"
00078         "       To convert any dataset into a .amat, .pmat, .dmat, .vmat, .csv or .arff format. \n"
00079         "       The extension of the destination is used to determine the format you want. \n"
00080         "       WARNING: In dmat format, all double are currently casted to float!\n"
00081         "       If the option --cols is specified, it requests to keep only the given columns\n"
00082         "       (no space between the commas and the columns); columns can be given either as a\n"
00083         "       number (zero-based) or a column name (string).  You can also specify a range,\n"
00084         "       such as 0-18, or any combination thereof, e.g. 5,3,8-18,Date,74-85\n"
00085         "       If the option --mat_to_mem is specified, we load the original matrix into memory\n"
00086         "       If the option --save_vmat is specified, we save the source vmat in the destination metadatadir\n"
00087         "       If the option --update is specified, we generate the <destination> only when the <source> file is newer\n"
00088         "         then the destination file or when the destination file is missing\n"
00089         "       If .pmat is specified as the destination file, the option --force_float will save the data in float format\n"
00090         "       If .csv (Comma-Separated Value) is specified as the destination file, the \n"
00091         "       following additional options are also supported:\n"
00092         "         --skip-missings: if a row (after selecting the appropriate columns) contains\n"
00093         "                          one or more missing values, it is skipped during export\n"
00094         "         --precision=N:   a maximum of N digits is printed after the decimal point\n"
00095         "         --delimiter=C:   use character C as the field delimiter (default = ',')\n"
00096         "         --convert-date:  first column is assumed to be in CYYMMDD format; it is\n"
00097         "                          exported as YYYYMMDD in the .csv file (19000000 is added)\n"
00098         "       If .arff (Attribute-Relation File Format) is specified as the destination file, the \n"
00099         "       following additional options are also supported:\n"
00100         "         --skip-missings: if a row (after selecting the appropriate columns) contains\n"
00101         "                          one or more missing values, it is skipped during export\n"
00102         "         --precision=N:   a maximum of N digits is printed after the decimal point\n"
00103         "         --date-cols=col1,col2,...:  we flag the specified columns as a date\n"
00104         "                                     we also convert the date from CYYMMDD to YYYYMMDD (if necessary)\n"
00105         "   or: vmat gendef <source> [binnum1 binnum2 ...] \n"
00106         "       Generate stats for dataset (will put them in its associated metadatadir). \n"
00107         "   or: vmat genvmat <source_dataset> <dest_vmat> [binned{num} | onehot{num} | normalized]\n"
00108         "       Will generate a template .vmat file with all the fields of the source preprocessed\n"
00109         "       with the processing you specify\n"
00110         "   or: vmat genkfold <source_dataset> <fileprefix> <kvalue>\n"
00111         "       Will generate <kvalue> pairs of .vmat that are splitted so they can be used for kfold trainings\n"
00112         "       The first .vmat-pair will be named <fileprefix>_train_1.vmat (all source_dataset except the first 1/k)\n"
00113         "       and <fileprefix>_test_1.vmat (the first 1/k of <source_dataset>\n"
00114         "   or: vmat diff <dataset1> <dataset2> [<tolerance> [<verbose>]]\n"
00115         "       Will report all elements that differ by more than tolerance (default = 1e-6),\n"
00116         "       in an absolute way for numbers less than 1, and in a relative way otherwise.\n"
00117         "       If verbose==0 then print only total number of differences.\n"
00118         "   or: vmat cdf <dataset> [<dataset> ...] \n"
00119         "       To interactively display cumulative density function for each field \n"
00120         "       along with its basic statistics \n"
00121         // "   or: vmat cond <dataset> <condfield#> \n"
00122         // "       Interactive display of coditional statistics conditioned on the \n"
00123         // "       conditioning field <condfield#> \n"
00124         "   or: vmat diststat <dataset> <inputsize>\n"
00125         "       Will compute and output basic statistics on the euclidean distance \n"
00126         "       between two consecutive input points \n"
00127         "   or: vmat dictionary <dataset>\n"
00128         "       Will create <dataset>.field#.dict, where # is the\n"
00129         "       field (column) number, starting at 0. Those files contain the plearn\n"
00130         "       scripts of the Dictionary objets for each field.\n"
00131         "   or: vmat catstr <dataset>\n"
00132         "       Will output the content of <dataset>, using its string mappings\n"
00133         "   or: vmat compare_stats <dataset1> <dataset2> [stdev threshold] [missing threshold]\n"
00134         "       Will compare stats from dataset1 to dataset2\n\n"
00135         "   or: vmat compare_stats_ks <dataset1> <dataset2> [--mat_to_mem]\n"
00136         "       Will compare stats from dataset2 to dataset2 with "
00137         "       Kolmogorov-Smirnov 2 samples statistic\n\n"
00138         "   or: vmat mtime <dataset>\n"
00139         "       Print the mtime of a dataset\n"
00140         "   or: vmat pmat_float_save <dataset>...\n"
00141         "       Print the size of the new pmat file in float format, the size of saved bytes, the maximum value difference, the maximum relative difference in value and the column of the maximum value and maximum relative value."
00142         "<dataset> is a parameter understandable by getDataSet: \n"
00143 
00144         + getDataSetHelp()
00145         ) 
00146 {}
00147 
00148 
00150 void VMatCommand::run(const vector<string>& args)
00151 {
00152     // new vmat sub-commands
00153     string command = args[0];
00154     if(command=="bbox")
00155     {
00156         string dataspec = args[1];
00157         real extra_percent = 0.00;
00158         if(args.size()==3)
00159             extra_percent = toreal(args[2]);
00160       
00161         VMat vm = getDataSet(dataspec);
00162         TVec< pair<real, real> > bbox = vm->getBoundingBox(extra_percent);
00163         for(int k=0; k<bbox.length(); k++)
00164             cout << bbox[k].first << " : " << bbox[k].second << endl;
00165     }
00166     else if (command == "view")
00167     {
00168         // The 'view' command has been moved to VMatViewCommand (to avoid
00169         // a forced dependency on the curses library).
00170         vector<string> new_args(args.size() - 1);
00171         for (size_t i = 1; i < args.size(); i++)
00172             new_args[i - 1] = args[i];
00173         PLearnCommandRegistry::run("vmat_view", new_args);
00174     }
00175     else if (command == "pmat_float_save")
00176     {
00177 #ifdef USEFLOAT
00178         PLERROR("vmat pmat_float_save don't work correctly when compiled in float.");
00179 #endif
00180         PLCHECK(args.size()>1);
00181         for(uint f=1;f<args.size();f++){
00182             PPath dataspec = args[f];
00183             if(!isfile(dataspec)){
00184                 PLWARNING("%s is not a file!",dataspec.c_str());
00185                 continue;
00186             }
00187                 
00188             VMat vm = getDataSet(dataspec);
00189             int64_t orig_size = vm->getSizeOnDisk();
00190             int saved_size=-1;
00191             int new_size=-1;
00192             if(orig_size!=-1){
00193                 FileVMatrix n = FileVMatrix(dataspec+"dummy",vm.length(),vm.width(),true,false);
00194                 new_size=n.getSizeOnDisk();
00195                 saved_size=orig_size-new_size;
00196             }
00197 
00198             Vec v(vm->width());
00199             double max_diff=0;
00200             double max_rel_diff=0;
00201             int col_max = -1;
00202             int col_max_rel = -1;
00203             for(int i=0;i<vm->length();i++){
00204                 vm->getRow(i,v);
00205                 for(int j=0;j<vm->width();j++){
00206                     double diff = v[j]-float(v[j]);
00207                     if(max_diff<diff){
00208                         max_diff=diff;
00209                         col_max = j;
00210                     }
00211                     double rel_diff = diff/v[j];
00212                     if(max_rel_diff<rel_diff){
00213                         max_rel_diff=rel_diff;
00214                         col_max_rel = j;
00215                     }
00216                 }
00217             }
00218             cout<<"new_size="<<new_size<<" saved_size="<< saved_size <<" max_difference="<< max_diff <<" max_relatif_difference="<<max_rel_diff<<" col_max="<<col_max<<" col_max_rel="<<col_max_rel<<endl;
00219         }
00220     }
00221     else
00222     {
00223         // Dirty hack to plug into old vmatmain code
00224         // Eventually, should get vmatmain code in here and clean
00225 
00226         int argc = (int)args.size()+1;
00227         char** argv = new char*[argc];
00228         string commandname = "vmat";
00229         argv[0] = const_cast<char*>(commandname.c_str());
00230         for(int i=1 ; i<argc; i++)
00231             argv[i] = const_cast<char*>(args[i-1].c_str());
00232         vmatmain(argc, argv);
00233         delete[] argv;
00234     }
00235 }
00236 
00237 } // end of namespace PLearn
00238 
00239 
00240 /*
00241   Local Variables:
00242   mode:c++
00243   c-basic-offset:4
00244   c-file-style:"stroustrup"
00245   c-file-offsets:((innamespace . 0)(inline-open . 0))
00246   indent-tabs-mode:nil
00247   fill-column:79
00248   End:
00249 */
00250 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines