PLearn 0.1
|
00001 00002 // -*- C++ -*- 00003 00004 // VMatCommand.cc 00005 // 00006 // Copyright (C) 2003 Pascal Vincent 00007 // 00008 // Redistribution and use in source and binary forms, with or without 00009 // modification, are permitted provided that the following conditions are met: 00010 // 00011 // 1. Redistributions of source code must retain the above copyright 00012 // notice, this list of conditions and the following disclaimer. 00013 // 00014 // 2. Redistributions in binary form must reproduce the above copyright 00015 // notice, this list of conditions and the following disclaimer in the 00016 // documentation and/or other materials provided with the distribution. 00017 // 00018 // 3. The name of the authors may not be used to endorse or promote 00019 // products derived from this software without specific prior written 00020 // permission. 00021 // 00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00032 // 00033 // This file is part of the PLearn library. For more information on the PLearn 00034 // library, go to the PLearn Web site at www.plearn.org 00035 00036 /* ******************************************************* 00037 * $Id: VMatCommand.cc 10303 2009-10-06 20:33:55Z nouiz $ 00038 ******************************************************* */ 00039 00041 #include <plearn/misc/vmatmain.h> 00042 #include "VMatCommand.h" 00043 #include <plearn/db/getDataSet.h> 00044 #include <plearn/base/lexical_cast.h> 00045 #include <plearn/vmat/FileVMatrix.h> 00046 #include <plearn/base/plerror.h> 00047 #include <plearn/io/fileutils.h> 00048 00049 namespace PLearn { 00050 using namespace std; 00051 00053 PLearnCommandRegistry VMatCommand::reg_(new VMatCommand); 00054 00055 VMatCommand::VMatCommand(): 00056 PLearnCommand( 00057 "vmat", 00058 "Examination and manipulation of vmat datasets", 00059 "Usage: vmat info <dataset>... \n" 00060 " Will info about dataset (size, etc..)\n" 00061 " or: vmat fields <dataset> [name_only] [transpose] \n" 00062 " To list the fields with their names (if 'name_only' is specified, the indexes won't be displayed,\n" 00063 " and if 'transpose' is also added, the fields will be listed on a single line)\n" 00064 " or: vmat fieldinfo <dataset> <fieldname_or_num> [--bin]\n" 00065 " To display statistics for that field \n" 00066 " or: vmat bbox <dataset> [<extra_percent>] \n" 00067 " To display the data bounding box (i.e., for each field, its min and max, possibly extended by +-extra_percent ex: 0.10 for +-10% of the data range )\n" 00068 " or: vmat cat <dataset>... [--precision=N] [<optional_vpl_filtering_code>]\n" 00069 " To display the dataset \n" 00070 " or: vmat sascat <dataset.vmat> <dataset.txt>\n" 00071 " To output in <dataset.txt> the dataset in SAS-like tab-separated format with field names on the first line\n" 00072 " or: vmat view <dataset>...\n" 00073 " Interactive display to browse on the data. \n" 00074 " ( will work only if your executable includes commands/PLearnCommands/VMatViewCommand.h )\n" 00075 " or: vmat stats <dataset> \n" 00076 " Will display basic statistics for each field \n" 00077 " or: vmat convert <source> <destination> [--cols=col1,col2,col3,...] [--mat_to_mem] [--save_vmat] [--force_float]\n" 00078 " To convert any dataset into a .amat, .pmat, .dmat, .vmat, .csv or .arff format. \n" 00079 " The extension of the destination is used to determine the format you want. \n" 00080 " WARNING: In dmat format, all double are currently casted to float!\n" 00081 " If the option --cols is specified, it requests to keep only the given columns\n" 00082 " (no space between the commas and the columns); columns can be given either as a\n" 00083 " number (zero-based) or a column name (string). You can also specify a range,\n" 00084 " such as 0-18, or any combination thereof, e.g. 5,3,8-18,Date,74-85\n" 00085 " If the option --mat_to_mem is specified, we load the original matrix into memory\n" 00086 " If the option --save_vmat is specified, we save the source vmat in the destination metadatadir\n" 00087 " If the option --update is specified, we generate the <destination> only when the <source> file is newer\n" 00088 " then the destination file or when the destination file is missing\n" 00089 " If .pmat is specified as the destination file, the option --force_float will save the data in float format\n" 00090 " If .csv (Comma-Separated Value) is specified as the destination file, the \n" 00091 " following additional options are also supported:\n" 00092 " --skip-missings: if a row (after selecting the appropriate columns) contains\n" 00093 " one or more missing values, it is skipped during export\n" 00094 " --precision=N: a maximum of N digits is printed after the decimal point\n" 00095 " --delimiter=C: use character C as the field delimiter (default = ',')\n" 00096 " --convert-date: first column is assumed to be in CYYMMDD format; it is\n" 00097 " exported as YYYYMMDD in the .csv file (19000000 is added)\n" 00098 " If .arff (Attribute-Relation File Format) is specified as the destination file, the \n" 00099 " following additional options are also supported:\n" 00100 " --skip-missings: if a row (after selecting the appropriate columns) contains\n" 00101 " one or more missing values, it is skipped during export\n" 00102 " --precision=N: a maximum of N digits is printed after the decimal point\n" 00103 " --date-cols=col1,col2,...: we flag the specified columns as a date\n" 00104 " we also convert the date from CYYMMDD to YYYYMMDD (if necessary)\n" 00105 " or: vmat gendef <source> [binnum1 binnum2 ...] \n" 00106 " Generate stats for dataset (will put them in its associated metadatadir). \n" 00107 " or: vmat genvmat <source_dataset> <dest_vmat> [binned{num} | onehot{num} | normalized]\n" 00108 " Will generate a template .vmat file with all the fields of the source preprocessed\n" 00109 " with the processing you specify\n" 00110 " or: vmat genkfold <source_dataset> <fileprefix> <kvalue>\n" 00111 " Will generate <kvalue> pairs of .vmat that are splitted so they can be used for kfold trainings\n" 00112 " The first .vmat-pair will be named <fileprefix>_train_1.vmat (all source_dataset except the first 1/k)\n" 00113 " and <fileprefix>_test_1.vmat (the first 1/k of <source_dataset>\n" 00114 " or: vmat diff <dataset1> <dataset2> [<tolerance> [<verbose>]]\n" 00115 " Will report all elements that differ by more than tolerance (default = 1e-6),\n" 00116 " in an absolute way for numbers less than 1, and in a relative way otherwise.\n" 00117 " If verbose==0 then print only total number of differences.\n" 00118 " or: vmat cdf <dataset> [<dataset> ...] \n" 00119 " To interactively display cumulative density function for each field \n" 00120 " along with its basic statistics \n" 00121 // " or: vmat cond <dataset> <condfield#> \n" 00122 // " Interactive display of coditional statistics conditioned on the \n" 00123 // " conditioning field <condfield#> \n" 00124 " or: vmat diststat <dataset> <inputsize>\n" 00125 " Will compute and output basic statistics on the euclidean distance \n" 00126 " between two consecutive input points \n" 00127 " or: vmat dictionary <dataset>\n" 00128 " Will create <dataset>.field#.dict, where # is the\n" 00129 " field (column) number, starting at 0. Those files contain the plearn\n" 00130 " scripts of the Dictionary objets for each field.\n" 00131 " or: vmat catstr <dataset>\n" 00132 " Will output the content of <dataset>, using its string mappings\n" 00133 " or: vmat compare_stats <dataset1> <dataset2> [stdev threshold] [missing threshold]\n" 00134 " Will compare stats from dataset1 to dataset2\n\n" 00135 " or: vmat compare_stats_ks <dataset1> <dataset2> [--mat_to_mem]\n" 00136 " Will compare stats from dataset2 to dataset2 with " 00137 " Kolmogorov-Smirnov 2 samples statistic\n\n" 00138 " or: vmat mtime <dataset>\n" 00139 " Print the mtime of a dataset\n" 00140 " or: vmat pmat_float_save <dataset>...\n" 00141 " Print the size of the new pmat file in float format, the size of saved bytes, the maximum value difference, the maximum relative difference in value and the column of the maximum value and maximum relative value." 00142 "<dataset> is a parameter understandable by getDataSet: \n" 00143 00144 + getDataSetHelp() 00145 ) 00146 {} 00147 00148 00150 void VMatCommand::run(const vector<string>& args) 00151 { 00152 // new vmat sub-commands 00153 string command = args[0]; 00154 if(command=="bbox") 00155 { 00156 string dataspec = args[1]; 00157 real extra_percent = 0.00; 00158 if(args.size()==3) 00159 extra_percent = toreal(args[2]); 00160 00161 VMat vm = getDataSet(dataspec); 00162 TVec< pair<real, real> > bbox = vm->getBoundingBox(extra_percent); 00163 for(int k=0; k<bbox.length(); k++) 00164 cout << bbox[k].first << " : " << bbox[k].second << endl; 00165 } 00166 else if (command == "view") 00167 { 00168 // The 'view' command has been moved to VMatViewCommand (to avoid 00169 // a forced dependency on the curses library). 00170 vector<string> new_args(args.size() - 1); 00171 for (size_t i = 1; i < args.size(); i++) 00172 new_args[i - 1] = args[i]; 00173 PLearnCommandRegistry::run("vmat_view", new_args); 00174 } 00175 else if (command == "pmat_float_save") 00176 { 00177 #ifdef USEFLOAT 00178 PLERROR("vmat pmat_float_save don't work correctly when compiled in float."); 00179 #endif 00180 PLCHECK(args.size()>1); 00181 for(uint f=1;f<args.size();f++){ 00182 PPath dataspec = args[f]; 00183 if(!isfile(dataspec)){ 00184 PLWARNING("%s is not a file!",dataspec.c_str()); 00185 continue; 00186 } 00187 00188 VMat vm = getDataSet(dataspec); 00189 int64_t orig_size = vm->getSizeOnDisk(); 00190 int saved_size=-1; 00191 int new_size=-1; 00192 if(orig_size!=-1){ 00193 FileVMatrix n = FileVMatrix(dataspec+"dummy",vm.length(),vm.width(),true,false); 00194 new_size=n.getSizeOnDisk(); 00195 saved_size=orig_size-new_size; 00196 } 00197 00198 Vec v(vm->width()); 00199 double max_diff=0; 00200 double max_rel_diff=0; 00201 int col_max = -1; 00202 int col_max_rel = -1; 00203 for(int i=0;i<vm->length();i++){ 00204 vm->getRow(i,v); 00205 for(int j=0;j<vm->width();j++){ 00206 double diff = v[j]-float(v[j]); 00207 if(max_diff<diff){ 00208 max_diff=diff; 00209 col_max = j; 00210 } 00211 double rel_diff = diff/v[j]; 00212 if(max_rel_diff<rel_diff){ 00213 max_rel_diff=rel_diff; 00214 col_max_rel = j; 00215 } 00216 } 00217 } 00218 cout<<"new_size="<<new_size<<" saved_size="<< saved_size <<" max_difference="<< max_diff <<" max_relatif_difference="<<max_rel_diff<<" col_max="<<col_max<<" col_max_rel="<<col_max_rel<<endl; 00219 } 00220 } 00221 else 00222 { 00223 // Dirty hack to plug into old vmatmain code 00224 // Eventually, should get vmatmain code in here and clean 00225 00226 int argc = (int)args.size()+1; 00227 char** argv = new char*[argc]; 00228 string commandname = "vmat"; 00229 argv[0] = const_cast<char*>(commandname.c_str()); 00230 for(int i=1 ; i<argc; i++) 00231 argv[i] = const_cast<char*>(args[i-1].c_str()); 00232 vmatmain(argc, argv); 00233 delete[] argv; 00234 } 00235 } 00236 00237 } // end of namespace PLearn 00238 00239 00240 /* 00241 Local Variables: 00242 mode:c++ 00243 c-basic-offset:4 00244 c-file-style:"stroustrup" 00245 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00246 indent-tabs-mode:nil 00247 fill-column:79 00248 End: 00249 */ 00250 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :