PLearn 0.1
TxtmatCommand.cc
Go to the documentation of this file.
00001 
00002 // -*- C++ -*-
00003 
00004 // TxtmatCommand.cc
00005 //
00006 // Copyright (C) 2003-2004 ApSTAT Technologies Inc. 
00007 // 
00008 // Redistribution and use in source and binary forms, with or without
00009 // modification, are permitted provided that the following conditions are met:
00010 // 
00011 //  1. Redistributions of source code must retain the above copyright
00012 //     notice, this list of conditions and the following disclaimer.
00013 // 
00014 //  2. Redistributions in binary form must reproduce the above copyright
00015 //     notice, this list of conditions and the following disclaimer in the
00016 //     documentation and/or other materials provided with the distribution.
00017 // 
00018 //  3. The name of the authors may not be used to endorse or promote
00019 //     products derived from this software without specific prior written
00020 //     permission.
00021 // 
00022 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR
00023 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
00024 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
00025 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00026 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
00027 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
00028 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
00029 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
00030 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
00031 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00032 // 
00033 // This file is part of the PLearn library. For more information on the PLearn
00034 // library, go to the PLearn Web site at www.plearn.org
00035 
00036 // Authors: Pascal Vincent
00037 
00038 /* *******************************************************      
00039  * $Id: TxtmatCommand.cc 3995 2005-08-25 13:58:23Z chapados $ 
00040  ******************************************************* */
00041 
00043 #include "TxtmatCommand.h"
00044 #include <plearn/base/stringutils.h>
00045 #include <plearn/base/ProgressBar.h>
00046 #include <curses.h>
00047 
00048 namespace PLearn {
00049 using namespace std;
00050 
00052 PLearnCommandRegistry TxtmatCommand::reg_(new TxtmatCommand);
00053 
00054 TxtmatCommand::TxtmatCommand() :
00055     PLearnCommand("txtmat",
00056 
00057                   "Examination of text matrices (.txtmat files containing a TextFilesVMatrix object)",
00058 
00059                   "Usage: txtmat info <file.txtmat> \n"
00060                   "       prints basic info: length, width, ...\n"
00061                   "   or: txtmat fields <file.txtmat> \n"
00062                   "       prints the list of fields \n"
00063                   "   or: txtmat lines <file.txtmat> [<start_row>] [<nrows>] \n"
00064                   "       prints the raw lines (raw meaning not even split in fields) \n"
00065                   "   or: txtmat view <file.txtmat> \n"
00066                   "       displays the txtmat as text fields using curses \n" 
00067                   "   or: txtmat row <file.txtmat> <rownum> [+] \n"
00068                   "       prints all the fields of the given row. \n"
00069                   "       If + is specified, it will also print their conversion to real \n"
00070                   "   or: txtmat col <file.txtmat> <colnum or fieldname> [<start_row> <nrows>] [+]\n"
00071                   "       prints the text content of the specified column or field. \n"
00072                   "       If + is specified, it will also print their conversion to real \n"
00073                   "   or: txtmat scan <file.txtmat> \n"
00074                   "       Will call getRow on every row, and then save the string mappings. \n"
00075                   "       This insures that all is checked and mappings are built if needed. \n"
00076                   "   or: txtmat counts <file.txtmat> \n"
00077                   "       Will generate a counts/ directory in the metadatadir and record, \n"
00078                   "       for each field with a mapping, how many times a particular mapped string was encountered \n"
00079                   "   or: txtmat savemap <file.txtmat> \n"
00080                   "       Will generate and save standard VMatrix string mapping from the txtmat \n"
00081                   "       specific mapping. \n"
00082                   "   or: txtmat checkstuff <file.txtmat> Ex. de petite fonction pour Charles\n"
00083         ) 
00084 {}
00085 
00086 void TxtmatCommand::checkstuff(PP<TextFilesVMatrix> vm)
00087 {
00088     int l = vm->length();
00089     TVec< pair<string, string> > fieldspec = vm->fieldspec;
00090     
00091     int NOPOL = vm->getIndexOfTextField("NOPOL");
00092     int NOVEH = vm->getIndexOfTextField("NOVEH");
00093     int DTEFPO = vm->getIndexOfTextField("DTEFPO");
00094     int UNTA1997 = vm->getIndexOfTextField("UNTA1997");
00095     int UNTA1998 = vm->getIndexOfTextField("UNTA1998");
00096     int UNTA1999 = vm->getIndexOfTextField("UNTA1999");
00097     int UNTA2000 = vm->getIndexOfTextField("UNTA2000");
00098     int UNTA2001 = vm->getIndexOfTextField("UNTA2001");
00099 
00100     double sum_1997 = 0;
00101     double sum_1998 = 0;
00102     double sum_1999 = 0;
00103     double sum_2000 = 0;
00104     double sum_2001 = 0;
00105 
00106     TVec<string> v = vm->getTextFields(0);
00107     string prev_NOPOL = v[NOPOL];
00108     string prev_NOVEH = v[NOVEH];
00109     string prev_DTEFPO = v[DTEFPO];
00110 
00111     for(int i=0; i<l; i++)
00112     {
00113         v = vm->getTextFields(i);
00114         if(prev_NOPOL!=v[NOPOL] || prev_NOVEH!=v[NOVEH] || prev_DTEFPO!=v[DTEFPO] || i==l-1)
00115         {
00116             cout << prev_NOPOL << " " << prev_NOVEH << " " << prev_DTEFPO << " \t" 
00117                  << sum_1997 << " " << sum_1998 << " " << sum_1999 << " " << sum_2000 << " " << sum_2001 << " \t"
00118                  << sum_1997+sum_1998+sum_1999+sum_2000+sum_2001 << endl;
00119             sum_1997 = sum_1998 = sum_1999 = sum_2000 = sum_2001 = 0;
00120             prev_NOPOL = v[NOPOL];
00121             prev_NOVEH = v[NOVEH];
00122             prev_DTEFPO = v[DTEFPO];
00123         }
00124         else
00125         {
00126             if(v[UNTA1997]!="")
00127                 sum_1997 += atof(v[UNTA1997].c_str());
00128             if(v[UNTA1998]!="")
00129                 sum_1998 += atof(v[UNTA1998].c_str());
00130             if(v[UNTA1999]!="")
00131                 sum_1999 += atof(v[UNTA1999].c_str());
00132             if(v[UNTA2000]!="")
00133                 sum_2000 += atof(v[UNTA2000].c_str());
00134             if(v[UNTA2001]!="")
00135                 sum_2001 += atof(v[UNTA2001].c_str());
00136         }
00137     }
00138 }
00139 
00140 void TxtmatCommand::view(PP<TextFilesVMatrix> vm, int lin, int col)
00141 {
00142     int nfields = vm->fieldspec.size();
00143     int l = vm->length();
00144 
00145     initscr();
00146     cbreak();
00147     noecho();
00148     keypad(stdscr,TRUE);
00149   
00150     int key= 0, curl= 0, curc= 0;
00151     bool view_vec = false;
00152   
00153     while(key != 'q' && key != 'Q')
00154     {
00155         erase();
00156 
00157         for(int j= 0; (j+2)*10 <= COLS && j+col<nfields; ++j)
00158         {
00159             string s= vm->fieldspec[j+col].first;
00160             mvprintw(0,(j+1)*10-1," %9s", s.substr(0,9).c_str());
00161             if(s.length() > 9)
00162                 mvprintw(1,(j+1)*10-1," %9s", s.substr(9,9).c_str());
00163         }
00164         for(int i= 0; i < LINES-3 && i+lin<l; ++i)
00165         {
00166             TVec<string> fields = vm->getTextFields(i+lin);
00167 
00168             mvprintw(i+2,0,"%d", i+lin);
00169             for(int j= 0; (j+2)*10 <= COLS && j+col<nfields; ++j)
00170             {
00171                 if(i == curl || j == curc)
00172                     attron(A_REVERSE);
00173 
00174                 // real x= vm(i+lin, j+col);
00175                 string s = fields[j+col];
00176                 mvprintw(i+2,(j+1)*10-1," %9s", s.substr(0,9).c_str());
00177 
00178                 attroff(A_REVERSE);
00179             }
00180         }
00181 
00182         //real x= vm(curl+lin, curc+col);
00183         //string strval= vm->getValString(curc+col, x);
00184         //if(!view_vec || strval == "")
00185         //strval= tostring(x);
00186 
00187         int fieldnum = curc+col;
00188         string strval = vm->getTextFields(curl+lin)[fieldnum];
00189         string fieldname = vm->fieldspec[fieldnum].first;
00190         string fieldtype = vm->fieldspec[fieldnum].second;
00191         pair<int,int> ra = vm->colrange[fieldnum];
00192         string transformed;
00193         if(view_vec)
00194         {
00195             try 
00196             { 
00197                 if(ra.second>0)
00198                 {
00199                     Vec dest(ra.second);
00200                     vm->transformStringToValue(fieldnum, strval, dest); 
00201                     for(int j=0; j<dest.length(); j++)
00202                     {
00203                         if(is_missing(dest[j]))
00204                             transformed += "MISSING ";
00205                         else
00206                             transformed += tostring(dest[j]) + " ";
00207                     }
00208                 }
00209             }
00210             catch(const PLearnError& e)
00211             {
00212                 mvprintw(1,1," %s", e.message().c_str());
00213                 transformed = "ERROR! See message at top of screen.";
00214             }
00215         }
00216         else
00217             transformed = "[ Press n to view as reals ]";
00218 
00219         mvprintw(LINES-1,0," %dx%d  (%d, %d) [ %s : %s ]  \"%s\" --> %s", 
00220                  l, nfields,
00221                  curl+lin, fieldnum, fieldname.c_str(), fieldtype.c_str(), strval.c_str(), transformed.c_str());
00222 
00223         refresh();
00224         key= getch();
00225 
00226         switch(key)
00227         {
00228         case KEY_UP: 
00229             if(0 < curl) --curl; 
00230             else if(lin>0) --lin; 
00231             break;
00232         case KEY_DOWN: 
00233             if(curl < LINES-4 && curl+lin < l-1) ++curl; 
00234             else if(lin  < l-1) ++lin; 
00235             if(curl+lin >= l) curl= l-lin-1;
00236             break;
00237         case KEY_PPAGE: 
00238             lin-=LINES-3; 
00239             if(lin < 0) lin= 0; 
00240             break;
00241         case KEY_NPAGE: 
00242             lin+=LINES-3; 
00243             if(lin >= l) lin= l-1; 
00244             if(curl+lin >= l) curl= l-lin-1;
00245             break;
00246         case KEY_LEFT: 
00247             if(0 < curc) --curc; 
00248             else if(col>0) --col; 
00249             break;
00250         case KEY_RIGHT: 
00251             if(curc < COLS/10-2 && curc+col < nfields-1) ++curc; 
00252             else if(col < nfields-1) ++col;
00253             if(curc+col >= nfields) curc= nfields-col-1;
00254             break;
00255         case (int)'l': case (int)'L': 
00256         {
00257             echo();
00258             mvprintw(LINES-1,0,"Goto line:                                                                            ");
00259             move(LINES-1, 11);
00260             char li[10];
00261             getnstr(li, 10);
00262             if(!pl_isnumber(li) || toint(li) < 0 || toint(li)>=l)
00263             {
00264                 mvprintw(LINES-1,0,"*** Invalid line number ***");
00265                 refresh();
00266                 sleep(1);
00267             }
00268             else
00269                 lin= toint(li);
00270             noecho();
00271         }
00272         break;
00273         case (int)'c': case (int)'C': 
00274         {
00275             echo();
00276             mvprintw(LINES-1,0,"Goto column:                                                                           ");
00277             move(LINES-1, 13);
00278             char c[10];
00279             getnstr(c, 10);
00280             if(!pl_isnumber(c) || toint(c) < 0 || toint(c)>=nfields)
00281             {
00282                 mvprintw(LINES-1,0,"*** Invalid column number ***");
00283                 refresh();
00284                 sleep(1);
00285             }
00286             else
00287                 col= toint(c);
00288             noecho();
00289         }
00290         break;
00291 
00292         case (int)'s': case (int)'S': 
00293             if(!view_vec)
00294             {
00295                 mvprintw(LINES-1,0,"*** Strings already shown ***");
00296                 refresh();
00297                 sleep(1);
00298             }
00299             else
00300                 view_vec= false;
00301             break;
00302 
00303         case (int)'n': case (int)'N': 
00304             if(view_vec)
00305             {
00306                 mvprintw(LINES-1,0,"*** Numbers already shown ***");
00307                 refresh();
00308                 sleep(1);
00309             }
00310             else
00311                 view_vec= true;
00312             break;
00313 
00314         case (int)'h': case (int)'H':
00315             erase();
00316 
00317             mvprintw(0,COLS/2-6,"*** HELP ***");
00318 
00319             mvprintw(2,10,"KEYS:");
00320             mvprintw(3,10," - up: move up one line");
00321             mvprintw(4,10," - down: move down one line");
00322             mvprintw(5,10," - right: move right one column");
00323             mvprintw(6,10," - left: move left one column");
00324             mvprintw(7,10," - page up: move up one screen");
00325             mvprintw(8,10," - page down: move down one screen");
00326             mvprintw(9,10," - 'l' or 'L': prompt for a line number and go to that line");
00327             mvprintw(10,10," - 'c' or 'C': prompt for a column number and go to that column");
00328             mvprintw(11,10," - 's' or 'S': display string fields as strings");
00329             mvprintw(12,10," - 'n' or 'N': display string fields as numbers");
00330             mvprintw(13,10," - 'h' or 'H': display this screen");
00331             mvprintw(14,10," - 'q' or 'Q': quit program");
00332 
00333             mvprintw(16,COLS/2-13,"(press any key to continue)");
00334 
00335             refresh();
00336             getch();
00337           
00338             break;
00339 
00340         case (int)'q': case (int)'Q': 
00341             break;
00342  
00343         default:
00344             mvprintw(LINES-1,0,"*** Invalid command (type 'h' for help) ***");
00345             refresh();
00346             sleep(1);
00347             break;
00348         }
00349     }
00350   
00351     endwin();
00352 }
00353 
00355 void TxtmatCommand::run(const vector<string>& args)
00356 {
00357     int nargs = args.size();
00358     string com = args[0];
00359     string fname = args[1];
00360     PP<TextFilesVMatrix> m = dynamic_cast<TextFilesVMatrix*>(macroLoadObject(fname));
00361 
00362     if(!m)
00363         PLERROR("File %s does not appear to contain a TextFilesVMatrix object", fname.c_str());
00364 
00365     if(com=="info")
00366     {
00367         cout << "length: " << m->length() << endl;
00368         cout << "width: " << m->width() << endl;
00369     }
00370     else if(com=="fields")
00371     {
00372         int n = m->fieldspec.size();
00373         cout << n << " fields:" << endl;
00374         for(int j=0; j<n; j++)
00375             cout << j << ": \t" << m->fieldspec[j].first << "\t" << m->fieldspec[j].second << endl;
00376     }
00377     else if(com=="lines")
00378     {
00379         int startrow = 0;
00380         int l = 1;
00381         if(nargs>=3)
00382             startrow = toint(args[2]);
00383         if(nargs>=4)
00384             l = toint(args[3]);
00385         for(int i=startrow; i<startrow+l; i++)
00386             cout << i << ": \t" << m->getTextRow(i) << endl;
00387     }
00388     else if(com=="view")
00389     {
00390         view(m);
00391     }
00392     else if(com=="row")
00393     {
00394         int i = toint(args[2]);
00395         bool tonum = false;
00396         if(nargs>=4 && args[3]=="+")
00397             tonum = true;
00398         TVec<string> txtfields = m->getTextFields(i);
00399         Vec v;
00400         if(tonum)
00401         {
00402             v.resize(m->width());
00403             m->getRow(i,v);
00404         }
00405         int n = m->fieldspec.size();
00406         for(int j=0; j<n; j++)
00407         {
00408             cout << j << " " << m->fieldspec[j].first << ": \t";
00409             cout << txtfields[j] << "\t";
00410             if(tonum)
00411                 cout << v[j];
00412             cout << endl;
00413         }
00414     }
00415     else if(com=="col")
00416     {
00417         string colname = args[2];
00418         int startrow = 0;
00419         int end = m->length();
00420         if(nargs>=4)
00421             startrow = toint(args[3]);
00422         if(nargs>=5)
00423             end = startrow+toint(args[4]);
00424         else
00425             end = 1;
00426         bool tonum = false;
00427         if(nargs>=6 && args[5]=="+")
00428             tonum = true;
00429         // find column with colname
00430         int col;
00431         for(col=0; col<m->fieldspec.size(); col++)
00432             if(m->fieldspec[col].first==colname)
00433                 break;
00434         if(col>=m->fieldspec.size()) // not a valid name
00435             col = toint(colname);
00436         for(int i=startrow; i<end; i++)
00437         {
00438             string strval = m->getTextFields(i)[col];
00439             cout << i << ": \t\"" << strval << "\"\t";
00440             if(tonum)
00441             {
00442                 string fieldtype = m->fieldspec[col].second;
00443                 cout << "[" << fieldtype << "]";
00444                 pair<int,int> ra = m->colrange[col];
00445                 if(ra.second>0)
00446                 {
00447                     cout << " --> ";
00448                     Vec dest(ra.second);
00449                     m->transformStringToValue(col, strval, dest); 
00450                     for(int j=0; j<dest.length(); j++)
00451                     {
00452                         if(is_missing(dest[j]))
00453                             cout << "MISSING_VALUE ";
00454                         else
00455                             cout << dest[j] << " ";
00456                     }
00457                 }
00458             }
00459             cout << endl;
00460         }
00461     }
00462     else if(com=="scan")
00463     {
00464         int l = m->length();
00465         Vec v(m->width());
00466         ProgressBar pg("Scanning lines",l);
00467         for(int i=0; i<l; i++)
00468         {
00469             m->getRow(i,v);
00470             pg(i);
00471         }
00472     }
00473     else if(com=="counts")
00474     {
00475         m->generateMapCounts();
00476     }
00477     else if(com=="savemap")
00478     {
00479         m->buildVMatrixStringMapping();
00480         m->saveAllStringMappings();
00481     }
00482     else if(com=="checkstuff")
00483     {
00484         checkstuff(m);
00485     }
00486     else 
00487         PLERROR("Unknown command: txtmat %s", com.c_str());
00488 }
00489 
00490 } // end of namespace PLearn
00491 
00492 
00493 /*
00494   Local Variables:
00495   mode:c++
00496   c-basic-offset:4
00497   c-file-style:"stroustrup"
00498   c-file-offsets:((innamespace . 0)(inline-open . 0))
00499   indent-tabs-mode:nil
00500   fill-column:79
00501   End:
00502 */
00503 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines