PLearn 0.1
|
00001 // -*- C++ -*- 00002 00003 // TestDependenciesCommand.cc 00004 // 00005 // Copyright (C) 2003 Pascal Vincent 00006 // 00007 // Redistribution and use in source and binary forms, with or without 00008 // modification, are permitted provided that the following conditions are met: 00009 // 00010 // 1. Redistributions of source code must retain the above copyright 00011 // notice, this list of conditions and the following disclaimer. 00012 // 00013 // 2. Redistributions in binary form must reproduce the above copyright 00014 // notice, this list of conditions and the following disclaimer in the 00015 // documentation and/or other materials provided with the distribution. 00016 // 00017 // 3. The name of the authors may not be used to endorse or promote 00018 // products derived from this software without specific prior written 00019 // permission. 00020 // 00021 // THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 00022 // IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 00023 // OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN 00024 // NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 00025 // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 00026 // TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 00027 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 00028 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 00029 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 00030 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 00031 // 00032 // This file is part of the PLearn library. For more information on the PLearn 00033 // library, go to the PLearn Web site at www.plearn.org 00034 00035 /* ******************************************************* 00036 * $Id: TestDependenciesCommand.cc 8917 2008-04-30 14:12:58Z nouiz $ 00037 ******************************************************* */ 00038 00040 #include "TestDependenciesCommand.h" 00041 #include <plearn/math/TMat_maths.h> 00042 #include <plearn/db/getDataSet.h> 00043 #include <plearn/math/stats_utils.h> 00044 #include <plearn/vmat/VMat_basic_stats.h> 00045 00046 // norman: sorry, no memory check yet! 00047 #ifdef WIN32 00048 #include <windows.h> 00049 // undef min and max macros to avoid conflict with the plearn min and max 00050 #undef min 00051 #undef max 00052 #else 00053 #include <plearn/sys/procinfo.h> 00054 #endif 00055 00056 namespace PLearn { 00057 using namespace std; 00058 00059 TestDependenciesCommand::TestDependenciesCommand() 00060 : PLearnCommand("test-dependencies", 00061 "Compute dependency statistics between input and target variables.", 00062 " test-dependencies <VMat> [<inputsize> <targetsize> [<datablocksize>]]\n" 00063 "Reads a VMatrix (or any matrix format) and computes dependency statistics between each\n" 00064 "of the input variables and each of the target variables. A dependency score is then\n" 00065 "computed and a report is produced, listing the input variables in decreasing value of\n" 00066 "that score. The current implementation only computes the Spearman rank correlation\n" 00067 "and the linear correlation. If <datablocksize> is provided, it is used to\n" 00068 "divide the data row-wise in blocks of <datablocksize> rows. The statistics\n" 00069 "are computed separately in each block, and then some statistics of these\n" 00070 "statistics (min, max, mean, stdev) are reported.\n" 00071 "Missing values are ignored in the Spearman rank correlation.\n" 00072 ) 00073 {} 00074 00076 PLearnCommandRegistry TestDependenciesCommand::reg_(new TestDependenciesCommand); 00077 00079 void TestDependenciesCommand::run(const vector<string>& args) 00080 { 00081 if(args.size()<1 || args.size()>4) 00082 PLERROR("test-dependencies expects 1 to 4 arguments, check the help"); 00083 00084 VMat data = getDataSet(args[0]); 00085 int inputsize = (args.size()>1)?toint(args[1]):data->inputsize(); 00086 int targetsize = (args.size()>2)?toint(args[2]):data->targetsize(); 00087 int row_blocksize = (args.size()>3)?toint(args[3]):data.length(); 00088 if (args.size()>1) 00089 data->defineSizes(inputsize,targetsize,data->weightsize()); 00090 00091 #ifdef WIN32 00092 MEMORYSTATUS stat; 00093 GlobalMemoryStatus (&stat); 00094 // Total available memory in bytes 00095 int memory_size = int(stat.dwAvailVirtual); 00096 #else 00097 int memory_size = int(getSystemTotalMemory()); 00098 #endif 00099 int n_rowblocks = int(ceil(data.length() / real(row_blocksize))); 00100 00101 // statistics computed for each variable, and for each rowblock 00102 // rank in "bestness" 00103 // score in "bestness" 00104 // rank correlation 00105 // rank correlation p-value 00106 // linear correlation 00107 // linear correlation p-value 00108 Mat var_rank(n_rowblocks,inputsize); 00109 Mat var_score(n_rowblocks,inputsize); 00110 Mat var_rank_corr(n_rowblocks,inputsize*targetsize); 00111 Mat var_rc_pvalue(n_rowblocks,inputsize*targetsize); 00112 Mat var_lin_corr(n_rowblocks,inputsize*targetsize); 00113 Mat var_lc_pvalue(n_rowblocks,inputsize*targetsize); 00114 int rowblockstart = 0; 00115 int n=data->length(); 00116 00117 for (int rowblock=0;rowblock<n_rowblocks;rowblock++, rowblockstart += row_blocksize) 00118 { 00119 int rowblocklen = (rowblock<n_rowblocks-1)?row_blocksize:(n-rowblockstart); 00120 VMat x = data.subMat(rowblockstart,0,rowblocklen,inputsize); 00121 VMat y = data.subMat(rowblockstart,inputsize,rowblocklen,targetsize); 00122 Mat r = var_rank_corr(rowblock).toMat(inputsize,targetsize); 00123 Mat pvalues = var_rc_pvalue(rowblock).toMat(inputsize,targetsize); 00124 int col_blocksize = memory_size/int(2*sizeof(real)*rowblocklen); 00125 if (col_blocksize>=inputsize) // everything fits in half the memory 00126 { 00127 x = VMat(x.toMat()); 00128 testSpearmanRankCorrelation(x,y,r,pvalues, true); 00129 } 00130 else // work by column blocks 00131 { 00132 int n_col_blocks = int(ceil(inputsize/real(col_blocksize))); 00133 cout << "work with " << n_col_blocks << " of " << col_blocksize << " columns each (except the last)." << endl; 00134 int bstart=0; 00135 for (int b=0;b<n_col_blocks;b++,bstart+=col_blocksize) 00136 { 00137 int bsize= (b<n_col_blocks-1)?col_blocksize:inputsize-bstart; 00138 VMat block = VMat(x.subMatColumns(bstart,bsize).toMat()); 00139 Mat rb = r.subMatRows(bstart,bsize); 00140 Mat pb = pvalues.subMatRows(bstart,bsize); 00141 cout << "compute rank correlation for variables " << bstart << " - " << bstart+bsize-1 << endl; 00142 testSpearmanRankCorrelation(block,y,rb,pb, true); 00143 } 00144 } 00145 // linear correlations and corresponding p-values 00146 Mat lr = var_lin_corr(rowblock).toMat(inputsize,targetsize); 00147 Mat lpvalues = var_lc_pvalue(rowblock).toMat(inputsize,targetsize); 00148 correlations(x, y, lr, lpvalues, true); 00149 Mat scores(inputsize,2); 00150 for (int i=0;i<inputsize;i++) 00151 { 00152 Vec r_i = r(i); 00153 real s =0; 00154 for (int j=0;j<targetsize;j++) 00155 { 00156 real abs_r = fabs(r_i[j]); 00157 if (abs_r>s) s=abs_r; 00158 } 00159 scores(i,0) = s; 00160 scores(i,1) = i; 00161 } 00162 sortRows(scores,0,false); 00163 cout << "Results for " << rowblock << "-th row block, from row " << rowblockstart << " to " << rowblockstart+rowblocklen-1 << " inclusively" << endl; 00164 for (int k=0;k<inputsize;k++) 00165 { 00166 int i = int(scores(k,1)); 00167 var_rank(rowblock,i) = k; 00168 var_score(rowblock,i) = scores(k,0); 00169 cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")"; 00170 if (targetsize==1) 00171 cout << " with rank correlation = " << r(i,0) << " {p-value = " << pvalues(i,0) 00172 << "}, linear corr. = " 00173 << lr(i,0) 00174 << " {p-value= " << lpvalues(i,0) << "}" << endl; 00175 if (targetsize>1) 00176 { 00177 cout << " (rank corr., rank p-value, lin. corr., lin. p-value) for individual targets: "; 00178 for (int j=0;j<targetsize;j++) 00179 cout << "(" << r(i,j) << ", " << pvalues(i,j) << "," << lr(i,j) << ", " 00180 << lpvalues(i,j) << ") "; 00181 cout << endl; 00182 } 00183 } 00184 } 00185 // compute mean var_score for each variable and sort them accordingly 00186 Mat mean_score(inputsize,2); 00187 for (int i=0;i<inputsize;i++) 00188 { 00189 mean_score(i,0) = mean(var_score.column(i)); 00190 mean_score(i,1) = i; 00191 } 00192 sortRows(mean_score,0,false); 00193 // compute statistics across row blocks 00194 cout << "For each block statistic print (mean,stdev,min,max)\n" << endl; 00195 for (int k=0;k<inputsize;k++) 00196 { 00197 int i = int(mean_score(k,1)); 00198 Mat varrank = var_rank.column(i); 00199 Mat varscore = var_score.column(i); 00200 Mat varrc = var_rank_corr.column(i); 00201 Mat varrcpv = var_rc_pvalue.column(i); 00202 Mat varlc = var_lin_corr.column(i); 00203 Mat varlcpv = var_lc_pvalue.column(i); 00204 Vec rankm(1),rankdev(1),scorem(1),scoredev(1),rcm(1),rcdev(1),rcpvm(1),rcpvdev(1), 00205 lcm(1),lcdev(1),lcpvm(1),lcpvdev(1); 00206 computeMeanAndStddev(varrank,rankm,rankdev); 00207 computeMeanAndStddev(varscore,scorem,scoredev); 00208 computeMeanAndStddev(varrc,rcm,rcdev); 00209 computeMeanAndStddev(varrcpv,rcpvm,rcpvdev); 00210 computeMeanAndStddev(varlc,lcm,lcdev); 00211 computeMeanAndStddev(varlcpv,lcpvm,lcpvdev); 00212 cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")"; 00213 if (targetsize==1) 00214 { 00215 cout << " rank corr (" << rcm[0] << "," << rcdev[0] << "," << min(varrc) << "," << max(varrc) << " ) "; 00216 cout << " var rank (" << rankm[0] << "," << rankdev[0] << "," << min(varrank) << "," << max(varrank) << " ) "; 00217 cout << " rank cor pval(" << rcpvm[0] << "," << rcpvdev[0] << "," << min(varrcpv) << "," << max(varrcpv) << " ) "; 00218 cout << " lin corr (" << lcm[0] << "," << lcdev[0] << "," << min(varlc) << "," << max(varlc) << " ) "; 00219 cout << " lin cor pval (" << lcpvm[0] << "," << lcpvdev[0] << "," << min(varlcpv) << "," << max(varlcpv) << " ) " << endl; 00220 } 00221 else PLWARNING("In TestDependenciesCommand::run - The case 'targetsize > 1' is not implemented yet"); 00222 } 00223 } 00224 00225 } // end of namespace PLearn 00226 00227 00228 /* 00229 Local Variables: 00230 mode:c++ 00231 c-basic-offset:4 00232 c-file-style:"stroustrup" 00233 c-file-offsets:((innamespace . 0)(inline-open . 0)) 00234 indent-tabs-mode:nil 00235 fill-column:79 00236 End: 00237 */ 00238 // vim: filetype=cpp:expandtab:shiftwidth=4:tabstop=8:softtabstop=4:encoding=utf-8:textwidth=79 :