PLearn 0.1
|
#include <TestDependenciesCommand.h>
Public Member Functions | |
TestDependenciesCommand () | |
virtual void | run (const vector< string > &args) |
The actual implementation of the 'TestDependenciesCommand' command. | |
Static Protected Attributes | |
static PLearnCommandRegistry | reg_ |
This allows to register the 'TestDependenciesCommand' command in the command registry. |
Definition at line 49 of file TestDependenciesCommand.h.
PLearn::TestDependenciesCommand::TestDependenciesCommand | ( | ) |
Definition at line 59 of file TestDependenciesCommand.cc.
: PLearnCommand("test-dependencies", "Compute dependency statistics between input and target variables.", " test-dependencies <VMat> [<inputsize> <targetsize> [<datablocksize>]]\n" "Reads a VMatrix (or any matrix format) and computes dependency statistics between each\n" "of the input variables and each of the target variables. A dependency score is then\n" "computed and a report is produced, listing the input variables in decreasing value of\n" "that score. The current implementation only computes the Spearman rank correlation\n" "and the linear correlation. If <datablocksize> is provided, it is used to\n" "divide the data row-wise in blocks of <datablocksize> rows. The statistics\n" "are computed separately in each block, and then some statistics of these\n" "statistics (min, max, mean, stdev) are reported.\n" "Missing values are ignored in the Spearman rank correlation.\n" ) {}
void PLearn::TestDependenciesCommand::run | ( | const vector< string > & | args | ) | [virtual] |
The actual implementation of the 'TestDependenciesCommand' command.
Implements PLearn::PLearnCommand.
Definition at line 79 of file TestDependenciesCommand.cc.
References b, PLearn::TMat< T >::column(), PLearn::computeMeanAndStddev(), PLearn::correlations(), PLearn::endl(), PLearn::VMat::fieldName(), PLearn::getDataSet(), PLearn::getSystemTotalMemory(), i, j, PLearn::VMat::length(), PLearn::max(), PLearn::mean(), PLearn::min(), n, PLERROR, PLWARNING, PLearn::sortRows(), PLearn::VMat::subMat(), PLearn::VMat::subMatColumns(), PLearn::TMat< T >::subMatRows(), PLearn::testSpearmanRankCorrelation(), PLearn::toint(), PLearn::VMat::toMat(), and x.
{ if(args.size()<1 || args.size()>4) PLERROR("test-dependencies expects 1 to 4 arguments, check the help"); VMat data = getDataSet(args[0]); int inputsize = (args.size()>1)?toint(args[1]):data->inputsize(); int targetsize = (args.size()>2)?toint(args[2]):data->targetsize(); int row_blocksize = (args.size()>3)?toint(args[3]):data.length(); if (args.size()>1) data->defineSizes(inputsize,targetsize,data->weightsize()); #ifdef WIN32 MEMORYSTATUS stat; GlobalMemoryStatus (&stat); // Total available memory in bytes int memory_size = int(stat.dwAvailVirtual); #else int memory_size = int(getSystemTotalMemory()); #endif int n_rowblocks = int(ceil(data.length() / real(row_blocksize))); // statistics computed for each variable, and for each rowblock // rank in "bestness" // score in "bestness" // rank correlation // rank correlation p-value // linear correlation // linear correlation p-value Mat var_rank(n_rowblocks,inputsize); Mat var_score(n_rowblocks,inputsize); Mat var_rank_corr(n_rowblocks,inputsize*targetsize); Mat var_rc_pvalue(n_rowblocks,inputsize*targetsize); Mat var_lin_corr(n_rowblocks,inputsize*targetsize); Mat var_lc_pvalue(n_rowblocks,inputsize*targetsize); int rowblockstart = 0; int n=data->length(); for (int rowblock=0;rowblock<n_rowblocks;rowblock++, rowblockstart += row_blocksize) { int rowblocklen = (rowblock<n_rowblocks-1)?row_blocksize:(n-rowblockstart); VMat x = data.subMat(rowblockstart,0,rowblocklen,inputsize); VMat y = data.subMat(rowblockstart,inputsize,rowblocklen,targetsize); Mat r = var_rank_corr(rowblock).toMat(inputsize,targetsize); Mat pvalues = var_rc_pvalue(rowblock).toMat(inputsize,targetsize); int col_blocksize = memory_size/int(2*sizeof(real)*rowblocklen); if (col_blocksize>=inputsize) // everything fits in half the memory { x = VMat(x.toMat()); testSpearmanRankCorrelation(x,y,r,pvalues, true); } else // work by column blocks { int n_col_blocks = int(ceil(inputsize/real(col_blocksize))); cout << "work with " << n_col_blocks << " of " << col_blocksize << " columns each (except the last)." << endl; int bstart=0; for (int b=0;b<n_col_blocks;b++,bstart+=col_blocksize) { int bsize= (b<n_col_blocks-1)?col_blocksize:inputsize-bstart; VMat block = VMat(x.subMatColumns(bstart,bsize).toMat()); Mat rb = r.subMatRows(bstart,bsize); Mat pb = pvalues.subMatRows(bstart,bsize); cout << "compute rank correlation for variables " << bstart << " - " << bstart+bsize-1 << endl; testSpearmanRankCorrelation(block,y,rb,pb, true); } } // linear correlations and corresponding p-values Mat lr = var_lin_corr(rowblock).toMat(inputsize,targetsize); Mat lpvalues = var_lc_pvalue(rowblock).toMat(inputsize,targetsize); correlations(x, y, lr, lpvalues, true); Mat scores(inputsize,2); for (int i=0;i<inputsize;i++) { Vec r_i = r(i); real s =0; for (int j=0;j<targetsize;j++) { real abs_r = fabs(r_i[j]); if (abs_r>s) s=abs_r; } scores(i,0) = s; scores(i,1) = i; } sortRows(scores,0,false); cout << "Results for " << rowblock << "-th row block, from row " << rowblockstart << " to " << rowblockstart+rowblocklen-1 << " inclusively" << endl; for (int k=0;k<inputsize;k++) { int i = int(scores(k,1)); var_rank(rowblock,i) = k; var_score(rowblock,i) = scores(k,0); cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")"; if (targetsize==1) cout << " with rank correlation = " << r(i,0) << " {p-value = " << pvalues(i,0) << "}, linear corr. = " << lr(i,0) << " {p-value= " << lpvalues(i,0) << "}" << endl; if (targetsize>1) { cout << " (rank corr., rank p-value, lin. corr., lin. p-value) for individual targets: "; for (int j=0;j<targetsize;j++) cout << "(" << r(i,j) << ", " << pvalues(i,j) << "," << lr(i,j) << ", " << lpvalues(i,j) << ") "; cout << endl; } } } // compute mean var_score for each variable and sort them accordingly Mat mean_score(inputsize,2); for (int i=0;i<inputsize;i++) { mean_score(i,0) = mean(var_score.column(i)); mean_score(i,1) = i; } sortRows(mean_score,0,false); // compute statistics across row blocks cout << "For each block statistic print (mean,stdev,min,max)\n" << endl; for (int k=0;k<inputsize;k++) { int i = int(mean_score(k,1)); Mat varrank = var_rank.column(i); Mat varscore = var_score.column(i); Mat varrc = var_rank_corr.column(i); Mat varrcpv = var_rc_pvalue.column(i); Mat varlc = var_lin_corr.column(i); Mat varlcpv = var_lc_pvalue.column(i); Vec rankm(1),rankdev(1),scorem(1),scoredev(1),rcm(1),rcdev(1),rcpvm(1),rcpvdev(1), lcm(1),lcdev(1),lcpvm(1),lcpvdev(1); computeMeanAndStddev(varrank,rankm,rankdev); computeMeanAndStddev(varscore,scorem,scoredev); computeMeanAndStddev(varrc,rcm,rcdev); computeMeanAndStddev(varrcpv,rcpvm,rcpvdev); computeMeanAndStddev(varlc,lcm,lcdev); computeMeanAndStddev(varlcpv,lcpvm,lcpvdev); cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")"; if (targetsize==1) { cout << " rank corr (" << rcm[0] << "," << rcdev[0] << "," << min(varrc) << "," << max(varrc) << " ) "; cout << " var rank (" << rankm[0] << "," << rankdev[0] << "," << min(varrank) << "," << max(varrank) << " ) "; cout << " rank cor pval(" << rcpvm[0] << "," << rcpvdev[0] << "," << min(varrcpv) << "," << max(varrcpv) << " ) "; cout << " lin corr (" << lcm[0] << "," << lcdev[0] << "," << min(varlc) << "," << max(varlc) << " ) "; cout << " lin cor pval (" << lcpvm[0] << "," << lcpvdev[0] << "," << min(varlcpv) << "," << max(varlcpv) << " ) " << endl; } else PLWARNING("In TestDependenciesCommand::run - The case 'targetsize > 1' is not implemented yet"); } }
PLearnCommandRegistry PLearn::TestDependenciesCommand::reg_ [static, protected] |
This allows to register the 'TestDependenciesCommand' command in the command registry.
Definition at line 57 of file TestDependenciesCommand.h.