PLearn 0.1
Public Member Functions | Static Protected Attributes
PLearn::TestDependenciesCommand Class Reference

#include <TestDependenciesCommand.h>

Inheritance diagram for PLearn::TestDependenciesCommand:
Inheritance graph
[legend]
Collaboration diagram for PLearn::TestDependenciesCommand:
Collaboration graph
[legend]

List of all members.

Public Member Functions

 TestDependenciesCommand ()
virtual void run (const vector< string > &args)
 The actual implementation of the 'TestDependenciesCommand' command.

Static Protected Attributes

static PLearnCommandRegistry reg_
 This allows to register the 'TestDependenciesCommand' command in the command registry.

Detailed Description

Definition at line 49 of file TestDependenciesCommand.h.


Constructor & Destructor Documentation

PLearn::TestDependenciesCommand::TestDependenciesCommand ( )

Definition at line 59 of file TestDependenciesCommand.cc.

    : PLearnCommand("test-dependencies",
                    "Compute dependency statistics between input and target variables.",
                    "  test-dependencies <VMat> [<inputsize> <targetsize> [<datablocksize>]]\n"
                    "Reads a VMatrix (or any matrix format) and computes dependency statistics between each\n"
                    "of the input variables and each of the target variables. A dependency score is then\n"
                    "computed and a report is produced, listing the input variables in decreasing value of\n"
                    "that score. The current implementation only computes the Spearman rank correlation\n"
                    "and the linear correlation. If <datablocksize> is provided, it is used to\n"
                    "divide the data row-wise in blocks of <datablocksize> rows. The statistics\n"
                    "are computed separately in each block, and then some statistics of these\n"
                    "statistics (min, max, mean, stdev) are reported.\n"
                    "Missing values are ignored in the Spearman rank correlation.\n"
        )
{}

Member Function Documentation

void PLearn::TestDependenciesCommand::run ( const vector< string > &  args) [virtual]

The actual implementation of the 'TestDependenciesCommand' command.

Implements PLearn::PLearnCommand.

Definition at line 79 of file TestDependenciesCommand.cc.

References b, PLearn::TMat< T >::column(), PLearn::computeMeanAndStddev(), PLearn::correlations(), PLearn::endl(), PLearn::VMat::fieldName(), PLearn::getDataSet(), PLearn::getSystemTotalMemory(), i, j, PLearn::VMat::length(), PLearn::max(), PLearn::mean(), PLearn::min(), n, PLERROR, PLWARNING, PLearn::sortRows(), PLearn::VMat::subMat(), PLearn::VMat::subMatColumns(), PLearn::TMat< T >::subMatRows(), PLearn::testSpearmanRankCorrelation(), PLearn::toint(), PLearn::VMat::toMat(), and x.

{
    if(args.size()<1 || args.size()>4)
        PLERROR("test-dependencies expects 1 to 4 arguments, check the help");

    VMat data = getDataSet(args[0]);
    int inputsize = (args.size()>1)?toint(args[1]):data->inputsize();
    int targetsize = (args.size()>2)?toint(args[2]):data->targetsize();
    int row_blocksize = (args.size()>3)?toint(args[3]):data.length();
    if (args.size()>1)
        data->defineSizes(inputsize,targetsize,data->weightsize());

#ifdef WIN32
    MEMORYSTATUS stat;
    GlobalMemoryStatus (&stat);
    // Total available memory in bytes
    int memory_size = int(stat.dwAvailVirtual);
#else
    int memory_size = int(getSystemTotalMemory());
#endif 
    int n_rowblocks = int(ceil(data.length() / real(row_blocksize)));
  
    // statistics computed for each variable, and for each rowblock
    // rank in "bestness"
    // score in "bestness"
    // rank correlation
    // rank correlation p-value
    // linear correlation
    // linear correlation p-value
    Mat var_rank(n_rowblocks,inputsize);
    Mat var_score(n_rowblocks,inputsize);
    Mat var_rank_corr(n_rowblocks,inputsize*targetsize);
    Mat var_rc_pvalue(n_rowblocks,inputsize*targetsize);
    Mat var_lin_corr(n_rowblocks,inputsize*targetsize);
    Mat var_lc_pvalue(n_rowblocks,inputsize*targetsize);
    int rowblockstart = 0;
    int n=data->length();

    for (int rowblock=0;rowblock<n_rowblocks;rowblock++, rowblockstart += row_blocksize)
    {
        int rowblocklen = (rowblock<n_rowblocks-1)?row_blocksize:(n-rowblockstart);
        VMat x = data.subMat(rowblockstart,0,rowblocklen,inputsize);
        VMat y = data.subMat(rowblockstart,inputsize,rowblocklen,targetsize);
        Mat r = var_rank_corr(rowblock).toMat(inputsize,targetsize);
        Mat pvalues = var_rc_pvalue(rowblock).toMat(inputsize,targetsize);
        int col_blocksize = memory_size/int(2*sizeof(real)*rowblocklen);
        if (col_blocksize>=inputsize) // everything fits in half the memory
        {
            x = VMat(x.toMat());
            testSpearmanRankCorrelation(x,y,r,pvalues, true);
        }
        else // work by column blocks
        {
            int n_col_blocks = int(ceil(inputsize/real(col_blocksize)));
            cout << "work with " << n_col_blocks << " of " << col_blocksize << " columns each (except the last)." << endl;
            int bstart=0;
            for (int b=0;b<n_col_blocks;b++,bstart+=col_blocksize)
            {
                int bsize= (b<n_col_blocks-1)?col_blocksize:inputsize-bstart;
                VMat block = VMat(x.subMatColumns(bstart,bsize).toMat());
                Mat rb = r.subMatRows(bstart,bsize);
                Mat pb = pvalues.subMatRows(bstart,bsize);
                cout << "compute rank correlation for variables " << bstart << " - " << bstart+bsize-1 << endl;
                testSpearmanRankCorrelation(block,y,rb,pb, true);
            }
        }
        // linear correlations and corresponding p-values
        Mat lr = var_lin_corr(rowblock).toMat(inputsize,targetsize);
        Mat lpvalues = var_lc_pvalue(rowblock).toMat(inputsize,targetsize);
        correlations(x, y, lr, lpvalues, true);
        Mat scores(inputsize,2);
        for (int i=0;i<inputsize;i++)
        {
            Vec r_i = r(i);
            real s =0;
            for (int j=0;j<targetsize;j++)
            {
                real abs_r = fabs(r_i[j]);
                if (abs_r>s) s=abs_r;
            }
            scores(i,0) = s;
            scores(i,1) = i;
        }
        sortRows(scores,0,false);
        cout << "Results for " << rowblock << "-th row block, from row " << rowblockstart << " to " << rowblockstart+rowblocklen-1 << " inclusively" << endl;
        for (int k=0;k<inputsize;k++)
        {
            int i = int(scores(k,1));
            var_rank(rowblock,i) = k;
            var_score(rowblock,i) = scores(k,0);
            cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")";
            if (targetsize==1)
                cout << " with rank correlation = " << r(i,0) << " {p-value = " << pvalues(i,0)
                     << "}, linear corr. = " 
                     << lr(i,0)
                     << " {p-value= " << lpvalues(i,0) << "}" << endl;
            if (targetsize>1)
            {
                cout << " (rank corr., rank p-value, lin. corr., lin. p-value) for individual targets: ";
                for (int j=0;j<targetsize;j++)
                    cout << "(" << r(i,j) << ", " << pvalues(i,j) << "," << lr(i,j) << ", " 
                         << lpvalues(i,j) << ") ";
                cout << endl;
            }
        }
    }
    // compute mean var_score for each variable and sort them accordingly
    Mat mean_score(inputsize,2);
    for (int i=0;i<inputsize;i++)
    {
        mean_score(i,0) = mean(var_score.column(i));
        mean_score(i,1) = i;
    }
    sortRows(mean_score,0,false);
    // compute statistics across row blocks
    cout << "For each block statistic print (mean,stdev,min,max)\n" << endl;
    for (int k=0;k<inputsize;k++)
    {
        int i = int(mean_score(k,1));
        Mat varrank = var_rank.column(i);
        Mat varscore = var_score.column(i);
        Mat varrc = var_rank_corr.column(i);
        Mat varrcpv = var_rc_pvalue.column(i);
        Mat varlc = var_lin_corr.column(i);
        Mat varlcpv = var_lc_pvalue.column(i);
        Vec rankm(1),rankdev(1),scorem(1),scoredev(1),rcm(1),rcdev(1),rcpvm(1),rcpvdev(1),
            lcm(1),lcdev(1),lcpvm(1),lcpvdev(1);
        computeMeanAndStddev(varrank,rankm,rankdev);
        computeMeanAndStddev(varscore,scorem,scoredev);
        computeMeanAndStddev(varrc,rcm,rcdev);
        computeMeanAndStddev(varrcpv,rcpvm,rcpvdev);
        computeMeanAndStddev(varlc,lcm,lcdev);
        computeMeanAndStddev(varlcpv,lcpvm,lcpvdev);
        cout << k << "-th best variable is " << data->fieldName(i) << " (col. " << i << ")";
        if (targetsize==1)
        {
            cout << " rank corr (" << rcm[0] << "," << rcdev[0] << "," << min(varrc) << "," << max(varrc) << " ) ";
            cout << " var rank (" << rankm[0] << "," << rankdev[0] << "," << min(varrank) << "," << max(varrank) << " ) ";
            cout << " rank cor pval(" << rcpvm[0] << "," << rcpvdev[0] << "," << min(varrcpv) << "," << max(varrcpv) << " ) ";
            cout << " lin corr (" << lcm[0] << "," << lcdev[0] << "," << min(varlc) << "," << max(varlc) << " ) ";
            cout << " lin cor pval (" << lcpvm[0] << "," << lcpvdev[0] << "," << min(varlcpv) << "," << max(varlcpv) << " ) " << endl;
        }
        else PLWARNING("In TestDependenciesCommand::run - The case 'targetsize > 1' is not implemented yet");
    }
}

Here is the call graph for this function:


Member Data Documentation

This allows to register the 'TestDependenciesCommand' command in the command registry.

Definition at line 57 of file TestDependenciesCommand.h.


The documentation for this class was generated from the following files:
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines