|
PLearn 0.1
|
Stores and learns the parameters between two linear layers of an RBM. More...
#include <RBMMatrixConnectionNatGrad.h>


Public Member Functions | |
| RBMMatrixConnectionNatGrad (real the_learning_rate=0) | |
| Default constructor. | |
| virtual void | update (const Mat &pos_down_values, const Mat &pos_up_values, const Mat &neg_down_values, const Mat &neg_up_values) |
| Not implemented. | |
| virtual void | bpropUpdate (const Mat &inputs, const Mat &outputs, Mat &input_gradients, const Mat &output_gradients, bool accumulate=false) |
| SOON TO BE DEPRECATED, USE bpropAccUpdate(const TVec<Mat*>& ports_value, const TVec<Mat*>& ports_gradient) | |
| virtual string | classname () const |
| virtual OptionList & | getOptionList () const |
| virtual OptionMap & | getOptionMap () const |
| virtual RemoteMethodMap & | getRemoteMethodMap () const |
| virtual RBMMatrixConnectionNatGrad * | deepCopy (CopiesMap &copies) const |
| virtual void | build () |
| Post-constructor. | |
| virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
| Transforms a shallow copy into a deep copy. | |
Static Public Member Functions | |
| static string | _classname_ () |
| static OptionList & | _getOptionList_ () |
| static RemoteMethodMap & | _getRemoteMethodMap_ () |
| static Object * | _new_instance_for_typemap_ () |
| static bool | _isa_ (const Object *o) |
| static void | _static_initialize_ () |
| static const PPath & | declaringFile () |
Public Attributes | |
| PP< NatGradEstimator > | natgrad_template |
| natural gradient estimator for neurons (if 0 then do not correct the gradient on neurons) | |
| TVec< PP< NatGradEstimator > > | cd_natgrad |
| TVec< PP< NatGradEstimator > > | bp_natgrad |
| Mat | weights_gradient |
| Vec | natural_gradient |
Static Public Attributes | |
| static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
| static void | declareOptions (OptionList &ol) |
| Declares the class options. | |
Private Types | |
| typedef RBMMatrixConnection | inherited |
Private Member Functions | |
| void | build_ () |
| This does the actual building. | |
Stores and learns the parameters between two linear layers of an RBM.
Definition at line 54 of file RBMMatrixConnectionNatGrad.h.
typedef RBMMatrixConnection PLearn::RBMMatrixConnectionNatGrad::inherited [private] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 56 of file RBMMatrixConnectionNatGrad.h.
| PLearn::RBMMatrixConnectionNatGrad::RBMMatrixConnectionNatGrad | ( | real | the_learning_rate = 0 | ) |
Default constructor.
Definition at line 55 of file RBMMatrixConnectionNatGrad.cc.
:
inherited(the_learning_rate)
{
}
| string PLearn::RBMMatrixConnectionNatGrad::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| OptionList & PLearn::RBMMatrixConnectionNatGrad::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| RemoteMethodMap & PLearn::RBMMatrixConnectionNatGrad::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| Object * PLearn::RBMMatrixConnectionNatGrad::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| StaticInitializer RBMMatrixConnectionNatGrad::_static_initializer_ & PLearn::RBMMatrixConnectionNatGrad::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| void PLearn::RBMMatrixConnectionNatGrad::bpropUpdate | ( | const Mat & | inputs, |
| const Mat & | outputs, | ||
| Mat & | input_gradients, | ||
| const Mat & | output_gradients, | ||
| bool | accumulate = false |
||
| ) | [virtual] |
SOON TO BE DEPRECATED, USE bpropAccUpdate(const TVec<Mat*>& ports_value, const TVec<Mat*>& ports_gradient)
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 138 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, PLearn::RBMConnection::down_size, i, PLearn::RBMConnection::learning_rate, PLearn::TMat< T >::length(), PLearn::multiplyAcc(), natural_gradient, PLASSERT, PLASSERT_MSG, PLearn::RBMConnection::pos_count, PLearn::product(), PLearn::productAcc(), PLearn::productScaleAcc(), PLearn::TMat< T >::resize(), PLearn::RBMConnection::up_size, PLearn::RBMMatrixConnection::weights, weights_gradient, and PLearn::TMat< T >::width().
{
PLASSERT( inputs.width() == down_size );
PLASSERT( outputs.width() == up_size );
PLASSERT( output_gradients.width() == up_size );
if( accumulate )
{
PLASSERT_MSG( input_gradients.width() == down_size &&
input_gradients.length() == inputs.length(),
"Cannot resize input_gradients and accumulate into it" );
// input_gradients += output_gradient * weights
productAcc(input_gradients, output_gradients, weights);
}
else
{
input_gradients.resize(inputs.length(), down_size);
// input_gradients = output_gradient * weights
product(input_gradients, output_gradients, weights);
}
// weights_gradient = 1/n * output_gradients' * inputs
productScaleAcc(weights_gradient, output_gradients, true, inputs, false,
1. / inputs.length(), 0.);
for (int i=0;i<up_size;i++)
{
(*bp_natgrad[i])(pos_count,weights_gradient(i),natural_gradient);
multiplyAcc(weights(i),natural_gradient,-learning_rate);
}
pos_count++;
}

| void PLearn::RBMMatrixConnectionNatGrad::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 86 of file RBMMatrixConnectionNatGrad.cc.
References PLearn::RBMMatrixConnection::build(), and build_().
{
inherited::build();
build_();
}

| void PLearn::RBMMatrixConnectionNatGrad::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 73 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, cd_natgrad, deepCopy(), PLearn::RBMConnection::down_size, i, natgrad_template, natural_gradient, PLearn::TMat< T >::resize(), PLearn::TVec< T >::resize(), PLearn::RBMConnection::up_size, and weights_gradient.
Referenced by build().
{
cd_natgrad.resize(up_size);
bp_natgrad.resize(up_size);
for (int i=0;i<up_size;i++)
{
cd_natgrad[i] = PLearn::deepCopy(natgrad_template);
bp_natgrad[i] = PLearn::deepCopy(natgrad_template);
}
weights_gradient.resize(up_size,down_size);
natural_gradient.resize(down_size);
}


| string PLearn::RBMMatrixConnectionNatGrad::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| void PLearn::RBMMatrixConnectionNatGrad::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares the class options.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 60 of file RBMMatrixConnectionNatGrad.cc.
References PLearn::declareOption(), PLearn::RBMMatrixConnection::declareOptions(), PLearn::OptionBase::learntoption, and natgrad_template.
{
declareOption(ol, "natgrad_template", &RBMMatrixConnectionNatGrad::natgrad_template,
OptionBase::learntoption,
"An object of type NatGradEstimator which will be copied for each row of the\n"
"weights matrix; each will compute the adjustment to the update direction\n"
"based on the natural gradient.\n");
// Now call the parent class' declareOptions
inherited::declareOptions(ol);
}

| static const PPath& PLearn::RBMMatrixConnectionNatGrad::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 95 of file RBMMatrixConnectionNatGrad.h.
:
//##### Protected Member Functions ######################################
| RBMMatrixConnectionNatGrad * PLearn::RBMMatrixConnectionNatGrad::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
Referenced by build_().

| OptionList & PLearn::RBMMatrixConnectionNatGrad::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| OptionMap & PLearn::RBMMatrixConnectionNatGrad::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| RemoteMethodMap & PLearn::RBMMatrixConnectionNatGrad::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
| void PLearn::RBMMatrixConnectionNatGrad::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 93 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, cd_natgrad, PLearn::deepCopyField(), PLearn::RBMMatrixConnection::makeDeepCopyFromShallowCopy(), natgrad_template, natural_gradient, and weights_gradient.
{
inherited::makeDeepCopyFromShallowCopy(copies);
deepCopyField(natgrad_template, copies);
deepCopyField(cd_natgrad, copies);
deepCopyField(bp_natgrad, copies);
deepCopyField(weights_gradient, copies);
deepCopyField(natural_gradient, copies);
}

| void PLearn::RBMMatrixConnectionNatGrad::update | ( | const Mat & | pos_down_values, |
| const Mat & | pos_up_values, | ||
| const Mat & | neg_down_values, | ||
| const Mat & | neg_up_values | ||
| ) | [virtual] |
Not implemented.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 104 of file RBMMatrixConnectionNatGrad.cc.
References cd_natgrad, i, PLearn::RBMConnection::learning_rate, PLearn::TMat< T >::length(), PLearn::RBMConnection::momentum, PLearn::multiplyAcc(), natural_gradient, PLASSERT, PLERROR, PLearn::RBMConnection::pos_count, PLearn::productScaleAcc(), PLearn::RBMConnection::up_size, PLearn::RBMMatrixConnection::weights, weights_gradient, and PLearn::TMat< T >::width().
{
// weights -= learning_rate * ( h_0 v_0' - h_1 v_1' );
// or:
// weights[i][j] += learning_rate * (h_1[i] v_1[j] - h_0[i] v_0[j]);
PLASSERT( pos_up_values.width() == weights.length() );
PLASSERT( neg_up_values.width() == weights.length() );
PLASSERT( pos_down_values.width() == weights.width() );
PLASSERT( neg_down_values.width() == weights.width() );
if( momentum == real(0.) )
{
// We use the average gradient over a mini-batch.
real mbnorm = 1. / pos_down_values.length();
productScaleAcc(weights_gradient, pos_up_values, true, pos_down_values, false,
mbnorm, 0.);
productScaleAcc(weights_gradient, neg_up_values, true, neg_down_values, false,
-mbnorm, 1.);
for (int i=0;i<up_size;i++)
{
(*cd_natgrad[i])(pos_count,weights_gradient(i),natural_gradient);
multiplyAcc(weights(i),natural_gradient,-learning_rate);
}
pos_count++;
}
else
PLERROR("RBMMatrixConnectionNatGrad::update with momentum - Not implemented");
}

Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 95 of file RBMMatrixConnectionNatGrad.h.
Definition at line 67 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), and makeDeepCopyFromShallowCopy().
Definition at line 66 of file RBMMatrixConnectionNatGrad.h.
Referenced by build_(), makeDeepCopyFromShallowCopy(), and update().
natural gradient estimator for neurons (if 0 then do not correct the gradient on neurons)
Definition at line 63 of file RBMMatrixConnectionNatGrad.h.
Referenced by build_(), declareOptions(), and makeDeepCopyFromShallowCopy().
Definition at line 72 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), makeDeepCopyFromShallowCopy(), and update().
Definition at line 71 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), makeDeepCopyFromShallowCopy(), and update().
1.7.4