PLearn 0.1
|
Stores and learns the parameters between two linear layers of an RBM. More...
#include <RBMMatrixConnectionNatGrad.h>
Public Member Functions | |
RBMMatrixConnectionNatGrad (real the_learning_rate=0) | |
Default constructor. | |
virtual void | update (const Mat &pos_down_values, const Mat &pos_up_values, const Mat &neg_down_values, const Mat &neg_up_values) |
Not implemented. | |
virtual void | bpropUpdate (const Mat &inputs, const Mat &outputs, Mat &input_gradients, const Mat &output_gradients, bool accumulate=false) |
SOON TO BE DEPRECATED, USE bpropAccUpdate(const TVec<Mat*>& ports_value, const TVec<Mat*>& ports_gradient) | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual RBMMatrixConnectionNatGrad * | deepCopy (CopiesMap &copies) const |
virtual void | build () |
Post-constructor. | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
Static Public Member Functions | |
static string | _classname_ () |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Public Attributes | |
PP< NatGradEstimator > | natgrad_template |
natural gradient estimator for neurons (if 0 then do not correct the gradient on neurons) | |
TVec< PP< NatGradEstimator > > | cd_natgrad |
TVec< PP< NatGradEstimator > > | bp_natgrad |
Mat | weights_gradient |
Vec | natural_gradient |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares the class options. | |
Private Types | |
typedef RBMMatrixConnection | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. |
Stores and learns the parameters between two linear layers of an RBM.
Definition at line 54 of file RBMMatrixConnectionNatGrad.h.
typedef RBMMatrixConnection PLearn::RBMMatrixConnectionNatGrad::inherited [private] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 56 of file RBMMatrixConnectionNatGrad.h.
PLearn::RBMMatrixConnectionNatGrad::RBMMatrixConnectionNatGrad | ( | real | the_learning_rate = 0 | ) |
Default constructor.
Definition at line 55 of file RBMMatrixConnectionNatGrad.cc.
: inherited(the_learning_rate) { }
string PLearn::RBMMatrixConnectionNatGrad::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
OptionList & PLearn::RBMMatrixConnectionNatGrad::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
RemoteMethodMap & PLearn::RBMMatrixConnectionNatGrad::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
Object * PLearn::RBMMatrixConnectionNatGrad::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
StaticInitializer RBMMatrixConnectionNatGrad::_static_initializer_ & PLearn::RBMMatrixConnectionNatGrad::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
void PLearn::RBMMatrixConnectionNatGrad::bpropUpdate | ( | const Mat & | inputs, |
const Mat & | outputs, | ||
Mat & | input_gradients, | ||
const Mat & | output_gradients, | ||
bool | accumulate = false |
||
) | [virtual] |
SOON TO BE DEPRECATED, USE bpropAccUpdate(const TVec<Mat*>& ports_value, const TVec<Mat*>& ports_gradient)
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 138 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, PLearn::RBMConnection::down_size, i, PLearn::RBMConnection::learning_rate, PLearn::TMat< T >::length(), PLearn::multiplyAcc(), natural_gradient, PLASSERT, PLASSERT_MSG, PLearn::RBMConnection::pos_count, PLearn::product(), PLearn::productAcc(), PLearn::productScaleAcc(), PLearn::TMat< T >::resize(), PLearn::RBMConnection::up_size, PLearn::RBMMatrixConnection::weights, weights_gradient, and PLearn::TMat< T >::width().
{ PLASSERT( inputs.width() == down_size ); PLASSERT( outputs.width() == up_size ); PLASSERT( output_gradients.width() == up_size ); if( accumulate ) { PLASSERT_MSG( input_gradients.width() == down_size && input_gradients.length() == inputs.length(), "Cannot resize input_gradients and accumulate into it" ); // input_gradients += output_gradient * weights productAcc(input_gradients, output_gradients, weights); } else { input_gradients.resize(inputs.length(), down_size); // input_gradients = output_gradient * weights product(input_gradients, output_gradients, weights); } // weights_gradient = 1/n * output_gradients' * inputs productScaleAcc(weights_gradient, output_gradients, true, inputs, false, 1. / inputs.length(), 0.); for (int i=0;i<up_size;i++) { (*bp_natgrad[i])(pos_count,weights_gradient(i),natural_gradient); multiplyAcc(weights(i),natural_gradient,-learning_rate); } pos_count++; }
void PLearn::RBMMatrixConnectionNatGrad::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 86 of file RBMMatrixConnectionNatGrad.cc.
References PLearn::RBMMatrixConnection::build(), and build_().
{ inherited::build(); build_(); }
void PLearn::RBMMatrixConnectionNatGrad::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 73 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, cd_natgrad, deepCopy(), PLearn::RBMConnection::down_size, i, natgrad_template, natural_gradient, PLearn::TMat< T >::resize(), PLearn::TVec< T >::resize(), PLearn::RBMConnection::up_size, and weights_gradient.
Referenced by build().
{ cd_natgrad.resize(up_size); bp_natgrad.resize(up_size); for (int i=0;i<up_size;i++) { cd_natgrad[i] = PLearn::deepCopy(natgrad_template); bp_natgrad[i] = PLearn::deepCopy(natgrad_template); } weights_gradient.resize(up_size,down_size); natural_gradient.resize(down_size); }
string PLearn::RBMMatrixConnectionNatGrad::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
void PLearn::RBMMatrixConnectionNatGrad::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares the class options.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 60 of file RBMMatrixConnectionNatGrad.cc.
References PLearn::declareOption(), PLearn::RBMMatrixConnection::declareOptions(), PLearn::OptionBase::learntoption, and natgrad_template.
{ declareOption(ol, "natgrad_template", &RBMMatrixConnectionNatGrad::natgrad_template, OptionBase::learntoption, "An object of type NatGradEstimator which will be copied for each row of the\n" "weights matrix; each will compute the adjustment to the update direction\n" "based on the natural gradient.\n"); // Now call the parent class' declareOptions inherited::declareOptions(ol); }
static const PPath& PLearn::RBMMatrixConnectionNatGrad::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 95 of file RBMMatrixConnectionNatGrad.h.
:
//##### Protected Member Functions ######################################
RBMMatrixConnectionNatGrad * PLearn::RBMMatrixConnectionNatGrad::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
Referenced by build_().
OptionList & PLearn::RBMMatrixConnectionNatGrad::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
OptionMap & PLearn::RBMMatrixConnectionNatGrad::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
RemoteMethodMap & PLearn::RBMMatrixConnectionNatGrad::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 53 of file RBMMatrixConnectionNatGrad.cc.
void PLearn::RBMMatrixConnectionNatGrad::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 93 of file RBMMatrixConnectionNatGrad.cc.
References bp_natgrad, cd_natgrad, PLearn::deepCopyField(), PLearn::RBMMatrixConnection::makeDeepCopyFromShallowCopy(), natgrad_template, natural_gradient, and weights_gradient.
{ inherited::makeDeepCopyFromShallowCopy(copies); deepCopyField(natgrad_template, copies); deepCopyField(cd_natgrad, copies); deepCopyField(bp_natgrad, copies); deepCopyField(weights_gradient, copies); deepCopyField(natural_gradient, copies); }
void PLearn::RBMMatrixConnectionNatGrad::update | ( | const Mat & | pos_down_values, |
const Mat & | pos_up_values, | ||
const Mat & | neg_down_values, | ||
const Mat & | neg_up_values | ||
) | [virtual] |
Not implemented.
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 104 of file RBMMatrixConnectionNatGrad.cc.
References cd_natgrad, i, PLearn::RBMConnection::learning_rate, PLearn::TMat< T >::length(), PLearn::RBMConnection::momentum, PLearn::multiplyAcc(), natural_gradient, PLASSERT, PLERROR, PLearn::RBMConnection::pos_count, PLearn::productScaleAcc(), PLearn::RBMConnection::up_size, PLearn::RBMMatrixConnection::weights, weights_gradient, and PLearn::TMat< T >::width().
{ // weights -= learning_rate * ( h_0 v_0' - h_1 v_1' ); // or: // weights[i][j] += learning_rate * (h_1[i] v_1[j] - h_0[i] v_0[j]); PLASSERT( pos_up_values.width() == weights.length() ); PLASSERT( neg_up_values.width() == weights.length() ); PLASSERT( pos_down_values.width() == weights.width() ); PLASSERT( neg_down_values.width() == weights.width() ); if( momentum == real(0.) ) { // We use the average gradient over a mini-batch. real mbnorm = 1. / pos_down_values.length(); productScaleAcc(weights_gradient, pos_up_values, true, pos_down_values, false, mbnorm, 0.); productScaleAcc(weights_gradient, neg_up_values, true, neg_down_values, false, -mbnorm, 1.); for (int i=0;i<up_size;i++) { (*cd_natgrad[i])(pos_count,weights_gradient(i),natural_gradient); multiplyAcc(weights(i),natural_gradient,-learning_rate); } pos_count++; } else PLERROR("RBMMatrixConnectionNatGrad::update with momentum - Not implemented"); }
Reimplemented from PLearn::RBMMatrixConnection.
Definition at line 95 of file RBMMatrixConnectionNatGrad.h.
Definition at line 67 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), and makeDeepCopyFromShallowCopy().
Definition at line 66 of file RBMMatrixConnectionNatGrad.h.
Referenced by build_(), makeDeepCopyFromShallowCopy(), and update().
natural gradient estimator for neurons (if 0 then do not correct the gradient on neurons)
Definition at line 63 of file RBMMatrixConnectionNatGrad.h.
Referenced by build_(), declareOptions(), and makeDeepCopyFromShallowCopy().
Definition at line 72 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), makeDeepCopyFromShallowCopy(), and update().
Definition at line 71 of file RBMMatrixConnectionNatGrad.h.
Referenced by bpropUpdate(), build_(), makeDeepCopyFromShallowCopy(), and update().