PLearn 0.1
|
Layer in an RBM formed with binomial units. More...
#include <RBMTruncExpLayer.h>
Public Member Functions | |
RBMTruncExpLayer () | |
Default constructor. | |
RBMTruncExpLayer (int the_size) | |
Constructor from the number of units. | |
virtual void | getUnitActivations (int i, PP< RBMParameters > rbmp, int offset=0) |
Uses "rbmp" to obtain the activations of unit "i" of this layer. | |
virtual void | getAllActivations (PP< RBMParameters > rbmp, int offset=0) |
Uses "rbmp" to obtain the activations of all units in this layer. | |
virtual void | generateSample () |
generate a sample, and update the sample field | |
virtual void | computeExpectation () |
compute the expectation | |
virtual void | bpropUpdate (const Vec &input, const Vec &output, Vec &input_gradient, const Vec &output_gradient) |
back-propagates the output gradient to the input | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual RBMTruncExpLayer * | deepCopy (CopiesMap &copies) const |
virtual void | build () |
Post-constructor. | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
RBMTruncExpLayer (real the_learning_rate=0.) | |
Default constructor. | |
RBMTruncExpLayer (int the_size, real the_learning_rate=0.) | |
Constructor from the number of units. | |
virtual void | generateSample () |
generate a sample, and update the sample field | |
virtual void | generateSamples () |
Generate a mini-batch set of samples. | |
virtual void | computeExpectation () |
compute the expectation | |
virtual void | computeExpectations () |
Not implemented. | |
virtual void | fprop (const Vec &input, Vec &output) const |
forward propagation | |
virtual void | bpropUpdate (const Vec &input, const Vec &output, Vec &input_gradient, const Vec &output_gradient, bool accumulate=false) |
back-propagates the output gradient to the input | |
virtual void | bpropUpdate (const Mat &inputs, const Mat &outputs, Mat &input_gradients, const Mat &output_gradients, bool accumulate=false) |
Back-propagate the output gradient to the input, and update parameters. | |
virtual string | classname () const |
virtual OptionList & | getOptionList () const |
virtual OptionMap & | getOptionMap () const |
virtual RemoteMethodMap & | getRemoteMethodMap () const |
virtual RBMTruncExpLayer * | deepCopy (CopiesMap &copies) const |
virtual void | build () |
Post-constructor. | |
virtual void | makeDeepCopyFromShallowCopy (CopiesMap &copies) |
Transforms a shallow copy into a deep copy. | |
Static Public Member Functions | |
static string | _classname_ () |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
static string | _classname_ () |
static OptionList & | _getOptionList_ () |
static RemoteMethodMap & | _getRemoteMethodMap_ () |
static Object * | _new_instance_for_typemap_ () |
static bool | _isa_ (const Object *o) |
static void | _static_initialize_ () |
static const PPath & | declaringFile () |
Static Public Attributes | |
static StaticInitializer | _static_initializer_ |
Static Protected Member Functions | |
static void | declareOptions (OptionList &ol) |
Declares the class options. | |
static void | declareOptions (OptionList &ol) |
Declares the class options. | |
Private Types | |
typedef RBMLayer | inherited |
typedef RBMLayer | inherited |
Private Member Functions | |
void | build_ () |
This does the actual building. | |
void | build_ () |
This does the actual building. |
Layer in an RBM formed with binomial units.
Layer in an RBM formed with truncated exponential units.
Definition at line 54 of file DEPRECATED/RBMTruncExpLayer.h.
typedef RBMLayer PLearn::RBMTruncExpLayer::inherited [private] |
Reimplemented from PLearn::RBMLayer.
Definition at line 56 of file DEPRECATED/RBMTruncExpLayer.h.
typedef RBMLayer PLearn::RBMTruncExpLayer::inherited [private] |
Reimplemented from PLearn::RBMLayer.
Definition at line 55 of file RBMTruncExpLayer.h.
PLearn::RBMTruncExpLayer::RBMTruncExpLayer | ( | ) |
PLearn::RBMTruncExpLayer::RBMTruncExpLayer | ( | int | the_size | ) |
Constructor from the number of units.
Definition at line 55 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::TVec< T >::resize(), and PLearn::sample().
{ size = the_size; units_types = string( the_size, 'l' ); activations.resize( the_size ); sample.resize( the_size ); expectation.resize( the_size ); expectation_is_up_to_date = false; }
PLearn::RBMTruncExpLayer::RBMTruncExpLayer | ( | real | the_learning_rate = 0. | ) |
Default constructor.
Definition at line 53 of file RBMTruncExpLayer.cc.
: inherited( the_learning_rate ) { }
Constructor from the number of units.
Definition at line 58 of file RBMTruncExpLayer.cc.
References PLearn::RBMLayer::activation, PLearn::RBMLayer::bias, PLearn::RBMLayer::bias_neg_stats, PLearn::RBMLayer::bias_pos_stats, PLearn::RBMLayer::expectation, PLearn::TVec< T >::resize(), PLearn::RBMLayer::sample, and PLearn::RBMLayer::size.
: inherited( the_learning_rate ) { size = the_size; activation.resize( the_size ); sample.resize( the_size ); expectation.resize( the_size ); bias.resize( the_size ); bias_pos_stats.resize( the_size ); bias_neg_stats.resize( the_size ); }
string PLearn::RBMTruncExpLayer::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
static string PLearn::RBMTruncExpLayer::_classname_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
OptionList & PLearn::RBMTruncExpLayer::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
static OptionList& PLearn::RBMTruncExpLayer::_getOptionList_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
static RemoteMethodMap& PLearn::RBMTruncExpLayer::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
RemoteMethodMap & PLearn::RBMTruncExpLayer::_getRemoteMethodMap_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
Reimplemented from PLearn::RBMLayer.
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
static Object* PLearn::RBMTruncExpLayer::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Object * PLearn::RBMTruncExpLayer::_new_instance_for_typemap_ | ( | ) | [static] |
Reimplemented from PLearn::Object.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
StaticInitializer RBMTruncExpLayer::_static_initializer_ & PLearn::RBMTruncExpLayer::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
static void PLearn::RBMTruncExpLayer::_static_initialize_ | ( | ) | [static] |
Reimplemented from PLearn::RBMLayer.
void PLearn::RBMTruncExpLayer::bpropUpdate | ( | const Vec & | input, |
const Vec & | output, | ||
Vec & | input_gradient, | ||
const Vec & | output_gradient | ||
) | [virtual] |
back-propagates the output gradient to the input
Implements PLearn::RBMLayer.
Definition at line 120 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::exp(), i, PLASSERT, PLearn::TVec< T >::resize(), and PLearn::TVec< T >::size().
{ PLASSERT( input.size() == size ); PLASSERT( output.size() == size ); PLASSERT( output_gradient.size() == size ); input_gradient.resize( size ); // df/da = exp(a)/(1-exp(a))^2 - 1/a^2 for( int i=0 ; i<size ; i++ ) { real a_i = input[i]; real ea_i = exp( a_i ); input_gradient[i] = ea_i/( (1 - ea_i) * (1 - ea_i) ) + 1/(a_i * a_i); } }
void PLearn::RBMTruncExpLayer::bpropUpdate | ( | const Vec & | input, |
const Vec & | output, | ||
Vec & | input_gradient, | ||
const Vec & | output_gradient, | ||
bool | accumulate = false |
||
) | [virtual] |
back-propagates the output gradient to the input
Implements PLearn::RBMLayer.
Definition at line 194 of file RBMTruncExpLayer.cc.
References PLearn::RBMLayer::applyBiasDecay(), PLearn::RBMLayer::bias, PLearn::RBMLayer::bias_inc, PLearn::TVec< T >::clear(), PLearn::exp(), i, PLearn::RBMLayer::learning_rate, PLearn::RBMLayer::momentum, PLASSERT, PLASSERT_MSG, PLearn::TVec< T >::resize(), PLearn::RBMLayer::size, and PLearn::TVec< T >::size().
{ PLASSERT( input.size() == size ); PLASSERT( output.size() == size ); PLASSERT( output_gradient.size() == size ); if( accumulate ) { PLASSERT_MSG( input_gradient.size() == size, "Cannot resize input_gradient AND accumulate into it" ); } else { input_gradient.resize( size ); input_gradient.clear(); } if( momentum != 0. ) bias_inc.resize( size ); // df/da = exp(a)/(1-exp(a))^2 - 1/a^2 for( int i=0 ; i<size ; i++ ) { real a_i = input[i] + bias[i]; real in_grad_i; // Polynomial approximation to avoid numerical instability // df/da = -1/12 + a^2/240 + O(a^4) if( fabs( a_i ) <= 0.01 ) { in_grad_i = output_gradient[i] * ( -1./12. + a_i * a_i / 240. ); } else { real ea_i = exp( a_i ); in_grad_i = output_gradient[i] * ( ea_i/( (1 - ea_i) * (1 - ea_i) ) + 1/(a_i * a_i) ); } input_gradient[i] += in_grad_i; if( momentum == 0. ) { // update the bias: bias -= learning_rate * input_gradient bias[i] -= learning_rate * in_grad_i; } else { // The update rule becomes: // bias_inc = momentum * bias_inc - learning_rate * input_gradient // bias += bias_inc bias_inc[i] = momentum * bias_inc[i] - learning_rate * in_grad_i; bias[i] += bias_inc[i]; } } applyBiasDecay(); }
virtual void PLearn::RBMTruncExpLayer::bpropUpdate | ( | const Mat & | inputs, |
const Mat & | outputs, | ||
Mat & | input_gradients, | ||
const Mat & | output_gradients, | ||
bool | accumulate = false |
||
) | [inline, virtual] |
Back-propagate the output gradient to the input, and update parameters.
Implements PLearn::RBMLayer.
Definition at line 92 of file RBMTruncExpLayer.h.
References PLASSERT_MSG.
{ PLASSERT_MSG(false, "Not implemented"); }
void PLearn::RBMTruncExpLayer::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::RBMLayer.
Definition at line 165 of file DEPRECATED/RBMTruncExpLayer.cc.
{ inherited::build(); build_(); }
virtual void PLearn::RBMTruncExpLayer::build | ( | ) | [virtual] |
Post-constructor.
The normal implementation should call simply inherited::build(), then this class's build_(). This method should be callable again at later times, after modifying some option fields to change the "architecture" of the object.
Reimplemented from PLearn::RBMLayer.
void PLearn::RBMTruncExpLayer::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::RBMLayer.
Definition at line 152 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::TVec< T >::resize(), and PLearn::sample().
{ if( size < 0 ) size = int(units_types.size()); if( size != (int) units_types.size() ) units_types = string( size, 'l' ); activations.resize( size ); sample.resize( size ); expectation.resize( size ); expectation_is_up_to_date = false; }
void PLearn::RBMTruncExpLayer::build_ | ( | ) | [private] |
This does the actual building.
Reimplemented from PLearn::RBMLayer.
string PLearn::RBMTruncExpLayer::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
virtual string PLearn::RBMTruncExpLayer::classname | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
void PLearn::RBMTruncExpLayer::computeExpectation | ( | ) | [virtual] |
compute the expectation
Implements PLearn::RBMLayer.
Definition at line 102 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::exp(), and i.
{ if( expectation_is_up_to_date ) return; /* Conditional expectation: * E[u|x] = 1/(1-exp(a)) + 1/a */ for( int i=0 ; i<size ; i++ ) { real a_i = activations[i]; expectation[i] = 1/(1-exp(a_i)) + 1/a_i; } expectation_is_up_to_date = true; }
virtual void PLearn::RBMTruncExpLayer::computeExpectation | ( | ) | [virtual] |
compute the expectation
Implements PLearn::RBMLayer.
void PLearn::RBMTruncExpLayer::computeExpectations | ( | ) | [virtual] |
Not implemented.
Implements PLearn::RBMLayer.
Definition at line 146 of file RBMTruncExpLayer.cc.
References PLearn::RBMLayer::activations, PLearn::RBMLayer::batch_size, PLearn::exp(), PLearn::RBMLayer::expectations, PLearn::RBMLayer::expectations_are_up_to_date, i, PLearn::TMat< T >::length(), PLASSERT, PLearn::RBMLayer::size, and PLearn::TMat< T >::width().
{ if( expectations_are_up_to_date ) return; /* Conditional expectation: * E[u|x] = 1/(1-exp(-a)) - 1/a */ PLASSERT( expectations.width() == size && expectations.length() == batch_size ); for (int k = 0; k < batch_size; k++) for( int i=0 ; i<size ; i++ ) { real a_i = activations(k,i); // Polynomial approximation to avoid numerical instability // f(a) = 1/2 + a/12 - a^3/720 + O(a^5) if( fabs( a_i ) <= 0.01 ) expectations(k,i) = 0.5 + a_i*(1./12. - a_i*a_i/720.); else expectations(k,i) = 1/(1-exp(-a_i)) - 1/a_i; } expectations_are_up_to_date = true; }
static void PLearn::RBMTruncExpLayer::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares the class options.
Reimplemented from PLearn::RBMLayer.
void PLearn::RBMTruncExpLayer::declareOptions | ( | OptionList & | ol | ) | [static, protected] |
Declares the class options.
Reimplemented from PLearn::RBMLayer.
Definition at line 141 of file DEPRECATED/RBMTruncExpLayer.cc.
{ /* declareOption(ol, "size", &RBMTruncExpLayer::size, OptionBase::buildoption, "Number of units."); */ // Now call the parent class' declareOptions inherited::declareOptions(ol); }
static const PPath& PLearn::RBMTruncExpLayer::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 98 of file DEPRECATED/RBMTruncExpLayer.h.
:
//##### Not Options #####################################################
static const PPath& PLearn::RBMTruncExpLayer::declaringFile | ( | ) | [inline, static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 103 of file RBMTruncExpLayer.h.
:
//##### Not Options #####################################################
virtual RBMTruncExpLayer* PLearn::RBMTruncExpLayer::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RBMLayer.
RBMTruncExpLayer * PLearn::RBMTruncExpLayer::deepCopy | ( | CopiesMap & | copies | ) | const [virtual] |
Reimplemented from PLearn::RBMLayer.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
forward propagation
Reimplemented from PLearn::RBMLayer.
Definition at line 175 of file RBMTruncExpLayer.cc.
References PLearn::RBMLayer::bias, PLearn::exp(), i, PLearn::OnlineLearningModule::input_size, PLearn::OnlineLearningModule::output_size, PLASSERT, PLearn::TVec< T >::resize(), PLearn::TVec< T >::size(), and PLearn::RBMLayer::size.
{ PLASSERT( input.size() == input_size ); output.resize( output_size ); for( int i=0 ; i<size ; i++ ) { real a_i = input[i] + bias[i]; // Polynomial approximation to avoid numerical instability // f(a) = 1/(1-exp(-a) - 1/a // f(a) = 1/2 + a/12 - a^3/720 + O(a^5) if( fabs( a_i ) <= 0.01 ) output[i] = 0.5 + a_i*(1./12. - a_i*a_i/720.); else output[i] = 1/(1-exp(-a_i)) - 1/a_i; } }
virtual void PLearn::RBMTruncExpLayer::generateSample | ( | ) | [virtual] |
generate a sample, and update the sample field
Implements PLearn::RBMLayer.
void PLearn::RBMTruncExpLayer::generateSample | ( | ) | [virtual] |
generate a sample, and update the sample field
Implements PLearn::RBMLayer.
Definition at line 83 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::exp(), i, pl_log, and PLearn::sample().
{ /* The cumulative is : * C(U) = P(u<U | x) = (1 - exp(-U a)) / (1 - exp(-a)) if 0 < U < 1, * 0 if U <= 0 and * 1 if 1 <= U * * And the inverse, if 0 <= s <=1: * C^{-1}(s) = - log(1 - s*(1 - exp(-a)) / a */ for( int i=0 ; i<size ; i++ ) { real s = random_gen->uniform_sample(); real a_i = activations[i]; sample[i] = - pl_log( 1. - s*( 1 - exp(-a_i) ) ) / a_i; } }
void PLearn::RBMTruncExpLayer::generateSamples | ( | ) | [virtual] |
Generate a mini-batch set of samples.
Implements PLearn::RBMLayer.
Definition at line 98 of file RBMTruncExpLayer.cc.
References PLearn::RBMLayer::activations, PLearn::RBMLayer::batch_size, PLearn::RBMLayer::expectations_are_up_to_date, i, PLearn::TMat< T >::length(), PLearn::logadd(), pl_log, PLASSERT, PLASSERT_MSG, PLCHECK_MSG, PLearn::RBMLayer::random_gen, PLearn::RBMLayer::samples, PLearn::RBMLayer::size, and PLearn::TMat< T >::width().
{ PLASSERT_MSG(random_gen, "random_gen should be initialized before generating samples"); PLCHECK_MSG(expectations_are_up_to_date, "Expectations should be computed " "before calling generateSamples()"); PLASSERT( samples.width() == size && samples.length() == batch_size ); for (int k = 0; k < batch_size; k++) for (int i=0 ; i<size ; i++) { real s = random_gen->uniform_sample(); real a_i = activations(k,i); if( fabs( a_i ) <= 1e-5 ) samples(k, i) = s + a_i*( s*(1 - s)/2 ); else samples(k, i) = logadd( pl_log( 1-s ), pl_log(s) + a_i ) / a_i; } }
void PLearn::RBMTruncExpLayer::getAllActivations | ( | PP< RBMParameters > | rbmp, |
int | offset = 0 |
||
) | [virtual] |
Uses "rbmp" to obtain the activations of all units in this layer.
Unit 0 of this layer corresponds to unit "offset" of "rbmp".
Implements PLearn::RBMLayer.
Definition at line 77 of file DEPRECATED/RBMTruncExpLayer.cc.
{ rbmp->computeUnitActivations( offset, size, activations ); expectation_is_up_to_date = false; }
virtual OptionList& PLearn::RBMTruncExpLayer::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
OptionList & PLearn::RBMTruncExpLayer::getOptionList | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
OptionMap & PLearn::RBMTruncExpLayer::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
virtual OptionMap& PLearn::RBMTruncExpLayer::getOptionMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
RemoteMethodMap & PLearn::RBMTruncExpLayer::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
Definition at line 49 of file DEPRECATED/RBMTruncExpLayer.cc.
virtual RemoteMethodMap& PLearn::RBMTruncExpLayer::getRemoteMethodMap | ( | ) | const [virtual] |
Reimplemented from PLearn::Object.
void PLearn::RBMTruncExpLayer::getUnitActivations | ( | int | i, |
PP< RBMParameters > | rbmp, | ||
int | offset = 0 |
||
) | [virtual] |
Uses "rbmp" to obtain the activations of unit "i" of this layer.
This activation vector is computed by the "i+offset"-th unit of "rbmp"
Implements PLearn::RBMLayer.
Definition at line 67 of file DEPRECATED/RBMTruncExpLayer.cc.
References PLearn::TVec< T >::subVec().
{ Vec activation = activations.subVec( i, 1 ); rbmp->computeUnitActivations( i+offset, 1, activation ); expectation_is_up_to_date = false; }
void PLearn::RBMTruncExpLayer::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::RBMLayer.
Definition at line 172 of file DEPRECATED/RBMTruncExpLayer.cc.
{ inherited::makeDeepCopyFromShallowCopy(copies); }
virtual void PLearn::RBMTruncExpLayer::makeDeepCopyFromShallowCopy | ( | CopiesMap & | copies | ) | [virtual] |
Transforms a shallow copy into a deep copy.
Reimplemented from PLearn::RBMLayer.
static StaticInitializer PLearn::RBMTruncExpLayer::_static_initializer_ [static] |
Reimplemented from PLearn::RBMLayer.
Definition at line 98 of file DEPRECATED/RBMTruncExpLayer.h.