Logo ROOT   6.21/01
Reference Guide
TMVA::DNN::TDenseLayer< Architecture_t > Class Template Reference

template<typename Architecture_t>
class TMVA::DNN::TDenseLayer< Architecture_t >

Generic layer class.

This generic layer class represents a dense layer of a neural network with a given width n and activation function f. The activation function of each layer is given by \(\mathbf{u} = \mathbf{W}\mathbf{x} + \boldsymbol{\theta}\).

In addition to the weight and bias matrices, each layer allocates memory for its activations and the corresponding input tensor before evaluation of the activation function as well as the gradients of the weights and biases.

The layer provides member functions for the forward propagation of activations through the given layer.

Definition at line 57 of file DenseLayer.h.

Public Types

using Matrix_t = typename Architecture_t::Matrix_t
 
using Scalar_t = typename Architecture_t::Scalar_t
 
using Tensor_t = typename Architecture_t::Tensor_t
 

Public Member Functions

 TDenseLayer (size_t BatchSize, size_t InputWidth, size_t Width, EInitialization init, Scalar_t DropoutProbability, EActivationFunction f, ERegularization reg, Scalar_t weightDecay)
 Constructor. More...
 
 TDenseLayer (TDenseLayer< Architecture_t > *layer)
 Copy the dense layer provided as a pointer. More...
 
 TDenseLayer (const TDenseLayer &)
 Copy Constructor. More...
 
 ~TDenseLayer ()
 Destructor. More...
 
virtual void AddWeightsXMLTo (void *parent)
 Writes the information and the weights about the layer in an XML node. More...
 
void Backward (Tensor_t &gradients_backward, const Tensor_t &activations_backward)
 Compute weight, bias and activation gradients. More...
 
void Forward (Tensor_t &input, bool applyDropout=false)
 Compute activation of the layer for the given input. More...
 
EActivationFunction GetActivationFunction () const
 
Scalar_t GetDropoutProbability () const
 Getters. More...
 
const Tensor_tGetInputActivation () const
 
Tensor_tGetInputActivation ()
 
ERegularization GetRegularization () const
 
Scalar_t GetWeightDecay () const
 
void Print () const
 std::vector<Matrix_t> &inp1, std::vector<Matrix_t> &inp2); More...
 
virtual void ReadWeightsFromXML (void *parent)
 Read the information and the weights about the layer from XML node. More...
 
virtual void SetDropoutProbability (Scalar_t dropoutProbability)
 Set dropout probabilities. More...
 
- Public Member Functions inherited from TMVA::DNN::VGeneralLayer< Architecture_t >
 VGeneralLayer (size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, size_t Height, size_t Width, size_t WeightsNSlices, size_t WeightsNRows, size_t WeightsNCols, size_t BiasesNSlices, size_t BiasesNRows, size_t BiasesNCols, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, EInitialization Init)
 Constructor. More...
 
 VGeneralLayer (size_t BatchSize, size_t InputDepth, size_t InputHeight, size_t InputWidth, size_t Depth, size_t Height, size_t Width, size_t WeightsNSlices, std::vector< size_t > WeightsNRows, std::vector< size_t > WeightsNCols, size_t BiasesNSlices, std::vector< size_t > BiasesNRows, std::vector< size_t > BiasesNCols, size_t OutputNSlices, size_t OutputNRows, size_t OutputNCols, EInitialization Init)
 General Constructor with different weights dimension. More...
 
 VGeneralLayer (VGeneralLayer< Architecture_t > *layer)
 Copy the layer provided as a pointer. More...
 
 VGeneralLayer (const VGeneralLayer &)
 Copy Constructor. More...
 
virtual ~VGeneralLayer ()
 Virtual Destructor. More...
 
void CopyBiases (const std::vector< Matrix_t > &otherBiases)
 Copies the biases provided as an input. More...
 
template<typename Arch >
void CopyParameters (const VGeneralLayer< Arch > &layer)
 Copy all trainable weight and biases from another equivalent layer but with different architecture The function can copy also extra parameters in addition to weights and biases if they are return by the function GetExtraLayerParameters. More...
 
void CopyWeights (const std::vector< Matrix_t > &otherWeights)
 Copies the weights provided as an input. More...
 
const Tensor_tGetActivationGradients () const
 
Tensor_tGetActivationGradients ()
 
Matrix_t GetActivationGradientsAt (size_t i)
 
const Matrix_tGetActivationGradientsAt (size_t i) const
 
size_t GetBatchSize () const
 Getters. More...
 
const std::vector< Matrix_t > & GetBiases () const
 
std::vector< Matrix_t > & GetBiases ()
 
const Matrix_tGetBiasesAt (size_t i) const
 
Matrix_tGetBiasesAt (size_t i)
 
const std::vector< Matrix_t > & GetBiasGradients () const
 
std::vector< Matrix_t > & GetBiasGradients ()
 
const Matrix_tGetBiasGradientsAt (size_t i) const
 
Matrix_tGetBiasGradientsAt (size_t i)
 
size_t GetDepth () const
 
virtual std::vector< Matrix_tGetExtraLayerParameters () const
 
size_t GetHeight () const
 
EInitialization GetInitialization () const
 
size_t GetInputDepth () const
 
size_t GetInputHeight () const
 
size_t GetInputWidth () const
 
const Tensor_tGetOutput () const
 
Tensor_tGetOutput ()
 
Matrix_t GetOutputAt (size_t i)
 
const Matrix_tGetOutputAt (size_t i) const
 
const std::vector< Matrix_t > & GetWeightGradients () const
 
std::vector< Matrix_t > & GetWeightGradients ()
 
const Matrix_tGetWeightGradientsAt (size_t i) const
 
Matrix_tGetWeightGradientsAt (size_t i)
 
const std::vector< Matrix_t > & GetWeights () const
 
std::vector< Matrix_t > & GetWeights ()
 
const Matrix_tGetWeightsAt (size_t i) const
 
Matrix_tGetWeightsAt (size_t i)
 
size_t GetWidth () const
 
virtual void Initialize ()
 Initialize the weights and biases according to the given initialization method. More...
 
bool IsTraining () const
 
void ReadMatrixXML (void *node, const char *name, Matrix_t &matrix)
 
virtual void ResetTraining ()
 Reset some training flags after a loop on all batches Some layer (e.g. More...
 
void SetBatchSize (size_t batchSize)
 Setters. More...
 
void SetDepth (size_t depth)
 
virtual void SetExtraLayerParameters (const std::vector< Matrix_t > &)
 
void SetHeight (size_t height)
 
void SetInputDepth (size_t inputDepth)
 
void SetInputHeight (size_t inputHeight)
 
void SetInputWidth (size_t inputWidth)
 
void SetIsTraining (bool isTraining)
 
void SetWidth (size_t width)
 
void Update (const Scalar_t learningRate)
 Updates the weights and biases, given the learning rate. More...
 
void UpdateBiases (const std::vector< Matrix_t > &biasGradients, const Scalar_t learningRate)
 Updates the biases, given the gradients and the learning rate. More...
 
void UpdateBiasGradients (const std::vector< Matrix_t > &biasGradients, const Scalar_t learningRate)
 Updates the bias gradients, given some other weight gradients and learning rate. More...
 
void UpdateWeightGradients (const std::vector< Matrix_t > &weightGradients, const Scalar_t learningRate)
 Updates the weight gradients, given some other weight gradients and learning rate. More...
 
void UpdateWeights (const std::vector< Matrix_t > &weightGradients, const Scalar_t learningRate)
 Updates the weights, given the gradients and the learning rate,. More...
 
void WriteMatrixToXML (void *node, const char *name, const Matrix_t &matrix)
 
void WriteTensorToXML (void *node, const char *name, const std::vector< Matrix_t > &tensor)
 helper functions for XML More...
 

Private Attributes

Architecture_t::ActivationDescriptor_t fActivationDesc
 
Tensor_t fDerivatives
 output of GEMM and input to activation function More...
 
Scalar_t fDropoutProbability
 activation functgion gradient More...
 
EActivationFunction fF
 Activation function of the layer. More...
 
Tensor_t fInputActivation
 
ERegularization fReg
 The regularization method. More...
 
Scalar_t fWeightDecay
 The weight decay. More...
 

Additional Inherited Members

- Protected Attributes inherited from TMVA::DNN::VGeneralLayer< Architecture_t >
Tensor_t fActivationGradients
 Gradients w.r.t. the activations of this layer. More...
 
size_t fBatchSize
 Batch size used for training and evaluation. More...
 
std::vector< Matrix_tfBiases
 The biases associated to the layer. More...
 
std::vector< Matrix_tfBiasGradients
 Gradients w.r.t. the bias values of the layer. More...
 
size_t fDepth
 The depth of the layer. More...
 
size_t fHeight
 The height of the layer. More...
 
EInitialization fInit
 The initialization method. More...
 
size_t fInputDepth
 The depth of the previous layer or input. More...
 
size_t fInputHeight
 The height of the previous layer or input. More...
 
size_t fInputWidth
 The width of the previous layer or input. More...
 
bool fIsTraining
 Flag indicating the mode. More...
 
Tensor_t fOutput
 Activations of this layer. More...
 
std::vector< Matrix_tfWeightGradients
 Gradients w.r.t. the weights of the layer. More...
 
std::vector< Matrix_tfWeights
 The weights associated to the layer. More...
 
size_t fWidth
 The width of this layer. More...
 

#include <TMVA/DNN/DenseLayer.h>

Inheritance diagram for TMVA::DNN::TDenseLayer< Architecture_t >:
[legend]

Member Typedef Documentation

◆ Matrix_t

template<typename Architecture_t>
using TMVA::DNN::TDenseLayer< Architecture_t >::Matrix_t = typename Architecture_t::Matrix_t

Definition at line 61 of file DenseLayer.h.

◆ Scalar_t

template<typename Architecture_t>
using TMVA::DNN::TDenseLayer< Architecture_t >::Scalar_t = typename Architecture_t::Scalar_t

Definition at line 60 of file DenseLayer.h.

◆ Tensor_t

template<typename Architecture_t>
using TMVA::DNN::TDenseLayer< Architecture_t >::Tensor_t = typename Architecture_t::Tensor_t

Definition at line 62 of file DenseLayer.h.

Constructor & Destructor Documentation

◆ TDenseLayer() [1/3]

template<typename Architecture_t >
TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer ( size_t  BatchSize,
size_t  InputWidth,
size_t  Width,
EInitialization  init,
Scalar_t  DropoutProbability,
EActivationFunction  f,
ERegularization  reg,
Scalar_t  weightDecay 
)

Constructor.

Definition at line 134 of file DenseLayer.h.

◆ TDenseLayer() [2/3]

template<typename Architecture_t >
TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer ( TDenseLayer< Architecture_t > *  layer)

Copy the dense layer provided as a pointer.

Definition at line 150 of file DenseLayer.h.

◆ TDenseLayer() [3/3]

template<typename Architecture_t >
TMVA::DNN::TDenseLayer< Architecture_t >::TDenseLayer ( const TDenseLayer< Architecture_t > &  layer)

Copy Constructor.

Definition at line 162 of file DenseLayer.h.

◆ ~TDenseLayer()

template<typename Architecture_t >
TMVA::DNN::TDenseLayer< Architecture_t >::~TDenseLayer ( )

Destructor.

Definition at line 174 of file DenseLayer.h.

Member Function Documentation

◆ AddWeightsXMLTo()

template<typename Architecture_t >
void TMVA::DNN::TDenseLayer< Architecture_t >::AddWeightsXMLTo ( void parent)
virtual

Writes the information and the weights about the layer in an XML node.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 247 of file DenseLayer.h.

◆ Backward()

template<typename Architecture_t >
auto TMVA::DNN::TDenseLayer< Architecture_t >::Backward ( Tensor_t gradients_backward,
const Tensor_t activations_backward 
)
virtual

Compute weight, bias and activation gradients.

Uses the precomputed first partial derviatives of the activation function computed during forward propagation and modifies them. Must only be called directly a the corresponding call to Forward(...).

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 204 of file DenseLayer.h.

◆ Forward()

template<typename Architecture_t >
auto TMVA::DNN::TDenseLayer< Architecture_t >::Forward ( Tensor_t input,
bool  applyDropout = false 
)
virtual

Compute activation of the layer for the given input.

The input must be in 3D tensor form with the different matrices corresponding to different events in the batch. Computes activations as well as the first partial derivative of the activation function at those activations.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 185 of file DenseLayer.h.

◆ GetActivationFunction()

template<typename Architecture_t>
EActivationFunction TMVA::DNN::TDenseLayer< Architecture_t >::GetActivationFunction ( ) const
inline

Definition at line 124 of file DenseLayer.h.

◆ GetDropoutProbability()

template<typename Architecture_t>
Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::GetDropoutProbability ( ) const
inline

Getters.

Definition at line 118 of file DenseLayer.h.

◆ GetInputActivation() [1/2]

template<typename Architecture_t>
const Tensor_t& TMVA::DNN::TDenseLayer< Architecture_t >::GetInputActivation ( ) const
inline

Definition at line 121 of file DenseLayer.h.

◆ GetInputActivation() [2/2]

template<typename Architecture_t>
Tensor_t& TMVA::DNN::TDenseLayer< Architecture_t >::GetInputActivation ( )
inline

Definition at line 122 of file DenseLayer.h.

◆ GetRegularization()

template<typename Architecture_t>
ERegularization TMVA::DNN::TDenseLayer< Architecture_t >::GetRegularization ( ) const
inline

Definition at line 125 of file DenseLayer.h.

◆ GetWeightDecay()

template<typename Architecture_t>
Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::GetWeightDecay ( ) const
inline

Definition at line 126 of file DenseLayer.h.

◆ Print()

template<typename Architecture_t >
void TMVA::DNN::TDenseLayer< Architecture_t >::Print ( ) const
virtual

std::vector<Matrix_t> &inp1, std::vector<Matrix_t> &inp2);

Printing the layer info.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 229 of file DenseLayer.h.

◆ ReadWeightsFromXML()

template<typename Architecture_t >
void TMVA::DNN::TDenseLayer< Architecture_t >::ReadWeightsFromXML ( void parent)
virtual

Read the information and the weights about the layer from XML node.

Implements TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 265 of file DenseLayer.h.

◆ SetDropoutProbability()

template<typename Architecture_t>
virtual void TMVA::DNN::TDenseLayer< Architecture_t >::SetDropoutProbability ( Scalar_t  dropoutProbability)
inlinevirtual

Set dropout probabilities.

Reimplemented from TMVA::DNN::VGeneralLayer< Architecture_t >.

Definition at line 115 of file DenseLayer.h.

Member Data Documentation

◆ fActivationDesc

template<typename Architecture_t>
Architecture_t::ActivationDescriptor_t TMVA::DNN::TDenseLayer< Architecture_t >::fActivationDesc
private

Definition at line 75 of file DenseLayer.h.

◆ fDerivatives

template<typename Architecture_t>
Tensor_t TMVA::DNN::TDenseLayer< Architecture_t >::fDerivatives
private

output of GEMM and input to activation function

Definition at line 67 of file DenseLayer.h.

◆ fDropoutProbability

template<typename Architecture_t>
Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::fDropoutProbability
private

activation functgion gradient

Probability that an input is active.

Definition at line 69 of file DenseLayer.h.

◆ fF

template<typename Architecture_t>
EActivationFunction TMVA::DNN::TDenseLayer< Architecture_t >::fF
private

Activation function of the layer.

Definition at line 71 of file DenseLayer.h.

◆ fInputActivation

template<typename Architecture_t>
Tensor_t TMVA::DNN::TDenseLayer< Architecture_t >::fInputActivation
private

Definition at line 66 of file DenseLayer.h.

◆ fReg

template<typename Architecture_t>
ERegularization TMVA::DNN::TDenseLayer< Architecture_t >::fReg
private

The regularization method.

Definition at line 72 of file DenseLayer.h.

◆ fWeightDecay

template<typename Architecture_t>
Scalar_t TMVA::DNN::TDenseLayer< Architecture_t >::fWeightDecay
private

The weight decay.

Definition at line 73 of file DenseLayer.h.


The documentation for this class was generated from the following file: