template<typename Architecture_t, typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>
class TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >

Adam Optimizer class.

This class represents the Adam Optimizer.

Definition at line 44 of file Adam.h.

Public Types
using	Matrix_t = typename Architecture_t::Matrix_t

using	Scalar_t = typename Architecture_t::Scalar_t

Public Types inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
using	Matrix_t = typename Architecture_t::Matrix_t

using	Scalar_t = typename Architecture_t::Scalar_t

Public Member Functions
	TAdam (DeepNet_t &deepNet, Scalar_t learningRate=0.001, Scalar_t beta1=0.9, Scalar_t beta2=0.999, Scalar_t epsilon=1e-8)
	Constructor. More...

	~TAdam ()=default
	Destructor. More...

Scalar_t	GetBeta1 () const
	Getters. More...

Scalar_t	GetBeta2 () const

Scalar_t	GetEpsilon () const

std::vector< std::vector< Matrix_t > > &	GetFirstMomentBiases ()

std::vector< Matrix_t > &	GetFirstMomentBiasesAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetFirstMomentWeights ()

std::vector< Matrix_t > &	GetFirstMomentWeightsAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetSecondMomentBiases ()

std::vector< Matrix_t > &	GetSecondMomentBiasesAt (size_t i)

std::vector< std::vector< Matrix_t > > &	GetSecondMomentWeights ()

std::vector< Matrix_t > &	GetSecondMomentWeightsAt (size_t i)

Public Member Functions inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
	VOptimizer (Scalar_t learningRate, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > &deepNet)
	Constructor. More...

virtual	~VOptimizer ()=default
	Virtual Destructor. More...

size_t	GetGlobalStep () const

VGeneralLayer< Architecture_t > *	GetLayerAt (size_t i)

std::vector< VGeneralLayer< Architecture_t > * > &	GetLayers ()

Scalar_t	GetLearningRate () const
	Getters. More...

void	IncrementGlobalStep ()
	Increments the global step. More...

void	SetLearningRate (size_t learningRate)
	Setters. More...

void	Step ()
	Performs one step of optimization. More...

Protected Member Functions
void	UpdateBiases (size_t layerIndex, std::vector< Matrix_t > &biases, const std::vector< Matrix_t > &biasGradients)
	Update the biases, given the current bias gradients. More...

void	UpdateWeights (size_t layerIndex, std::vector< Matrix_t > &weights, const std::vector< Matrix_t > &weightGradients)
	Update the weights, given the current weight gradients. More...

virtual void	UpdateBiases (size_t layerIndex, std::vector< Matrix_t > &biases, const std::vector< Matrix_t > &biasGradients)=0
	Update the biases, given the current bias gradients. More...

virtual void	UpdateWeights (size_t layerIndex, std::vector< Matrix_t > &weights, const std::vector< Matrix_t > &weightGradients)=0
	Update the weights, given the current weight gradients. More...

Protected Attributes
Scalar_t	fBeta1
	The Beta1 constant used by the optimizer. More...

Scalar_t	fBeta2
	The Beta2 constant used by the optimizer. More...

Scalar_t	fEpsilon
	The Smoothing term used to avoid division by zero. More...

std::vector< std::vector< Matrix_t > >	fFirstMomentBiases
	The decaying average of the first moment of the past bias gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fFirstMomentWeights
	The decaying average of the first moment of the past weight gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fSecondMomentBiases
	The decaying average of the second moment of the past bias gradients associated with the deep net. More...

std::vector< std::vector< Matrix_t > >	fSecondMomentWeights
	The decaying average of the second moment of the past weight gradients associated with the deep net. More...

Protected Attributes inherited from TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >
TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > &	fDeepNet
	The reference to the deep net. More...

size_t	fGlobalStep
	The current global step count during training. More...

Scalar_t	fLearningRate
	The learning rate used for training. More...

#include <TMVA/DNN/Adam.h>

Inheritance diagram for TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >:

[legend]

Member Typedef Documentation

◆ Matrix_t

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

using TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::Matrix_t = typename Architecture_t::Matrix_t

Definition at line 46 of file Adam.h.

◆ Scalar_t

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

using TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::Scalar_t = typename Architecture_t::Scalar_t

Definition at line 47 of file Adam.h.

Constructor & Destructor Documentation

◆ TAdam()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::TAdam	(	DeepNet_t &	deepNet,
		Scalar_t	learningRate = `0.001`,
		Scalar_t	beta1 = `0.9`,
		Scalar_t	beta2 = `0.999`,
		Scalar_t	epsilon = `1e-8`
	)

Constructor.

Definition at line 101 of file Adam.h.

◆ ~TAdam()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::~TAdam ( )

default

Destructor.

Member Function Documentation

◆ GetBeta1()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetBeta1 ( ) const

inline

Getters.

Definition at line 79 of file Adam.h.

◆ GetBeta2()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetBeta2 ( ) const

inline

Definition at line 80 of file Adam.h.

◆ GetEpsilon()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetEpsilon ( ) const

inline

Definition at line 81 of file Adam.h.

◆ GetFirstMomentBiases()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentBiases ( )

inline

Definition at line 86 of file Adam.h.

◆ GetFirstMomentBiasesAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentBiasesAt ( size_t i )

inline

Definition at line 87 of file Adam.h.

◆ GetFirstMomentWeights()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentWeights ( )

inline

Definition at line 83 of file Adam.h.

◆ GetFirstMomentWeightsAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetFirstMomentWeightsAt ( size_t i )

inline

Definition at line 84 of file Adam.h.

◆ GetSecondMomentBiases()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentBiases ( )

inline

Definition at line 92 of file Adam.h.

◆ GetSecondMomentBiasesAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentBiasesAt ( size_t i )

inline

Definition at line 93 of file Adam.h.

◆ GetSecondMomentWeights()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< std::vector< Matrix_t > > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentWeights ( )

inline

Definition at line 89 of file Adam.h.

◆ GetSecondMomentWeightsAt()

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector< Matrix_t > & TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::GetSecondMomentWeightsAt ( size_t i )

inline

Definition at line 90 of file Adam.h.

◆ UpdateBiases()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

auto TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::UpdateBiases	(	size_t	layerIndex,
		std::vector< Matrix_t > &	biases,
		const std::vector< Matrix_t > &	biasGradients
	)

protectedvirtual

Update the biases, given the current bias gradients.

Implements TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >.

Definition at line 168 of file Adam.h.

◆ UpdateWeights()

template<typename Architecture_t , typename Layer_t , typename DeepNet_t >

auto TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::UpdateWeights	(	size_t	layerIndex,
		std::vector< Matrix_t > &	weights,
		const std::vector< Matrix_t > &	weightGradients
	)

protectedvirtual

Update the weights, given the current weight gradients.

Implements TMVA::DNN::VOptimizer< Architecture_t, VGeneralLayer< Architecture_t >, TDeepNet< Architecture_t, VGeneralLayer< Architecture_t > > >.

Definition at line 144 of file Adam.h.

Member Data Documentation

◆ fBeta1

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fBeta1

protected

The Beta1 constant used by the optimizer.

Definition at line 50 of file Adam.h.

◆ fBeta2

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fBeta2

protected

The Beta2 constant used by the optimizer.

Definition at line 51 of file Adam.h.

◆ fEpsilon

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

Scalar_t TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fEpsilon

protected

The Smoothing term used to avoid division by zero.

Definition at line 52 of file Adam.h.

◆ fFirstMomentBiases

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fFirstMomentBiases

protected

The decaying average of the first moment of the past bias gradients associated with the deep net.

Definition at line 56 of file Adam.h.

◆ fFirstMomentWeights

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fFirstMomentWeights

protected

The decaying average of the first moment of the past weight gradients associated with the deep net.

Definition at line 54 of file Adam.h.

◆ fSecondMomentBiases

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fSecondMomentBiases

protected

The decaying average of the second moment of the past bias gradients associated with the deep net.

Definition at line 61 of file Adam.h.

◆ fSecondMomentWeights

template<typename Architecture_t , typename Layer_t = VGeneralLayer<Architecture_t>, typename DeepNet_t = TDeepNet<Architecture_t, Layer_t>>

std::vector<std::vector<Matrix_t> > TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >::fSecondMomentWeights

protected

The decaying average of the second moment of the past weight gradients associated with the deep net.

Definition at line 59 of file Adam.h.

Libraries for TMVA::DNN::TAdam< Architecture_t, Layer_t, DeepNet_t >:

[legend]

Public Types

Public Member Functions

Protected Member Functions

Protected Attributes

Member Typedef Documentation

◆ Matrix_t

◆ Scalar_t

Constructor & Destructor Documentation

◆ TAdam()

◆ ~TAdam()

Member Function Documentation

◆ GetBeta1()

◆ GetBeta2()

◆ GetEpsilon()

◆ GetFirstMomentBiases()

◆ GetFirstMomentBiasesAt()

◆ GetFirstMomentWeights()

◆ GetFirstMomentWeightsAt()

◆ GetSecondMomentBiases()

◆ GetSecondMomentBiasesAt()

◆ GetSecondMomentWeights()

◆ GetSecondMomentWeightsAt()

◆ UpdateBiases()

◆ UpdateWeights()

Member Data Documentation

◆ fBeta1

◆ fBeta2

◆ fEpsilon

◆ fFirstMomentBiases

◆ fFirstMomentWeights

◆ fSecondMomentBiases

◆ fSecondMomentWeights