Logo ROOT   6.08/07
Reference Guide
Namespaces | Classes | Typedefs | Enumerations | Functions | Variables
TMVA::DNN Namespace Reference

Namespaces

 Blas
 

Classes

class  Batch
 The Batch class encapsulates one mini-batch. More...
 
class  ClassificationSettings
 Settings for classificationused to distinguish between different function signatures. More...
 
class  Layer
 Layer defines the layout of a layer. More...
 
class  LayerData
 LayerData holds the data of one layer. More...
 
class  MeanVariance
 
class  Net
 neural net More...
 
class  Settings
 Settings for the training of the neural net. More...
 
class  Steepest
 Steepest Gradient Descent algorithm (SGD) More...
 
class  TBatch
 TBatch. More...
 
class  TBatchIterator
 TBatchIterator. More...
 
class  TCpu
 The TCpu architecture class. More...
 
class  TCpuBuffer
 TCpuBuffer. More...
 
class  TCpuMatrix
 The TCpuMatrix class. More...
 
class  TCuda
 The TCuda architecture class. More...
 
class  TCudaDeviceBuffer
 TCudaDeviceBuffer. More...
 
class  TCudaDeviceReference
 TCudaDeviceReference. More...
 
class  TCudaHostBuffer
 TCudaHostBuffer. More...
 
class  TCudaMatrix
 TCudaMatrix Class. More...
 
class  TDataLoader
 TDataLoader. More...
 
class  TDevice
 TDevice. More...
 
class  TGradientDescent
 
class  TLayer
 Generic layer class. More...
 
class  TNet
 Generic neural network class. More...
 
class  TReference
 The reference architecture class. More...
 
class  TSharedLayer
 Layer class width shared weight and bias layers. More...
 

Typedefs

typedef std::vector< char > DropContainer
 
using IndexIterator_t = typename std::vector< size_t >::iterator
 
using MatrixInput_t = std::pair< const TMatrixT< Double_t > &, const TMatrixT< Double_t > & >
 
typedef std::tuple< Settings &, Batch &, DropContainer & > pass_through_type
 
using TMVAInput_t = std::vector< Event * >
 

Enumerations

enum  EActivationFunction {
  EActivationFunction::kIdentity = 0, EActivationFunction::kRelu = 1, EActivationFunction::kSigmoid = 2, EActivationFunction::kTanh = 3,
  EActivationFunction::kSymmRelu = 4, EActivationFunction::kSoftSign = 5, EActivationFunction::kGauss = 6
}
 Enum that represents layer activation functions. More...
 
enum  EInitialization { EInitialization::kGauss = 'G', EInitialization::kUniform = 'U', EInitialization::kIdentity = 'I', EInitialization::kZero = 'Z' }
 
enum  ELossFunction { ELossFunction::kCrossEntropy = 'C', ELossFunction::kMeanSquaredError = 'R', ELossFunction::kSoftmaxCrossEntropy = 'S' }
 Enum that represents objective functions for the net, i.e. More...
 
enum  EnumFunction {
  EnumFunction::ZERO = '0', EnumFunction::LINEAR = 'L', EnumFunction::TANH = 'T', EnumFunction::RELU = 'R',
  EnumFunction::SYMMRELU = 'r', EnumFunction::TANHSHIFT = 't', EnumFunction::SIGMOID = 's', EnumFunction::SOFTSIGN = 'S',
  EnumFunction::GAUSS = 'G', EnumFunction::GAUSSCOMPLEMENT = 'C'
}
 
enum  EnumRegularization { EnumRegularization::NONE, EnumRegularization::L1, EnumRegularization::L2, EnumRegularization::L1MAX }
 
enum  EOutputFunction { EOutputFunction::kIdentity = 'I', EOutputFunction::kSigmoid = 'S', EOutputFunction::kSoftmax = 'M' }
 Enum that represents output functions. More...
 
enum  ERegularization { ERegularization::kNone = '0', ERegularization::kL1 = '1', ERegularization::kL2 = '2' }
 Enum representing the regularization type applied for a given layer. More...
 
enum  MinimizerType { fSteepest }
 < list all the minimizer types More...
 
enum  ModeErrorFunction { ModeErrorFunction::SUMOFSQUARES = 'S', ModeErrorFunction::CROSSENTROPY = 'C', ModeErrorFunction::CROSSENTROPY_MUTUALEXCLUSIVE = 'M' }
 error functions to be chosen from More...
 
enum  ModeOutput { ModeOutput::FETCH }
 
enum  ModeOutputValues : int { ModeOutputValues::DIRECT = 0x01, ModeOutputValues::SIGMOID = 0x02, ModeOutputValues::SOFTMAX = 0x04, ModeOutputValues::BATCHNORMALIZATION = 0x08 }
 
enum  WeightInitializationStrategy { WeightInitializationStrategy::XAVIER, WeightInitializationStrategy::TEST, WeightInitializationStrategy::LAYERSIZE, WeightInitializationStrategy::XAVIERUNIFORM }
 weight initialization strategies to be chosen from More...
 

Functions

template<typename Architecture_t >
void addRegularizationGradients (typename Architecture_t::Matrix_t &A, const typename Architecture_t::Matrix_t &W, typename Architecture_t::Scalar_t weightDecay, ERegularization R)
 Add the regularization gradient corresponding to weight matrix W, to the matrix A. More...
 
template<typename ItValue , typename Fnc >
void applyFunctions (ItValue itValue, ItValue itValueEnd, Fnc fnc)
 apply the activation functions More...
 
template<typename ItValue , typename Fnc , typename InvFnc , typename ItGradient >
void applyFunctions (ItValue itValue, ItValue itValueEnd, Fnc fnc, InvFnc invFnc, ItGradient itGradient)
 apply the activation functions and compute the gradient More...
 
template<typename ItValue , typename ItFunction >
void applyFunctions (ItValue itValue, ItValue itValueEnd, ItFunction itFunction)
 
template<typename ItValue , typename ItFunction , typename ItInverseFunction , typename ItGradient >
void applyFunctions (ItValue itValue, ItValue itValueEnd, ItFunction itFunction, ItInverseFunction itInverseFunction, ItGradient itGradient)
 
template<typename AMatrix , typename F >
void applyMatrix (AMatrix &X, F f)
 Apply functional to each element in the matrix. More...
 
template<bool HasDropOut, typename ItSource , typename ItWeight , typename ItTarget , typename ItDrop >
void applyWeights (ItSource itSourceBegin, ItSource itSourceEnd, ItWeight itWeight, ItTarget itTargetBegin, ItTarget itTargetEnd, ItDrop itDrop)
 apply weights using drop-out; for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true" More...
 
template<typename ItSource , typename ItWeight , typename ItTarget >
void applyWeights (ItSource itSourceBegin, ItSource itSourceEnd, ItWeight itWeight, ItTarget itTargetBegin, ItTarget itTargetEnd)
 
template<bool HasDropOut, typename ItSource , typename ItWeight , typename ItPrev , typename ItDrop >
void applyWeightsBackwards (ItSource itCurrBegin, ItSource itCurrEnd, ItWeight itWeight, ItPrev itPrevBegin, ItPrev itPrevEnd, ItDrop itDrop)
 apply weights backwards (for backprop); for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true" More...
 
template<typename ItSource , typename ItWeight , typename ItPrev >
void applyWeightsBackwards (ItSource itCurrBegin, ItSource itCurrEnd, ItWeight itWeight, ItPrev itPrevBegin, ItPrev itPrevEnd)
 
template<typename LAYERDATA >
void backward (LAYERDATA &prevLayerData, LAYERDATA &currLayerData)
 backward application of the weights (back-propagation of the error) More...
 
template<EnumRegularization Regularization>
double computeRegularization (double weight, const double &factorWeightDecay)
 compute the regularization (L1, L2) More...
 
template<>
double computeRegularization< EnumRegularization::L1 > (double weight, const double &factorWeightDecay)
 
template<>
double computeRegularization< EnumRegularization::L2 > (double weight, const double &factorWeightDecay)
 
template<typename AArchitecture >
void constructRandomLinearNet (TNet< AArchitecture > &net)
 Construct a random linear neural network with up to five layers. More...
 
template<typename AMatrix >
void copyMatrix (AMatrix &X, const AMatrix &Y)
 Generate a random batch as input for a neural net. More...
 
template<typename ItProbability , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double crossEntropy (ItProbability itProbabilityBegin, ItProbability itProbabilityEnd, ItTruth itTruthBegin, ItTruth, ItDelta itDelta, ItDelta itDeltaEnd, ItInvActFnc, double patternWeight)
 cross entropy error function More...
 
void cudaError (cudaError_t code, const char *file, int line, bool abort=true)
 Function to check cuda return code. More...
 
template<typename Architecture_t >
void evaluate (typename Architecture_t::Matrix_t &A, EActivationFunction f)
 Apply the given activation function to each value in the given matrix A. More...
 
template<typename Architecture_t >
void evaluate (typename Architecture_t::Matrix_t &A, EOutputFunction f, const typename Architecture_t::Matrix_t &X)
 Apply the given output function to each value in the given matrix A. More...
 
template<typename Architecture_t >
auto evaluate (ELossFunction f, const typename Architecture_t::Matrix_t &Y, const typename Architecture_t::Matrix_t &output) -> decltype(Architecture_t::CrossEntropy(Y, output))
 Compute the value of the objective function f for given activations of the ouput layer and the truth Y. More...
 
template<typename Architecture_t >
void evaluateDerivative (typename Architecture_t::Matrix_t &B, EActivationFunction f, const typename Architecture_t::Matrix_t &A)
 Compute the first partial derivative of the activation function for the values given in matrix A and write the results into B. More...
 
template<typename Architecture_t >
void evaluateGradients (typename Architecture_t::Matrix_t &dY, ELossFunction f, const typename Architecture_t::Matrix_t &Y, const typename Architecture_t::Matrix_t &output)
 Compute the gradient of the given output function f for given activations output of the output layer and truth Y and write the results into dY. More...
 
template<typename F , typename AFloat >
AFloat finiteDifference (F f, AFloat dx)
 Numerically compute the derivative of the functional f using finite differences. More...
 
template<typename LAYERDATA >
void forward (const LAYERDATA &prevLayerData, LAYERDATA &currLayerData)
 apply the weights (and functions) in forward direction of the DNN More...
 
double gaussDouble (double mean, double sigma)
 
template<typename AMatrix >
void identityMatrix (AMatrix &X)
 Set matrix to the identity matrix. More...
 
template<typename Architecture_t >
void initialize (typename Architecture_t::Matrix_t &A, EInitialization m)
 
template<typename T >
bool isFlagSet (T flag, T value)
 
template<typename Matrix1 , typename Matrix2 >
auto maximumRelativeError (const Matrix1 &X, const Matrix2 &Y) -> decltype(X(0, 0))
 Compute the maximum, element-wise relative error of the matrices X and Y normalized by the element of Y. More...
 
ModeOutputValues operator & (ModeOutputValues lhs, ModeOutputValues rhs)
 
ModeOutputValues operator &= (ModeOutputValues &lhs, ModeOutputValues rhs)
 
ModeOutputValues operator| (ModeOutputValues lhs, ModeOutputValues rhs)
 
ModeOutputValues operator|= (ModeOutputValues &lhs, ModeOutputValues rhs)
 
template<typename AFloat >
std::string print_error (AFloat &e)
 Color code error. More...
 
template<typename AMatrix >
void randomBatch (AMatrix &X)
 Generate a random batch as input for a neural net. More...
 
int randomInt (int maxValue)
 
template<typename AMatrix >
void randomMatrix (AMatrix &X)
 Fill matrix with random, Gaussian-distributed values. More...
 
template<typename AMatrix , typename AFloat , typename F >
AFloat reduce (F f, AFloat start, const AMatrix &X)
 Generate a random batch as input for a neural net. More...
 
template<typename AMatrix , typename AFloat , typename F >
AFloat reduceMean (F f, AFloat start, const AMatrix &X)
 Apply function to matrix element-wise and compute the mean of the resulting element values. More...
 
template<typename Architecture_t >
auto regularization (const typename Architecture_t::Matrix_t &A, ERegularization R) -> decltype(Architecture_t::L1Regularization(A))
 Evaluate the regularization functional for a given weight matrix. More...
 
template<typename T >
relativeError (const T &x, const T &y)
 Compute the relative error of x and y. More...
 
template<typename ItOutput , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double softMaxCrossEntropy (ItOutput itProbabilityBegin, ItOutput itProbabilityEnd, ItTruth itTruthBegin, ItTruth, ItDelta itDelta, ItDelta itDeltaEnd, ItInvActFnc, double patternWeight)
 soft-max-cross-entropy error function (for mutual exclusive cross-entropy) More...
 
double studenttDouble (double distributionParameter)
 
template<typename ItOutput , typename ItTruth , typename ItDelta , typename InvFnc >
double sumOfSquares (ItOutput itOutputBegin, ItOutput itOutputEnd, ItTruth itTruthBegin, ItTruth, ItDelta itDelta, ItDelta itDeltaEnd, InvFnc invFnc, double patternWeight)
 sum of squares error function More...
 
template<typename ItOutput , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double sumOfSquares (ItOutput itOutputBegin, ItOutput itOutputEnd, ItTruth itTruthBegin, ItTruth itTruthEnd, ItDelta itDelta, ItDelta itDeltaEnd, ItInvActFnc itInvActFnc, double patternWeight)
 
template<typename Architecture_t >
auto testIdentity () -> typename Architecture_t::Scalar_t
 Test the data loader by loading identical input and output data, running it through an identity neural network and computing the the mean squared error. More...
 
template<typename Architecture_t >
auto testSum () -> typename Architecture_t::Scalar_t
 Test that the data loader loads all data in the data set by summing up all elements batch wise and comparing to the result over the complete data set. More...
 
template<typename Container , typename T >
void uniformDouble (Container &container, T maxValue)
 
double uniformDouble (double minValue, double maxValue)
 
template<typename T >
uniformFromTo (T from, T to)
 
template<typename ItSource , typename ItDelta , typename ItTargetGradient , typename ItGradient >
void update (ItSource itSource, ItSource itSourceEnd, ItDelta itTargetDeltaBegin, ItDelta itTargetDeltaEnd, ItTargetGradient itTargetGradientBegin, ItGradient itGradient)
 update the gradients More...
 
template<EnumRegularization Regularization, typename ItSource , typename ItDelta , typename ItTargetGradient , typename ItGradient , typename ItWeight >
void update (ItSource itSource, ItSource itSourceEnd, ItDelta itTargetDeltaBegin, ItDelta itTargetDeltaEnd, ItTargetGradient itTargetGradientBegin, ItGradient itGradient, ItWeight itWeight, double weightDecay)
 update the gradients, using regularization More...
 
template<typename LAYERDATA >
void update (const LAYERDATA &prevLayerData, LAYERDATA &currLayerData, double factorWeightDecay, EnumRegularization regularization)
 update the node values More...
 
template<typename ItWeight >
double weightDecay (double error, ItWeight itWeight, ItWeight itWeightEnd, double factorWeightDecay, EnumRegularization eRegularization)
 compute the weight decay for regularization (L1 or L2) More...
 
template<typename AMatrix , typename F >
void zipWithMatrix (AMatrix &Z, F f, const AMatrix &X, const AMatrix &Y)
 Combine elements of two given matrices into a single matrix using the given function f. More...
 

Variables

static std::shared_ptr< std::function< double(double)> > Gauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return exp (-std::pow(value*s,2.0)); })
 
static std::shared_ptr< std::function< double(double)> > GaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return 1.0 - exp (-std::pow(value*s,2.0)); })
 
static std::shared_ptr< std::function< double(double)> > InvGauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return -2.0 * value * s*s * (*Gauss.get ()) (value); })
 
static std::shared_ptr< std::function< double(double)> > InvGaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return +2.0 * value * s*s * (*GaussComplement.get ()) (value); })
 
static std::shared_ptr< std::function< double(double)> > InvLinear = std::make_shared<std::function<double(double)>> ([](double ){ return 1.0; })
 
static std::shared_ptr< std::function< double(double)> > InvReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? 1.0 : 0; })
 
static std::shared_ptr< std::function< double(double)> > InvSigmoid = std::make_shared<std::function<double(double)>> ([](double value){ double s = (*Sigmoid.get ()) (value); return s*(1.0-s); })
 
static std::shared_ptr< std::function< double(double)> > InvSoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 / (1.0 + std::exp (-value)); })
 
static std::shared_ptr< std::function< double(double)> > InvSoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return std::pow ((1.0 - fabs (value)),2.0); })
 
static std::shared_ptr< std::function< double(double)> > InvSymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? 1.0 : value < -margin ? 1.0 : 0; })
 
static std::shared_ptr< std::function< double(double)> > InvTanh = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 - std::pow (value, 2.0); })
 
static std::shared_ptr< std::function< double(double)> > InvTanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return 0.3 + (1.0 - std::pow (value, 2.0)); })
 
static std::shared_ptr< std::function< double(double)> > Linear = std::make_shared<std::function<double(double)>> ([](double value){ return value; })
 
static std::shared_ptr< std::function< double(double)> > ReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? value-margin : 0; })
 
static std::shared_ptr< std::function< double(double)> > Sigmoid = std::make_shared<std::function<double(double)>> ([](double value){ value = std::max (-100.0, std::min (100.0,value)); return 1.0/(1.0 + std::exp (-value)); })
 
static std::shared_ptr< std::function< double(double)> > SoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return std::log (1.0+ std::exp (value)); })
 
static std::shared_ptr< std::function< double(double)> > SoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return value / (1.0 + fabs (value)); })
 
static std::shared_ptr< std::function< double(double)> > SymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? value-margin : value < -margin ? value+margin : 0; })
 
static std::shared_ptr< std::function< double(double)> > Tanh = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value); })
 
static std::shared_ptr< std::function< double(double)> > TanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value-0.3); })
 
static std::shared_ptr< std::function< double(double)> > ZeroFnc = std::make_shared<std::function<double(double)>> ([](double ){ return 0; })
 

Typedef Documentation

◆ DropContainer

typedef std::vector<char> TMVA::DNN::DropContainer

Definition at line 220 of file NeuralNet.h.

◆ IndexIterator_t

using TMVA::DNN::IndexIterator_t = typedef typename std::vector<size_t>::iterator

Definition at line 37 of file DataLoader.h.

◆ MatrixInput_t

using TMVA::DNN::MatrixInput_t = typedef std::pair<const TMatrixT<Double_t> &, const TMatrixT<Double_t> &>

Definition at line 34 of file DataLoader.h.

◆ pass_through_type

Definition at line 1301 of file NeuralNet.h.

◆ TMVAInput_t

using TMVA::DNN::TMVAInput_t = typedef std::vector<Event*>

Definition at line 35 of file DataLoader.h.

Enumeration Type Documentation

◆ EActivationFunction

Enum that represents layer activation functions.

Enumerator
kIdentity 
kRelu 
kSigmoid 
kTanh 
kSymmRelu 
kSoftSign 
kGauss 

Definition at line 31 of file Functions.h.

◆ EInitialization

Enumerator
kGauss 
kUniform 
kIdentity 
kZero 

Definition at line 70 of file Functions.h.

◆ ELossFunction

Enum that represents objective functions for the net, i.e.

functions that take the output from the last layer in the net together with the truths and return the objective function values that is to be minimized in the training process.

Enumerator
kCrossEntropy 
kMeanSquaredError 
kSoftmaxCrossEntropy 

Definition at line 54 of file Functions.h.

◆ EnumFunction

Enumerator
ZERO 
LINEAR 
TANH 
RELU 
SYMMRELU 
TANHSHIFT 
SIGMOID 
SOFTSIGN 
GAUSS 
GAUSSCOMPLEMENT 

Definition at line 157 of file NeuralNet.h.

◆ EnumRegularization

Enumerator
NONE 
L1 
L2 
L1MAX 

Definition at line 173 of file NeuralNet.h.

◆ EOutputFunction

Enum that represents output functions.

Enumerator
kIdentity 
kSigmoid 
kSoftmax 

Definition at line 43 of file Functions.h.

◆ ERegularization

Enum representing the regularization type applied for a given layer.

Enumerator
kNone 
kL1 
kL2 

Definition at line 62 of file Functions.h.

◆ MinimizerType

< list all the minimizer types

Enumerator
fSteepest 

SGD.

Definition at line 321 of file NeuralNet.h.

◆ ModeErrorFunction

error functions to be chosen from

Enumerator
SUMOFSQUARES 
CROSSENTROPY 
CROSSENTROPY_MUTUALEXCLUSIVE 

Definition at line 1045 of file NeuralNet.h.

◆ ModeOutput

enum TMVA::DNN::ModeOutput
strong
Enumerator
FETCH 

Definition at line 1036 of file NeuralNet.h.

◆ ModeOutputValues

enum TMVA::DNN::ModeOutputValues : int
strong
Enumerator
DIRECT 
SIGMOID 
SOFTMAX 
BATCHNORMALIZATION 

Definition at line 179 of file NeuralNet.h.

◆ WeightInitializationStrategy

weight initialization strategies to be chosen from

Enumerator
XAVIER 
TEST 
LAYERSIZE 
XAVIERUNIFORM 

Definition at line 1056 of file NeuralNet.h.

Function Documentation

◆ addRegularizationGradients()

template<typename Architecture_t >
void TMVA::DNN::addRegularizationGradients ( typename Architecture_t::Matrix_t &  A,
const typename Architecture_t::Matrix_t &  W,
typename Architecture_t::Scalar_t  weightDecay,
ERegularization  R 
)
inline

Add the regularization gradient corresponding to weight matrix W, to the matrix A.

Definition at line 233 of file Functions.h.

◆ applyFunctions() [1/4]

template<typename ItValue , typename Fnc >
void TMVA::DNN::applyFunctions ( ItValue  itValue,
ItValue  itValueEnd,
Fnc  fnc 
)

apply the activation functions

Definition at line 140 of file NeuralNet.icc.

◆ applyFunctions() [2/4]

template<typename ItValue , typename Fnc , typename InvFnc , typename ItGradient >
void TMVA::DNN::applyFunctions ( ItValue  itValue,
ItValue  itValueEnd,
Fnc  fnc,
InvFnc  invFnc,
ItGradient  itGradient 
)

apply the activation functions and compute the gradient

Definition at line 157 of file NeuralNet.icc.

◆ applyFunctions() [3/4]

template<typename ItValue , typename ItFunction >
void TMVA::DNN::applyFunctions ( ItValue  itValue,
ItValue  itValueEnd,
ItFunction  itFunction 
)

◆ applyFunctions() [4/4]

template<typename ItValue , typename ItFunction , typename ItInverseFunction , typename ItGradient >
void TMVA::DNN::applyFunctions ( ItValue  itValue,
ItValue  itValueEnd,
ItFunction  itFunction,
ItInverseFunction  itInverseFunction,
ItGradient  itGradient 
)

◆ applyMatrix()

template<typename AMatrix , typename F >
void TMVA::DNN::applyMatrix ( AMatrix &  X,
F  f 
)

Apply functional to each element in the matrix.

Definition at line 104 of file Utility.h.

◆ applyWeights() [1/2]

template<bool HasDropOut, typename ItSource , typename ItWeight , typename ItTarget , typename ItDrop >
void TMVA::DNN::applyWeights ( ItSource  itSourceBegin,
ItSource  itSourceEnd,
ItWeight  itWeight,
ItTarget  itTargetBegin,
ItTarget  itTargetEnd,
ItDrop  itDrop 
)

apply weights using drop-out; for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true"

itDrop correlates with itSourceBegin

Definition at line 84 of file NeuralNet.icc.

◆ applyWeights() [2/2]

template<typename ItSource , typename ItWeight , typename ItTarget >
void TMVA::DNN::applyWeights ( ItSource  itSourceBegin,
ItSource  itSourceEnd,
ItWeight  itWeight,
ItTarget  itTargetBegin,
ItTarget  itTargetEnd 
)

◆ applyWeightsBackwards() [1/2]

template<bool HasDropOut, typename ItSource , typename ItWeight , typename ItPrev , typename ItDrop >
void TMVA::DNN::applyWeightsBackwards ( ItSource  itCurrBegin,
ItSource  itCurrEnd,
ItWeight  itWeight,
ItPrev  itPrevBegin,
ItPrev  itPrevEnd,
ItDrop  itDrop 
)

apply weights backwards (for backprop); for no drop out, provide (&bool = true) to itDrop such that *itDrop becomes "true"

itDrop correlates with itPrev (to be in agreement with "applyWeights" where it correlates with itSources (same node as itTarget here in applyBackwards)

Definition at line 111 of file NeuralNet.icc.

◆ applyWeightsBackwards() [2/2]

template<typename ItSource , typename ItWeight , typename ItPrev >
void TMVA::DNN::applyWeightsBackwards ( ItSource  itCurrBegin,
ItSource  itCurrEnd,
ItWeight  itWeight,
ItPrev  itPrevBegin,
ItPrev  itPrevEnd 
)

◆ backward()

template<typename LAYERDATA >
void TMVA::DNN::backward ( LAYERDATA &  prevLayerData,
LAYERDATA &  currLayerData 
)

backward application of the weights (back-propagation of the error)

Definition at line 565 of file NeuralNet.icc.

◆ computeRegularization()

template<EnumRegularization Regularization>
double TMVA::DNN::computeRegularization ( double  weight,
const double &  factorWeightDecay 
)
inline

compute the regularization (L1, L2)

Definition at line 202 of file NeuralNet.icc.

◆ computeRegularization< EnumRegularization::L1 >()

template<>
double TMVA::DNN::computeRegularization< EnumRegularization::L1 > ( double  weight,
const double &  factorWeightDecay 
)
inline

Definition at line 212 of file NeuralNet.icc.

◆ computeRegularization< EnumRegularization::L2 >()

template<>
double TMVA::DNN::computeRegularization< EnumRegularization::L2 > ( double  weight,
const double &  factorWeightDecay 
)
inline

Definition at line 219 of file NeuralNet.icc.

◆ constructRandomLinearNet()

template<typename AArchitecture >
void TMVA::DNN::constructRandomLinearNet ( TNet< AArchitecture > &  net)

Construct a random linear neural network with up to five layers.

Definition at line 22 of file Utility.h.

◆ copyMatrix()

template<typename AMatrix >
void TMVA::DNN::copyMatrix ( AMatrix &  X,
const AMatrix &  Y 
)

Generate a random batch as input for a neural net.

Definition at line 88 of file Utility.h.

◆ crossEntropy()

template<typename ItProbability , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double TMVA::DNN::crossEntropy ( ItProbability  itProbabilityBegin,
ItProbability  itProbabilityEnd,
ItTruth  itTruthBegin,
ItTruth  ,
ItDelta  itDelta,
ItDelta  itDeltaEnd,
ItInvActFnc  ,
double  patternWeight 
)

cross entropy error function

Definition at line 405 of file NeuralNet.icc.

◆ cudaError()

void TMVA::DNN::cudaError ( cudaError_t  code,
const char *  file,
int  line,
bool  abort = true 
)
inline

Function to check cuda return code.

Taken from http://stackoverflow.com/questions/14038589/

Definition at line 177 of file CudaMatrix.h.

◆ evaluate() [1/3]

template<typename Architecture_t >
void TMVA::DNN::evaluate ( typename Architecture_t::Matrix_t &  A,
EActivationFunction  f 
)
inline

Apply the given activation function to each value in the given matrix A.

Definition at line 85 of file Functions.h.

◆ evaluate() [2/3]

template<typename Architecture_t >
void TMVA::DNN::evaluate ( typename Architecture_t::Matrix_t &  A,
EOutputFunction  f,
const typename Architecture_t::Matrix_t &  X 
)
inline

Apply the given output function to each value in the given matrix A.

Definition at line 142 of file Functions.h.

◆ evaluate() [3/3]

template<typename Architecture_t >
auto TMVA::DNN::evaluate ( ELossFunction  f,
const typename Architecture_t::Matrix_t &  Y,
const typename Architecture_t::Matrix_t &  output 
) -> decltype(Architecture_t::CrossEntropy(Y,output))
inline

Compute the value of the objective function f for given activations of the ouput layer and the truth Y.

Definition at line 165 of file Functions.h.

◆ evaluateDerivative()

template<typename Architecture_t >
void TMVA::DNN::evaluateDerivative ( typename Architecture_t::Matrix_t &  B,
EActivationFunction  f,
const typename Architecture_t::Matrix_t &  A 
)
inline

Compute the first partial derivative of the activation function for the values given in matrix A and write the results into B.

Definition at line 111 of file Functions.h.

◆ evaluateGradients()

template<typename Architecture_t >
void TMVA::DNN::evaluateGradients ( typename Architecture_t::Matrix_t &  dY,
ELossFunction  f,
const typename Architecture_t::Matrix_t &  Y,
const typename Architecture_t::Matrix_t &  output 
)
inline

Compute the gradient of the given output function f for given activations output of the output layer and truth Y and write the results into dY.

Definition at line 186 of file Functions.h.

◆ finiteDifference()

template<typename F , typename AFloat >
AFloat TMVA::DNN::finiteDifference ( F  f,
AFloat  dx 
)
inline

Numerically compute the derivative of the functional f using finite differences.

Definition at line 224 of file Utility.h.

◆ forward()

template<typename LAYERDATA >
void TMVA::DNN::forward ( const LAYERDATA &  prevLayerData,
LAYERDATA &  currLayerData 
)

apply the weights (and functions) in forward direction of the DNN

Definition at line 539 of file NeuralNet.icc.

◆ gaussDouble()

double TMVA::DNN::gaussDouble ( double  mean,
double  sigma 
)

Definition at line 13 of file NeuralNet.cxx.

◆ identityMatrix()

template<typename AMatrix >
void TMVA::DNN::identityMatrix ( AMatrix &  X)

Set matrix to the identity matrix.

Definition at line 40 of file Utility.h.

◆ initialize()

template<typename Architecture_t >
void TMVA::DNN::initialize ( typename Architecture_t::Matrix_t &  A,
EInitialization  m 
)
inline

Definition at line 257 of file Functions.h.

◆ isFlagSet()

template<typename T >
bool TMVA::DNN::isFlagSet ( flag,
value 
)

Definition at line 213 of file NeuralNet.h.

◆ maximumRelativeError()

template<typename Matrix1 , typename Matrix2 >
auto TMVA::DNN::maximumRelativeError ( const Matrix1 &  X,
const Matrix2 &  Y 
) -> decltype(X(0,0))

Compute the maximum, element-wise relative error of the matrices X and Y normalized by the element of Y.

Protected against division by zero.

Definition at line 200 of file Utility.h.

◆ operator &()

ModeOutputValues TMVA::DNN::operator& ( ModeOutputValues  lhs,
ModeOutputValues  rhs 
)
inline

Definition at line 200 of file NeuralNet.h.

◆ operator &=()

ModeOutputValues TMVA::DNN::operator&= ( ModeOutputValues lhs,
ModeOutputValues  rhs 
)
inline

Definition at line 205 of file NeuralNet.h.

◆ operator|()

ModeOutputValues TMVA::DNN::operator| ( ModeOutputValues  lhs,
ModeOutputValues  rhs 
)
inline

Definition at line 189 of file NeuralNet.h.

◆ operator|=()

ModeOutputValues TMVA::DNN::operator|= ( ModeOutputValues lhs,
ModeOutputValues  rhs 
)
inline

Definition at line 194 of file NeuralNet.h.

◆ print_error()

template<typename AFloat >
std::string TMVA::DNN::print_error ( AFloat &  e)

Color code error.

Definition at line 232 of file Utility.h.

◆ randomBatch()

template<typename AMatrix >
void TMVA::DNN::randomBatch ( AMatrix &  X)

Generate a random batch as input for a neural net.

Definition at line 80 of file Utility.h.

◆ randomInt()

int TMVA::DNN::randomInt ( int  maxValue)

Definition at line 30 of file NeuralNet.cxx.

◆ randomMatrix()

template<typename AMatrix >
void TMVA::DNN::randomMatrix ( AMatrix &  X)

Fill matrix with random, Gaussian-distributed values.

Definition at line 60 of file Utility.h.

◆ reduce()

template<typename AMatrix , typename AFloat , typename F >
AFloat TMVA::DNN::reduce ( F  f,
AFloat  start,
const AMatrix &  X 
)

Generate a random batch as input for a neural net.

Definition at line 140 of file Utility.h.

◆ reduceMean()

template<typename AMatrix , typename AFloat , typename F >
AFloat TMVA::DNN::reduceMean ( F  f,
AFloat  start,
const AMatrix &  X 
)

Apply function to matrix element-wise and compute the mean of the resulting element values.

Definition at line 160 of file Utility.h.

◆ regularization()

template<typename Architecture_t >
auto TMVA::DNN::regularization ( const typename Architecture_t::Matrix_t &  A,
ERegularization  R 
) -> decltype(Architecture_t::L1Regularization(A))
inline

Evaluate the regularization functional for a given weight matrix.

Definition at line 213 of file Functions.h.

◆ relativeError()

template<typename T >
T TMVA::DNN::relativeError ( const T &  x,
const T &  y 
)
inline

Compute the relative error of x and y.

Definition at line 179 of file Utility.h.

◆ softMaxCrossEntropy()

template<typename ItOutput , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double TMVA::DNN::softMaxCrossEntropy ( ItOutput  itProbabilityBegin,
ItOutput  itProbabilityEnd,
ItTruth  itTruthBegin,
ItTruth  ,
ItDelta  itDelta,
ItDelta  itDeltaEnd,
ItInvActFnc  ,
double  patternWeight 
)

soft-max-cross-entropy error function (for mutual exclusive cross-entropy)

Definition at line 451 of file NeuralNet.icc.

◆ studenttDouble()

double TMVA::DNN::studenttDouble ( double  distributionParameter)

Definition at line 38 of file NeuralNet.cxx.

◆ sumOfSquares() [1/2]

template<typename ItOutput , typename ItTruth , typename ItDelta , typename InvFnc >
double TMVA::DNN::sumOfSquares ( ItOutput  itOutputBegin,
ItOutput  itOutputEnd,
ItTruth  itTruthBegin,
ItTruth  ,
ItDelta  itDelta,
ItDelta  itDeltaEnd,
InvFnc  invFnc,
double  patternWeight 
)

sum of squares error function

Definition at line 375 of file NeuralNet.icc.

◆ sumOfSquares() [2/2]

template<typename ItOutput , typename ItTruth , typename ItDelta , typename ItInvActFnc >
double TMVA::DNN::sumOfSquares ( ItOutput  itOutputBegin,
ItOutput  itOutputEnd,
ItTruth  itTruthBegin,
ItTruth  itTruthEnd,
ItDelta  itDelta,
ItDelta  itDeltaEnd,
ItInvActFnc  itInvActFnc,
double  patternWeight 
)

◆ testIdentity()

template<typename Architecture_t >
auto TMVA::DNN::testIdentity ( ) -> typename Architecture_t::Scalar_t

Test the data loader by loading identical input and output data, running it through an identity neural network and computing the the mean squared error.

Should obviously be zero.

Definition at line 65 of file TestDataLoader.h.

◆ testSum()

template<typename Architecture_t >
auto TMVA::DNN::testSum ( ) -> typename Architecture_t::Scalar_t

Test that the data loader loads all data in the data set by summing up all elements batch wise and comparing to the result over the complete data set.

Definition at line 30 of file TestDataLoader.h.

◆ uniformDouble() [1/2]

template<typename Container , typename T >
void TMVA::DNN::uniformDouble ( Container &  container,
maxValue 
)

Definition at line 34 of file NeuralNet.icc.

◆ uniformDouble() [2/2]

double TMVA::DNN::uniformDouble ( double  minValue,
double  maxValue 
)

Definition at line 21 of file NeuralNet.cxx.

◆ uniformFromTo()

template<typename T >
T TMVA::DNN::uniformFromTo ( from,
to 
)

Definition at line 26 of file NeuralNet.icc.

◆ update() [1/3]

template<typename ItSource , typename ItDelta , typename ItTargetGradient , typename ItGradient >
void TMVA::DNN::update ( ItSource  itSource,
ItSource  itSourceEnd,
ItDelta  itTargetDeltaBegin,
ItDelta  itTargetDeltaEnd,
ItTargetGradient  itTargetGradientBegin,
ItGradient  itGradient 
)

update the gradients

Definition at line 176 of file NeuralNet.icc.

◆ update() [2/3]

template<EnumRegularization Regularization, typename ItSource , typename ItDelta , typename ItTargetGradient , typename ItGradient , typename ItWeight >
void TMVA::DNN::update ( ItSource  itSource,
ItSource  itSourceEnd,
ItDelta  itTargetDeltaBegin,
ItDelta  itTargetDeltaEnd,
ItTargetGradient  itTargetGradientBegin,
ItGradient  itGradient,
ItWeight  itWeight,
double  weightDecay 
)

update the gradients, using regularization

Definition at line 230 of file NeuralNet.icc.

◆ update() [3/3]

template<typename LAYERDATA >
void TMVA::DNN::update ( const LAYERDATA &  prevLayerData,
LAYERDATA &  currLayerData,
double  factorWeightDecay,
EnumRegularization  regularization 
)

update the node values

Definition at line 593 of file NeuralNet.icc.

◆ weightDecay()

template<typename ItWeight >
double TMVA::DNN::weightDecay ( double  error,
ItWeight  itWeight,
ItWeight  itWeightEnd,
double  factorWeightDecay,
EnumRegularization  eRegularization 
)

compute the weight decay for regularization (L1 or L2)

Definition at line 491 of file NeuralNet.icc.

◆ zipWithMatrix()

template<typename AMatrix , typename F >
void TMVA::DNN::zipWithMatrix ( AMatrix &  Z,
F  f,
const AMatrix &  X,
const AMatrix &  Y 
)

Combine elements of two given matrices into a single matrix using the given function f.

Definition at line 121 of file Utility.h.

Variable Documentation

◆ Gauss

std::shared_ptr<std::function<double(double)> > TMVA::DNN::Gauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return exp (-std::pow(value*s,2.0)); })
static

Definition at line 71 of file NeuralNet.icc.

◆ GaussComplement

std::shared_ptr<std::function<double(double)> > TMVA::DNN::GaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return 1.0 - exp (-std::pow(value*s,2.0)); })
static

Definition at line 74 of file NeuralNet.icc.

◆ InvGauss

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvGauss = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return -2.0 * value * s*s * (*Gauss.get ()) (value); })
static

Definition at line 72 of file NeuralNet.icc.

◆ InvGaussComplement

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvGaussComplement = std::make_shared<std::function<double(double)>> ([](double value){ const double s = 6.0; return +2.0 * value * s*s * (*GaussComplement.get ()) (value); })
static

Definition at line 75 of file NeuralNet.icc.

◆ InvLinear

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvLinear = std::make_shared<std::function<double(double)>> ([](double ){ return 1.0; })
static

Definition at line 54 of file NeuralNet.icc.

◆ InvReLU

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? 1.0 : 0; })
static

Definition at line 60 of file NeuralNet.icc.

◆ InvSigmoid

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvSigmoid = std::make_shared<std::function<double(double)>> ([](double value){ double s = (*Sigmoid.get ()) (value); return s*(1.0-s); })
static

Definition at line 48 of file NeuralNet.icc.

◆ InvSoftPlus

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvSoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 / (1.0 + std::exp (-value)); })
static

Definition at line 63 of file NeuralNet.icc.

◆ InvSoftSign

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvSoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return std::pow ((1.0 - fabs (value)),2.0); })
static

Definition at line 69 of file NeuralNet.icc.

◆ InvSymmReLU

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvSymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? 1.0 : value < -margin ? 1.0 : 0; })
static

Definition at line 57 of file NeuralNet.icc.

◆ InvTanh

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvTanh = std::make_shared<std::function<double(double)>> ([](double value){ return 1.0 - std::pow (value, 2.0); })
static

Definition at line 51 of file NeuralNet.icc.

◆ InvTanhShift

std::shared_ptr<std::function<double(double)> > TMVA::DNN::InvTanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return 0.3 + (1.0 - std::pow (value, 2.0)); })
static

Definition at line 66 of file NeuralNet.icc.

◆ Linear

std::shared_ptr<std::function<double(double)> > TMVA::DNN::Linear = std::make_shared<std::function<double(double)>> ([](double value){ return value; })
static

Definition at line 53 of file NeuralNet.icc.

◆ ReLU

std::shared_ptr<std::function<double(double)> > TMVA::DNN::ReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.0; return value > margin ? value-margin : 0; })
static

Definition at line 59 of file NeuralNet.icc.

◆ Sigmoid

std::shared_ptr<std::function<double(double)> > TMVA::DNN::Sigmoid = std::make_shared<std::function<double(double)>> ([](double value){ value = std::max (-100.0, std::min (100.0,value)); return 1.0/(1.0 + std::exp (-value)); })
static

Definition at line 47 of file NeuralNet.icc.

◆ SoftPlus

std::shared_ptr<std::function<double(double)> > TMVA::DNN::SoftPlus = std::make_shared<std::function<double(double)>> ([](double value){ return std::log (1.0+ std::exp (value)); })
static

Definition at line 62 of file NeuralNet.icc.

◆ SoftSign

std::shared_ptr<std::function<double(double)> > TMVA::DNN::SoftSign = std::make_shared<std::function<double(double)>> ([](double value){ return value / (1.0 + fabs (value)); })
static

Definition at line 68 of file NeuralNet.icc.

◆ SymmReLU

std::shared_ptr<std::function<double(double)> > TMVA::DNN::SymmReLU = std::make_shared<std::function<double(double)>> ([](double value){ const double margin = 0.3; return value > margin ? value-margin : value < -margin ? value+margin : 0; })
static

Definition at line 56 of file NeuralNet.icc.

◆ Tanh

std::shared_ptr<std::function<double(double)> > TMVA::DNN::Tanh = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value); })
static

Definition at line 50 of file NeuralNet.icc.

◆ TanhShift

std::shared_ptr<std::function<double(double)> > TMVA::DNN::TanhShift = std::make_shared<std::function<double(double)>> ([](double value){ return tanh (value-0.3); })
static

Definition at line 65 of file NeuralNet.icc.

◆ ZeroFnc

std::shared_ptr<std::function<double(double)> > TMVA::DNN::ZeroFnc = std::make_shared<std::function<double(double)>> ([](double ){ return 0; })
static

Definition at line 44 of file NeuralNet.icc.