template<typename AReal>
class TMVA::DNN::TReference< AReal >
The reference architecture class.
Class template that contains the reference implementation of the low-level interface for the DNN implementation. The reference implementation uses the TMatrixT class template to represent matrices.
- Template Parameters
-
AReal | The floating point type used to represent scalars. |
Definition at line 30 of file DataLoader.h.
|
static void | AddBiases (TMatrixT< AReal > &A, const TMatrixT< AReal > &biases) |
|
static void | ConvLayerForward (std::vector< TMatrixT< AReal >> &, std::vector< TMatrixT< AReal >> &, const std::vector< TMatrixT< AReal >> &, const TMatrixT< AReal > &, const TMatrixT< AReal > &, EActivationFunction, const std::vector< int > &, size_t, size_t, AReal, bool) |
| Forward propagation in the Convolutional layer. More...
|
|
static void | CorruptInput (TMatrixT< AReal > &input, TMatrixT< AReal > &corruptedInput, AReal corruptionLevel) |
|
static void | EncodeInput (TMatrixT< AReal > &input, TMatrixT< AReal > &compressedInput, TMatrixT< AReal > &Weights) |
|
static void | ForwardLogReg (TMatrixT< AReal > &input, TMatrixT< AReal > &p, TMatrixT< AReal > &fWeights) |
|
static void | ReconstructInput (TMatrixT< AReal > &compressedInput, TMatrixT< AReal > &reconstructedInput, TMatrixT< AReal > &fWeights) |
|
static void | SoftmaxAE (TMatrixT< AReal > &A) |
|
static void | SumColumns (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
| Sum columns of (m x n) matrixx A and write the results into the first m elements in A . More...
|
|
static void | UpdateParams (TMatrixT< AReal > &x, TMatrixT< AReal > &tildeX, TMatrixT< AReal > &y, TMatrixT< AReal > &z, TMatrixT< AReal > &fVBiases, TMatrixT< AReal > &fHBiases, TMatrixT< AReal > &fWeights, TMatrixT< AReal > &VBiasError, TMatrixT< AReal > &HBiasError, AReal learningRate, size_t fBatchSize) |
|
static void | UpdateParamsLogReg (TMatrixT< AReal > &input, TMatrixT< AReal > &output, TMatrixT< AReal > &difference, TMatrixT< AReal > &p, TMatrixT< AReal > &fWeights, TMatrixT< AReal > &fBiases, AReal learningRate, size_t fBatchSize) |
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | MultiplyTranspose (TMatrixT< Scalar_t > &output, const TMatrixT< Scalar_t > &input, const TMatrixT< Scalar_t > &weights) |
| Matrix-multiply input with the transpose of and write the results into output . More...
|
|
static void | AddRowWise (TMatrixT< Scalar_t > &output, const TMatrixT< Scalar_t > &biases) |
| Add the vectors biases row-wise to the matrix output. More...
|
|
|
Low-level functions required for the forward propagation of activations through the network.
|
static void | Backward (TMatrixT< Scalar_t > &activationGradientsBackward, TMatrixT< Scalar_t > &weightGradients, TMatrixT< Scalar_t > &biasGradients, TMatrixT< Scalar_t > &df, const TMatrixT< Scalar_t > &activationGradients, const TMatrixT< Scalar_t > &weights, const TMatrixT< Scalar_t > &activationBackward) |
| Perform the complete backward propagation step. More...
|
|
static Matrix_t & | RecurrentLayerBackward (TMatrixT< Scalar_t > &state_gradients_backward, TMatrixT< Scalar_t > &input_weight_gradients, TMatrixT< Scalar_t > &state_weight_gradients, TMatrixT< Scalar_t > &bias_gradients, TMatrixT< Scalar_t > &df, const TMatrixT< Scalar_t > &state, const TMatrixT< Scalar_t > &weights_input, const TMatrixT< Scalar_t > &weights_state, const TMatrixT< Scalar_t > &input, TMatrixT< Scalar_t > &input_gradient) |
| Backpropagation step for a Recurrent Neural Network. More...
|
|
static void | ScaleAdd (TMatrixT< Scalar_t > &A, const TMatrixT< Scalar_t > &B, Scalar_t beta=1.0) |
| Adds a the elements in matrix B scaled by c to the elements in the matrix A. More...
|
|
static void | Copy (TMatrixT< Scalar_t > &A, const TMatrixT< Scalar_t > &B) |
|
template<typename AMatrix_t > |
static void | CopyDiffArch (TMatrixT< Scalar_t > &A, const AMatrix_t &B) |
|
static void | ScaleAdd (std::vector< TMatrixT< Scalar_t >> &A, const std::vector< TMatrixT< Scalar_t >> &B, Scalar_t beta=1.0) |
| Above functions extended to vectors. More...
|
|
static void | Copy (std::vector< TMatrixT< Scalar_t >> &A, const std::vector< TMatrixT< Scalar_t >> &B) |
|
template<typename AMatrix_t > |
static void | CopyDiffArch (std::vector< TMatrixT< Scalar_t > > &A, const std::vector< AMatrix_t > &B) |
|
|
For each activation function, the low-level interface contains two routines.
One that applies the acitvation function to a matrix and one that evaluate the derivatives of the activation function at the elements of a given matrix and writes the results into the result matrix.
|
static void | Identity (TMatrixT< AReal > &B) |
|
static void | IdentityDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | Relu (TMatrixT< AReal > &B) |
|
static void | ReluDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | Sigmoid (TMatrixT< AReal > &B) |
|
static void | SigmoidDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | Tanh (TMatrixT< AReal > &B) |
|
static void | TanhDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | SymmetricRelu (TMatrixT< AReal > &B) |
|
static void | SymmetricReluDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | SoftSign (TMatrixT< AReal > &B) |
|
static void | SoftSignDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
static void | Gauss (TMatrixT< AReal > &B) |
|
static void | GaussDerivative (TMatrixT< AReal > &B, const TMatrixT< AReal > &A) |
|
|
Loss functions compute a scalar value given the output of the network for a given training input and the expected network prediction Y that quantifies the quality of the prediction.
For each function also a routing that computes the gradients (suffixed by Gradients) must be provided for the starting of the backpropagation algorithm.
|
static AReal | MeanSquaredError (const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
|
static void | MeanSquaredErrorGradients (TMatrixT< AReal > &dY, const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
|
static AReal | CrossEntropy (const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
| Sigmoid transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | CrossEntropyGradients (TMatrixT< AReal > &dY, const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
|
static AReal | SoftmaxCrossEntropy (const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
| Softmax transformation is implicitly applied, thus output should hold the linear activations of the last layer in the net. More...
|
|
static void | SoftmaxCrossEntropyGradients (TMatrixT< AReal > &dY, const TMatrixT< AReal > &Y, const TMatrixT< AReal > &output, const TMatrixT< AReal > &weights) |
|
|
Output functions transform the activations output of the output layer in the network to a valid prediction YHat for the desired usage of the network, e.g.
the identity function for regression or the sigmoid transformation for two-class classification.
|
static void | Sigmoid (TMatrixT< AReal > &YHat, const TMatrixT< AReal > &) |
|
static void | Softmax (TMatrixT< AReal > &YHat, const TMatrixT< AReal > &) |
|
|
For each regularization type two functions are required, one named <Type>Regularization that evaluates the corresponding regularization functional for a given weight matrix and the Add<Type>RegularizationGradients , that adds the regularization component in the gradients to the provided matrix.
|
static AReal | L1Regularization (const TMatrixT< AReal > &W) |
|
static void | AddL1RegularizationGradients (TMatrixT< AReal > &A, const TMatrixT< AReal > &W, AReal weightDecay) |
|
static AReal | L2Regularization (const TMatrixT< AReal > &W) |
|
static void | AddL2RegularizationGradients (TMatrixT< AReal > &A, const TMatrixT< AReal > &W, AReal weightDecay) |
|
|
For each initialization method, one function in the low-level interface is provided.
The naming scheme is
Initialize<Type>
for a given initialization method Type.
|
static void | InitializeGauss (TMatrixT< AReal > &A) |
|
static void | InitializeUniform (TMatrixT< AReal > &A) |
|
static void | InitializeIdentity (TMatrixT< AReal > &A) |
|
static void | InitializeZero (TMatrixT< AReal > &A) |
|
static void | InitializeGlorotUniform (TMatrixT< AReal > &A) |
| Sample from a uniform distribution in range [ -lim,+lim] where lim = sqrt(6/N_in+N_out). More...
|
|
static void | InitializeGlorotNormal (TMatrixT< AReal > &A) |
| Truncated normal initialization (Glorot, called also Xavier normal) The values are sample with a normal distribution with stddev = sqrt(2/N_input + N_output) and values larger than 2 * stddev are discarded See Glorot & Bengio, AISTATS 2010 - http://jmlr.org/proceedings/papers/v9/glorot10a/glorot10a.pdf. More...
|
|
static TRandom & | GetRandomGenerator () |
|
static void | SetRandomSeed (size_t seed) |
|
|
static void | Dropout (TMatrixT< AReal > &A, AReal dropoutProbability) |
| Apply dropout with activation probability p to the given matrix A and scale the result by reciprocal of p . More...
|
|
|
static void | Im2col (TMatrixT< AReal > &A, TMatrixT< AReal > &B, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols, size_t zeroPaddingHeight, size_t zeroPaddingWidth) |
| Transform the matrix B in local view format, suitable for convolution, and store it in matrix A . More...
|
|
static void | Im2colIndices (std::vector< int > &, const TMatrixT< AReal > &, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t) |
|
static void | Im2colFast (TMatrixT< AReal > &, const TMatrixT< AReal > &, const std::vector< int > &) |
|
static void | RotateWeights (TMatrixT< AReal > &A, const TMatrixT< AReal > &B, size_t filterDepth, size_t filterHeight, size_t filterWidth, size_t numFilters) |
| Rotates the matrix B , which is representing a weights, and stores them in the matrix A . More...
|
|
static void | AddConvBiases (TMatrixT< AReal > &output, const TMatrixT< AReal > &biases) |
| Add the biases in the Convolutional Layer. More...
|
|
|
static void | ConvLayerBackward (std::vector< TMatrixT< AReal >> &, TMatrixT< AReal > &, TMatrixT< AReal > &, std::vector< TMatrixT< AReal >> &, const std::vector< TMatrixT< AReal >> &, const TMatrixT< AReal > &, const std::vector< TMatrixT< AReal >> &, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t, size_t) |
| Perform the complete backward propagation step in a Convolutional Layer. More...
|
|
|
static void | Downsample (TMatrixT< AReal > &A, TMatrixT< AReal > &B, const TMatrixT< AReal > &C, size_t imgHeight, size_t imgWidth, size_t fltHeight, size_t fltWidth, size_t strideRows, size_t strideCols) |
| Downsample the matrix C to the matrix A , using max operation, such that the winning indices are stored in matrix B . More...
|
|
|
static void | MaxPoolLayerBackward (std::vector< TMatrixT< AReal >> &activationGradientsBackward, const std::vector< TMatrixT< AReal >> &activationGradients, const std::vector< TMatrixT< AReal >> &indexMatrix, size_t batchSize, size_t depth, size_t nLocalViews) |
| Perform the complete backward propagation step in a Max Pooling Layer. More...
|
|
|
static void | Reshape (TMatrixT< AReal > &A, const TMatrixT< AReal > &B) |
| Transform the matrix B to a matrix with different dimensions A . More...
|
|
static void | Flatten (TMatrixT< AReal > &A, const std::vector< TMatrixT< AReal >> &B, size_t size, size_t nRows, size_t nCols) |
| Flattens the tensor B , such that each matrix, is stretched in one row, resulting with a matrix A . More...
|
|
static void | Deflatten (std::vector< TMatrixT< AReal >> &A, const TMatrixT< Scalar_t > &B, size_t index, size_t nRows, size_t nCols) |
| Transforms each row of B to a matrix and stores it in the tensor B . More...
|
|
static void | Rearrange (std::vector< TMatrixT< AReal >> &out, const std::vector< TMatrixT< AReal >> &in) |
| Rearrage data accoring to time fill B x T x D out with T x B x D matrix in. More...
|
|