Logo ROOT   6.08/07
Reference Guide
Cuda.h
Go to the documentation of this file.
1 // @(#)root/tmva/tmva/dnn:$Id$
2 // Author: Simon Pfreundschuh 05/07/16
3 
4 /*************************************************************************
5  * Copyright (C) 2016, Simon Pfreundschuh *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 ///////////////////////////////////////////////////////////////////
13 // Definition of the TCuda architecture class, which provides an //
14 // implementation of the low-level functionality for neural //
15 // networks for the CUDA computing architectures. //
16 ///////////////////////////////////////////////////////////////////
17 
18 #ifndef TMVA_DNN_ARCHITECTURES_CUDA
19 #define TMVA_DNN_ARCHITECTURES_CUDA
20 
21 #include "cuda.h"
22 #include "Cuda/CudaBuffers.h"
23 #include "Cuda/CudaMatrix.h"
24 #include "TMVA/DNN/DataLoader.h"
25 #include <utility>
26 
27 namespace TMVA
28 {
29 namespace DNN
30 {
31 
32 /** The TCuda architecture class.
33  *
34  * Low-level interface class for CUDA computing architectures. Contains as
35  * public types the declaration of the scalar, matrix and buffer types
36  * for this architecture as well as the remaining functions in the low-level
37  * interface in the form of static members.
38  */
39 template<typename AFloat = Real_t>
40 class TCuda
41 {
42 
43 public:
44 
45  using Scalar_t = AFloat;
49 
50  //____________________________________________________________________________
51  //
52  // Propagation
53  //____________________________________________________________________________
54 
55  /** @name Forward Propagation
56  * Low-level functions required for the forward propagation of activations
57  * through the network.
58  */
59  ///@{
60  /** Matrix-multiply \p input with the transpose of \pweights and
61  * write the results into \p output. */
63  const TCudaMatrix<AFloat> &input,
64  const TCudaMatrix<AFloat> &weights);
65  /** Add the vectors biases row-wise to the matrix output */
67  const TCudaMatrix<AFloat> &biases);
68  ///@}
69 
70  /** @name Backward Propagation
71  * Low-level functions required for the forward propagation of activations
72  * through the network.
73  */
74  ///@{
75  /** Perform the complete backward propagation step. If the provided
76  * \p activationGradientsBackward matrix is not empty, compute the
77  * gradients of the objective function with respect to the activations
78  * of the previous layer (backward direction).
79  * Also compute the weight and the bias gradients. Modifies the values
80  * in \p df and thus produces only a valid result, if it is applied the
81  * first time after the corresponding forward propagation has been per-
82  * formed. */
83  static void Backward(TCudaMatrix<AFloat> & activationGradientsBackward,
84  TCudaMatrix<AFloat> & weightGradients,
85  TCudaMatrix<AFloat> & biasGradients,
87  const TCudaMatrix<AFloat> & activationGradients,
88  const TCudaMatrix<AFloat> & weights,
89  const TCudaMatrix<AFloat> & activationBackward);
90  /** Adds a the elements in matrix B scaled by c to the elements in
91  * the matrix A. This is required for the weight update in the gradient
92  * descent step.*/
93  static void ScaleAdd(TCudaMatrix<AFloat> & A,
94  const TCudaMatrix<AFloat> & B,
95  Scalar_t beta = 1.0);
96  /** Copy the elements of matrix A into matrix B. */
97  static void Copy(TCudaMatrix<AFloat> & B,
98  const TCudaMatrix<AFloat> & A);
99  ///@}
100 
101  //____________________________________________________________________________
102  //
103  // Activation Functions
104  //____________________________________________________________________________
105 
106  /** @name Activation Functions
107  * For each activation function, the low-level interface contains two routines.
108  * One that applies the acitvation function to a matrix and one that evaluate
109  * the derivatives of the activation function at the elements of a given matrix
110  * and writes the results into the result matrix.
111  */
112  ///@{
113  static void Identity(TCudaMatrix<AFloat> & B);
115  const TCudaMatrix<AFloat> & A);
116 
117  static void Relu(TCudaMatrix<AFloat> & B);
118  static void ReluDerivative(TCudaMatrix<AFloat> & B,
119  const TCudaMatrix<AFloat> & A);
120 
121  static void Sigmoid(TCudaMatrix<AFloat> & B);
123  const TCudaMatrix<AFloat> & A);
124 
125  static void Tanh(TCudaMatrix<AFloat> & B);
126  static void TanhDerivative(TCudaMatrix<AFloat> & B,
127  const TCudaMatrix<AFloat> & A);
128 
129  static void SymmetricRelu(TCudaMatrix<AFloat> & B);
131  const TCudaMatrix<AFloat> & A);
132 
133  static void SoftSign(TCudaMatrix<AFloat> & B);
135  const TCudaMatrix<AFloat> & A);
136 
137  static void Gauss(TCudaMatrix<AFloat> & B);
138  static void GaussDerivative(TCudaMatrix<AFloat> & B,
139  const TCudaMatrix<AFloat> & A);
140  ///@}
141 
142  //____________________________________________________________________________
143  //
144  // Loss Functions
145  //____________________________________________________________________________
146 
147  /** @name Loss Functions
148  * Loss functions compute a scalar value given the \p output of the network
149  * for a given training input and the expected network prediction \p Y that
150  * quantifies the quality of the prediction. For each function also a routing
151  * that computes the gradients (suffixed by Gradients) must be provided for
152  * the starting of the backpropagation algorithm.
153  */
154  ///@{
155 
156  static AFloat MeanSquaredError(const TCudaMatrix<AFloat> &Y,
157  const TCudaMatrix<AFloat> &output);
159  const TCudaMatrix<AFloat> &Y,
160  const TCudaMatrix<AFloat> &output);
161 
162  /** Sigmoid transformation is implicitly applied, thus \p output should
163  * hold the linear activations of the last layer in the net. */
164  static AFloat CrossEntropy(const TCudaMatrix<AFloat> &Y,
165  const TCudaMatrix<AFloat> &output);
166 
168  const TCudaMatrix<AFloat> & Y,
169  const TCudaMatrix<AFloat> & output);
170 
171  /** Softmax transformation is implicitly applied, thus \p output should
172  * hold the linear activations of the last layer in the net. */
173  static AFloat SoftmaxCrossEntropy(const TCudaMatrix<AFloat> &Y,
174  const TCudaMatrix<AFloat> &output);
176  const TCudaMatrix<AFloat> & Y,
177  const TCudaMatrix<AFloat> & output);
178  ///@}
179 
180  //____________________________________________________________________________
181  //
182  // Output Functions
183  //____________________________________________________________________________
184 
185  /** @name Output Functions
186  * Output functions transform the activations \p output of the
187  * output layer in the network to a valid prediction \p YHat for
188  * the desired usage of the network, e.g. the identity function
189  * for regression or the sigmoid transformation for two-class
190  * classification.
191  */
192  ///@{
193  static void Sigmoid(TCudaMatrix<AFloat> &YHat,
194  const TCudaMatrix<AFloat> & );
195  static void Softmax(TCudaMatrix<AFloat> &YHat,
196  const TCudaMatrix<AFloat> & );
197  ///@}
198 
199  //____________________________________________________________________________
200  //
201  // Regularization
202  //____________________________________________________________________________
203 
204  /** @name Regularization
205  * For each regularization type two functions are required, one named
206  * <tt><Type>Regularization</tt> that evaluates the corresponding
207  * regularization functional for a given weight matrix and the
208  * <tt>Add<Type>RegularizationGradients</tt>, that adds the regularization
209  * component in the gradients to the provided matrix.
210  */
211  ///@{
212 
213  static AFloat L1Regularization(const TCudaMatrix<AFloat> & W);
215  const TCudaMatrix<AFloat> & W,
216  AFloat weightDecay);
217 
218  static AFloat L2Regularization(const TCudaMatrix<AFloat> & W);
220  const TCudaMatrix<AFloat> & W,
221  AFloat weightDecay);
222  ///@}
223 
224  //____________________________________________________________________________
225  //
226  // Initialization
227  //____________________________________________________________________________
228 
229  /** @name Initialization
230  * For each initialization method, one function in the low-level interface
231  * is provided. The naming scheme is <p>Initialize<Type></p> for a given
232  * initialization method Type.
233  */
234  ///@{
235 
236  static void InitializeGauss(TCudaMatrix<AFloat> & A);
237  static void InitializeUniform(TCudaMatrix<AFloat> & A);
239  static void InitializeZero(TCudaMatrix<AFloat> & A);
240 
241  ///@}
242 
243  //____________________________________________________________________________
244  //
245  // Dropout
246  //____________________________________________________________________________
247 
248  /** @name Dropout
249  */
250  ///@{
251 
252  /** Apply dropout with activation probability \p p to the given
253  * matrix \p A and scale the result by reciprocal of \p p. */
254  static void Dropout(TCudaMatrix<AFloat> & A, AFloat p);
255 
256  ///@}
257 
258  //____________________________________________________________________________
259  //
260  // Additional Arithmetic Functions
261  //____________________________________________________________________________
262 
263  /** @name Additional Arithmetic Functions
264  *
265  * Additional arithmetic on CUDA matrices used to implement the low-level
266  * interface.
267  */
268  ///@{
269 
270  /** Standard multiplication of two matrices \p A and \p B with the result being
271  * written into C.
272  */
273  static void Multiply(TCudaMatrix<AFloat> & C,
274  const TCudaMatrix<AFloat> & A,
275  const TCudaMatrix<AFloat> & B);
276  /** Matrix multiplication of two matrices \p A and \p B^T (transposed) with the
277  * result being written into C.
278  */
280  const TCudaMatrix<AFloat> & input,
281  const TCudaMatrix<AFloat> & Weights);
282  /** In-place Hadamard (element-wise) product of matrices \p A and \p B
283  * with the result being written into \p A.
284  */
285  static void Hadamard(TCudaMatrix<AFloat> & A, const TCudaMatrix<AFloat> & B);
286 
287  /** Sum columns of (m x n) matrixx \p A and write the results into the first
288  * m elements in \p A.
289  */
290  static void SumColumns(TCudaMatrix<AFloat> & B, const TCudaMatrix<AFloat> & A);
291 
292  /** Compute the sum of all elements in \p A */
293  static AFloat Sum(const TCudaMatrix<AFloat> &A);
294 };
295 
296 } // namespace DNN
297 } // namespace TMVA
298 
299 #endif
static double B[]
static AFloat SoftmaxCrossEntropy(const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
Softmax transformation is implicitly applied, thus output should hold the linear activations of the l...
static void MeanSquaredErrorGradients(TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
static void SymmetricReluDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void GaussDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void SoftmaxCrossEntropyGradients(TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
TCudaDeviceBuffer.
Definition: CudaBuffers.h:27
static void MultiplyTranspose(TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &weights)
Matrix-multiply input with the transpose of and write the results into output.
static void Softmax(TCudaMatrix< AFloat > &YHat, const TCudaMatrix< AFloat > &)
static AFloat MeanSquaredError(const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
The TCuda architecture class.
Definition: Cuda.h:40
static void ScaleAdd(TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B, Scalar_t beta=1.0)
Adds a the elements in matrix B scaled by c to the elements in the matrix A.
static double A[]
static void Hadamard(TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B)
In-place Hadamard (element-wise) product of matrices A and B with the result being written into A...
double beta(double x, double y)
Calculates the beta function.
AFloat Scalar_t
Definition: Cuda.h:45
static void IdentityDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void Dropout(TCudaMatrix< AFloat > &A, AFloat p)
Apply dropout with activation probability p to the given matrix A and scale the result by reciprocal ...
double weightDecay(double error, ItWeight itWeight, ItWeight itWeightEnd, double factorWeightDecay, EnumRegularization eRegularization)
compute the weight decay for regularization (L1 or L2)
Definition: NeuralNet.icc:491
static void Relu(TCudaMatrix< AFloat > &B)
static void AddRowWise(TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &biases)
Add the vectors biases row-wise to the matrix output.
TCudaHostBuffer.
Definition: CudaBuffers.h:41
static void SoftSignDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void Gauss(TCudaMatrix< AFloat > &B)
static void SymmetricRelu(TCudaMatrix< AFloat > &B)
static AFloat CrossEntropy(const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
Sigmoid transformation is implicitly applied, thus output should hold the linear activations of the l...
static double C[]
static void TanhDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void Tanh(TCudaMatrix< AFloat > &B)
static void Multiply(TCudaMatrix< AFloat > &C, const TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &B)
Standard multiplication of two matrices A and B with the result being written into C...
static void InitializeUniform(TCudaMatrix< AFloat > &A)
static void AddL2RegularizationGradients(TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay)
static void Sigmoid(TCudaMatrix< AFloat > &B)
static AFloat L2Regularization(const TCudaMatrix< AFloat > &W)
static void InitializeIdentity(TCudaMatrix< AFloat > &A)
static void ReluDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static AFloat Sum(const TCudaMatrix< AFloat > &A)
Compute the sum of all elements in A.
static void Backward(TCudaMatrix< AFloat > &activationGradientsBackward, TCudaMatrix< AFloat > &weightGradients, TCudaMatrix< AFloat > &biasGradients, TCudaMatrix< AFloat > &df, const TCudaMatrix< AFloat > &activationGradients, const TCudaMatrix< AFloat > &weights, const TCudaMatrix< AFloat > &activationBackward)
Perform the complete backward propagation step.
static void AddL1RegularizationGradients(TCudaMatrix< AFloat > &A, const TCudaMatrix< AFloat > &W, AFloat weightDecay)
static void SigmoidDerivative(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
static void InitializeZero(TCudaMatrix< AFloat > &A)
static void SoftSign(TCudaMatrix< AFloat > &B)
Abstract ClassifierFactory template that handles arbitrary types.
static void TransposeMultiply(TCudaMatrix< AFloat > &output, const TCudaMatrix< AFloat > &input, const TCudaMatrix< AFloat > &Weights)
Matrix multiplication of two matrices A and B^T (transposed) with the result being written into C...
static void Identity(TCudaMatrix< AFloat > &B)
static void SumColumns(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
Sum columns of (m x n) matrixx A and write the results into the first m elements in A...
static AFloat L1Regularization(const TCudaMatrix< AFloat > &W)
static void CrossEntropyGradients(TCudaMatrix< AFloat > &dY, const TCudaMatrix< AFloat > &Y, const TCudaMatrix< AFloat > &output)
static void Copy(TCudaMatrix< AFloat > &B, const TCudaMatrix< AFloat > &A)
Copy the elements of matrix A into matrix B.
TCudaMatrix Class.
Definition: CudaMatrix.h:98
static void InitializeGauss(TCudaMatrix< AFloat > &A)