Logo ROOT   master
Reference Guide
ActivationFunctions.hxx
Go to the documentation of this file.
1 // @(#)root/tmva/tmva/dnn:$Id$
2 // Author: Simon Pfreundschuh 19/07/16
3 
4 /*************************************************************************
5  * Copyright (C) 2016, Simon Pfreundschuh *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12  ///////////////////////////////////////////////////////////////////
13  // Implementation of the activation functions for multi-threaded //
14  // CPU architectures using Roots TThreadExecutor and BLAS. //
15  ///////////////////////////////////////////////////////////////////
16 
18 #include <math.h>
19 
20 #ifdef R__HAS_VDT
21 #include "vdt/tanh.h"
22 #endif
23 
24 
25 namespace TMVA
26 {
27 namespace DNN
28 {
29 
30 //______________________________________________________________________________
31 template<typename AFloat>
33  const ActivationDescriptor_t /* activationDescr */,
34  const double /* coef */, const AFloat /*alpha */, const AFloat /*beta*/)
35 {
36  // scaling and translation is not yet implemented
37  TMVA::DNN::evaluate<TCpu<AFloat>>( X, activFunct);
38 }
39 //______________________________________________________________________________
40 template<typename AFloat>
42  const Tensor_t & dY, const Tensor_t & X,
43  EActivationFunction activFunct,
44  const ActivationDescriptor_t /* activationDescr */,
45  const AFloat /* alpha */, const AFloat /* beta */)
46 {
47  // scaling and translation not yet implemented
48  // output tensor (Y) could also be used to speed up derivative calculation
49  // compute dx = f'(x)
50  TMVA::DNN::evaluateDerivative<TCpu<AFloat>>(dX, activFunct, X);
51  // Compute element-wise product. dx = f'(x) * dY
52  Hadamard(dX, dY);
53 }
54 //______________________________________________________________________________
55 template<typename AFloat>
57  const TCpuTensor<AFloat> &/*A*/)
58 {
59  auto f = [](AFloat) {return 1.0;};
60  B.Map(f);
61 }
62 
63 //______________________________________________________________________________
64 template<typename AFloat>
66 {
67  auto f = [](AFloat x) {return (x < 0.0) ? 0.0 : x;};
68  B.Map(f);
69 }
70 
71 //______________________________________________________________________________
72 template<typename AFloat>
74  const TCpuTensor<AFloat> &A)
75 {
76  auto f = [](AFloat x) {return (x < 0.0) ? 0.0 : 1.0;};
77  B.MapFrom(f, A);
78 }
79 
80 //______________________________________________________________________________
81 template<typename AFloat>
83 {
84  auto f = [](AFloat x) {return 1.0 / (1.0 + exp(-x));};
85  B.Map(f);
86 }
87 
88 //______________________________________________________________________________
89 template<typename AFloat>
91  const TCpuTensor<AFloat> &A)
92 {
93  auto f = [](AFloat x) {
94  AFloat sig = 1.0 / (1.0 + exp(-x));
95  return sig * (1.0 - sig);
96  };
97  B.MapFrom(f, A);
98 }
99 
100 //______________________________________________________________________________
101 template<typename AFloat>
103 {
104  auto f = [](AFloat x) {return tanh(x);};
105  B.Map(f);
106 }
107 
108 //______________________________________________________________________________
109 template<typename AFloat>
111  const TCpuTensor<AFloat> &A)
112 {
113  auto f = [](AFloat x) {
114  AFloat t = tanh(x);
115  return 1 - t * t;
116  };
117  B.MapFrom(f, A);
118 }
119 
120 #ifdef R__HAS_VDT
121 //______________________________________________________________________________
122 template <>
124 {
125  auto f = [](float x) { return vdt::fast_tanhf(x); };
126  B.Map(f);
127 }
128 template <>
129 void TCpu<double>::FastTanh(TCpuTensor<double> &B)
130 {
131  auto f = [](double x) { return vdt::fast_tanh(x); };
132  B.Map(f);
133 }
134 
135 //______________________________________________________________________________
136 template <>
137 void TCpu<float>::FastTanhDerivative(TCpuTensor<float> &B, const TCpuTensor<float> &A)
138 {
139  auto f = [](float x) {
140  double t = vdt::fast_tanhf(x);
141  return 1 - t * t;
142  };
143  B.MapFrom(f, A);
144 }
145 template <>
146 void TCpu<double>::FastTanhDerivative(TCpuTensor<double> &B, const TCpuTensor<double> &A)
147 {
148  auto f = [](double x) {
149  double t = vdt::fast_tanh(x);
150  return 1 - t * t;
151  };
152  B.MapFrom(f, A);
153 }
154 
155 #else // when VDT is not available
156 //______________________________________________________________________________
157 template <typename AFloat>
159 {
161 }
162 
163 //______________________________________________________________________________
164 template <typename AFloat>
166 {
168 }
169 #endif
170 
171 //______________________________________________________________________________
172 template<typename AFloat>
174 {
175  auto f = [](AFloat x) {return fabs(x);};
176  B.Map(f);
177 }
178 
179 //______________________________________________________________________________
180 template<typename AFloat>
182  const TCpuTensor<AFloat> &A)
183 {
184  auto f = [](AFloat x) {
185  return (x < 0.0) ? -1.0 : 1.0;
186  };
187  B.MapFrom(f, A);
188 }
189 
190 //______________________________________________________________________________
191 template<typename AFloat>
193 {
194  auto f = [](AFloat x) {return x / (1 + fabs(x));};
195  B.Map(f);
196 }
197 
198 //______________________________________________________________________________
199 template<typename AFloat>
201  const TCpuTensor<AFloat> &A)
202 {
203  auto f = [](AFloat x) {
204  x = 1.0 + fabs(x);
205  x = 1.0 / (x * x);
206  return x;
207  };
208  B.MapFrom(f, A);
209 }
210 
211 //______________________________________________________________________________
212 template<typename AFloat>
214 {
215  auto f = [](AFloat x) {return exp(- x * x);};
216  B.Map(f);
217 }
218 
219 //______________________________________________________________________________
220 template<typename AFloat>
222  const TCpuTensor<AFloat> &A)
223 {
224  auto f = [](AFloat x) {return - 2.0 * x * exp(- x * x);};
225  B.MapFrom(f, A);
226 }
227 
228 } // namespace DNN
229 } // namespace TMVA
static void ActivationFunctionBackward(Tensor_t &dX, const Tensor_t &Y, const Tensor_t &dY, const Tensor_t &X, EActivationFunction activFunct, const ActivationDescriptor_t activationDescr, const Scalar_t alpha=1, const Scalar_t beta=0)
Computes the gradient of the activation function.
static double B[]
double tanh(double)
static void FastTanhDerivative(Tensor_t &B, const Tensor_t &A)
static void SoftSign(Tensor_t &B)
#define f(i)
Definition: RSha256.hxx:104
static double A[]
Double_t x[n]
Definition: legend1.C:17
static void GaussDerivative(Tensor_t &B, const Tensor_t &A)
static void TanhDerivative(Tensor_t &B, const Tensor_t &A)
static void FastTanh(Tensor_t &B)
static void Relu(Tensor_t &B)
static void Sigmoid(Tensor_t &B)
static void SoftSignDerivative(Tensor_t &B, const Tensor_t &A)
VecExpr< UnaryOp< Fabs< T >, VecExpr< A, T, D >, T >, T, D > fabs(const VecExpr< A, T, D > &rhs)
static void ReluDerivative(Tensor_t &B, const Tensor_t &A)
static void Gauss(Tensor_t &B)
static void SymmetricRelu(Tensor_t &B)
static void Tanh(Tensor_t &B)
static void ActivationFunctionForward(Tensor_t &X, EActivationFunction activFunct, const ActivationDescriptor_t activationDescr, const double coef=0.0, const Scalar_t alpha=1, const Scalar_t beta=0)
static void SymmetricReluDerivative(Tensor_t &B, const Tensor_t &A)
create variable transformations
EActivationFunction
Enum that represents layer activation functions.
Definition: Functions.h:31
double exp(double)
static void IdentityDerivative(Tensor_t &B, const Tensor_t &A)
static void SigmoidDerivative(Tensor_t &B, const Tensor_t &A)