Logo ROOT  

Reference Guide
 
Loading...
Searching...
No Matches
ROperator_LSTM.hxx
Go to the documentation of this file.
1#ifndef TMVA_SOFIE_ROPERATOR_LSTM
2#define TMVA_SOFIE_ROPERATOR_LSTM
3
4#include "TMVA/RModel.hxx"
5#include "TMVA/ROperator.hxx"
6#include "TMVA/SOFIE_common.hxx"
7
8#include <memory>
9#include <sstream>
10#include <string>
11#include <vector>
12
13namespace TMVA::Experimental::SOFIE {
14
15/*! \brief Long Short-Term Memory operator
16 *
17 * Inference code generation for one-layer LSTM. Supports forward, reverse and bidirectional LSTM.
18 * See the <a href="https://github.com/onnx/onnx/blob/master/docs/Operators.md#LSTM">ONNX documentation</a>
19 * for details about the supported LSTM architectures.
20 */
21template <typename T> class ROperator_LSTM final : public ROperator {
22 private:
23 std::vector<float> fAttrActivationAlpha; ///< Sacling values used by some activation functions
24 std::vector<float> fAttrActivationBeta; ///< Scaling values used by some activation functions
25 std::vector<std::string> fAttrActivations; ///< Activation functions
26 float fAttrClip; ///< Clip threshold
27 std::string fAttrDirection; ///< Direction of processing
28 size_t fAttrHiddenSize; ///< Number of the hidden layers
29 size_t fAttrInputForget; ///< Forget gate
30 size_t fAttrLayout; ///< Data layout
31
32 std::string fNX; ///< Name of the input
33 std::string fNW; ///< Name of the weights
34 std::string fNR; ///< Name of the recurrence
35 std::string fNB; ///< Name of the bias
36 std::string fNSequence_lens; ///< Name of length of the sequences
37 std::string fNInitial_h; ///< Name of the initial value of the hidden states
38 std::string fNInitial_c; ///< Name of the initial value of the cell states
39 std::string fNP; ///< Name of peepholes
40 std::string fNY; ///< Name of the output
41 std::string fNY_h; ///< Name of the last sequence of the output
42 std::string fNY_c; ///< Name of the last sequence of the cell states
43
44 std::vector<size_t> fShapeX; ///< Shape of the input
45 std::vector<size_t> fShapeW; ///< Shape of the weights
46 std::vector<size_t> fShapeR; ///< Shape of the recurrence
47 std::vector<size_t> fShapeB; ///< Shape of the bias
48 std::vector<size_t> fShapeSequence_lens; ///< Shape of the length of the sequences
49 std::vector<size_t> fShapeInitial_h; ///< Shape of the initial value of the hidden states
50 std::vector<size_t> fShapeInitial_c; ///< Shape of the initial value of the cell states
51 std::vector<size_t> fShapeP; ///< Shape of the peepholes
52 std::vector<size_t> fShapeY; ///< Shape of the output
53 std::vector<size_t> fShapeY_h; ///< Shape of the last sequence of the output
54 std::vector<size_t> fShapeY_c; ///< Shape of the last sequence of the cell states
55
56 std::string fType; ///< Type of the tensors
57
58 public:
59 /*! Default constructor of ROperator_LSTM */
60 ROperator_LSTM() {}
61
62 /*! \brief Constructor of ROperator_LSTM from the attributes
63 *
64 * \param activation_alpha scaling values used by some activation functions
65 * \param activation_beta scaling values used by some activation functions
66 * \param activations activation functions
67 * \param clip clip threshold
68 * \param direction direction of processing of the sequneces
69 * \param hidden_size number of hidden layers
70 * \param input_forget forget gate
71 * \param layout data layout
72 * \param nameX name of the input tensor
73 * \param nameW name of the weight tensor
74 * \param nameR name of the recurrence tensor
75 * \param nameB name of the bias tensor
76 * \param nameSequence_lens name of the length of the sequences
77 * \param nameInitial_h name of the initial value of the hidden states
78 * \param nameInitial_c name of the initial value of the cell states
79 * \param nameP name of the peepholes tensor
80 * \param nameY name of the output
81 * \param nameY_h name of the last sequence of the output
82 * \param nameY_c name of the last sequence of the cell states
83 */
84 ROperator_LSTM(std::vector<float> activation_alpha,
85 std::vector<float> activation_beta,
86 std::vector<std::string> activations, float clip,
87 std::string direction, size_t hidden_size,
88 size_t input_forget, size_t layout,
89 std::string nameX, std::string nameW, std::string nameR,
90 std::string nameB, std::string nameSequence_lens,
91 std::string nameInitial_h, std::string nameInitial_c, std::string nameP,
92 std::string nameY, std::string nameY_h, std::string nameY_c)
93 : fAttrActivationAlpha(activation_alpha),
94 fAttrActivationBeta(activation_beta), fAttrActivations(activations),
95 fAttrClip(clip), fAttrDirection(direction), fAttrHiddenSize(hidden_size),
96 fAttrInputForget(input_forget), fAttrLayout(layout),
97 fNX(UTILITY::Clean_name(nameX)), fNW(UTILITY::Clean_name(nameW)),
98 fNR(UTILITY::Clean_name(nameR)), fNB(UTILITY::Clean_name(nameB)),
99 fNSequence_lens(UTILITY::Clean_name(nameSequence_lens)),
100 fNInitial_h(UTILITY::Clean_name(nameInitial_h)),
101 fNInitial_c(UTILITY::Clean_name(nameInitial_c)), fNP(UTILITY::Clean_name(nameP)),
102 fNY(UTILITY::Clean_name(nameY)), fNY_h(UTILITY::Clean_name(nameY_h)),
103 fNY_c(UTILITY::Clean_name(nameY_c)) {
104 if (std::is_same<T, float>::value) {
105 fType = "float";
106 } else {
107 throw std::runtime_error(
108 "TMVA SOFIE Encountered unsupported type parsing a LSTM operator");
109 }
110
111 fInputTensorNames = { fNX, fNW, fNR };
112 if (!fNB.empty()){
113 fInputTensorNames.emplace_back(fNB);
114 }
115 if (!fNSequence_lens.empty()){
116 fInputTensorNames.emplace_back(fNSequence_lens);
117 }
118 if (!fNInitial_h.empty()){
119 fInputTensorNames.emplace_back(fNInitial_h);
120 }
121 if (!fNInitial_c.empty()){
122 fInputTensorNames.emplace_back(fNInitial_c);
123 }
124 if (!fNP.empty()){
125 fInputTensorNames.emplace_back(fNP);
126 }
127
128 fOutputTensorNames = { };
129 if (!fNY.empty()){
130 fOutputTensorNames.emplace_back(fNY);
131 }
132 if (!fNY_h.empty()){
133 fOutputTensorNames.emplace_back(fNY_h);
134 }
135 if (!fNY_c.empty()){
136 fOutputTensorNames.emplace_back(fNY_c);
137 }
138 }
139
140 /*! \brief Infers the shape of the output tensors
141 *
142 * \param input shape of the input tensors
143 */
144 std::vector<std::vector<size_t>>
145 ShapeInference(std::vector<std::vector<size_t>> input);
146
147 /*! \brief Initialize the model
148 *
149 * \param model Model
150 */
151 void Initialize(RModel &) override;
152
153 /*! \brief Generate the inference code
154 *
155 * \param OpName name of the operator
156 */
157 std::string Generate(std::string OpName) override;
158
159 /*! \brief Generate the code for the Session internal data vectors
160 *
161 * \param opName name of the operator
162 */
163 std::string GenerateSessionMembersCode(std::string opName) override;
164
165 /*! \brief Returns the blas routines needed to compile the generated code
166 */
167 std::vector<std::string> GetBlasRoutines() override { return { std::string("Gemm"), std::string("Axpy") }; }
168};
169
170template <typename T>
171auto ROperator_LSTM<T>::ShapeInference(std::vector<std::vector<size_t>> input) -> std::vector<std::vector<size_t>>
172{
173 size_t num_directions = input[1][0];
174 size_t hidden_size = input[1][1] / 4;
175 if (fAttrLayout == 0) {
176 size_t seq_length = input[0][0];
177 size_t batch_size = input[0][1];
178 std::vector<std::vector<size_t>> ret({{seq_length, num_directions, batch_size, hidden_size},
179 {num_directions, batch_size, hidden_size},
180 {num_directions, batch_size, hidden_size}});
181 return ret;
182 } else {
183 size_t batch_size = input[0][0];
184 size_t seq_length = input[0][1];
185 std::vector<std::vector<size_t>> ret({{batch_size, seq_length, num_directions, hidden_size},
186 {batch_size, num_directions, hidden_size},
187 {batch_size, num_directions, hidden_size}});
188 return ret;
189 }
190}
191
192template <typename T>
193auto ROperator_LSTM<T>::Initialize(RModel &model) -> void
194{
195 // Check the input and output tensors
196 if (!model.CheckIfTensorAlreadyExist(fNX)) {
197 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNX + " is not found in model.");
198 }
199 fShapeX = model.GetTensorShape(fNX);
200 if (fShapeX.size() != 3) {
201 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNX + " is not of 3 dimensions.");
202 }
203 if (!model.CheckIfTensorAlreadyExist(fNW)) {
204 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNW + " is not found in model.");
205 }
206 fShapeW = model.GetTensorShape(fNW);
207 if (fShapeW.size() != 3) {
208 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNW + " is not of 3 dimensions.");
209 }
210 if (!model.CheckIfTensorAlreadyExist(fNR)) {
211 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNR + " is not found in model.");
212 }
213 fShapeR = model.GetTensorShape(fNR);
214 if (fShapeR.size() != 3) {
215 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNR + " is not of 3 dimensions.");
216 }
217 if (!fNB.empty()) {
218 if (!model.CheckIfTensorAlreadyExist(fNB)) {
219 throw std::runtime_error("TMVA SOFIE LSTM op input tensor " + fNB + " is not found in model.");
220 }
221 fShapeB = model.GetTensorShape(fNB);
222 if (fShapeB.size() != 2 && fShapeB.size() != 5) {
223 throw std::runtime_error("TMVA SOFIE LSTM op input tensor " + fNB + " is not of 2 or 5 dimensions.");
224 }
225 if (fShapeB.size() == 2) {
226 // Broadcasting the bias
227 auto original_data = model.GetInitializedTensorData(fNB);
228 size_t num_directions = fShapeW[0];
229 size_t seq_length = (fAttrLayout == 0) ? fShapeX[0] : fShapeX[1];
230 size_t batch_size = (fAttrLayout == 0) ? fShapeX[1] : fShapeX[0];
231 if (fType == "float") {
232 float *original_bias = static_cast<float *>(original_data.get());
233 float *new_bias = new float[4 * num_directions * seq_length * batch_size * fAttrHiddenSize];
234 for (size_t gate = 0; gate < 4; gate++) {
235 std::vector<float> sum(fAttrHiddenSize);
236 for (size_t direction = 0; direction < num_directions; direction++) {
237 size_t offset = direction * 8 * fAttrHiddenSize + gate * fAttrHiddenSize;
238 for (size_t h = 0; h < fAttrHiddenSize; h++) {
239 sum[h] = original_bias[offset + h] + original_bias[offset + h + 4 * fAttrHiddenSize];
240 }
241 for (size_t seq = 0; seq < seq_length; seq++) {
242 for (size_t batch = 0; batch < batch_size; batch++) {
243 size_t bias_offset = gate * num_directions * seq_length * batch_size * fAttrHiddenSize +
244 direction * seq_length * batch_size * fAttrHiddenSize +
245 seq * batch_size * fAttrHiddenSize + batch * fAttrHiddenSize;
246 std::copy(sum.begin(), sum.end(), new_bias + bias_offset);
247 }
248 }
249 }
250 }
251 std::vector<size_t> new_bias_shape = {4, num_directions, seq_length, batch_size, fAttrHiddenSize};
252 std::shared_ptr<void> new_bias_ptr(new_bias, std::default_delete<float[]>());
253 model.UpdateInitializedTensor(fNB, model.GetTensorType(fNB), new_bias_shape, new_bias_ptr);
254 fShapeB = model.GetTensorShape(fNB);
255 }
256 }
257 }
258 if (!fNSequence_lens.empty()) {
259 if (!model.CheckIfTensorAlreadyExist(fNSequence_lens)) {
260 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNSequence_lens + "is not found in model.");
261 }
262 fShapeSequence_lens = model.GetTensorShape(fNSequence_lens);
263 if (fShapeSequence_lens.size() != 1) {
264 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNSequence_lens + " is not of 1 dimension.");
265 }
266 }
267 if (!fNInitial_h.empty()) {
268 if (!model.CheckIfTensorAlreadyExist(fNInitial_h)) {
269 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNInitial_h + " is not found in model.");
270 }
271 fShapeInitial_h = model.GetTensorShape(fNInitial_h);
272 if (fShapeInitial_h.size() != 3) {
273 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNInitial_h + " is not of 3 dimensions.");
274 }
275 }
276 if (!fNInitial_c.empty()) {
277 if (!model.CheckIfTensorAlreadyExist(fNInitial_c)) {
278 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNInitial_c + " is not found in model.");
279 }
280 fShapeInitial_c = model.GetTensorShape(fNInitial_c);
281 if (fShapeInitial_c.size() != 3) {
282 throw std::runtime_error("TMVA SOFIE LSTM Op input tensor " + fNInitial_c + " is not of 3 dimensions.");
283 }
284 }
285 if (!fNP.empty()) {
286 if (!model.CheckIfTensorAlreadyExist(fNP)) {
287 throw std::runtime_error("TMVA SOFIE LSTM op input tensor " + fNP + " is not found in model.");
288 }
289 fShapeP = model.GetTensorShape(fNP);
290 if (fShapeP.size() != 2 && fShapeP.size() != 4) {
291 throw std::runtime_error("TMVA SOFIE LSTM op input tensor " + fNP + " is not of 2 or 4 dimensions.");
292 }
293 if (fShapeP.size() == 2) {
294 // Broadcasting the weight for peepholes
295 auto original_data = model.GetInitializedTensorData(fNP);
296 size_t num_directions = fShapeW[0];
297 size_t batch_size = (fAttrLayout == 0) ? fShapeX[1] : fShapeX[0];
298 if (fType == "float") {
299 float *original_p = static_cast<float *>(original_data.get());
300 float *new_p = new float[num_directions * 3 * batch_size * fAttrHiddenSize];
301 for (size_t direction = 0; direction < num_directions; direction++) {
302 for (size_t gate = 0; gate < 3; gate++) {
303 size_t p_offset = direction * 3 * fAttrHiddenSize + gate * fAttrHiddenSize;
304 for (size_t batch = 0; batch < batch_size; batch++) {
305 size_t offset = direction * 3 * batch_size * fAttrHiddenSize +
306 gate * batch_size * fAttrHiddenSize + batch * fAttrHiddenSize;
307 std::copy(original_p + p_offset, original_p + p_offset + fAttrHiddenSize, new_p + offset);
308 }
309 }
310 }
311 std::vector<size_t> new_p_shape = {num_directions, 3, batch_size, fAttrHiddenSize};
312 std::shared_ptr<void> new_p_ptr(new_p, std::default_delete<float[]>());
313 model.UpdateInitializedTensor(fNP, model.GetTensorType(fNP), new_p_shape, new_p_ptr);
314 fShapeP = model.GetTensorShape(fNP);
315 }
316 }
317 }
318 if (!fNY.empty()) {
319 fShapeY = ShapeInference({fShapeX, fShapeW})[0];
320 if (!model.CheckIfTensorAlreadyExist(fNY)) {
321 model.AddIntermediateTensor(fNY, model.GetTensorType(fNX), fShapeY);
322 }
323 }
324 if (!fNY_h.empty()) {
325 fShapeY_h = ShapeInference({fShapeX, fShapeW})[1];
326 if (!model.CheckIfTensorAlreadyExist(fNY_h)) {
327 model.AddIntermediateTensor(fNY_h, model.GetTensorType(fNX), fShapeY_h);
328 }
329 }
330 if (!fNY_c.empty()) {
331 fShapeY_c = ShapeInference({fShapeX, fShapeW})[2];
332 if (!model.CheckIfTensorAlreadyExist(fNY_c)) {
333 model.AddIntermediateTensor(fNY_c, model.GetTensorType(fNX), fShapeY_c);
334 }
335 }
336 // Check the attributes
337 for (auto &activation : fAttrActivations) {
338 if (activation != "Relu" && activation != "Tanh" && activation != "Sigmoid" && activation != "Affine" &&
339 activation != "LeakyRelu" && activation != "ThresholdRelu" && activation != "ScaledTanh" &&
340 activation != "HardSigmoid" && activation != "Elu" && activation != "Softsign" && activation != "Softplus") {
341 throw std::runtime_error("TMVA SOFIE - Activation function " + activation + " not implemented");
342 }
343 }
344 if (fAttrDirection != "forward" && fAttrDirection != "backward" && fAttrDirection != "bidirectional") {
345 throw std::runtime_error("TMVA SOFIE - Invalid LSTM direction fAttrDirection = " + fAttrDirection);
346 }
347 if (4 * fAttrHiddenSize != fShapeW[1]) {
348 throw std::runtime_error("TMVA SOFIE - fAttrHiddenSize must be equal to " + std::to_string(fShapeW[1] / 4));
349 }
350 if (fAttrInputForget > 1) {
351 throw std::runtime_error("TMVA SOFIE - fAttrInputForget = " + std::to_string(fAttrInputForget) +
352 " must be 0 or 1.");
353 }
354 if (fAttrLayout > 1) {
355 throw std::runtime_error("TMVA SOFIE - Layout fAttrLayout = " + std::to_string(fAttrLayout) +
356 " must be 0 (timewise) or 1 (batchwise)");
357 }
358 if (fAttrActivations.empty()) {
359 if (fAttrDirection == "bidirectional") {
360 fAttrActivations = {"Sigmoid", "Tanh", "Tanh", "Sigmoid", "Tanh", "Tanh"};
361 } else {
362 fAttrActivations = {"Sigmoid", "Tanh", "Tanh"};
363 }
364 }
365}
366
367// generate code for Session data members (e.g. internal vectors)
368template <typename T>
369std::string ROperator_LSTM<T>::GenerateSessionMembersCode(std::string opName)
370{
371 opName = "op_" + opName;
372 std::stringstream out;
373
374 size_t num_directions = fShapeW[0];
375 size_t seq_length = (fAttrLayout == 0) ? fShapeX[0] : fShapeX[1];
376 size_t batch_size = (fAttrLayout == 0) ? fShapeX[1] : fShapeX[0];
377 size_t input_size = fShapeX[2];
378
379 struct Block {
380 std::string name;
381 size_t size;
382 };
383
384 std::vector<Block> blocks;
385
386 size_t ff_size = seq_length * batch_size * fAttrHiddenSize;
387 size_t hs_size = seq_length * num_directions * batch_size * fAttrHiddenSize;
388
389 // Layout-dependent buffers
390 if (fAttrLayout != 0) {
391 blocks.push_back({"input", seq_length * batch_size * input_size});
392 blocks.push_back({"initial_hidden_state", num_directions * batch_size * fAttrHiddenSize});
393 blocks.push_back({"initial_cell_state", num_directions * batch_size * fAttrHiddenSize});
394 }
395
396 // Feedforward gates
397 blocks.push_back({"ff_input_gate", ff_size});
398 blocks.push_back({"ff_output_gate", ff_size});
399 blocks.push_back({"ff_cell_gate", ff_size});
400 if (fAttrInputForget == 0)
401 blocks.push_back({"ff_forget_gate", ff_size});
402
403 // Gate outputs
404 blocks.push_back({"input_gate", hs_size});
405 blocks.push_back({"output_gate", hs_size});
406 blocks.push_back({"cell_gate", hs_size});
407 if (fAttrInputForget == 0)
408 blocks.push_back({"forget_gate", hs_size});
409
410 // Cell state
411 blocks.push_back({"cell_state", hs_size});
412 blocks.push_back({"new_cell_state", hs_size});
413
414 // Hidden state (conditional)
415 if (fAttrLayout != 0 || fNY.empty()) {
416 blocks.push_back({"hidden_state", hs_size});
417 }
418
419 // Compute total size
420 size_t total_size = 0;
421 for (const auto &b : blocks) {
422 total_size += b.size;
423 }
424
425 // Backing storage
426 out << "std::vector<" << fType << "> fVec_" << opName << "_buffer = std::vector<" << fType << ">(" << total_size
427 << ");\n";
428
429 // Emit pointers
430 std::size_t offset = 0;
431 for (const auto &b : blocks) {
432 out << fType << "* fVec_" << opName << "_" << b.name << " = fVec_" << opName << "_buffer.data() + " << offset
433 << ";\n";
434 offset += b.size;
435 }
436
437 out << "\n";
438
439 return out.str();
440}
441
442template <typename T>
443auto ROperator_LSTM<T>::Generate(std::string OpName) -> std::string
444{
445 OpName = "op_" + OpName;
446 std::stringstream out;
447
448 size_t seq_length = (fAttrLayout == 0) ? fShapeX[0] : fShapeX[1];
449 size_t batch_size = (fAttrLayout == 0) ? fShapeX[1] : fShapeX[0];
450 size_t input_size = fShapeX[2];
451 size_t num_directions = fShapeW[0];
452
453 // set the input
454 if (fAttrLayout == 0) {
455 out << SP << fType << " const *" << OpName << "_input = tensor_" << fNX << ";\n";
456 } else {
457 out << SP << fType << " * " << OpName << "_input = this->fVec_" << OpName << "_input;\n";
458
459 out << SP << "for(size_t seq = 0; seq < " << seq_length << "; seq++) {\n";
460 out << SP << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
461 out << SP << SP << SP << "for(size_t i = 0; i < " << input_size << "; i++) {\n";
462 out << SP << SP << SP << SP << OpName << "_input[seq * " << batch_size * input_size << " + batch * " << input_size
463 << " + i] = " << "tensor_" << fNX << "[batch * " << seq_length * input_size << " + seq * " << input_size
464 << " + i];\n";
465 out << SP << SP << SP << "}\n";
466 out << SP << SP << "}\n";
467 out << SP << "}\n";
468 }
469
470 // Set the initial hidden state
471 if (!fNInitial_h.empty()) {
472 if (fAttrLayout == 0) {
473 out << SP << fType << " const*" << OpName << "_initial_hidden_state = " << " tensor_" << fNInitial_h << ";\n";
474 } else {
475 out << SP << fType << " const* " << OpName << "_initial_hidden_state = this->fVec_" << OpName
476 << "_initial_hidden_state;\n";
477
478 for (size_t direction = 0; direction < num_directions; direction++) {
479 out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
480 out << SP << SP << "for(size_t h = 0; h < " << fAttrHiddenSize << "; h++) {\n";
481 out << SP << SP << SP << OpName << "_initial_hidden_state[" << direction * batch_size * fAttrHiddenSize
482 << " + batch * " << fAttrHiddenSize << " + h] = tensor_" << fNInitial_h << "[batch * "
483 << num_directions * fAttrHiddenSize << " + " << direction * fAttrHiddenSize << " + h];\n";
484 out << SP << SP << "}\n";
485 out << SP << "}\n";
486 }
487 }
488 }
489
490 // Set the initial cell state
491 if (!fNInitial_c.empty()) {
492 if (fAttrLayout == 0) {
493 out << SP << fType << " const*" << OpName << "_initial_cell_state = " << " tensor_" << fNInitial_c << ";\n";
494 } else {
495 out << SP << fType << " const* " << OpName << "_initial_cell_state = this->fVec_" << OpName
496 << "_initial_cell_state;\n";
497
498 for (size_t direction = 0; direction < num_directions; direction++) {
499 out << SP << "for(size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
500 out << SP << SP << "for(size_t h = 0; h < " << fAttrHiddenSize << "; h++) {\n";
501 out << SP << SP << SP << OpName << "_initial_cell_state[" << direction * batch_size * fAttrHiddenSize
502 << " + batch * " << fAttrHiddenSize << " + h] = tensor_" << fNInitial_c << "[batch * "
503 << num_directions * fAttrHiddenSize << " + " << direction * fAttrHiddenSize << " + h];\n";
504 out << SP << SP << "}\n";
505 out << SP << "}\n";
506 }
507 }
508 }
509
510 // Set the feedforward
511 out << SP << fType << " * " << OpName << "_ff_input_gate = this->fVec_" << OpName << "_ff_input_gate;\n";
512 out << SP << fType << " * " << OpName << "_ff_output_gate = this->fVec_" << OpName << "_ff_output_gate;\n";
513 out << SP << fType << " * " << OpName << "_ff_cell_gate = this->fVec_" << OpName << "_ff_cell_gate;\n";
514 if (fAttrInputForget == 0) {
515 out << SP << fType << " * " << OpName << "_ff_forget_gate = this->fVec_" << OpName << "_ff_forget_gate;\n";
516 }
517 // Set the gates
518 out << SP << fType << " * " << OpName << "_input_gate = this->fVec_" << OpName << "_input_gate;\n";
519 out << SP << fType << " * " << OpName << "_output_gate = this->fVec_" << OpName << "_output_gate;\n";
520 out << SP << fType << " * " << OpName << "_cell_gate = this->fVec_" << OpName << "_cell_gate;\n";
521 if (fAttrInputForget == 0) {
522 out << SP << fType << " * " << OpName << "_forget_gate = this->fVec_" << OpName << "_forget_gate;\n";
523 }
524 // Set the cell state and the new cell state = h(cell state)
525 out << SP << fType << " * " << OpName << "_cell_state = this->fVec_" << OpName << "_cell_state;\n";
526 out << SP << fType << " * " << OpName << "_new_cell_state = this->fVec_" << OpName << "_new_cell_state;\n";
527
528 // Set the hidden state
529 if (fAttrLayout == 0 && !fNY.empty()) {
530 out << SP << fType << " *" << OpName << "_hidden_state = tensor_" << fNY << ";\n";
531 } else {
532 out << SP << fType << " * " << OpName << "_hidden_state = this->fVec_" << OpName << "_hidden_state;\n";
533 }
534
535 out << SP << "char " << OpName << "_transA = 'N';\n";
536 out << SP << "char " << OpName << "_transB = 'T';\n";
537 out << SP << "int " << OpName << "_m = " << seq_length * batch_size << ";\n";
538 out << SP << "int " << OpName << "_n = " << fAttrHiddenSize << ";\n";
539 out << SP << "int " << OpName << "_k = " << input_size << ";\n";
540 if (fType == "float") {
541 out << SP << fType << " " << OpName << "_alpha = 1.;\n";
542 out << SP << fType << " " << OpName << "_beta = 0.;\n";
543 }
544 if (!fNB.empty()) {
545 out << SP << "int " << OpName << "_bias_size = " << seq_length * batch_size * fAttrHiddenSize << ";\n";
546 out << SP << "int " << OpName << "_incx = 1;\n";
547 out << SP << "int " << OpName << "_incy = 1;\n";
548 }
549
550 auto emit_sgemm = [&](const std::string &out_name, size_t offset) -> std::string {
551 std::stringstream ss;
552 ss << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName << "_n, &" << OpName
553 << "_m, &" << OpName << "_k, &" << OpName << "_alpha, tensor_" << fNW;
554
555 if (offset != 0)
556 ss << " + " << offset;
557
558 ss << ", &" << OpName << "_k, " << OpName << "_input, &" << OpName << "_k, &" << OpName << "_beta, " << OpName
559 << "_" << out_name << ", &" << OpName << "_n);\n";
560 return ss.str();
561 };
562
563 for (size_t direction = 0; direction < num_directions; direction++) {
564 if (direction == 0) {
565 if (fType == "float") {
566 // input_gate = input * weight_i^T
567 out << SP << emit_sgemm("ff_input_gate", 0);
568 // output_gate = input * weight_o^T
569 size_t wo_offset = fAttrHiddenSize * input_size;
570 out << SP << emit_sgemm("ff_output_gate", wo_offset);
571 // cell_gate = input * weight_c^T
572 size_t wc_offset = 3 * fAttrHiddenSize * input_size;
573 out << SP << emit_sgemm("ff_cell_gate", wc_offset);
574 }
575 } else {
576 if (fType == "float") {
577 // input_gate = input * weight_i^T
578 out << SP << emit_sgemm("ff_input_gate", 4 * fAttrHiddenSize * input_size);
579 // output_gate = input * weight_o^T
580 size_t wo_offset = 4 * fAttrHiddenSize * input_size + 1 * fAttrHiddenSize * input_size;
581 out << SP << emit_sgemm("ff_output_gate", wo_offset);
582 // cell_gate = input * weight_c^T
583 size_t wc_offset = 4 * fAttrHiddenSize * input_size + 3 * fAttrHiddenSize * input_size;
584 out << SP << emit_sgemm("ff_cell_gate", wc_offset);
585 }
586 }
587 if (fAttrInputForget == 0) {
588 // forget_gate = input * weight_f^T
589 if (direction == 0) {
590 if (fType == "float") {
591 size_t wf_offset = 2 * fAttrHiddenSize * input_size;
592 out << SP << emit_sgemm("ff_forget_gate", wf_offset);
593 }
594 } else {
595 if (fType == "float") {
596 size_t wf_offset = 4 * fAttrHiddenSize * input_size + 2 * fAttrHiddenSize * input_size;
597 out << SP << emit_sgemm("ff_forget_gate", wf_offset);
598 }
599 }
600 }
601
602 // Add the bias
603 if (!fNB.empty()) {
604 if (direction == 0) {
605 if (fType == "float") {
606 // ff_input_gate += bias_i
607 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << ", &"
608 << OpName << "_incx, " << OpName << "_ff_input_gate, &" << OpName << "_incy);\n";
609 // ff_output_gate += bias_o
610 size_t bo_offset = seq_length * batch_size * fAttrHiddenSize;
611 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + "
612 << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_output_gate, &" << OpName
613 << "_incy);\n";
614 // ff_cell_gate += bias_c
615 size_t bc_offset = 3 * seq_length * batch_size * fAttrHiddenSize;
616 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + "
617 << bc_offset << ", &" << OpName << "_incx, " << OpName << "_ff_cell_gate, &" << OpName
618 << "_incy);\n";
619 }
620 } else {
621 if (fType == "float") {
622 // ff_input_gate += bias_i
623 size_t bi_offset = 4 * seq_length * batch_size * fAttrHiddenSize;
624 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + "
625 << bi_offset << ", &" << OpName << "_incx, " << OpName << "_ff_input_gate, &" << OpName
626 << "_incy);\n";
627 // ff_output_gate += bias_o
628 size_t bo_offset =
629 4 * seq_length * batch_size * fAttrHiddenSize + seq_length * batch_size * fAttrHiddenSize;
630 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + "
631 << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_output_gate, &" << OpName
632 << "_incy);\n";
633 // ff_cell_gate += bias_c
634 size_t bc_offset = 4 * num_directions * seq_length * batch_size * fAttrHiddenSize +
635 3 * seq_length * batch_size * fAttrHiddenSize;
636 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB << " + "
637 << bc_offset << ", &" << OpName << "_incx, " << OpName << "_ff_cell_gate, &" << OpName
638 << "_incy);\n";
639 }
640 }
641 if (fAttrInputForget == 0) {
642 // ff_forget_gate += bias_f
643 if (direction == 0) {
644 if (fType == "float") {
645 size_t bo_offset = 2 * seq_length * batch_size * fAttrHiddenSize;
646 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB
647 << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_forget_gate, &" << OpName
648 << "_incy);\n";
649 }
650 } else {
651 if (fType == "float") {
652 size_t bo_offset =
653 4 * seq_length * batch_size * fAttrHiddenSize + 2 * seq_length * batch_size * fAttrHiddenSize;
654 out << SP << "BLAS::saxpy_(&" << OpName << "_bias_size, &" << OpName << "_alpha, tensor_" << fNB
655 << " + " << bo_offset << ", &" << OpName << "_incx, " << OpName << "_ff_forget_gate, &" << OpName
656 << "_incy);\n";
657 }
658 }
659 }
660 }
661
662 // Copy ff_input_gate, ff_output_gate, ff_cell_gate and ff_forget_gate into input_gate, output_gate,
663 // cell_gate and forget_gate
664 out << SP << "for (size_t seq = 0; seq < " << seq_length << "; seq++) {\n";
665 out << SP << SP << "size_t ff_offset = seq * " << batch_size * fAttrHiddenSize << ";\n";
666 if (direction == 0) {
667 out << SP << SP << "size_t gate_offset = seq * " << num_directions * batch_size * fAttrHiddenSize << ";\n";
668 } else {
669 out << SP << SP << "size_t gate_offset = seq * " << num_directions * batch_size * fAttrHiddenSize << " + "
670 << batch_size * fAttrHiddenSize << ";\n";
671 }
672 size_t ff_seq_size = batch_size * fAttrHiddenSize;
673 out << SP << SP << "std::copy(" << OpName << "_ff_input_gate + ff_offset, " << OpName
674 << "_ff_input_gate + ff_offset + " << ff_seq_size << ", " << OpName << "_input_gate + gate_offset);\n";
675 out << SP << SP << "std::copy(" << OpName << "_ff_output_gate + ff_offset, " << OpName
676 << "_ff_output_gate + ff_offset + " << ff_seq_size << ", " << OpName << "_output_gate + gate_offset);\n";
677 out << SP << SP << "std::copy(" << OpName << "_ff_cell_gate + ff_offset, " << OpName
678 << "_ff_cell_gate + ff_offset + " << ff_seq_size << ", " << OpName << "_cell_gate + gate_offset);\n";
679 if (fAttrInputForget == 0) {
680 out << SP << SP << "std::copy(" << OpName << "_ff_forget_gate + ff_offset, " << OpName
681 << "_ff_forget_gate + ff_offset + " << ff_seq_size << ", " << OpName << "_forget_gate + gate_offset);\n";
682 }
683 out << SP << "}\n";
684
685 out << SP << "for (size_t seq = 0; seq < " << seq_length << "; seq++) {\n";
686 if (fAttrDirection == "backward" || direction == 1) {
687 out << SP << SP << "size_t index = " << seq_length - 1 << " - seq;\n";
688 } else {
689 out << SP << SP << "size_t index = seq;\n";
690 }
691 out << SP << SP << "int m2 = " << batch_size << ";\n";
692 if (direction == 0) {
693 out << SP << SP << "size_t offset = index * " << num_directions * batch_size * fAttrHiddenSize << ";\n";
694 } else {
695 out << SP << SP << "size_t offset = index * " << num_directions * batch_size * fAttrHiddenSize << " + "
696 << batch_size * fAttrHiddenSize << ";\n";
697 }
698 size_t size = batch_size * fAttrHiddenSize;
699 // gate = gate + initial_hidden_state * Recurrence^T
700 out << SP << SP << "if (seq == 0) {\n";
701 if (!fNInitial_h.empty()) {
702 if (direction == 0) {
703 if (fType == "float") {
704 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
705 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName
706 << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName << "_alpha, "
707 << OpName << "_input_gate + offset, &" << OpName << "_n);\n";
708 size_t ro_offset = fAttrHiddenSize * fAttrHiddenSize;
709 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
710 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset
711 << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName
712 << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n";
713 size_t rc_offset = 3 * fAttrHiddenSize * fAttrHiddenSize;
714 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
715 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset
716 << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName
717 << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n";
718 if (fAttrInputForget == 0) {
719 size_t rf_offset = 2 * fAttrHiddenSize * fAttrHiddenSize;
720 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &"
721 << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + "
722 << rf_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName
723 << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n";
724 }
725 }
726 } else { // direction=1
727 if (fType == "float") {
728 size_t ri_offset = 4 * fAttrHiddenSize * fAttrHiddenSize;
729 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
730 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ri_offset
731 << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName
732 << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n";
733 size_t ro_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 1 * fAttrHiddenSize * fAttrHiddenSize;
734 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
735 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset
736 << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName
737 << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n";
738 size_t rc_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 3 * fAttrHiddenSize * fAttrHiddenSize;
739 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
740 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset
741 << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName << "_n, &" << OpName
742 << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n";
743 if (fAttrInputForget == 0) {
744 size_t rf_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 2 * fAttrHiddenSize * fAttrHiddenSize;
745 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &"
746 << OpName << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + "
747 << rf_offset << ", &" << OpName << "_n, " << OpName << "_initial_hidden_state, &" << OpName
748 << "_n, &" << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n";
749 }
750 }
751 }
752 }
753 out << SP << SP << "} else {\n";
754 // gate = gate + previous_hidden_state * Recurrence^T
755 if (direction == 0) {
756 if (fAttrDirection == "backward") {
757 out << SP << SP << SP << "size_t previous_offset = (index + 1) * "
758 << num_directions * batch_size * fAttrHiddenSize << ";\n";
759 } else {
760 out << SP << SP << SP << "size_t previous_offset = (seq - 1) * "
761 << num_directions * batch_size * fAttrHiddenSize << ";\n";
762 }
763 if (fType == "float") {
764 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
765 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << ", &" << OpName << "_n, "
766 << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &" << OpName << "_alpha, " << OpName
767 << "_input_gate + offset, &" << OpName << "_n);\n";
768 size_t ro_offset = 1 * fAttrHiddenSize * fAttrHiddenSize;
769 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
770 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset
771 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
772 << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n";
773 size_t rc_offset = 3 * fAttrHiddenSize * fAttrHiddenSize;
774 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
775 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset
776 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
777 << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n";
778 if (fAttrInputForget == 0) {
779 size_t rf_offset = 2 * fAttrHiddenSize * fAttrHiddenSize;
780 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
781 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset
782 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
783 << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n";
784 }
785 }
786 } else {
787 out << SP << SP << SP << "size_t previous_offset = (index + 1) * "
788 << num_directions * batch_size * fAttrHiddenSize << " + " << batch_size * fAttrHiddenSize << ";\n";
789 if (fType == "float") {
790 size_t ri_offset = 4 * fAttrHiddenSize * fAttrHiddenSize;
791 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
792 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ri_offset
793 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
794 << OpName << "_alpha, " << OpName << "_input_gate + offset, &" << OpName << "_n);\n";
795 size_t ro_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + fAttrHiddenSize * fAttrHiddenSize;
796 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
797 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << ro_offset
798 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
799 << OpName << "_alpha, " << OpName << "_output_gate + offset, &" << OpName << "_n);\n";
800 size_t rc_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 3 * fAttrHiddenSize * fAttrHiddenSize;
801 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
802 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rc_offset
803 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
804 << OpName << "_alpha, " << OpName << "_cell_gate + offset, &" << OpName << "_n);\n";
805 if (fAttrInputForget == 0) {
806 size_t rf_offset = 4 * fAttrHiddenSize * fAttrHiddenSize + 2 * fAttrHiddenSize * fAttrHiddenSize;
807 out << SP << SP << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
808 << "_n, &m2, &" << OpName << "_n, &" << OpName << "_alpha, tensor_" << fNR << " + " << rf_offset
809 << ", &" << OpName << "_n, " << OpName << "_hidden_state + previous_offset, &" << OpName << "_n, &"
810 << OpName << "_alpha, " << OpName << "_forget_gate + offset, &" << OpName << "_n);\n";
811 }
812 }
813 }
814 out << SP << SP << "}\n";
815
816 // Clip the elements of the cell gate into the range [-fAttrClip, fAttrClip]
817 if (fAttrClip > .0) {
818 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
819 if (fType == "float") {
820 out << SP << SP << SP << "float x = (" << OpName << "_cell_gate[i] > " << -fAttrClip << ") ? " << OpName
821 << "_cell_gate[i] : " << -fAttrClip << ";\n";
822 }
823 out << SP << SP << SP << OpName << "_cell_gate[i] = (x < " << fAttrClip << ") ? x : " << fAttrClip << ";\n";
824 out << SP << SP << "}\n";
825 }
826 // Apply the activation function to the cell gate, cell_gate = g(cell_gate)
827 if (fAttrActivations[direction * 3 + 1] == "Relu") {
828 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
829 out << SP << SP << SP << "if (" << OpName << "_cell_gate[i] < 0.)\n";
830 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = 0.;\n";
831 out << SP << SP << "}\n";
832 } else if (fAttrActivations[direction * 3 + 1] == "Tanh") {
833 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
834 if (fType == "float") {
835 out << SP << SP << SP << "float ex = exp(-2 * " << OpName << "_cell_gate[i]);\n";
836 }
837 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = (1. - ex) / (1. + ex);\n";
838 out << SP << SP << "}\n";
839 } else if (fAttrActivations[direction * 3 + 1] == "Sigmoid") {
840 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
841 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = 1. / (1. + exp(-" << OpName << "_cell_gate[i]));\n";
842 out << SP << SP << "}\n";
843 } else if (fAttrActivations[direction * 3 + 1] == "Affine") {
844 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
845 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = " << fAttrActivationAlpha[direction * 3 + 1] << " * "
846 << OpName << "_cell_gate[i] + " << fAttrActivationBeta[direction * 3 + 1] << ";\n";
847 out << SP << SP << "}\n";
848 } else if (fAttrActivations[direction * 3 + 1] == "ScaledTanh") {
849 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
850 if (fType == "float") {
851 out << SP << SP << SP << "float ex = exp(-2 * " << fAttrActivationBeta[direction * 3 + 1] << " * " << OpName
852 << "_cell_gate[i]);\n";
853 }
854 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = " << fAttrActivationAlpha[direction * 3 + 1]
855 << " * (1. - ex) / (1. + ex);\n";
856 out << SP << SP << "}\n";
857 } else if (fAttrActivations[direction * 3 + 1] == "HardSigmoid") {
858 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
859 if (fType == "float") {
860 out << SP << SP << SP << "float a = " << fAttrActivationAlpha[direction * 3 + 1] << " * " << OpName
861 << "_cell_gate[i] + " << fAttrActivationBeta[direction * 3 + 1] << ";\n";
862 out << SP << SP << SP << "float b = (a > 0.) ? a : 0.;\n";
863 }
864 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = (b < 1.) ? b : 1.;\n";
865 out << SP << SP << "}\n";
866 } else if (fAttrActivations[direction * 3 + 1] == "LeakyRelu") {
867 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
868 out << SP << SP << SP << "if (" << OpName << "_cell_gate[i] < 0.)\n";
869 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = " << fAttrActivationAlpha[direction * 3 + 1] << " * "
870 << OpName << "_cell_gate[i];\n";
871 out << SP << SP << "}\n";
872 } else if (fAttrActivations[direction * 3 + 1] == "ThresholdRelu") {
873 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
874 out << SP << SP << SP << "if (" << OpName << "_cell_gate[i] < " << fAttrActivationAlpha[direction * 3 + 1]
875 << ")\n";
876 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = 0.;\n";
877 out << SP << SP << "}";
878 } else if (fAttrActivations[direction * 3 + 1] == "Elu") {
879 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
880 out << SP << SP << SP << "if (" << OpName << "_cell_gate[i] < 0.)\n";
881 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = " << fAttrActivationAlpha[direction * 3 + 1]
882 << " * exp(" << OpName << "_cell_gate[i] - 1.);\n";
883 out << SP << SP << "}\n";
884 } else if (fAttrActivations[direction * 3 + 1] == "Softsign") {
885 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
886 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = " << OpName << "_cell_gate[i] / (1. + abs(" << OpName
887 << "_cell_gate[i]));\n";
888 out << SP << SP << "}\n";
889 } else { // fAttrActivations[direction * 3 + 1] = Softplus
890 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
891 out << SP << SP << SP << SP << OpName << "_cell_gate[i] = log(1. + exp(" << OpName << "_cell_gate[i]));\n";
892 out << SP << SP << "}\n";
893 }
894
895 // Peephole connections for the input gate and the forget gate
896 if (!fNP.empty()) {
897 // gate = 1.0 * gate + previous_cell_state * P^T
898 out << SP << SP << "if (seq == 0) {\n";
899 if (!fNInitial_c.empty()) {
900 if (direction == 0) {
901 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
902 out << SP << SP << SP << SP << OpName << "_input_gate[i + offset] += tensor_" << fNP << "[i] * "
903 << OpName << "_initial_cell_state[i];\n";
904 out << SP << SP << SP << "}\n";
905 if (fAttrInputForget == 0) {
906 size_t pf_offset = batch_size * fAttrHiddenSize;
907 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
908 out << SP << SP << SP << SP << OpName << "_forget_gate[i + offset] += tensor_" << fNP << "[i + "
909 << pf_offset << "] * " << OpName << "_initial_cell_state[i];\n";
910 out << SP << SP << SP << "}\n";
911 }
912 } else {
913 size_t pi_offset = 3 * batch_size * fAttrHiddenSize;
914 size_t initial_c_offset = batch_size * fAttrHiddenSize;
915 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
916 out << SP << SP << SP << SP << OpName << "_input_gate[i + offset] += tensor_" << fNP << "[i + "
917 << pi_offset << "] * " << OpName << "_initial_cell_state[i + " << initial_c_offset << "];\n";
918 out << SP << SP << SP << "}\n";
919 if (fAttrInputForget == 0) {
920 size_t pf_offset = 3 * batch_size * fAttrHiddenSize + batch_size * fAttrHiddenSize;
921 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
922 out << SP << SP << SP << SP << OpName << "_forget_gate[i + offset] += tensor_" << fNP << "[i + "
923 << pf_offset << "] * " << OpName << "_initial_cell_state[i + " << initial_c_offset << "];\n";
924 out << SP << SP << SP << "}\n";
925 }
926 }
927 }
928 out << SP << SP << "} else {\n";
929 if (direction == 0) {
930 if (fAttrDirection == "backward") {
931 out << SP << SP << SP << "size_t c_offset = (index + 1) * "
932 << num_directions * batch_size * fAttrHiddenSize << ";\n";
933 } else {
934 out << SP << SP << SP << "size_t c_offset = (seq - 1) * "
935 << num_directions * batch_size * fAttrHiddenSize << ";\n";
936 }
937 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
938 out << SP << SP << SP << SP << OpName << "_input_gate[i + offset] += tensor_" << fNP << "[i] * " << OpName
939 << "_cell_state[i + c_offset];\n";
940 out << SP << SP << SP << "}\n";
941 if (fAttrInputForget == 0) {
942 size_t pf_offset = batch_size * fAttrHiddenSize;
943 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
944 out << SP << SP << SP << SP << OpName << "_forget_gate[i + offset] += tensor_" << fNP << "[i + "
945 << pf_offset << "] * " << OpName << "_cell_state[i + c_offset];\n";
946 out << SP << SP << SP << "}\n";
947 }
948 } else { // direction=1
949 size_t pi_offset = 3 * batch_size * fAttrHiddenSize;
950 out << SP << SP << SP << "size_t c_offset = (index + 1) * " << num_directions * batch_size * fAttrHiddenSize
951 << " + " << batch_size * fAttrHiddenSize << ";\n";
952 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
953 out << SP << SP << SP << SP << OpName << "_input_gate[i + offset] += tensor_" << fNP << "[i + " << pi_offset
954 << "] * " << OpName << "_cell_state[i + c_offset];\n";
955 out << SP << SP << SP << "}\n";
956 if (fAttrInputForget == 0) {
957 size_t pf_offset = 3 * batch_size * fAttrHiddenSize + batch_size * fAttrHiddenSize;
958 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
959 out << SP << SP << SP << SP << OpName << "_forget_gate[i + offset] += tensor_" << fNP << "[i + "
960 << pf_offset << "] * " << OpName << "_cell_state[i + c_offset];\n";
961 out << SP << SP << SP << "}\n";
962 }
963 }
964 out << SP << SP << "}\n";
965 }
966
967 // Clip the elements of the input gate into the range [-fAttrClip, fAttrClip]
968 if (fAttrClip > .0) {
969 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
970 if (fType == "float") {
971 out << SP << SP << SP << "float x = (" << OpName << "_input_gate[i] > " << -fAttrClip << ") ? " << OpName
972 << "_input_gate[i] : " << -fAttrClip << ";\n";
973 }
974 out << SP << SP << SP << OpName << "_input_gate[i] = (x < " << fAttrClip << ") ? x : " << fAttrClip << ";\n";
975 out << SP << SP << "}\n";
976 }
977 // Apply the activation function to the input gate
978 if (fAttrActivations[direction * 3] == "Relu") {
979 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
980 out << SP << SP << SP << "if (" << OpName << "_input_gate[i] < 0.)\n";
981 out << SP << SP << SP << SP << OpName << "_input_gate[i] = 0.;\n";
982 out << SP << SP << "}\n";
983 } else if (fAttrActivations[direction * 3] == "Tanh") {
984 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
985 if (fType == "float") {
986 out << SP << SP << SP << "float ex = exp(-2 * " << OpName << "_input_gate[i]);\n";
987 }
988 out << SP << SP << SP << SP << OpName << "_input_gate[i] = (1. - ex) / (1. + ex);\n";
989 out << SP << SP << "}\n";
990 } else if (fAttrActivations[direction * 3] == "Sigmoid") {
991 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
992 out << SP << SP << SP << SP << OpName << "_input_gate[i] = 1. / (1. + exp(-" << OpName
993 << "_input_gate[i]));\n";
994 out << SP << SP << "}\n";
995 } else if (fAttrActivations[direction * 3] == "Affine") {
996 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
997 out << SP << SP << SP << SP << OpName << "_input_gate[i] = " << fAttrActivationAlpha[direction * 3] << " * "
998 << OpName << "_input_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
999 out << SP << SP << "}\n";
1000 } else if (fAttrActivations[direction * 3] == "ScaledTanh") {
1001 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1002 if (fType == "float") {
1003 out << SP << SP << SP << "float ex = exp(-2 * " << fAttrActivationBeta[direction * 3] << " * " << OpName
1004 << "_input_gate[i]);\n";
1005 }
1006 out << SP << SP << SP << SP << OpName << "_input_gate[i] = " << fAttrActivationAlpha[direction * 3]
1007 << " * (1. - ex) / (1. + ex);\n";
1008 out << SP << SP << "}\n";
1009 } else if (fAttrActivations[direction * 3] == "HardSigmoid") {
1010 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1011 if (fType == "float") {
1012 out << SP << SP << SP << "float a = " << fAttrActivationAlpha[direction * 3] << " * " << OpName
1013 << "_input_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
1014 out << SP << SP << SP << "float b = (a > 0.) ? a : 0.;\n";
1015 }
1016 out << SP << SP << SP << SP << OpName << "_input_gate[i] = (b < 1.) ? b : 1.;\n";
1017 out << SP << SP << "}\n";
1018 } else if (fAttrActivations[direction * 3] == "LeakyRelu") {
1019 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1020 out << SP << SP << SP << "if (" << OpName << "_input_gate[i] < 0.)\n";
1021 out << SP << SP << SP << SP << OpName << "_input_gate[i] = " << fAttrActivationAlpha[direction * 3] << " * "
1022 << OpName << "_input_gate[i];\n";
1023 out << SP << SP << "}\n";
1024 } else if (fAttrActivations[direction * 3] == "ThresholdRelu") {
1025 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1026 out << SP << SP << SP << "if (" << OpName << "_input_gate[i] < " << fAttrActivationAlpha[direction * 3]
1027 << ")\n";
1028 out << SP << SP << SP << SP << OpName << "_input_gate[i] = 0.;\n";
1029 out << SP << SP << "}";
1030 } else if (fAttrActivations[direction * 3] == "Elu") {
1031 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1032 out << SP << SP << SP << "if (" << OpName << "_input_gate[i] < 0.)\n";
1033 out << SP << SP << SP << SP << OpName << "_input_gate[i] = " << fAttrActivationAlpha[direction * 3]
1034 << " * exp(" << OpName << "_input_gate[i] - 1.);\n";
1035 out << SP << SP << "}\n";
1036 } else if (fAttrActivations[direction * 3] == "Softsign") {
1037 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1038 out << SP << SP << SP << SP << OpName << "_input_gate[i] = " << OpName << "_input_gate[i] / (1. + abs("
1039 << OpName << "_input_gate[i]));\n";
1040 out << SP << SP << "}\n";
1041 } else { // fAttrActivations[direction * 3] = Softplus
1042 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1043 out << SP << SP << SP << SP << OpName << "_input_gate[i] = log(1. + exp(" << OpName << "_input_gate[i]));\n";
1044 out << SP << SP << "}\n";
1045 }
1046
1047 if (fAttrInputForget == 0) {
1048 // Clip the elements of the forget gate into the range [-fAttrClip, fAttrClip]
1049 if (fAttrClip > .0) {
1050 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1051 if (fType == "float") {
1052 out << SP << SP << SP << "float x = (" << OpName << "_forget_gate[i] > " << -fAttrClip << ") ? "
1053 << OpName << "_forget_gate[i] : " << -fAttrClip << ";\n";
1054 }
1055 out << SP << SP << SP << OpName << "_forget_gate[i] = (x < " << fAttrClip << ") ? x : " << fAttrClip
1056 << ";\n";
1057 out << SP << SP << "}\n";
1058 }
1059 // Apply the activation function to the forget gate, cell_gate = g(cell_gate)
1060 if (fAttrActivations[direction * 3] == "Relu") {
1061 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1062 out << SP << SP << SP << "if (" << OpName << "_forget_gate[i] < 0.)\n";
1063 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = 0.;\n";
1064 out << SP << SP << "}\n";
1065 } else if (fAttrActivations[direction * 3] == "Tanh") {
1066 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1067 if (fType == "float") {
1068 out << SP << SP << SP << "float ex = exp(-2 * " << OpName << "_forget_gate[i]);\n";
1069 }
1070 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = (1. - ex) / (1. + ex);\n";
1071 out << SP << SP << "}\n";
1072 } else if (fAttrActivations[direction * 3] == "Sigmoid") {
1073 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1074 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = 1. / (1. + exp(-" << OpName
1075 << "_forget_gate[i]));\n";
1076 out << SP << SP << "}\n";
1077 } else if (fAttrActivations[direction * 3] == "Affine") {
1078 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1079 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = " << fAttrActivationAlpha[direction * 3]
1080 << " * " << OpName << "_forget_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
1081 out << SP << SP << "}\n";
1082 } else if (fAttrActivations[direction * 3] == "ScaledTanh") {
1083 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1084 if (fType == "float") {
1085 out << SP << SP << SP << "float ex = exp(-2 * " << fAttrActivationBeta[direction * 3] << " * " << OpName
1086 << "_forget_gate[i]);\n";
1087 }
1088 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = " << fAttrActivationAlpha[direction * 3]
1089 << " * (1. - ex) / (1. + ex);\n";
1090 out << SP << SP << "}\n";
1091 } else if (fAttrActivations[direction * 3] == "HardSigmoid") {
1092 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1093 if (fType == "float") {
1094 out << SP << SP << SP << "float a = " << fAttrActivationAlpha[direction * 3] << " * " << OpName
1095 << "_forget_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
1096 out << SP << SP << SP << "float b = (a > 0.) ? a : 0.;\n";
1097 }
1098 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = (b < 1.) ? b : 1.;\n";
1099 out << SP << SP << "}\n";
1100 } else if (fAttrActivations[direction * 3] == "LeakyRelu") {
1101 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1102 out << SP << SP << SP << "if (" << OpName << "_forget_gate[i] < 0.)\n";
1103 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = " << fAttrActivationAlpha[direction * 3]
1104 << " * " << OpName << "_forget_gate[i];\n";
1105 out << SP << SP << "}\n";
1106 } else if (fAttrActivations[direction * 3] == "ThresholdRelu") {
1107 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1108 out << SP << SP << SP << "if (" << OpName << "_forget_gate[i] < " << fAttrActivationAlpha[direction * 3]
1109 << ")\n";
1110 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = 0.;\n";
1111 out << SP << SP << "}";
1112 } else if (fAttrActivations[direction * 3] == "Elu") {
1113 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1114 out << SP << SP << SP << "if (" << OpName << "_forget_gate[i] < 0.)\n";
1115 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = " << fAttrActivationAlpha[direction * 3]
1116 << " * exp(" << OpName << "_forget_gate[i] - 1.);\n";
1117 out << SP << SP << "}\n";
1118 } else if (fAttrActivations[direction * 3] == "Softsign") {
1119 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1120 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = " << OpName << "_forget_gate[i] / (1. + abs("
1121 << OpName << "_forget_gate[i]));\n";
1122 out << SP << SP << "}\n";
1123 } else { // fAttrActivations[direction * 3] = Softplus
1124 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1125 out << SP << SP << SP << SP << OpName << "_forget_gate[i] = log(1. + exp(" << OpName
1126 << "_forget_gate[i]));\n";
1127 out << SP << SP << "}\n";
1128 }
1129 }
1130
1131 // cell_state = input_gate o cell_gate
1132 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1133 out << SP << SP << SP << OpName << "_cell_state[i] = " << OpName << "_input_gate[i] * " << OpName
1134 << "_cell_gate[i];\n";
1135 out << SP << SP << "}\n";
1136
1137 if (fAttrInputForget == 0) {
1138 out << SP << SP << "if (seq == 0) {\n";
1139 if (!fNInitial_c.empty()) {
1140 // cell_state += forget_gate o initial_cell_state
1141 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
1142 out << SP << SP << SP << SP << OpName << "_cell_state[i + offset] += " << OpName
1143 << "_forget_gate[i + offset] * " << OpName << "_initial_cell_state[i];\n";
1144 out << SP << SP << SP << "}\n";
1145 }
1146 out << SP << SP << "} else {\n";
1147 // cell_state += forget_gate o previous_cell_state
1148 if (direction == 0) {
1149 if (fAttrDirection == "backward") {
1150 out << SP << SP << SP << "size_t previous_offset = (index + 1) * "
1151 << num_directions * batch_size * fAttrHiddenSize << ";\n";
1152 } else {
1153 out << SP << SP << SP << "size_t previous_offset = (seq - 1) * "
1154 << num_directions * batch_size * fAttrHiddenSize << ";\n";
1155 }
1156 } else { // direction=1
1157 out << SP << SP << SP << "size_t previous_offset = (index + 1) * "
1158 << num_directions * batch_size * fAttrHiddenSize << " + " << batch_size * fAttrHiddenSize << ";\n";
1159 }
1160 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
1161 out << SP << SP << SP << SP << OpName << "_cell_state[i + offset] += " << OpName
1162 << "_forget_gate[i + offset] * " << OpName << "_cell_state[i + previous_offset];\n";
1163 out << SP << SP << SP << "}\n";
1164 out << SP << SP << "}\n";
1165 }
1166
1167 if (!fNP.empty()) {
1168 // Peephole connection for the output gate
1169 if (direction == 0) {
1170 size_t p_offset = 2 * batch_size * fAttrHiddenSize;
1171 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
1172 out << SP << SP << SP << SP << OpName << "_output_gate[i + offset] += tensor_" << fNP << "[i + " << p_offset
1173 << "] * " << OpName << "_cell_state[i + offset];\n";
1174 out << SP << SP << SP << "}\n";
1175 } else { // direction=1
1176 size_t p_offset = 3 * batch_size * fAttrHiddenSize + 2 * batch_size * fAttrHiddenSize;
1177 out << SP << SP << SP << "for (size_t i = 0; i < " << size << "; i++) {\n";
1178 out << SP << SP << SP << SP << OpName << "_output_gate[i + offset] += tensor_" << fNP << "[i + " << p_offset
1179 << "] * " << OpName << "_cell_state[i + offset];\n";
1180 out << SP << SP << SP << "}\n";
1181 }
1182 }
1183
1184 // Clip the elements of the output gate into the range [-fAttrClip, fAttrClip]
1185 if (fAttrClip > .0) {
1186 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1187 if (fType == "float") {
1188 out << SP << SP << SP << "float x = (" << OpName << "_output_gate[i] > " << -fAttrClip << ") ? " << OpName
1189 << "_output_gate[i] : " << -fAttrClip << ";\n";
1190 }
1191 out << SP << SP << SP << OpName << "_output_gate[i] = (x < " << fAttrClip << ") ? x : " << fAttrClip << ";\n";
1192 out << SP << SP << "}\n";
1193 }
1194 // Apply the activation function to the output gate
1195 if (fAttrActivations[direction * 3] == "Relu") {
1196 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1197 out << SP << SP << SP << "if (" << OpName << "_output_gate[i] < 0.)\n";
1198 out << SP << SP << SP << SP << OpName << "_output_gate[i] = 0.;\n";
1199 out << SP << SP << "}\n";
1200 } else if (fAttrActivations[direction * 3] == "Tanh") {
1201 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1202 if (fType == "float") {
1203 out << SP << SP << SP << "float ex = exp(-2 * " << OpName << "_output_gate[i]);\n";
1204 }
1205 out << SP << SP << SP << SP << OpName << "_output_gate[i] = (1. - ex) / (1. + ex);\n";
1206 out << SP << SP << "}\n";
1207 } else if (fAttrActivations[direction * 3] == "Sigmoid") {
1208 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1209 out << SP << SP << SP << SP << OpName << "_output_gate[i] = 1. / (1. + exp(-" << OpName
1210 << "_output_gate[i]));\n";
1211 out << SP << SP << "}\n";
1212 } else if (fAttrActivations[direction * 3] == "Affine") {
1213 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1214 out << SP << SP << SP << SP << OpName << "_output_gate[i] = " << fAttrActivationAlpha[direction * 3] << " * "
1215 << OpName << "_output_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
1216 out << SP << SP << "}\n";
1217 } else if (fAttrActivations[direction * 3] == "ScaledTanh") {
1218 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1219 if (fType == "float") {
1220 out << SP << SP << SP << "float ex = exp(-2 * " << fAttrActivationBeta[direction * 3] << " * " << OpName
1221 << "_output_gate[i]);\n";
1222 }
1223 out << SP << SP << SP << SP << OpName << "_output_gate[i] = " << fAttrActivationAlpha[direction * 3]
1224 << " * (1. - ex) / (1. + ex);\n";
1225 out << SP << SP << "}\n";
1226 } else if (fAttrActivations[direction * 3] == "HardSigmoid") {
1227 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1228 if (fType == "float") {
1229 out << SP << SP << SP << "float a = " << fAttrActivationAlpha[direction * 3] << " * " << OpName
1230 << "_output_gate[i] + " << fAttrActivationBeta[direction * 3] << ";\n";
1231 out << SP << SP << SP << "float b = (a > 0.) ? a : 0.;\n";
1232 }
1233 out << SP << SP << SP << SP << OpName << "_output_gate[i] = (b < 1.) ? b : 1.;\n";
1234 out << SP << SP << "}\n";
1235 } else if (fAttrActivations[direction * 3] == "LeakyRelu") {
1236 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1237 out << SP << SP << SP << "if (" << OpName << "_output_gate[i] < 0.)\n";
1238 out << SP << SP << SP << SP << OpName << "_output_gate[i] = " << fAttrActivationAlpha[direction * 3] << " * "
1239 << OpName << "_output_gate[i];\n";
1240 out << SP << SP << "}\n";
1241 } else if (fAttrActivations[direction * 3] == "ThresholdRelu") {
1242 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1243 out << SP << SP << SP << "if (" << OpName << "_output_gate[i] < " << fAttrActivationAlpha[direction * 3]
1244 << ")\n";
1245 out << SP << SP << SP << SP << OpName << "_output_gate[i] = 0.;\n";
1246 out << SP << SP << "}";
1247 } else if (fAttrActivations[direction * 3] == "Elu") {
1248 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1249 out << SP << SP << SP << "if (" << OpName << "_output_gate[i] < 0.)\n";
1250 out << SP << SP << SP << SP << OpName << "_output_gate[i] = " << fAttrActivationAlpha[direction * 3]
1251 << " * exp(" << OpName << "_output_gate[i] - 1.);\n";
1252 out << SP << SP << "}\n";
1253 } else if (fAttrActivations[direction * 3] == "Softsign") {
1254 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1255 out << SP << SP << SP << SP << OpName << "_output_gate[i] = " << OpName << "_output_gate[i] / (1. + abs("
1256 << OpName << "_output_gate[i]));\n";
1257 out << SP << SP << "}\n";
1258 } else { // fAttrActivations[direction * 3] = Softplus
1259 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1260 out << SP << SP << SP << SP << OpName << "_output_gate[i] = log(1. + exp(" << OpName << "_output_gate[i]));\n";
1261 out << SP << SP << "}\n";
1262 }
1263
1264 // copy cell_state into new_cell_state
1265 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName << "_cell_state + offset + "
1266 << size << ", " << OpName << "_new_cell_state + offset);\n";
1267 // Clip the elements of the new_cell_state into the range [-fAttrClip, fAttrClip]
1268 if (fAttrClip > .0) {
1269 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1270 if (fType == "float") {
1271 out << SP << SP << SP << "float x = (" << OpName << "_new_cell_state[i] > " << -fAttrClip << ") ? "
1272 << OpName << "_new_cell_state[i] : " << -fAttrClip << ";\n";
1273 }
1274 out << SP << SP << SP << OpName << "_new_cell_state[i] = (x < " << fAttrClip << ") ? x : " << fAttrClip
1275 << ";\n";
1276 out << SP << SP << "}\n";
1277 }
1278 // Apply the activation function to the new cell state
1279 if (fAttrActivations[direction * 3 + 2] == "Relu") {
1280 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1281 out << SP << SP << SP << "if (" << OpName << "_new_cell_state[i] < 0.)\n";
1282 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = 0.;\n";
1283 out << SP << SP << "}\n";
1284 } else if (fAttrActivations[direction * 3 + 2] == "Tanh") {
1285 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1286 if (fType == "float") {
1287 out << SP << SP << SP << "float ex = exp(-2 * " << OpName << "_new_cell_state[i]);\n";
1288 }
1289 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = (1. - ex) / (1. + ex);\n";
1290 out << SP << SP << "}\n";
1291 } else if (fAttrActivations[direction * 3 + 2] == "Sigmoid") {
1292 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1293 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = 1. / (1. + exp(-" << OpName
1294 << "_new_cell_state[i]));\n";
1295 out << SP << SP << "}\n";
1296 } else if (fAttrActivations[direction * 3 + 2] == "Affine") {
1297 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1298 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = " << fAttrActivationAlpha[direction * 3 + 2]
1299 << " * " << OpName << "_new_cell_state[i] + " << fAttrActivationBeta[direction * 3 + 2] << ";\n";
1300 out << SP << SP << "}\n";
1301 } else if (fAttrActivations[direction * 3 + 2] == "ScaledTanh") {
1302 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1303 if (fType == "float") {
1304 out << SP << SP << SP << "float ex = exp(-2 * " << fAttrActivationBeta[direction * 3 + 2] << " * " << OpName
1305 << "_new_cell_state[i]);\n";
1306 }
1307 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = " << fAttrActivationAlpha[direction * 3 + 2]
1308 << " * (1. - ex) / (1. + ex);\n";
1309 out << SP << SP << "}\n";
1310 } else if (fAttrActivations[direction * 3 + 2] == "HardSigmoid") {
1311 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1312 if (fType == "float") {
1313 out << SP << SP << SP << "float a = " << fAttrActivationAlpha[direction * 3 + 2] << " * " << OpName
1314 << "_new_cell_state[i] + " << fAttrActivationBeta[direction * 3 + 2] << ";\n";
1315 out << SP << SP << SP << "float b = (a > 0.) ? a : 0.;\n";
1316 }
1317 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = (b < 1.) ? b : 1.;\n";
1318 out << SP << SP << "}\n";
1319 } else if (fAttrActivations[direction * 3 + 2] == "LeakyRelu") {
1320 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1321 out << SP << SP << SP << "if (" << OpName << "_new_cell_state[i] < 0.)\n";
1322 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = " << fAttrActivationAlpha[direction * 3 + 2]
1323 << " * " << OpName << "_new_cell_state[i];\n";
1324 out << SP << SP << "}\n";
1325 } else if (fAttrActivations[direction * 3 + 2] == "ThresholdRelu") {
1326 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1327 out << SP << SP << SP << "if (" << OpName << "_new_cell_state[i] < " << fAttrActivationAlpha[direction * 3 + 2]
1328 << ")\n";
1329 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = 0.;\n";
1330 out << SP << SP << "}";
1331 } else if (fAttrActivations[direction * 3 + 2] == "Elu") {
1332 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1333 out << SP << SP << SP << "if (" << OpName << "_new_cell_state[i] < 0.)\n";
1334 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = " << fAttrActivationAlpha[direction * 3 + 2]
1335 << " * exp(" << OpName << "_new_cell_state[i] - 1.);\n";
1336 out << SP << SP << "}\n";
1337 } else if (fAttrActivations[direction * 3 + 2] == "Softsign") {
1338 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1339 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = " << OpName << "_new_cell_state[i] / (1. + abs("
1340 << OpName << "_new_cell_state[i]));\n";
1341 out << SP << SP << "}\n";
1342 } else { // fAttrActivations[direction * 3 + 2] = Softplus
1343 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1344 out << SP << SP << SP << SP << OpName << "_new_cell_state[i] = log(1. + exp(" << OpName
1345 << "_new_cell_state[i]));\n";
1346 out << SP << SP << "}\n";
1347 }
1348
1349 // hidden_state = output_gate o new_cell_state
1350 out << SP << SP << "for (size_t i = offset; i < offset + " << size << "; i++) {\n";
1351 out << SP << SP << SP << OpName << "_hidden_state[i] = " << OpName << "_output_gate[i] * " << OpName
1352 << "_new_cell_state[i];\n";
1353 out << SP << SP << "}\n";
1354 out << SP << "}\n";
1355 }
1356
1357 // Padding the hidden state for LSTM with different sequence lengths
1358 if (!fNSequence_lens.empty()) {
1359 out << SP << "for (size_t seq = 0; seq < " << seq_length << "; seq++) {\n";
1360 out << SP << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1361 out << SP << SP << SP << "if (seq >= tensor_" << fNSequence_lens << "[batch]) {\n";
1362 for (size_t direction = 0; direction < num_directions; direction++) {
1363 out << SP << SP << SP << SP << SP << "for (size_t h = 0; h < " << fAttrHiddenSize << "; h++) {\n";
1364 out << SP << SP << SP << SP << SP << SP << "size_t idx = seq * "
1365 << num_directions * batch_size * fAttrHiddenSize + direction * batch_size * fAttrHiddenSize
1366 << " + batch * " << fAttrHiddenSize << " + h;\n";
1367 out << SP << SP << SP << SP << SP << SP << OpName << "_cell_state[idx] = 0.;\n";
1368 out << SP << SP << SP << SP << SP << SP << OpName << "_hidden_state[idx] = 0.;\n";
1369 out << SP << SP << SP << SP << SP << "}\n";
1370 }
1371 out << SP << SP << SP << "}\n";
1372 out << SP << SP << "}\n";
1373 out << SP << "}\n";
1374 }
1375
1376 // Copy the hidden state into y and y_h and copy cell_state into y_c
1377 if (fAttrLayout == 0) {
1378 if (!fNY_h.empty()) {
1379 // Copy hidden_state into Y_h
1380 if (fNSequence_lens.empty()) {
1381 size_t y_h_size = batch_size * fAttrHiddenSize;
1382 if (fAttrDirection == "backward") {
1383 out << SP << "std::copy(" << OpName << "_hidden_state, " << OpName << "_hidden_state + " << y_h_size
1384 << ", tensor_" << fNY_h << ");\n";
1385 } else {
1386 size_t offset = (seq_length - 1) * num_directions * batch_size * fAttrHiddenSize;
1387 out << SP << "std::copy(" << OpName << "_hidden_state + " << offset << ", " << OpName
1388 << "_hidden_state + " << offset << " + " << y_h_size << ", tensor_" << fNY_h << ");\n";
1389 }
1390 if (num_directions == 2) {
1391 out << SP << "std::copy(" << OpName << "_hidden_state + " << y_h_size << ", " << OpName
1392 << "_hidden_state + " << 2 * y_h_size << ", tensor_" << fNY_h << " + " << y_h_size << ");\n";
1393 }
1394 } else { // LSTM with different sequence lengths
1395 if (fAttrDirection == "backward") {
1396 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1397 out << SP << SP << "size_t offset = batch * " << fAttrHiddenSize << ";\n";
1398 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1399 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + offset);\n";
1400 out << SP << "}\n";
1401 } else {
1402 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1403 out << SP << SP << "size_t seq = " << "tensor_" << fNSequence_lens << "[batch] - 1;\n";
1404 out << SP << SP << "size_t offset = seq * " << num_directions * batch_size * fAttrHiddenSize
1405 << " + batch * " << fAttrHiddenSize << ";\n";
1406 out << SP << SP << "size_t y_h_offset = batch * " << fAttrHiddenSize << ";\n";
1407 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1408 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + y_h_offset);\n";
1409 out << SP << "}\n";
1410 }
1411 if (num_directions == 2) {
1412 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1413 out << SP << SP << "size_t offset = " << batch_size * fAttrHiddenSize << " + batch * " << fAttrHiddenSize
1414 << ";\n";
1415 out << SP << SP << "size_t y_h_offset = " << batch_size * fAttrHiddenSize << " + batch * "
1416 << fAttrHiddenSize << ";\n";
1417 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1418 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + y_h_offset);\n";
1419 out << SP << "}\n";
1420 }
1421 }
1422 }
1423 if (!fNY_c.empty()) {
1424 // Copy cell_state into Y_c
1425 if (fNSequence_lens.empty()) {
1426 size_t y_h_size = batch_size * fAttrHiddenSize;
1427 if (fAttrDirection == "backward") {
1428 out << SP << "std::copy(" << OpName << "_cell_state, " << OpName << "_hidden_state + " << y_h_size
1429 << ", tensor_" << fNY_c << ");\n";
1430 } else {
1431 size_t offset = (seq_length - 1) * num_directions * batch_size * fAttrHiddenSize;
1432 out << SP << "std::copy(" << OpName << "_cell_state + " << offset << ", " << OpName << "_cell_state + "
1433 << offset << " + " << y_h_size << ", tensor_" << fNY_c << ");\n";
1434 }
1435 if (num_directions == 2) {
1436 out << SP << "std::copy(" << OpName << "_cell_state + " << y_h_size << ", " << OpName << "_cell_state + "
1437 << 2 * y_h_size << ", tensor_" << fNY_c << " + " << y_h_size << ");\n";
1438 }
1439 } else { // LSTM with different sequence lengths
1440 if (fAttrDirection == "backward") {
1441 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1442 out << SP << SP << "size_t offset = batch * " << fAttrHiddenSize << ";\n";
1443 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName
1444 << "_cell_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_c << " + offset);\n";
1445 out << SP << "}\n";
1446 } else {
1447 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1448 out << SP << SP << "size_t seq = " << "tensor_" << fNSequence_lens << "[batch] - 1;\n";
1449 out << SP << SP << "size_t offset = seq * " << num_directions * batch_size * fAttrHiddenSize
1450 << " + batch * " << fAttrHiddenSize << ";\n";
1451 out << SP << SP << "size_t y_h_offset = batch * " << fAttrHiddenSize << ";\n";
1452 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName
1453 << "_cell_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_c << " + y_h_offset);\n";
1454 out << SP << "}\n";
1455 }
1456 if (num_directions == 2) {
1457 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1458 out << SP << SP << "size_t offset = " << batch_size * fAttrHiddenSize << " + batch * " << fAttrHiddenSize
1459 << ";\n";
1460 out << SP << SP << "size_t y_h_offset = " << batch_size * fAttrHiddenSize << " + batch * "
1461 << fAttrHiddenSize << ";\n";
1462 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName
1463 << "_cell_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_c << " + y_h_offset);\n";
1464 out << SP << "}\n";
1465 }
1466 }
1467 }
1468 } else { // fAttrLayout=1
1469 if (!fNY.empty()) {
1470 // Copy hidden_state into Y
1471 for (size_t direction = 0; direction < num_directions; direction++) {
1472 out << SP << "for (size_t seq = 0; seq < " << seq_length << "; seq++) {\n";
1473 out << SP << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1474 out << SP << SP << SP << "size_t offset = seq * " << num_directions * batch_size * fAttrHiddenSize << " + "
1475 << direction * batch_size * fAttrHiddenSize << " + batch * " << fAttrHiddenSize << ";\n";
1476 out << SP << SP << SP << "size_t y_offset = batch * " << seq_length * num_directions * fAttrHiddenSize
1477 << " + seq * " << num_directions * fAttrHiddenSize << " + " << direction * fAttrHiddenSize << ";\n";
1478 out << SP << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1479 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY << " + y_offset);\n";
1480 out << SP << SP << "}\n";
1481 out << SP << "}\n";
1482 }
1483 }
1484 if (!fNY_h.empty()) {
1485 // Copy the hidden_state into Y_h
1486 if (fAttrDirection == "backward") {
1487 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1488 out << SP << SP << "size_t offset = batch * " << fAttrHiddenSize << ";\n";
1489 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << ";\n";
1490 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1491 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + y_h_offset);\n";
1492 out << SP << "}\n";
1493 } else {
1494 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1495 if (fNSequence_lens.empty()) {
1496 out << SP << SP << "size_t seq = " << seq_length - 1 << ";\n";
1497 } else {
1498 out << SP << SP << "size_t seq = " << "tensor_" << fNSequence_lens << "[batch] - 1;\n";
1499 }
1500 out << SP << SP << "size_t offset = seq * " << num_directions * batch_size * fAttrHiddenSize
1501 << " + batch * " << fAttrHiddenSize << ";\n";
1502 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << ";\n";
1503 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1504 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + y_h_offset);\n";
1505 out << SP << "}\n";
1506 }
1507 if (num_directions == 2) {
1508 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1509 out << SP << SP << "size_t offset = " << batch_size * fAttrHiddenSize << " + batch * " << fAttrHiddenSize
1510 << ";\n";
1511 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << " + "
1512 << fAttrHiddenSize << ";\n";
1513 out << SP << SP << "std::copy(" << OpName << "_hidden_state + offset, " << OpName
1514 << "_hidden_state + offset + " << fAttrHiddenSize << ", tensor_" << fNY_h << " + y_h_offset);\n";
1515 out << SP << "}\n";
1516 }
1517 }
1518
1519 if (!fNY_c.empty()) {
1520 // copy the cell_state into Y_c
1521 if (fAttrDirection == "backward") {
1522 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1523 out << SP << SP << "size_t offset = batch * " << fAttrHiddenSize << ";\n";
1524 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << ";\n";
1525 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName << "_cell_state + offset + "
1526 << fAttrHiddenSize << ", tensor_" << fNY_c << " + y_h_offset);\n";
1527 out << SP << "}\n";
1528 } else {
1529 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1530 if (fNSequence_lens.empty()) {
1531 out << SP << SP << "size_t seq = " << seq_length - 1 << ";\n";
1532 } else {
1533 out << SP << SP << "size_t seq = " << "tensor_" << fNSequence_lens << "[batch] - 1;\n";
1534 }
1535 out << SP << SP << "size_t offset = seq * " << num_directions * batch_size * fAttrHiddenSize
1536 << " + batch * " << fAttrHiddenSize << ";\n";
1537 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << ";\n";
1538 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName << "_cell_state + offset + "
1539 << fAttrHiddenSize << ", tensor_" << fNY_c << " + y_h_offset);\n";
1540 out << SP << "}\n";
1541 }
1542 if (num_directions == 2) {
1543 out << SP << "for (size_t batch = 0; batch < " << batch_size << "; batch++) {\n";
1544 out << SP << SP << "size_t offset = " << batch_size * fAttrHiddenSize << " + batch * " << fAttrHiddenSize
1545 << ";\n";
1546 out << SP << SP << "size_t y_h_offset = batch * " << num_directions * fAttrHiddenSize << " + "
1547 << fAttrHiddenSize << ";\n";
1548 out << SP << SP << "std::copy(" << OpName << "_cell_state + offset, " << OpName << "_cell_state + offset + "
1549 << fAttrHiddenSize << ", tensor_" << fNY_c << " + y_h_offset);\n";
1550 out << SP << "}\n";
1551 }
1552 }
1553 }
1554
1555 return out.str();
1556}
1557
1558} // namespace TMVA::Experimental::SOFIE
1559
1560#endif
b
#define b(i)
Definition RSha256.hxx:100
h
#define h(i)
Definition RSha256.hxx:106
size
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
SOFIE_common.hxx
input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Definition TGWin32VirtualXProxy.cxx:145
offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Definition TGWin32VirtualXProxy.cxx:248
name
char name[80]
Definition TGX11.cxx:142
TMVA::Experimental::SOFIE::ROperator_LSTM
Long Short-Term Memory operator.
Definition ROperator_LSTM.hxx:21
TMVA::Experimental::SOFIE::ROperator_LSTM::GenerateSessionMembersCode
std::string GenerateSessionMembersCode(std::string opName) override
Generate the code for the Session internal data vectors.
Definition ROperator_LSTM.hxx:369
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeR
std::vector< size_t > fShapeR
Shape of the recurrence.
Definition ROperator_LSTM.hxx:46
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeInitial_c
std::vector< size_t > fShapeInitial_c
Shape of the initial value of the cell states.
Definition ROperator_LSTM.hxx:50
TMVA::Experimental::SOFIE::ROperator_LSTM::fNP
std::string fNP
Name of peepholes.
Definition ROperator_LSTM.hxx:39
TMVA::Experimental::SOFIE::ROperator_LSTM::fNY_c
std::string fNY_c
Name of the last sequence of the cell states.
Definition ROperator_LSTM.hxx:42
TMVA::Experimental::SOFIE::ROperator_LSTM::fNX
std::string fNX
Name of the input.
Definition ROperator_LSTM.hxx:32
TMVA::Experimental::SOFIE::ROperator_LSTM::ROperator_LSTM
ROperator_LSTM(std::vector< float > activation_alpha, std::vector< float > activation_beta, std::vector< std::string > activations, float clip, std::string direction, size_t hidden_size, size_t input_forget, size_t layout, std::string nameX, std::string nameW, std::string nameR, std::string nameB, std::string nameSequence_lens, std::string nameInitial_h, std::string nameInitial_c, std::string nameP, std::string nameY, std::string nameY_h, std::string nameY_c)
Constructor of ROperator_LSTM from the attributes.
Definition ROperator_LSTM.hxx:84
TMVA::Experimental::SOFIE::ROperator_LSTM::fNR
std::string fNR
Name of the recurrence.
Definition ROperator_LSTM.hxx:34
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeY_h
std::vector< size_t > fShapeY_h
Shape of the last sequence of the output.
Definition ROperator_LSTM.hxx:53
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrActivationAlpha
std::vector< float > fAttrActivationAlpha
Sacling values used by some activation functions.
Definition ROperator_LSTM.hxx:23
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeInitial_h
std::vector< size_t > fShapeInitial_h
Shape of the initial value of the hidden states.
Definition ROperator_LSTM.hxx:49
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrLayout
size_t fAttrLayout
Data layout.
Definition ROperator_LSTM.hxx:30
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrHiddenSize
size_t fAttrHiddenSize
Number of the hidden layers.
Definition ROperator_LSTM.hxx:28
TMVA::Experimental::SOFIE::ROperator_LSTM::GetBlasRoutines
std::vector< std::string > GetBlasRoutines() override
Returns the blas routines needed to compile the generated code.
Definition ROperator_LSTM.hxx:167
TMVA::Experimental::SOFIE::ROperator_LSTM::fNInitial_c
std::string fNInitial_c
Name of the initial value of the cell states.
Definition ROperator_LSTM.hxx:38
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeB
std::vector< size_t > fShapeB
Shape of the bias.
Definition ROperator_LSTM.hxx:47
TMVA::Experimental::SOFIE::ROperator_LSTM::ShapeInference
std::vector< std::vector< size_t > > ShapeInference(std::vector< std::vector< size_t > > input)
Infers the shape of the output tensors.
Definition ROperator_LSTM.hxx:171
TMVA::Experimental::SOFIE::ROperator_LSTM::fNW
std::string fNW
Name of the weights.
Definition ROperator_LSTM.hxx:33
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrClip
float fAttrClip
Clip threshold.
Definition ROperator_LSTM.hxx:26
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeY
std::vector< size_t > fShapeY
Shape of the output.
Definition ROperator_LSTM.hxx:52
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeP
std::vector< size_t > fShapeP
Shape of the peepholes.
Definition ROperator_LSTM.hxx:51
TMVA::Experimental::SOFIE::ROperator_LSTM::fType
std::string fType
Type of the tensors.
Definition ROperator_LSTM.hxx:56
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrDirection
std::string fAttrDirection
Direction of processing.
Definition ROperator_LSTM.hxx:27
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrActivationBeta
std::vector< float > fAttrActivationBeta
Scaling values used by some activation functions.
Definition ROperator_LSTM.hxx:24
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeX
std::vector< size_t > fShapeX
Shape of the input.
Definition ROperator_LSTM.hxx:44
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeW
std::vector< size_t > fShapeW
Shape of the weights.
Definition ROperator_LSTM.hxx:45
TMVA::Experimental::SOFIE::ROperator_LSTM::fNSequence_lens
std::string fNSequence_lens
Name of length of the sequences.
Definition ROperator_LSTM.hxx:36
TMVA::Experimental::SOFIE::ROperator_LSTM::fNY_h
std::string fNY_h
Name of the last sequence of the output.
Definition ROperator_LSTM.hxx:41
TMVA::Experimental::SOFIE::ROperator_LSTM::fNY
std::string fNY
Name of the output.
Definition ROperator_LSTM.hxx:40
TMVA::Experimental::SOFIE::ROperator_LSTM::Initialize
void Initialize(RModel &) override
Initialize the model.
Definition ROperator_LSTM.hxx:193
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrActivations
std::vector< std::string > fAttrActivations
Activation functions.
Definition ROperator_LSTM.hxx:25
TMVA::Experimental::SOFIE::ROperator_LSTM::fAttrInputForget
size_t fAttrInputForget
Forget gate.
Definition ROperator_LSTM.hxx:29
TMVA::Experimental::SOFIE::ROperator_LSTM::Generate
std::string Generate(std::string OpName) override
Generate the inference code.
Definition ROperator_LSTM.hxx:443
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeSequence_lens
std::vector< size_t > fShapeSequence_lens
Shape of the length of the sequences.
Definition ROperator_LSTM.hxx:48
TMVA::Experimental::SOFIE::ROperator_LSTM::fShapeY_c
std::vector< size_t > fShapeY_c
Shape of the last sequence of the cell states.
Definition ROperator_LSTM.hxx:54
TMVA::Experimental::SOFIE::ROperator_LSTM::fNInitial_h
std::string fNInitial_h
Name of the initial value of the hidden states.
Definition ROperator_LSTM.hxx:37
TMVA::Experimental::SOFIE::ROperator_LSTM::fNB
std::string fNB
Name of the bias.
Definition ROperator_LSTM.hxx:35
TMVA::Experimental::SOFIE::ROperator_LSTM::ROperator_LSTM
ROperator_LSTM()
Default constructor of ROperator_LSTM.
Definition ROperator_LSTM.hxx:60
TMVA::Experimental::SOFIE::ROperator::fInputTensorNames
std::vector< std::string_view > fInputTensorNames
Definition ROperator.hxx:44
TMVA::Experimental::SOFIE::ROperator::fOutputTensorNames
std::vector< std::string_view > fOutputTensorNames
Definition ROperator.hxx:45
sum
static uint64_t sum(uint64_t i)
Definition Factory.cxx:2335