Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RModel.cxx
Go to the documentation of this file.
1#include <limits>
2#include <algorithm>
3#include <cctype>
4#include <cstdint>
5#include <cstdio>
6#include <memory>
7#include <string>
8#include <cstdlib>
9
10#include "TMVA/RModel.hxx"
11#include "TMVA/ROperator.hxx"
12
14
15// Out-of-line because ROperator is forward-declared in the public header.
17{
18 std::default_delete<ROperator>()(ptr);
19}
20
21RModel::~RModel() = default;
22RModel::RModel(RModel &&) = default;
23RModel &RModel::operator=(RModel &&) = default;
24
25namespace {
26
27const std::string SP = " ";
28
29void ReplaceAll(std::string &str, const std::string &from, const std::string &to)
30{
31 size_t pos = 0;
32 while ((pos = str.find(from, pos)) != std::string::npos) {
33 str.replace(pos, from.length(), to);
34 pos += to.length();
35 }
36}
37
38bool IsIdentifierChar(char c)
39{
40 return std::isalnum(static_cast<unsigned char>(c)) || c == '_';
41}
42
43// Returns true if s is a valid C++ identifier (can be used as a variable name).
44// Dim::param can be either a plain name (e.g. "W") or a computed expression
45// (e.g. "((W+-3)/2+1)"); only the former can be used as a C++ variable name.
46bool IsIdentifier(const std::string &s)
47{
48 if (s.empty() || std::isdigit(static_cast<unsigned char>(s[0])))
49 return false;
50 for (char c : s)
51 if (!IsIdentifierChar(c))
52 return false;
53 return true;
54}
55
56// Get the data member name corresponding to a tensor with a given name.
57std::string TensorMember(std::string const &name)
58{
59 return "tensor_" + name;
60}
61
62// Safetensors dtype token (https://huggingface.co/docs/safetensors) for a
63// tensor type.
64std::string SafetensorsDType(ETensorType type)
65{
66 switch (type) {
67 case ETensorType::FLOAT: return "F32";
68 case ETensorType::DOUBLE: return "F64";
69 case ETensorType::INT64: return "I64";
70 default:
71 throw std::runtime_error("tmva-sofie tensor with type " + ConvertTypeToString(type) +
72 " cannot be written to a safetensors file");
73 }
74}
75
76// Escape a string for inclusion in the safetensors JSON header. Only what
77// JSON requires: quotation marks, backslashes and control characters.
78std::string JsonEscape(const std::string &s)
79{
80 std::string out;
81 out.reserve(s.size() + 2);
82 for (const char c : s) {
83 switch (c) {
84 case '"': out += "\\\""; break;
85 case '\\': out += "\\\\"; break;
86 default:
87 if (static_cast<unsigned char>(c) < 0x20) {
88 char buf[8];
89 std::snprintf(buf, sizeof(buf), "\\u%04x", c);
90 out += buf;
91 } else {
92 out.push_back(c);
93 }
94 }
95 }
96 return out;
97}
98
99} // namespace
100
101std::underlying_type_t<Options> operator|(Options opA, Options opB) {
102 return static_cast<std::underlying_type_t<Options>>(opA) | static_cast<std::underlying_type_t<Options>>(opB);
103}
104std::underlying_type_t<Options> operator|(std::underlying_type_t<Options> opA, Options opB) {
105 return opA | static_cast<std::underlying_type_t<Options>>(opB);
106}
107
108std::vector<size_t> RModel::GetTensorShape(const std::string & name) const {
109 auto f = fReadyInputTensorInfos.find(name);
110 if (f != fReadyInputTensorInfos.end()) {
111 return f->second.shape;
112 }
113 auto f2 = fInitializedTensors.find(name);
114 if (f2 != fInitializedTensors.end()) {
115 return f2->second.shape();
116 }
117 auto f3 = fInputTensorInfos.find(name);
118 if (f3 != fInputTensorInfos.end()) {
119 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] is an input tensor with unspecified dimension parameter");
120 }
121 auto f4 = fIntermediateTensorInfos.find(name);
122 if (f4 != fIntermediateTensorInfos.end()) {
123 return f4->second.shape;
124 }
125 // case of shape tensors
126 auto f5 = fShapeTensors.find(name);
127 if (f5 != fShapeTensors.end()) {
128 // shape is vector of size 1 with size of shape values or just a scalar
129 if (f5->second.second) // check scalar flag
130 return std::vector<size_t>{};
131 else
132 return std::vector<size_t>{f5->second.first.size()};
133 }
134
136 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] is a dynamic tensor. Use GetDynamicTensorShape instead of GetTensorShape");
137
140
141 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not found");
142}
143
144std::vector<Dim> RModel::GetDimTensorShape(const std::string & name) const {
145 if (auto f = fDynamicTensorInfos.find(name); f != fDynamicTensorInfos.end()) {
146 return f->second.shape;
147 }
148 if (auto f = fInputTensorInfos.find(name); f != fInputTensorInfos.end()) {
149 return f->second.shape;
150 }
151 // in case is not a dynamic tensor convert normal shape to Dim one
152 // for this we need to return the vector by value
154}
155std::vector<Dim> RModel::GetDynamicTensorShape(const std::string & name) const {
156 if (auto f = fDynamicTensorInfos.find(name); f != fDynamicTensorInfos.end()) {
157 return f->second.shape;
158 }
159 if (auto f = fInputTensorInfos.find(name); f != fInputTensorInfos.end()) {
160 return f->second.shape;
161 }
162 // throw error if shape is not dynamic
163 if (!IsDynamicTensor(name))
164 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not dynamic");
165
166 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the shape is requested is not found");
167}
168
170 auto f = fReadyInputTensorInfos.find(name);
171 if (f != fReadyInputTensorInfos.end()) {
172 return f->second.type;
173 }
174 auto f2 = fInitializedTensors.find(name);
175 if (f2 != fInitializedTensors.end()) {
176 return f2->second.type();
177 }
178 auto f3 = fInputTensorInfos.find(name);
179 if (f3 != fInputTensorInfos.end()) {
180 return f3->second.type;
181 }
182 auto f4 = fIntermediateTensorInfos.find(name);
183 if (f4 != fIntermediateTensorInfos.end()) {
184 return f4->second.type;
185 }
186 auto f5 = fDynamicTensorInfos.find(name);
187 if (f5 != fDynamicTensorInfos.end()){
188 return f5->second.type;
189 }
190 // case of shape tensor type is INT64
191 if (fShapeTensors.find(name) != fShapeTensors.end()){
192 return ETensorType::INT64;
193 }
194
197
198 throw std::runtime_error("TMVA SOFIE tensor [" + name + "] for which the type is requested is not found, model name: " + fName);
199}
200
201bool RModel::CheckIfTensorAlreadyExist(std::string tensor_name) {
202 if (fReadyInputTensorInfos.find(tensor_name) != fReadyInputTensorInfos.end()) return true;
203 if (fInputTensorInfos.find(tensor_name) != fInputTensorInfos.end()) return true;
204 if (fInitializedTensors.find(tensor_name) != fInitializedTensors.end()) return true;
205 if (fIntermediateTensorInfos.find(tensor_name) != fIntermediateTensorInfos.end()) return true;
206 if (fDynamicTensorInfos.find(tensor_name) != fDynamicTensorInfos.end()) return true;
207 if (fShapeTensors.find(tensor_name) != fShapeTensors.end()) return true;
209 return false;
210}
211
212void RModel::AddInputTensorInfo(std::string input_name, ETensorType type, std::vector<Dim> shape) {
213 input_name = UTILITY::Clean_name(input_name);
214 if (CheckIfTensorAlreadyExist(input_name)) {
215 throw std::runtime_error("TMVA-SOFIE: input tensor with name " + input_name + " already exists \n");
216 }
217
218 InputTensorInfo inputInfo { type, shape };
219 fInputTensorInfos[input_name] = inputInfo;
220}
221
222void RModel::AddInputTensorInfo(std::string input_name, ETensorType type, std::vector<size_t> shape) {
223 input_name = UTILITY::Clean_name(input_name);
224 if (CheckIfTensorAlreadyExist(input_name)) {
225 throw std::runtime_error("TMVA-SOFIE: input tensor with name " + input_name + " already exists \n");
226 }
227 TensorInfo inputInfo { type, shape };
228 fReadyInputTensorInfos[input_name] = inputInfo;
229}
230
231void RModel::AddInputTensorName(std::string input_name) {
232 fInputTensorNames.emplace_back(UTILITY::Clean_name(input_name));
233}
234
235void RModel::AddOperator(std::unique_ptr<ROperator> op, int order_execution)
236{
237 AddBlasRoutines(op->GetBlasRoutines());
238 auto libs = op->GetStdLibs();
239 auto op_input_tensors = op->GetOpInputTensors();
240 for (auto &stdlib : libs) {
241 AddNeededStdLib(stdlib);
242 }
243 // Convert to the deleter used for storage (see ROperatorDeleter in the header)
244 std::unique_ptr<ROperator, ROperatorDeleter> opStored(op.release());
245 if (order_execution >= 0) {
246 fOperators.insert(fOperators.begin() + order_execution, std::move(opStored));
247 } else {
248 fOperators.push_back(std::move(opStored));
249 order_execution = fOperators.size() - 1;
250 }
251
252 // storing the last usage of tensors which are input to the operator
253 // (excluding tensors which are inputs to the model or the initialized (weights) tensors)
254 // We call this function during parsing so we don't have yet initialized the operators
255 for (size_t index = 0; index < op_input_tensors.size(); index++) {
256 if (!IsInitializedTensor(UTILITY::Clean_name(std::string(op_input_tensors[index]))) &&
257 std::find(fInputTensorNames.begin(), fInputTensorNames.end(),
258 UTILITY::Clean_name(std::string(op_input_tensors[index]))) == fInputTensorNames.end()) {
259
260 fIntermediateTensorFrequencyLookup[op_input_tensors[index]] = order_execution;
261 if (Verbose())
262 std::cout << "adding order execution for " << op_input_tensors[index] << " order " << order_execution
263 << std::endl;
264 }
265 }
266}
267
268void RModel::AddInitializedTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
269 tensor_name = UTILITY::Clean_name(tensor_name);
270 //NB: own data
271 if (CheckIfTensorAlreadyExist(tensor_name)) {
272 throw std::runtime_error("TMVA-SOFIE: initialized tensor with name " + tensor_name + " already exists \n");
273 }
274 InitializedTensor new_tensor {type, shape, data};
275 fInitializedTensors[tensor_name] = new_tensor;
276}
277
278void RModel::AddInitializedTensor(const std::string &tensor_name, ETensorType tensor_type,
279 const std::vector<std::size_t> &shape, void *raw_data)
280{
281 size_t size = ConvertShapeToLength(shape);
282 auto itemsize = GetTypeSize(tensor_type);
283 std::shared_ptr<void> data(malloc(size * itemsize), free);
284 std::memcpy(data.get(), raw_data, size * itemsize);
285 AddInitializedTensor(tensor_name, tensor_type, shape, data);
286}
287
288void RModel::AddConstantTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
289 tensor_name = UTILITY::Clean_name(tensor_name);
290 //NB: own data
291 if (CheckIfTensorAlreadyExist(tensor_name)) {
292 throw std::runtime_error("TMVA-SOFIE: constant tensor with name " + tensor_name + " already exists \n");
293 }
294 InitializedTensor new_tensor {type, shape, data, true}; // add here flag to specify is a constant tensor
295 fInitializedTensors[tensor_name] = new_tensor;
296}
297
298void RModel::AddShapeTensor(const std::string & name, const std::vector<Dim> & shape_values, bool scalar){
299 auto tensor_name = UTILITY::Clean_name(name);
300 if (fShapeTensors.count(tensor_name) != 0) {
301 throw std::runtime_error("TMVA-SOFIE: shape tensor with name " + tensor_name + " already exists \n");
302 }
303 fShapeTensors[tensor_name] = std::make_pair(shape_values, scalar);
304}
305
306bool RModel::AddAliasTensor(const std::string &name, const std::string &origin)
307{
308 // add an alias tensor to origin: the generated code points it to the memory of origin.
309 // The tensor must already be registered as an intermediate tensor.
310 auto tensor_name = UTILITY::Clean_name(name);
311 auto origin_name = UTILITY::Clean_name(origin);
312 // two operators writing the same tensor is a malformed graph rather than a case to refuse,
313 // so this comes before the checks below and the error does not depend on the optimization level
314 if (fAliasTensors.count(tensor_name) != 0) {
315 throw std::runtime_error("TMVA-SOFIE: alias tensor with name " + tensor_name + " already exists \n");
316 }
317 // an alias of an alias refers directly to the tensor owning the memory
318 if (auto it = fAliasTensors.find(origin_name); it != fAliasTensors.end())
319 origin_name = it->second;
320
321 // with kBasic every tensor keeps its own buffer, as the automatic differentiation of the
322 // generated code requires
324 return false;
325 // graph outputs are written into the buffers provided by the caller
326 if (std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), tensor_name) != fOutputTensorNames.end())
327 return false;
328 // only intermediate tensors can be shared: graph inputs are read-only and initialized tensors
329 // are managed by the session
330 if (fIntermediateTensorInfos.count(origin_name) == 0 && fDynamicTensorInfos.count(origin_name) == 0)
331 return false;
332 // boolean tensors are std::vector<uint8_t> members accessed directly: ROperator_SubGraph
333 // reads fTensor_<cond>[0] and writes fTensor_<out>.begin(), which an alias does not have
334 if (GetTensorType(origin_name) == ETensorType::BOOL || GetTensorType(tensor_name) != GetTensorType(origin_name))
335 return false;
336
337 fAliasTensors[tensor_name] = origin_name;
338 return true;
339}
340
341bool RModel::IsShapeTensor(const std::string & tensor_name) const {
342 return fShapeTensors.count(tensor_name) != 0;
343}
344
345bool RModel::IsAliasTensor(const std::string & tensor_name) const {
346 return fAliasTensors.count(tensor_name) != 0;
347}
348
349const std::vector<Dim> & RModel::GetShapeTensorValues(const std::string & tensor_name) const {
350 //if (!IsShapeTensor(tensor_name) ) return std::vector<Dim>{};
351 return fShapeTensors.at(tensor_name).first;
352}
353
354bool RModel::IsInitializedTensor(const std::string& tensorName) const {
355 std::string name = UTILITY::Clean_name(tensorName);
356 return fInitializedTensors.find(name) != fInitializedTensors.end();
357}
358bool RModel::IsConstantTensor(const std::string& tensorName) const {
359 // a constant tensor is an initialized tensor but has the constant flag set
360 std::string name = UTILITY::Clean_name(tensorName);
361 auto itr = fInitializedTensors.find(name);
362 if (itr == fInitializedTensors.end()) return false;
363 return itr->second.IsConstantTensor();
364}
365
366// dynamic tensors include also Dim input tensors
367bool RModel::IsDynamicTensor(const std::string& tensorName) const {
368 std::string name = UTILITY::Clean_name(tensorName);
369 bool ret = fDynamicTensorInfos.find(name) != fDynamicTensorInfos.end();
370 return (ret) ? true : IsDimInputTensor(tensorName);
371}
372bool RModel::IsDimInputTensor(const std::string& tensorName) const {
373 std::string name = UTILITY::Clean_name(tensorName);
374 return fInputTensorInfos.find(name) != fInputTensorInfos.end();
375}
376bool RModel::IsReadyInputTensor(const std::string& tensorName) const {
377 std::string name = UTILITY::Clean_name(tensorName);
379}
380
381// generic addition of a tensor
382void RModel::AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector<Dim> dim_shape) {
383 auto int_shape = ConvertShapeToInt(dim_shape);
384 if (!int_shape.empty())
385 AddIntermediateTensor(tensor_name, type, int_shape);
386 else
387 AddDynamicTensor(tensor_name, type, dim_shape);
388}
389
390void RModel::AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape) {
391 tensor_name = UTILITY::Clean_name(tensor_name);
392 if (CheckIfTensorAlreadyExist(tensor_name)) {
393 throw std::runtime_error("TMVA-SOFIE: intermediate tensor with name " + tensor_name + " already exists \n");
394 }
395 TensorInfo new_tensor {type, shape};
396 fIntermediateTensorInfos[tensor_name] = new_tensor;
397}
398
399void RModel::AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector<Dim> shape){
400 tensor_name = UTILITY::Clean_name(tensor_name);
401 if (CheckIfTensorAlreadyExist(tensor_name)){
402 throw std::runtime_error("TMVA-SOFIE: intermediate tensor with name " + tensor_name + " already exists \n");
403 }
404 DynamicTensorInfo new_tensor {type, shape};
405 fDynamicTensorInfos[tensor_name] = new_tensor;
406 // store shape parameter if not existing
407 for (auto &d : shape) {
408 if (d.isParam) {
409 if (d.dim != size_t(-1)) {
410 AddShapeParam(d.param, d.dim);
411 }
412 }
413 }
414}
415
416void RModel::AddShapeParam(const std::string & param, size_t default_value) {
417 // a parameter computed at run time by an operator is never a Session constructor argument
418 if (fComputedShapeParams.count(param) != 0)
419 return;
420 if (fShapeParams.count(param) == 0) {
421 fShapeParams[param] = std::to_string(default_value);
422 // add also in the vector list (used to keep the order)
423 fDimShapeNames.push_back(param);
424 }
425}
426
427void RModel::AddComputedShapeParam(const std::string &param)
428{
429 fComputedShapeParams.insert(param);
430 // it may already be registered as an argument, reached through a shape that broadcasting
431 // rebuilt from a string; the operator's own declaration is the only one that should remain
432 fShapeParams.erase(param);
433 fDimShapeNames.erase(std::remove(fDimShapeNames.begin(), fDimShapeNames.end(), param), fDimShapeNames.end());
434}
435
436void RModel::AddOutputTensorNameList(std::vector<std::string> outputtensornames) {
437 fOutputTensorNames.clear();
438 for(auto& it : outputtensornames) {
439 fOutputTensorNames.emplace_back(UTILITY::Clean_name(it));
440 }
441}
442
443void RModel::UpdateOutputTensorList(std::vector<std::string> curr_output_tensors, std::vector<std::string> new_output_tensors) {
444 for(auto& it:curr_output_tensors) {
445 fOutputTensorNames.erase(std::remove(fOutputTensorNames.begin(), fOutputTensorNames.end(), it), fOutputTensorNames.end());
446 }
447 fOutputTensorNames.insert(fOutputTensorNames.end(), new_output_tensors.begin(), new_output_tensors.end());
448}
449
450void RModel::UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector<std::size_t> shape, std::shared_ptr<void> data) {
451 tensor_name = UTILITY::Clean_name(tensor_name);
452 if (!CheckIfTensorAlreadyExist(tensor_name)) {
453 throw std::runtime_error("TMVA-SOFIE: tensor " + tensor_name + " not found when trying to update it");
454 }
455 InitializedTensor new_tensor {type, shape, data};
456 fInitializedTensors[tensor_name] = new_tensor;
457}
458
459std::shared_ptr<void> RModel::GetInitializedTensorData(std::string tensor_name) {
460 auto f = fInitializedTensors.find(tensor_name);
461 if (f == fInitializedTensors.end()) {
462 throw std::runtime_error("TMVA-SOFIE: tensor " + tensor_name + " not found when trying to get its data");
463 } else {
464 return f->second.sharedptr();
465 }
466}
467
468void RModel::SetNotWritableInitializedTensor(const std::string & tensor_name) {
469 auto t = fInitializedTensors.find(tensor_name);
470 if (t == fInitializedTensors.end()) {
471 throw std::runtime_error("TMVA-SOFIE: initialized tensor " + tensor_name + " not found when trying to get its info");
472 }
473 t->second.SetNotWritable();
474 }
475
476std::string RModel::AllocateIntermediateMemory(std::span<const std::string_view> op_output_tensors)
477{
478 std::stringstream code;
479
480 if (fVerbose) {
481 std::cout << "Total chunks allocated\n";
482 for (auto chunk = fIntermediateMemoryInfo.total_stack.begin(); chunk != fIntermediateMemoryInfo.total_stack.end(); ++chunk) {
483 std::cout << "..... chunk " << chunk->first << " size " << chunk->second.tensor_size << " " << chunk->second.tensor_name << std::endl;
484 }
485 }
486
487 auto declareIntermediateTensor = [this, &code](std::string const &name, size_t size, size_t location) {
488 std::string typeName = ConvertTypeToString(GetTensorType(name));
489 code << "\n // Allocating memory for intermediate tensor " << name << " with size " << size << " bytes";
490 code << "\n"
491 << typeName << "* " << TensorMember(name) << " = reinterpret_cast<" << typeName
492 << "*>(fIntermediateMemoryPool.data() + " << location << ");\n";
493 };
494
495 if (fVerbose) std::cout << "*** AllocateIntermediateMemory: Loop on op output tensors\n";
496 // order output tensors by size
497 std::vector<TensorMemoryInfo> ordered_output_tensors;
498
499 for (auto &it : op_output_tensors) {
500 auto name = std::string(it);
503 continue;
504
505 // case of alias tensor
506 if (IsAliasTensor(name)) {
507 continue;
508 }
509
511 // important fill the pair in the ordered output tensors with the string view and not the string
512 TensorMemoryInfo tmi = {it, tensor_size};
513 ordered_output_tensors.push_back(tmi);
514 }
515 std::sort(ordered_output_tensors.begin(), ordered_output_tensors.end(),
516 [](const TensorMemoryInfo &a, const TensorMemoryInfo &b) { return a.tensor_size > b.tensor_size; });
517
518 for (auto &it : ordered_output_tensors) {
519 bool allocated = false;
520 std::string name = std::string{it.tensor_name};
521 size_t tensor_size = it.tensor_size;
522 if (fVerbose)
523 std::cout << "output tensor " << name << " size " << tensor_size << std::endl;
524
525 for (auto chunk = fIntermediateMemoryInfo.available_stack.begin();
527
528 if (fVerbose) std::cout << ".. available chunk " << chunk->first << " with size = " << chunk->second;
529 // check if available memory chunks can accommodate the tensor
530 if (chunk->second >= tensor_size) {
531 // need to use here string_view (i.e it.tensor_name)
532 // split returns the new chunk with size of new tensor. The free chunk is before the used one
533 auto new_chunk = fIntermediateMemoryInfo.total_stack[chunk->first].split(it.tensor_name, tensor_size);
534 auto new_chunk_location = chunk->first + chunk->second - tensor_size;
535 fIntermediateMemoryInfo.total_stack[new_chunk_location] = new_chunk;
536
537 declareIntermediateTensor(name, tensor_size, new_chunk_location);
538 chunk->second -= tensor_size;
539
540 allocated = true;
541
542 if (fVerbose) std::cout << " is re-used and split in a new of size " << new_chunk.tensor_size << " at " << new_chunk_location;
543
544 if (chunk->second == 0) {
545 if (fVerbose) std::cout << " and deleted since size matches";
547 }
548 if (fVerbose) std::cout << std::endl;
549 break;
550 } else if (chunk->first == fIntermediateMemoryInfo.available_stack.rbegin()->first &&
551 fIntermediateMemoryInfo.total_stack.rbegin()->first == chunk->first) {
552 // case last available chunk is the last in the memory, we can increase that one
553 fIntermediateMemoryInfo.total_stack[chunk->first] = {it.tensor_name, tensor_size};
554 declareIntermediateTensor(name, tensor_size, chunk->first);
556 allocated = true;
557 if (fVerbose) std::cout << " is extended with a bigger one of size " << tensor_size << std::endl;
558 break;
559 }
560 ++chunk;
561 if (fVerbose) std::cout << std::endl;
562 }
563
564 if (!allocated) {
565 size_t chunk_idx = fIntermediateMemoryInfo.total_stack.empty()
566 ? 0
567 : fIntermediateMemoryInfo.total_stack.rbegin()->first +
568 fIntermediateMemoryInfo.total_stack.rbegin()->second.tensor_size;
569
570 fIntermediateMemoryInfo.total_stack[chunk_idx] = it;
571
572 declareIntermediateTensor(name, tensor_size, chunk_idx);
573
574 if (fVerbose) std::cout << "no chunk available - add in total stack a new chunk with size of tensor and idx : " << chunk_idx
575 << std::endl;
576 }
577 }
578 return code.str();
579}
580
581void RModel::CheckAndFlushIntermediateMemory(std::span<const std::string_view> op_input_tensors, const size_t& op_idx){
582 if (fVerbose) std::cout << "*** CheckAndFlushIntermediateMemory: Loop on input tensors for op " << op_idx << "\n";
583 //print available chunks
584 if (fVerbose) std::cout << "available chunks before freeing them : \n";
585 for (auto chunk = fIntermediateMemoryInfo.available_stack.begin();
586 chunk != fIntermediateMemoryInfo.available_stack.end(); chunk++) {
587 if (fVerbose) std::cout << "-- free chunk " << chunk->first << " size = " << chunk->second << std::endl;
588 }
589 for (auto &iv : op_input_tensors) {
590 // last occurrence of the tensor is reached => flush it from memory
591 if (fVerbose) std::cout << ".. input tensors : " << iv;
592
593 // for alias tensors the memory to flush is the one of the tensor they refer to
594 std::string it{iv}; // convert view to string
595 if (IsAliasTensor(it))
596 it = fAliasTensors[it];
597 // find(): operator[] would insert a key viewing the local string
598 auto lastUse = fIntermediateTensorFrequencyLookup.find(it);
599 if (lastUse != fIntermediateTensorFrequencyLookup.end() && lastUse->second == op_idx) {
600 if (fVerbose) std::cout << " flash condition is met - looping on chunks to find matching one \n";
601 for (auto chunk = fIntermediateMemoryInfo.total_stack.begin();
602 chunk != fIntermediateMemoryInfo.total_stack.end(); ++chunk) {
603 if (fVerbose) std::cout << "--- chunk " << chunk->first << " , " << chunk->second.tensor_name << " size " << chunk->second.tensor_size;
604 if (chunk->second.tensor_name == it) {
605 if (fVerbose) std::cout << " -- Found chunk corresponding to input tensor: " << chunk->first;
606 // check if nearby chunks in available memory can coalesce
607 auto first_greater = fIntermediateMemoryInfo.available_stack.upper_bound(
608 chunk->first); // smallest element greater than the flushed chunk idx
609 auto last_smaller = (first_greater == fIntermediateMemoryInfo.available_stack.begin())
611 : std::prev(first_greater); // largest element smaller than the flushed chunk idx
612
613 // check if the next stack entry is actually adjacent in memory
614
615 if (last_smaller != fIntermediateMemoryInfo.available_stack.end() &&
616 last_smaller->first + last_smaller->second == chunk->first) {
617 // merge chunk with previous one
618 last_smaller->second += chunk->second.tensor_size;
619 fIntermediateMemoryInfo.total_stack[last_smaller->first].merge(chunk->second);
620 if (fVerbose) std::cout << " is adjacent in memory with previous one - merge ";
621 if (first_greater != fIntermediateMemoryInfo.available_stack.end() &&
622 last_smaller->first + last_smaller->second == first_greater->first) {
623 // merge also with following one
624 last_smaller->second += first_greater->second;
625 fIntermediateMemoryInfo.total_stack[last_smaller->first].merge(
626 fIntermediateMemoryInfo.total_stack[first_greater->first]);
627 // delete merged one in available stack and in total stack
628 fIntermediateMemoryInfo.total_stack.erase(first_greater->first);
629 fIntermediateMemoryInfo.available_stack.erase(first_greater);
630 if (fVerbose) std::cout << " merge also with following that is free ";
631 }
632 fIntermediateMemoryInfo.total_stack.erase(chunk->first);
633 if (fVerbose) std::cout << std::endl;
634 break;
635 } else if (first_greater != fIntermediateMemoryInfo.available_stack.end() &&
636 chunk->first + chunk->second.tensor_size == first_greater->first) {
637 // merge with first greater
638 if (fVerbose) std::cout << " is adjacent in memory with following one - merge \n";
639 // cannot modify idx of first_greter. Insert a new one and delete previous one
640 size_t new_size = chunk->second.tensor_size + first_greater->second;
641 size_t first_greater_idx = first_greater->first;
642 fIntermediateMemoryInfo.available_stack.erase(first_greater);
643 // cannot use anymore first_greater
644 fIntermediateMemoryInfo.available_stack.insert({chunk->first, new_size});
645 fIntermediateMemoryInfo.total_stack[chunk->first].merge(
646 fIntermediateMemoryInfo.total_stack[first_greater_idx]);
647 fIntermediateMemoryInfo.total_stack.erase(first_greater_idx);
648 } else {
649 fIntermediateMemoryInfo.available_stack.insert({chunk->first, chunk->second.tensor_size});
650 if (fVerbose) std::cout << " insert in the available stack the chunk with size " << chunk->second.tensor_size << std::endl;
651 }
652 chunk->second.tensor_name = "free";
653 break;
654 }
655 }
656 } else {
657 if (fVerbose) std::cout << std::endl;
658 }
659 }
660}
661
662void RModel::Initialize(int batchSize, bool verbose) {
663 std::map<std::string, size_t> inputParams;
664 if (batchSize > 0) {
665 inputParams["input_size"] = batchSize;
666 inputParams["batch_size"] = batchSize;
667 inputParams["bs"] = batchSize;
668 }
669 Initialize(inputParams, verbose);
671}
672void RModel::Initialize(const std::map<std::string, size_t> & inputParams, bool verbose) {
673
674 fVerbose = int(verbose);
675
676 if (fIsInitialized) {
677 if (verbose)
678 std::cout << "Model is already initialized - skip initialization " << std::endl;
679 return;
680 }
682 fDynamicTensorInfos.clear();
683
684
685 // loop on inputs and see if shape can be full specified
686 // if the batch size is provided it can be used to specify the full shape
687 // Add the full specified tensors in fReadyInputTensors collection
688 auto originalInputTensorInfos = fInputTensorInfos; // need to copy because we may delete elements
689 // Visit the inputs in their model order, not in the platform-dependent
690 // iteration order of the unordered map: this loop registers the shape
691 // parameters, which defines the order of the Session constructor arguments.
692 for (auto const &inputName : fInputTensorNames) {
693 auto found = originalInputTensorInfos.find(inputName);
694 if (found == originalInputTensorInfos.end())
695 continue; // the shape of this input is already fully defined
696 auto &input = *found;
697 if (verbose) std::cout << "looking at the tensor " << input.first << std::endl;
698 // if a parameter (e.g. batch_size) is specified use for converting parametric shape in defined one
699 if (!inputParams.empty()) {
700 for (auto &d : input.second.shape) {
701 if (d.isParam) {
702 std::string pname = d.param;
703 if (pname == input.first + "_size") pname = "input_size";
704 auto itr = inputParams.find(pname);
705 if (itr != inputParams.end() ) {
706 d = Dim{ itr->second };
707 if (verbose)
708 std::cout << "Tensor: " << input.first << " - fix parametric shape " << itr->first << " to " << itr->second << std::endl;
709 }
710 }
711 }
712 }
713 // see if shape now is fully defined
714 auto shape = ConvertShapeToInt(input.second.shape);
715 if (verbose)
716 std::cout << "converting input shape for " << input.first << " " << ConvertShapeToString(shape) << " from "
717 << ConvertDimShapeToString(input.second.shape) << std::endl;
718 if (!shape.empty()) {
719 // case shape is defined (not parametric) we add the tensor in the fReadyInputTensorInfos map and
720 // we remove the tensor from the fInputTensorInfo where th eold parametric shape was stored
721 fInputTensorInfos.erase(input.first);
722 // add to the ready input tensor information the new fixed shape
723 AddInputTensorInfo(input.first, input.second.type, shape);
724 // check consistency
725 assert( fReadyInputTensorInfos.size() + fInputTensorInfos.size() == fInputTensorNames.size());
726 }
727 // store the parameters of the input tensors
728 else {
729 // store the found parametric shape parameters
730 for (auto &d : input.second.shape) {
731 if (d.isParam) {
732 // through AddShapeParam, which keeps out the parameters computed at run time
733 AddShapeParam(d.param, d.dim);
734 }
735 }
736 }
737 }
738
739 if (verbose) {
742 }
743
744 // Go through model and initialize each operator
745 int i = 0;
746
747 std::vector<size_t> temp_available_stack; // vector stores individual chunks of available memory that maybe reused
748
749 // Build set of initialized tensors consumed by at least one runtime operator (need for later)
750 std::unordered_set<std::string> runtimeInitializedInputs;
751 for(size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx){
752 if (verbose) {
753 auto& r = *fOperators[op_idx].get();
754 std::cout << "Initializing operator " << i << " " << typeid(r).name() << std::endl;
755 }
756 fOperators[op_idx]->Initialize(*this);
757 for(auto &it:fOperators[op_idx]->GetOpOutputTensors()){
758 std::string name = std::string{it};
759 // check if tensor is not an initialized or output tensor and it is not already in the list
761 std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), name) == fOutputTensorNames.end() &&
763 {
765 }
766 }
767 // loop for non-constant operators and flag the inputs which are initialized tensors to make sure they are writable
768 if (!fOperators[op_idx]->IsOutputConstant()) {
769 for (auto &it : fOperators[op_idx]->GetOpInputTensors()) {
770 std::string name = std::string{it};
771 if (fInitializedTensors.find(name) != fInitializedTensors.end()) {
772 runtimeInitializedInputs.insert(name);
773 }
774 }
775 }
776
777 i++;
778 }
779
780 // loop on initialized tensors and make the integers as constant to be
781 // not written in a weight file and check if the tensors flagged as not writable are really not writable,
782 // i.e. are not used by non constant operators
783 for (auto &it : fInitializedTensors) {
784 // check if not-writable tensors are really not writable, i.e. are not used by non constant operators
785 if (it.second.IsNotWritable() && runtimeInitializedInputs.find(it.first) != runtimeInitializedInputs.end()) {
786 it.second.SetWritable();
787 if (verbose) {
788 std::cout << "Initialized tensor " << it.first << " is flagged as not writable but is used by non constant operators, set it as writable \n";
789 }
790 }
791 // if the tensor is an integer we can flag it as constant since it will not be written in a weight file and it is considered equivalent as being created from a Constant operator
792 // only FLOAT tensors are written in a weight file
793 if (it.second.type() != ETensorType::FLOAT) {
794 it.second.SetConstant();
795 }
796 }
797
798 // check if there are initialized tensors to write in a weight file
799 if (fUseWeightFile) {
800 bool modelHasWeights = false;
801 for (auto &it : fInitializedTensors) {
802 if (it.second.IsWeightTensor()) {
803 modelHasWeights = true;
804 break;
805 }
806 }
807 if (!modelHasWeights)
808 fUseWeightFile = false;
809 }
810
811 // update fIntermediateTensorFrequencyLookup for alias tensors
812 for (auto & it : fAliasTensors) {
816 else {
817 // take the largest one
819 }
820 }
821
822 fIsInitialized = true;
823}
824
825void RModel::InitializeSubGraph(std::shared_ptr<RModel> graph) {
826 // add the subgraph to the list
827 fSubGraphs.push_back(graph);
828 //this needs to be done before initializing
829 graph->fParentGraph = this;
830 graph->fIsSubGraph = true;
831
832 graph->Initialize(fBatchSize, fVerbose);
833 // set the same options as parent model
834 graph->fWeightFile = fWeightFile;
835 graph->fUseWeightFile = fUseWeightFile;
836 // add needed blas routines and libs
837 std::vector<std::string> blasRoutines;
838 for (auto & e : graph->fNeededBlasRoutines)
839 blasRoutines.push_back(e);
840 AddBlasRoutines(blasRoutines);
841 for (auto e : graph->fNeededStdLib)
843 // helper functions used by the subgraph must be emitted in the top-level
844 // header, so propagate them to the parent model
845 for (auto const &h : graph->GetNeededHelperFunctions())
847
848 // add parent input tensors to current graph
849 for (auto & name : fInputTensorNames)
850 graph->fInputTensorNames.emplace_back(name);
851
852 // clean graph name
853 graph->fName = UTILITY::Clean_name(graph->fName);
854
855}
856
857// Function to generate the code for declaring and initializing constant tensors
858// This is for tensors which are not part of weight files and can be created from the Constant operator
859template <typename T>
860std::string GenerateConstantTensorCode(const std::pair<std::string, InitializedTensor> &t)
861{
862 std::stringstream strs;
863 std::string type = ConvertTypeToString(t.second.type());
864 size_t length = ConvertShapeToLength(t.second.shape());
865 // avoid using stack sizes for constant tensors to reduce compilation time
866 // also for weights which can be broadcasted do not use stack but allocate as a std::vector
867 bool allocateOnStack = (length > 100 || t.second.IsWeightTensor()) ? false : true;
868
869 const T *data = t.second.data<T>();
870
871 // and check if all values are the same
872 bool sameData = false;
873
874 // for non stack allocation check if data are the same
875 if (!allocateOnStack && length > 1) {
876 size_t idx = 1;
877 do {
878 sameData = (data[idx] == data[idx - 1]);
879 idx++;
880 } while (sameData && idx < length);
881 }
882 if (allocateOnStack) {
883 strs << type << " fTensor_" << t.first << "[" << length << "] = " << ConvertValuesToString(length, data) << ";\n";
884 strs << type << " * " << TensorMember(t.first) << " = fTensor_" + t.first + ";\n";
885 } else {
886 strs << "std::vector<" << type << "> fTensor_" << t.first << " = ";
887 if (sameData)
888 strs << "std::vector<" << type << ">(" << length << ", " << ConvertValToString(data[0]) << ");\n";
889 else {
890 strs << ConvertValuesToString(length, data) << ";\n";
891 }
892 strs << type << " * " << TensorMember(t.first) << " = fTensor_" + t.first + ".data();\n";
893 }
894 return strs.str();
895}
896
898{
899 if (!fInitializedTensors.empty())
900 fGC += "// initialized (weights and constant) tensors\n";
901
902 // here are constant tensor or initialized ones which are not weights (e.g. int64_t tensors )
903 for (auto &i : fInitializedTensors) {
904 if (i.second.IsNotWritable()) continue;
905 size_t length = ConvertShapeToLength(i.second.shape());
906 if (!fUseWeightFile || i.second.IsConstantTensor() || !i.second.IsWeightTensor() || i.second.type() != ETensorType::FLOAT ) {
907 if (i.second.type() == ETensorType::FLOAT) {
908 // check if NaN of Inf are inside tensor data
909 bool hasInfOrNaN = false;
910 const float *data = i.second.data<float>();
911 for (size_t idx = 0; idx < length; idx++) {
912 if (std::is_floating_point<float>::value) {
913 if (std::isinf(data[idx]) || std::isnan(data[idx])) {
914 hasInfOrNaN = true;
915 break;
916 }
917 }
918 }
919 if (hasInfOrNaN)
920 AddNeededStdLib("limits");
921 fGC += GenerateConstantTensorCode<float>(i);
922 fConstantTensorSize += length * sizeof(float);
923 } else if (i.second.type() == ETensorType::INT64) {
924 fGC += GenerateConstantTensorCode<int64_t>(i);
925 fConstantTensorSize += length * sizeof(int64_t);
926 } else if (i.second.type() == ETensorType::INT32) {
927 fGC += GenerateConstantTensorCode<int32_t>(i);
928 fConstantTensorSize += length * sizeof(int32_t);
929 } else if (i.second.type() == ETensorType::BOOL || i.second.type() == ETensorType::UINT8 ) {
930 fGC += GenerateConstantTensorCode<uint8_t>(i);
931 fConstantTensorSize += length * sizeof(uint8_t);
932 }
933
934
935 } else {
936 // case of tensors which are read from a file
937 if (i.second.type() == ETensorType::FLOAT) {
938 fGC += "std::vector<float> fTensor_" + i.first + " = std::vector<float>(" + std::to_string(length) + ");\n";
939 fGC += "float * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n";
940 fWeightsTensorSize += length * sizeof(float);
941 }
942 }
943 }
944}
945
947 if (fIntermediateMemoryInfo.total_stack.empty()) return;
948 fGC += "\n//--- Allocating session memory pool to be used for allocating intermediate tensors\n";
949
950 // char memory block is allocated since char takes 1 byte, thus easier to allocate tensors
951 // of other data types
952 auto const &totalStack = fIntermediateMemoryInfo.total_stack;
953 const size_t memPoolSize = totalStack.rbegin()->first + totalStack.rbegin()->second.tensor_size;
954 fGC += "std::vector<char> fIntermediateMemoryPool = std::vector<char>(" + std::to_string(memPoolSize) + ");\n\n";
955}
956
958 if (!fIntermediateTensorInfos.empty()) {
959 std::string tensor_declaration_block = "";
960 for (auto &i : fIntermediateTensorInfos) {
961 // alias tensors have no storage: the operator creating them declares a pointer to the
962 // memory of the tensor they refer to
963 if (IsAliasTensor(i.first))
964 continue;
965 if (i.second.type == ETensorType::BOOL) {
966 tensor_declaration_block += "std::vector<std::uint8_t> fTensor_" + i.first +
967 " = std::vector<std::uint8_t>(" +
968 std::to_string(ConvertShapeToLength(i.second.shape)) + ");\n";
969 tensor_declaration_block +=
970 "std::uint8_t * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n";
971 continue;
972 }
973 bool is_extended = (fOptimizationLevel == OptimizationLevel::kExtended);
974 bool not_in_freq_map =
976 bool not_in_output_names =
977 (std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), i.first) == fOutputTensorNames.end());
978
979 if ((not_in_freq_map && not_in_output_names) || (!not_in_freq_map && !is_extended && not_in_output_names)) {
980 size_t length = ConvertShapeToLength(i.second.shape);
981
982 if (i.second.type == ETensorType::FLOAT) {
983 tensor_declaration_block += "std::vector<float> fTensor_" + i.first + " = std::vector<float>(" + std::to_string(length) + ");\n";
984 tensor_declaration_block += "float * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n";
986 }
987 else if (i.second.type == ETensorType::DOUBLE) {
988 tensor_declaration_block += "std::vector<double> fTensor_" + i.first + " = std::vector<double>(" + std::to_string(length) + ");\n";
989 tensor_declaration_block += "double * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n";
991 }
992 else if (i.second.type == ETensorType::INT64) {
993 tensor_declaration_block += "std::vector<int64_t> fTensor_" + i.first + " = std::vector<int64_t>(" + std::to_string(length) + ");\n";
994 tensor_declaration_block += "int64_t * " + TensorMember(i.first) + " = fTensor_" + i.first + ".data();\n";
996 }
997 }
998 }
999
1000 if (tensor_declaration_block.length()) {
1001 fGC += "\n//--- declare and allocate the intermediate tensors\n" + tensor_declaration_block;
1002 }
1003 }
1004 // add also the dynamic tensors (only declarations, allocation will be done later)
1005 if (!fDynamicTensorInfos.empty()) {
1006 fGC += "//--- declare the dynamic tensors\n";
1007 for (auto &i : fDynamicTensorInfos) {
1008 if (IsAliasTensor(i.first))
1009 continue;
1010 fGC += ConvertTypeToString(i.second.type) + " * " + TensorMember(i.first) + " = nullptr;\n";
1011 }
1012 fGC += "//--- dynamic tensors pool\n";
1013 fGC += "std::vector<char> fDynamicMemoryPool;\n";
1014 }
1015}
1016
1017// generate code for specific operator declarations to be defined in the Session class
1019 std::string strcode;
1020 for (auto & op : fOperators) {
1021 strcode += op->GenerateDeclCode();
1022 }
1023 if (strcode.empty()) return;
1024 fGC += "\n//---- operator declarations \n";
1025 fGC += strcode;
1026 fGC += "\n";
1027}
1028
1030{
1031 // generate code for allocating dynamic tensors using the greedy memory allocations
1032 if (fDynamicTensorInfos.empty())
1033 return;
1034
1035 if (fVerbose) {
1036 std::cout << "generating code for dynamic tensor management" << std::endl;
1038 }
1039
1040 // the generated code uses the TensorLifeInfo / OrganizeMemory inference helpers
1041 AddNeededHelperFunction("DynamicMemory");
1042
1043 std::stringstream out;
1044 out << "// dynamic tensor memory management\n";
1045 out << SP << "std::vector<TensorLifeInfo> dynamicTensorInfos;\n";
1046 out << SP << "dynamicTensorInfos.reserve(" << fDynamicTensorInfos.size() << ");\n";
1047
1048 // loop on all the operators to find begin/end life of the tensors
1049 int op_index = 0;
1050 std::vector<std::pair<std::string, ETensorType>> tensors;
1051 tensors.reserve(fDynamicTensorInfos.size());
1052 for (auto & op : fOperators) {
1053 // loop on output tensors -
1054 for (auto &it : op->GetOpOutputTensors()) {
1055 if (fVerbose) {
1056 auto op_ptr = op.get();
1057 std::cout << "Looping on operator " << op_index << " " << typeid(*op_ptr).name() << std::endl;
1058 }
1059 // check if is a dynamic tensor and not an alias tensor or output tensor
1060 std::string name = std::string(it);
1062 && std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), name) == fOutputTensorNames.end()) {
1063 auto tensor_size = ConvertDimShapeToLength(GetDimTensorShape(name));
1064 auto type = GetTensorType(name);
1065 size_t type_size = GetTypeSize(type);
1066 int begin = op_index;
1067 int end = fOperators.size();
1068 // look for end
1069 auto it_lookup = fIntermediateTensorFrequencyLookup.find(name);
1070 if (it_lookup != fIntermediateTensorFrequencyLookup.end())
1071 end = it_lookup->second + 1; // end is last time used + 1
1072 // // some tensors (like xcol in convolutions) are just used within the operators
1073 // if (end == 0 && begin > 0) end = begin+1;
1074
1075 if (begin> end) {
1076 std::cout << "op " << op_index << "tensor_" << name << " begin " << begin << " " << " end " << end << std::endl;
1077 throw std::runtime_error("TMVA-SOFIE: RModel::GenerateDynamicTensorInfo: tensor_" + name + " has end before begin");
1078 }
1079
1080 // write in code
1081 out << SP << "dynamicTensorInfos.push_back( {" << begin << ", " << end << ", " << type_size << "* (" << tensor_size << ") });"
1082 << " // tensor_" << name << std::endl;
1083 tensors.push_back({name,type});
1084 }
1085 }
1086 op_index++; // increment operator index
1087 }
1088 out << "\n" << SP << "auto memory_result = OrganizeMemory(dynamicTensorInfos);\n\n";
1089 out << "// allocating now the memory\n";
1090 out << SP << "fDynamicMemoryPool = std::vector<char>(memory_result.total_bytes);\n";
1091 out << SP << "int idx = 0;\n";
1092 for (auto & it : tensors) {
1093 out << SP << "tensor_" << it.first << " = reinterpret_cast<" << ConvertTypeToString(it.second) << " *>(fDynamicMemoryPool.data() + memory_result.offsets[idx++]);\n";
1094 }
1095 // check that all dynamic tensors are covered
1096 bool missingTensor = false;
1097 for (auto &i : fDynamicTensorInfos) {
1098 if (IsAliasTensor(i.first)) continue;
1099 if (std::find(fOutputTensorNames.begin(), fOutputTensorNames.end(), i.first) != fOutputTensorNames.end()) continue;
1100 if (std::find(tensors.begin(), tensors.end(), std::pair<std::string,ETensorType>{i.first, i.second.type}) == tensors.end()) {
1101 std::cout << "Dynamic tensors " << i.first << " is not in list of operator input/output " << std::endl;
1102 missingTensor = true;
1103 }
1104 }
1105 if (missingTensor)
1106 throw std::runtime_error("TMVA-SOFIE: RModel::GenerateDynamicTensorInfo - some tensors are not in input/output list");
1107
1108 fGC += out.str();
1109}
1110
1111/// Check if a given parameter is used for the shape of an input tensor.
1112bool RModel::IsInputTensorShapeParam(std::string const &paramName) const
1113{
1114 for (auto &name : fInputTensorNames) {
1115 if (IsDimInputTensor(name)) {
1116 auto shape = GetDynamicTensorShape(name);
1117 for (auto &d : shape) {
1118 if (d.param == paramName)
1119 return true;
1120 }
1121 }
1122 }
1123 return false;
1124}
1125
1126/// Collects all identifiers starting with "tensor_" in the input code,
1127/// provided that the occurrence is not immediately preceded by a
1128/// character that is valid in a C++ identifier. Excludes input and output tensor names.
1129/// Returns a deduplicated std::vector<std::string>.
1130std::vector<std::string> RModel::CollectTensorMemberNames(const std::string &input)
1131{
1132 const std::string target = "tensor_";
1133
1134 std::vector<std::string> result;
1135
1136 for (size_t i = 0; i < input.size();) {
1137
1138 bool doCollect = false;
1139
1140 if (i + target.size() <= input.size() && input.compare(i, target.size(), target) == 0 &&
1141 (i == 0 || !IsIdentifierChar(input[i - 1]))) {
1142
1143 doCollect = true;
1144
1145 std::size_t j = i + target.size();
1146
1147 // Extend to full identifier
1148 while (j < input.size() && IsIdentifierChar(input[j]))
1149 ++j;
1150
1151 std::string fullName = input.substr(i, j - i);
1152
1153 // Exclude input tensor names
1154 for (std::string const &name : fInputTensorNames) {
1155 if (fullName == target + name) {
1156 doCollect = false;
1157 break;
1158 }
1159 }
1160
1161 // Exclude output tensor names
1162 if (doCollect) {
1163 for (std::string const &name : fOutputTensorNames) {
1164 if (fullName == target + name) {
1165 doCollect = false;
1166 break;
1167 }
1168 }
1169 }
1170
1171 if (doCollect) {
1172 result.push_back(fullName);
1173 }
1174
1175 i = j; // advance past the identifier
1176 } else {
1177 ++i;
1178 }
1179 }
1180
1181 // Deduplicate (order not preserved)
1182 std::sort(result.begin(), result.end());
1183 result.erase(std::unique(result.begin(), result.end()), result.end());
1184
1185 return result;
1186}
1187
1188std::string RModel::GenerateInferSignature(bool isdecl) {
1189 // generate the infer signature given the inputs: eg. "float * tensor1, float * tensor2"
1190 // if (decl = false) generate only calling signature (tensor1,tensor2,....)
1191 std::string rGC;
1192 std::unordered_map<std::string, int> inputParams;
1193 int i_input = 0;
1194 for (auto &name : fInputTensorNames) {
1195 // if is a dynamic tensor pass initial parameters
1196 if (IsDimInputTensor(name)) {
1197 auto shape = GetDynamicTensorShape(name);
1198 for (auto &d : shape) {
1199 std::string pName = d.param;
1200 // need to check if the input parameters is already existing in another input tensor
1201 if (d.isParam && inputParams.count(pName) == 0) {
1202 if (isdecl) rGC += "size_t ";
1203 rGC += d.param + ",";
1204 inputParams[pName] = i_input;
1205 }
1206 }
1207 }
1208 if (isdecl) {
1210 // ConvertTypeToString returns "other_<type number>" for the types it does not know
1211 if (type.rfind("other", 0) == 0)
1212 throw std::runtime_error("TMVA-SOFIE: input tensor " + name +
1213 " is of a data type which is not yet supported.");
1214 rGC += type + " const* ";
1215 }
1216 rGC += "tensor_" + name + ",";
1217 i_input++;
1218 }
1219
1220 if (fInputTensorNames.size() > 0) rGC.pop_back();// remove last ","
1221 return rGC;
1222}
1223
1224namespace {
1225
1226std::string typeForOutput(ETensorType t) {
1227 // The std::vector<bool> is a special type that is not wrapping continuous memory.
1228 // We don't want to use it as a return type.
1230 return ConvertTypeToString(t);
1231}
1232
1233std::string memberNameForDimShape(std::string name)
1234{
1235 if (!name.empty()) {
1236 name[0] = std::toupper(static_cast<unsigned char>(name[0]));
1237 }
1238 name = "f" + name;
1239 return name;
1240}
1241
1242}
1243
1245{
1246 size_t outputSize = fOutputTensorNames.size();
1247 // assume output types are all the same
1248
1249 bool sameOutputTypes = true;
1250 std::string inferReturnType; // type return by infer function
1251 ETensorType eFirstOutputType = GetTensorType(*fOutputTensorNames.begin());
1252 fGC += "\n\n";
1253 if (outputSize == 1) {
1254 fGC += "std::vector<" + typeForOutput(eFirstOutputType) + ">";
1255 } else {
1256 // if all output types are the same we return an std::vector - otherwise a tuple
1257 for (std::string const &name : fOutputTensorNames) {
1258 if (GetTensorType(name) != eFirstOutputType)
1259 sameOutputTypes = false;
1260 }
1261 if (sameOutputTypes)
1262 fGC += "std::vector<std::vector<" + typeForOutput(eFirstOutputType) + ">>";
1263 else {
1264 inferReturnType = "std::tuple<";
1265 for (size_t i = 0; i < outputSize; i++) {
1266 inferReturnType += "std::vector<" + typeForOutput(GetTensorType(fOutputTensorNames[i])) + ">";
1267 if (i < outputSize - 1)
1268 inferReturnType += ",";
1269 }
1270 inferReturnType += ">";
1271 fGC += inferReturnType;
1272 }
1273 }
1274
1275 fGC += " infer(" + GenerateInferSignature() + "){\n";
1276
1277 std::string doInferArgs = GenerateInferSignature(false);
1278 if (!doInferArgs.empty())
1279 doInferArgs += ",";
1280 // several outputs can share one run-time shape parameter: declare and pass it once
1281 std::unordered_set<std::string> emittedShapeParams;
1282 for (std::string const &name : fOutputTensorNames) {
1283 bool isDynamic = fDynamicTensorInfos.count(name) > 0;
1284 std::string n;
1285 if(!isDynamic) {
1286 n = std::to_string(ConvertShapeToLength(GetTensorShape(name)));
1287 } else {
1288 std::string dimLen = ConvertDimShapeToLength(GetDynamicTensorShape(name));
1289 // Use the session member (fXxx) when any dim is a runtime-computed identifier
1290 // (e.g. NonZero count). For expression-type dims derived from input shapes
1291 // (e.g. "((W+-3)/2+1)"), use the expression directly.
1292 // for input shape parameters we don't need to use the session member since it is passed as argument to the infer function and it is not a runtime computed value
1293 bool hasRuntimeParam = false;
1294 for (auto const &dim : GetDynamicTensorShape(name)) {
1295 if (dim.isParam && IsIdentifier(dim.param) && !IsInputTensorShapeParam(dim.param))
1296 hasRuntimeParam = true;
1297 }
1298 n = hasRuntimeParam ? memberNameForDimShape(dimLen) : dimLen;
1299 }
1300 std::string outputName = "output_tensor_" + name;
1301 fGC += SP + "std::vector<" + typeForOutput(GetTensorType(name)) + " > " + outputName + "(" + n + ");\n";
1302 doInferArgs += " " + outputName + ".data(),";
1303 if(isDynamic) {
1304 for (auto const &dim : GetDynamicTensorShape(name)) {
1305 if (dim.isParam && !IsInputTensorShapeParam(dim.param) && IsIdentifier(dim.param) &&
1306 emittedShapeParams.insert(dim.param).second) {
1307 fGC += SP + "size_t " + dim.param + " = 0;\n";
1308 doInferArgs += " " + dim.param + ",";
1309 }
1310 }
1311 }
1312 }
1313 if (!doInferArgs.empty())
1314 doInferArgs.back() = ' ';
1315
1316 // verifying if the dynamic parameters are within allowed range
1317 std::unordered_set<std::string> input_params_checked;
1318 std::string dynamic_parameters_check = "";
1319 for (auto &name : fInputTensorNames) {
1320 if (IsDimInputTensor(name)) {
1321 auto shape = GetDynamicTensorShape(name);
1322 for (auto &d : shape) {
1323 std::string pName = d.param;
1324 if (d.isParam && input_params_checked.count(pName) == 0) {
1325 std::string memberName = memberNameForDimShape(d.param);
1326 dynamic_parameters_check += d.param + " > " + memberName + " || ";
1327 input_params_checked.insert(pName);
1328 fGC += SP + "if (" + d.param + " > " + memberName + ") {\n";
1329 fGC += SP + SP + "throw std::runtime_error(\"TMVA-SOFIE: dynamic input tensor shape parameter " +
1330 d.param + " exceeds the initialized maximum allowed shape.\");\n";
1331 fGC += SP + "}\n";
1332 }
1333 }
1334 }
1335 }
1336
1337 fGC += SP + "doInfer(*this, " + doInferArgs + ");\n";
1338
1339 // If the output tensors have dynamic sizes, now is the time to set them
1340 for (std::string const &name : fOutputTensorNames) {
1341 bool isDynamic = fDynamicTensorInfos.count(name) > 0;
1342 if (isDynamic) {
1343 std::string outputName = "output_tensor_" + name;
1344 auto tensor_size = ConvertDimShapeToLength(GetDimTensorShape(name));
1345 fGC += SP + outputName + ".resize(" + tensor_size + ");\n";
1346 }
1347 }
1348
1349 fGC += SP + "return {";
1350 for (size_t i = 0; i < fOutputTensorNames.size(); i++) {
1351 fGC += "output_tensor_" + fOutputTensorNames[i];
1352 if (i < fOutputTensorNames.size() - 1)
1353 fGC += ",";
1354 }
1355 fGC += "};\n";
1356 fGC += "}\n"; // end of infer function scope
1357}
1358
1360{
1361 std::string sessionName = !fIsSubGraph ? "Session" : "Session_" + fName;
1362
1363 // forward declare session struct
1364 fGC += "struct " + sessionName + ";\n";
1365
1366 // Determine the signature of the actual inference function
1367 std::string doInferSignature = GenerateInferSignature();
1368 if (!doInferSignature.empty())
1369 doInferSignature += ", ";
1370 // one argument per shape parameter, even when several outputs share it
1371 std::unordered_set<std::string> signatureShapeParams;
1372 for (auto const &name : fOutputTensorNames) {
1373 bool isDynamic = fDynamicTensorInfos.count(name) > 0;
1374 doInferSignature += typeForOutput(GetTensorType(name)) + " *tensor_" + name + ",";
1375 if(isDynamic) {
1376 for (auto const &dim : GetDynamicTensorShape(name)) {
1377 if (dim.isParam && !IsInputTensorShapeParam(dim.param) && IsIdentifier(dim.param) &&
1378 signatureShapeParams.insert(dim.param).second)
1379 doInferSignature += " size_t &" + dim.param + "_output,";
1380 }
1381 }
1382 }
1383 doInferSignature.back() = ' ';
1384
1385 doInferSignature = sessionName + " const &session, " + doInferSignature;
1386
1387 doInferSignature = "inline void doInfer(" + doInferSignature + ")";
1388
1389 // forward declare inference implementation
1390 fGC += doInferSignature + ";\n";
1391
1392 // define the Session struct: copying is not allowed because some members
1393 // are pointers to memory owned by the std::vector members of the session
1394 // struct. Since the std::vector-owned memory is not relocated when moving,
1395 // move contruction and assignment is allowed.
1396 std::string sessionCode = R"(struct @SESSION@ {
1397
1398 ~@SESSION@() = default;
1399
1400 @SESSION@(@SESSION@ const&) = delete;
1401 @SESSION@(@SESSION@ &&) = default;
1402
1403 @SESSION@& operator=(@SESSION@ const&) = delete;
1404 @SESSION@& operator=(@SESSION@ &&) = default;
1405
1406 )";
1407 ReplaceAll(sessionCode, "@SESSION@", sessionName);
1408 fGC += sessionCode;
1409
1410 // generate code for declaring the initialized tensors
1412
1414 // evaluate total intermediate memory and position intermediate tensor addresses
1415 std::string intermediate_memory_alloc_string = "";
1416 intermediate_memory_alloc_string += "\n// --- Positioning intermediate tensor memory --";
1417 for (size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx) {
1418 if (fVerbose) {
1419 auto op = fOperators[op_idx].get();
1420 std::cout << "\n******************\n analyzing input/output operator " << op_idx << " "
1421 << typeid(*op).name() << std::endl;
1422 }
1423 intermediate_memory_alloc_string += AllocateIntermediateMemory(fOperators[op_idx]->GetOpOutputTensors());
1424 CheckAndFlushIntermediateMemory(fOperators[op_idx]->GetOpInputTensors(), op_idx);
1425 }
1426
1427 // to check remaining unused fragments after memory allocation (lesser the better)
1428 // for (const auto &it: fIntermediateMemoryInfo.available_stack){
1429 // std::cout<<"chunk_idx: "<<it.first<<", chunk_size: "<<it.second<<"\n";
1430 // }
1431
1432 // generate the memory pool to be used by intermediate tensors
1434
1435 // position intermediate tensors
1436 fGC += intermediate_memory_alloc_string;
1437 }
1438
1439 // generate the declaring the intermediate tensors
1441 // generate code for declarations of some specific operators
1443
1444 // storing the parameters for future checking to avoid mismatches
1445 if (!fDimShapeNames.empty()) {
1446 fGC += "\n// dynamic shape parameters\n";
1447 auto dimShapeNames = fDimShapeNames;
1448 std::sort(dimShapeNames.begin(), dimShapeNames.end());
1449 for (const auto &p : dimShapeNames) {
1450 fGC += "size_t " + memberNameForDimShape(p) + ";\n";
1451 }
1452 }
1453
1454 // add subgraph session
1455 if (!fSubGraphs.empty()) fGC += "// subgraph sessions\n";
1456 for (auto & graph : fSubGraphs) {
1457 fGC += "Session_" + graph->fName + " fSession_" + graph->fName + ";\n";
1458 }
1459
1460 // Generate code for Session constructor
1461 // add here specific operator code that needs to define session data members
1462 fGC += "\n";
1463 for (size_t id = 0; id < fOperators.size(); id++) {
1464 std::string opName = std::to_string(id);
1465 fGC += fOperators[id]->GenerateSessionMembersCode(opName);
1466 }
1467 fGC += "\n";
1468 // collect declaration of shape parameters (default values) and argument
1469 // forwarding for delegating constructors
1470 std::string dynParamDecls;
1471 std::string dynParamArgs;
1472 if (!fDimShapeNames.empty()) {
1473 // need to use same order as in infer function not alphabetical one
1474 for (auto &p : fDimShapeNames) {
1475 dynParamDecls += ",\n size_t " + p + " = " + fShapeParams[p];
1476 dynParamArgs += ", " + p;
1477 }
1478 }
1479 // here add initialization and reading of weight tensors
1481 // A Session is constructed from an in-memory safetensors blob; the
1482 // file-based constructor loads the payload fully into memory and
1483 // delegates. The temporary buffer is alive for the duration of the
1484 // delegated constructor call, and the blob constructor copies the
1485 // weights out of it.
1486 AddNeededHelperFunction("SafetensorsBlob");
1487 fGC += " static std::string LoadWeightsFromFile(const std::string &filename) {\n";
1488 fGC += " std::ifstream f(filename, std::ios::binary);\n";
1489 fGC += " if (!f.is_open()) {\n";
1490 fGC += " throw std::runtime_error(\"tmva-sofie failed to open file \" + filename + \" for input "
1491 "weights\");\n";
1492 fGC += " }\n";
1493 fGC += " return std::string(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());\n";
1494 fGC += " }\n\n";
1495 fGC += sessionName + "(std::string filename =\"" + fName + ".safetensors\"" + dynParamDecls + ")\n";
1496 fGC += " : " + sessionName + "(SafetensorsBlob{LoadWeightsFromFile(filename)}" + dynParamArgs + ") {}\n\n";
1497 fGC += sessionName + "(SafetensorsBlob weights_blob" + dynParamDecls + ") {\n";
1498 } else if (fUseWeightFile) {
1499 std::string fileName = fName + ".dat";
1500 fGC += sessionName + "(std::string filename =\"" + fileName + "\"" + dynParamDecls + ") {\n";
1501 } else {
1502 // no need to pass weight file since it is not used
1503 // keep passing a string for compatibility
1504 fGC += sessionName + "(std::string = \"\"" + dynParamDecls + ") {\n";
1505 }
1506
1507 // initializing dynamic parameters
1508 if (!fDimShapeNames.empty()) {
1509 fGC += "\n\n";
1510 // sort a copy: fDimShapeNames defines the constructor argument order
1511 auto dimShapeNames = fDimShapeNames;
1512 std::sort(dimShapeNames.begin(), dimShapeNames.end());
1513 for (const auto &p : dimShapeNames) {
1514 fGC += " " + memberNameForDimShape(p) + " = " + p + ";\n";
1515 }
1516 }
1517 // add some extra code needed for initialization of dynamic parameters
1519
1520 if (fUseWeightFile) {
1521 fGC += "\n//--- reading weights from file\n";
1523 fGC += "\n";
1524 // fUseWeightFile = fUseWeightFile;
1525 }
1526
1527 // now we have passed the parameters we can allocate the dynamic tensors
1529
1530 // add here initialization code for operator
1531 for (size_t id = 0; id < fOperators.size(); id++) {
1532 fGC += fOperators[id]->GenerateInitCode();
1533 }
1534
1535 fGC += "}\n\n";
1536
1538 // Models requested with a safetensors weight file but without weight
1539 // tensors also expose the blob constructor, which ignores the blob, so
1540 // that both construction modes exist (from a file and from an in-memory
1541 // blob) whatever the model.
1542 AddNeededHelperFunction("SafetensorsBlob");
1543 fGC += sessionName + "(SafetensorsBlob" + dynParamDecls + ")\n";
1544 fGC += " : " + sessionName + "(std::string{}" + dynParamArgs + ") {}\n\n";
1545 }
1546
1547 // Used to build the tangent Session objects needed to differentiate the
1548 // generated code with Clad: the derivatives of the (constant) weights
1549 // are zero, but a default-constructed Session holds the actual values.
1550 fGC += "// Set all weight and constant tensors to zero. This is useful to create\n"
1551 "// the tangent Session objects needed to differentiate the generated code\n"
1552 "// with Clad.\n"
1553 "void SetWeightsToZero() {\n";
1554 for (auto &i : fInitializedTensors) {
1555 // IsNotWritable tensors have no emitted fTensor_ member, and integer
1556 // tensors are structural (indices, shapes) that carry no tangent.
1557 if (i.second.IsNotWritable() ||
1558 (i.second.type() != ETensorType::FLOAT && i.second.type() != ETensorType::DOUBLE))
1559 continue;
1560 fGC += " for (auto &v : fTensor_" + i.first + ") v = 0;\n";
1561 }
1562 for (auto &graph : fSubGraphs) {
1563 fGC += " fSession_" + graph->fName + ".SetWeightsToZero();\n";
1564 }
1565 fGC += "}\n\n";
1566
1567 // generate the inference overload that returns an output struct
1569
1570 // end of session
1571 fGC += "}; // end of Session\n\n";
1572
1574
1575 fGC += doInferSignature + " {\n";
1576 fGC += "\n";
1577
1578 // generate the inference code
1579 if (fVerbose)
1580 std::cout << "Generating main inference code for " << fName << std::endl;
1581
1582 if (fOutputTensorNames.size() == 0)
1583 throw std::runtime_error("TMVA-SOFIE: output size=0 are not supported");
1584
1585 std::string allOperatorCode;
1586
1587 for (size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx) {
1588 if (fVerbose)
1589 std::cout << "Generating code for operator .... " << op_idx << std::endl;
1590 std::string operatorCode = fOperators[op_idx]->Generate(std::to_string(op_idx));
1591 allOperatorCode += operatorCode;
1592 }
1593
1594 // If the generated code users members of the session struct, use the
1595 // local variable name that we're using for the session:
1596 ReplaceAll(allOperatorCode, "this->", "session.");
1597
1598 // Collect all "tensor_*" data members that are not input or output tensors
1599 std::vector<std::string> tensorMemberNames = CollectTensorMemberNames(allOperatorCode);
1600 const std::string prefix = "tensor_";
1601 for (auto const& name: tensorMemberNames) {
1602 // alias tensors are not session members: the operator creating them declares a local pointer
1603 if (IsAliasTensor(name.substr(prefix.size())))
1604 continue;
1605 fGC += " auto &" + name + " = session." + name + ";\n";
1606 }
1607 fGC += "\n";
1608
1609 // an initialized tensor (constant, or a weight reached e.g. through an Identity) that is a
1610 // model output is not written by any operator: copy it into the output buffer before the
1611 // operator code, so that operators reading the weight also see its value (the output
1612 // parameter shadows the session member in doInfer)
1613 for (auto const &name : fOutputTensorNames) {
1615 std::string t = "session.tensor_" + name;
1617 fGC += " std::copy(" + t + ", " + t + " + " + std::to_string(length) + ", tensor_" + name + ");\n";
1618 }
1619 }
1620
1621 fGC += allOperatorCode;
1622
1623 std::unordered_set<std::string> assignedShapeParams;
1624 for (auto const& name: fOutputTensorNames) {
1625 bool isDynamic = fDynamicTensorInfos.count(name) > 0;
1626 if(isDynamic) {
1627 for (auto const &dim : GetDynamicTensorShape(name)) {
1628 if (dim.isParam && !IsInputTensorShapeParam(dim.param) && IsIdentifier(dim.param) &&
1629 assignedShapeParams.insert(dim.param).second)
1630 fGC += " " + dim.param + "_output = " + dim.param + ";\n";
1631 }
1632 }
1633 }
1634 fGC += "\n";
1635
1636 fGC += "}\n";
1637}
1638
1639void RModel::Generate(std::underlying_type_t<Options> options, int batchSize, bool verbose)
1640{
1641 fVerbose = verbose;
1642 fBatchSize = batchSize;
1643
1644 // weights are contained in the generated header when kNoWeightFile is used
1645 if (static_cast<std::underlying_type_t<Options>>(Options::kNoWeightFile) & options) {
1646 fUseWeightFile = false;
1648 }
1649 if (static_cast<std::underlying_type_t<Options>>(Options::kSafetensorsWeightFile) & options) {
1650 fUseWeightFile = true;
1652 }
1653
1654 // initialize the model including all operators and sub-graphs
1655 Initialize(batchSize, verbose);
1656
1657 std::string hgname;
1658 if (!fIsSubGraph) {
1659 fGC.clear();
1660 GenerateHeaderInfo(hgname);
1661 }
1662
1663 // generate first code for the subgraphs
1664 for (auto &graph : fSubGraphs) {
1665 if (fVerbose)
1666 std::cout << "generate session code for subgraph " << graph->fName << std::endl;
1667 graph->GenerateSessionCode();
1668 fGC += graph->fGC;
1669 }
1670
1671 if (fVerbose)
1672 std::cout << "generate Main session code - model " << fName << std::endl;
1673
1674 // generate main session code
1676
1677 if (!fIsSubGraph) {
1678 fGC += ("} //TMVA_SOFIE_" + fName + "\n");
1679 fGC += "\n#endif // " + hgname + "\n";
1680 // dump the standalone definitions of the helper functions this model uses
1681 // so that the generated header does not depend on TMVA/SOFIE_common.hxx
1683 }
1684}
1685
1687 // generate the code to read initialized tensors from a text data file
1689 // check if there are tensors to write
1690
1691 if (!fUseWeightFile) return;
1692
1693 fGC += " std::ifstream f;\n";
1694 fGC += " f.open(filename);\n";
1695 fGC += " if (!f.is_open()) {\n";
1696 fGC += " throw std::runtime_error(\"tmva-sofie failed to open file \" + filename + \" for input weights\");\n";
1697 fGC += " }\n";
1698
1699 // ReadTensorFromStream is emitted as a standalone helper in the header
1700 AddNeededHelperFunction("ReadTensorFromStream");
1701
1702 // loop on tensors and parse the file
1703 for (auto& i: fInitializedTensors) {
1704 // skip Constant and shape tensors (not written in a file)
1705 if (!i.second.IsWeightTensor()) continue;
1706 std::string tensor_name = "tensor_" + i.first;
1707 if (i.second.type() == ETensorType::FLOAT) {
1708 std::string length = std::to_string(ConvertShapeToLength(i.second.shape()));
1709 fGC += " ReadTensorFromStream(f, " + tensor_name + ", \"" + tensor_name + "\", " + length + ");\n";
1710 } else {
1711 throw std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be read from a file");
1712 }
1713 }
1714 fGC += " f.close();\n";
1715 }
1716
1717 // generate the code to read initialized tensors from a safetensors blob
1718 // in memory (the blob comes from a file or directly from the caller)
1720 // the SafetensorsBlob/SafetensorsReader helpers are emitted as
1721 // standalone helpers in the generated header
1722 AddNeededHelperFunction("SafetensorsReader");
1723
1724 fGC += " SafetensorsReader sofie_weights_reader(weights_blob);\n";
1725
1726 for (auto &i : fInitializedTensors) {
1727 // skip Constant and shape tensors (not written in a file)
1728 if (!i.second.IsWeightTensor())
1729 continue;
1730 std::string tensor_name = "tensor_" + i.first;
1731 std::string dtype;
1732 try {
1733 dtype = SafetensorsDType(i.second.type());
1734 } catch (const std::runtime_error &) {
1735 throw std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " +
1736 ConvertTypeToString(i.second.type()) +
1737 " cannot be read from a safetensors payload");
1738 }
1739 std::string length = std::to_string(ConvertShapeToLength(i.second.shape()));
1740 fGC += " sofie_weights_reader.Read(\"" + tensor_name + "\", fTensor_" + i.first + ", " + length + ", \"" +
1741 dtype + "\");\n";
1742 }
1743 }
1744}
1745
1747 // Determine the file extension based on the weight file type
1748 std::string fileExtension;
1749 switch (fWeightFile) {
1751 fileExtension = ".dat";
1752 break;
1754 fileExtension = ".safetensors";
1755 break;
1757 fileExtension = ".dat";
1758 break;
1759 }
1760
1761 // If filename is empty, use the model name as the base filename
1762 if (filename.empty()) {
1763 filename = fFileName + fileExtension;
1764 }
1765
1766 // Write the initialized tensors to the file
1768 std::ofstream f(filename, std::ios::binary);
1769 if (!f.is_open())
1770 throw std::runtime_error("tmva-sofie failed to open file " + filename + " for tensor weight data");
1771
1773
1774 long curr_pos = f.tellp();
1775 f.close();
1776 return curr_pos;
1777 } else if (fWeightFile == WeightFileType::Text) {
1778 std::ofstream f;
1779 f.open(filename);
1780 if (!f.is_open())
1781 throw
1782 std::runtime_error("tmva-sofie failed to open file " + filename + " for tensor weight data");
1783 for (auto& i: fInitializedTensors) {
1784 // skip Constant tensors and not writable tensors (e.g. shape tensors)
1785 if (!i.second.IsWeightTensor()) {
1786 continue;
1787 }
1788 size_t length = ConvertShapeToLength(i.second.shape());
1789 std::string tensor_name = "tensor_" + i.first;
1790 f << tensor_name << " " << length << "\n";
1791 if (i.second.type() == ETensorType::FLOAT) {
1792 const float * data = i.second.data<float>();
1793 for (size_t idx = 0; idx < length; idx++) {
1794 // round to zero sub-normal values
1795 float value = data[idx];
1796 if (value != 0. && std::abs(value) < std::numeric_limits<float>::min() ) value = 0;
1797 // handle non-finite values explicitly
1798 if (std::isinf(value))
1799 f << (value > 0 ? "inf" : "-inf");
1800 else if (std::isnan(value))
1801 f << "nan";
1802 else
1803 f << std::setprecision(std::numeric_limits<float>::max_digits10) << value;
1804 f << ( (idx < length-1) ? " " : "\n" );
1805 }
1806 }
1807 else {
1808 throw std::runtime_error("tmva-sofie tensor " + tensor_name + " with type " + ConvertTypeToString(i.second.type()) + " cannot be written to a file");
1809 }
1810 if (f.fail())
1811 throw std::runtime_error("tmva-sofie failed to write tensor data to file for " + tensor_name);
1812 }
1813 long curr_pos = f.tellp();
1814 f.close();
1815 return curr_pos;
1816 } else {
1817 return -1;
1818 }
1819}
1820
1822{
1823 // safetensors payloads are little-endian by specification; refuse to
1824 // write silently corrupted output on a big-endian host
1825 const std::uint16_t one = 1;
1826 if (!*reinterpret_cast<const std::uint8_t *>(&one))
1827 throw std::runtime_error("tmva-sofie: safetensors weights can only be written on a little-endian host");
1828
1829 // The safetensors layout (https://huggingface.co/docs/safetensors):
1830 // 8 bytes: little-endian unsigned size N of the JSON header
1831 // N bytes: JSON header {name: {"dtype", "shape", "data_offsets"}}
1832 // rest: the raw little-endian tensor payloads, concatenated;
1833 // data_offsets are relative to the start of this region
1834 std::uint64_t offset = 0;
1835 // sort the tensor names for a reproducible output
1836 std::vector<std::string> names;
1837 for (const auto &item : fInitializedTensors) {
1838 // skip Constant tensors and not writable tensors (e.g. shape tensors)
1839 if (item.second.IsWeightTensor())
1840 names.push_back(item.first);
1841 }
1842 std::sort(names.begin(), names.end());
1843 std::string headerStr = "{";
1844 for (size_t idx = 0; idx < names.size(); ++idx) {
1845 const auto &tensor = fInitializedTensors.at(names[idx]);
1846 const std::uint64_t nbytes = ConvertShapeToLength(tensor.shape()) * GetTypeSize(tensor.type());
1847 if (idx > 0)
1848 headerStr += ",";
1849 headerStr += "\"" + JsonEscape("tensor_" + names[idx]) + "\":{\"dtype\":\"" + SafetensorsDType(tensor.type()) +
1850 "\",\"shape\":[";
1851 for (size_t i = 0; i < tensor.shape().size(); ++i) {
1852 if (i > 0)
1853 headerStr += ",";
1854 headerStr += std::to_string(tensor.shape()[i]);
1855 }
1856 headerStr += "],\"data_offsets\":[" + std::to_string(offset) + "," + std::to_string(offset + nbytes) + "]}";
1857 offset += nbytes;
1858 }
1859 headerStr += "}";
1860 // 8-byte little-endian header length
1861 const std::uint64_t headerSize = headerStr.size();
1862 char sizestr[8];
1863 for (int i = 0; i < 8; ++i)
1864 sizestr[i] = static_cast<char>((headerSize >> (8 * i)) & 0xff);
1865 f.write(sizestr, 8);
1866 f.write(headerStr.data(), headerStr.size());
1867 for (const auto &name : names) {
1868 const auto &tensor = fInitializedTensors.at(name);
1869 const std::uint64_t nbytes = ConvertShapeToLength(tensor.shape()) * GetTypeSize(tensor.type());
1870 f.write(reinterpret_cast<const char *>(tensor.data<void>()), nbytes);
1871 }
1872 if (f.fail())
1873 throw std::runtime_error("tmva-sofie failed to write safetensors payload");
1874}
1875
1877{
1878 std::ostringstream buffer;
1880 return buffer.str();
1881}
1882
1884 std::cout << "Summary of model " << GetName() << std::endl;
1885 for(size_t op_idx = 0; op_idx < fOperators.size(); ++op_idx){
1886 auto& r = *fOperators[op_idx].get();
1887 std::string raw_name = typeid(r).name();
1888 // look for ROperator_NAME
1889 std::string name = raw_name.substr(raw_name.find("ROperator_")+10, raw_name.size());
1890 std::cout << op_idx << " " << name << " : ";
1891 for (auto & t_in : r.GetOpInputTensors()) std::cout << t_in << " ";
1892 std::cout << " ----> ";
1893 for (auto & t_out : r.GetOpOutputTensors()) std::cout << t_out << " ";
1894 std::cout << std::endl;
1895 }
1896}
1897
1898/// To emit the dimensions of the input tensors as a data member of a session,
1899/// which is helpful when validating the inference inputs.
1901{
1902 fGC += "\n// Input tensor dimensions\n";
1903 // SingleDim / TensorDims / makeDims are emitted as standalone helpers
1904 AddNeededHelperFunction("InputTensorDims");
1905 bool hasDynamicInputTensors = false;
1906
1907 for (std::size_t iInput = 0; iInput < fInputTensorNames.size(); ++iInput) {
1908 auto const &name = fInputTensorNames[iInput];
1909 if (IsDimInputTensor(name)) {
1910 hasDynamicInputTensors = true;
1911 }
1912 std::vector<Dim> shape = GetDimTensorShape(name);
1913 fGC += "constexpr std::array<SingleDim, " + std::to_string(shape.size()) + "> dim_" + name + "{";
1914 for (std::size_t iDim = 0; iDim < shape.size(); ++iDim) {
1915 auto const &dim = shape[iDim];
1916 if (dim.isParam) {
1917 fGC += "SingleDim{\"" + dim.GetVal() + "\"}";
1918 } else {
1919 fGC += "SingleDim{" + dim.GetVal() + "}";
1920 }
1921 if (iDim != shape.size() - 1) {
1922 fGC += ", ";
1923 }
1924 }
1925 fGC += "};\n";
1926 }
1927 fGC += "\nconstexpr std::array<TensorDims, " + std::to_string(fInputTensorNames.size()) + "> inputTensorDims{\n";
1928 for (std::size_t iInput = 0; iInput < fInputTensorNames.size(); ++iInput) {
1929 auto const &name = fInputTensorNames[iInput];
1930 fGC += SP + "makeDims(dim_" + name + ")";
1931 if (iInput == fInputTensorNames.size() - 1) {
1932 fGC += "\n";
1933 } else {
1934 fGC += ",\n";
1935 }
1936 }
1937 fGC += "};\n";
1938
1939 fGC +=
1940 "\nconstexpr bool hasDynamicInputTensors{" + std::string{hasDynamicInputTensors ? "true" : "false"} + "};\n\n";
1941
1942 fGC += "\n// Output tensor dimensions\n";
1943 bool hasDynamicOutputTensors = false;
1944 for (std::size_t iOutput = 0; iOutput < fOutputTensorNames.size(); ++iOutput) {
1945 auto const &name = fOutputTensorNames[iOutput];
1946 if (IsDynamicTensor(name)) {
1947 hasDynamicOutputTensors = true;
1948 }
1949 std::vector<Dim> shape = GetDimTensorShape(name);
1950 fGC += "constexpr std::array<SingleDim, " + std::to_string(shape.size()) + "> dim_" + name + "{";
1951 for (std::size_t iDim = 0; iDim < shape.size(); ++iDim) {
1952 auto const &dim = shape[iDim];
1953 if (dim.isParam) {
1954 fGC += "SingleDim{\"" + dim.GetVal() + "\"}";
1955 } else {
1956 fGC += "SingleDim{" + dim.GetVal() + "}";
1957 }
1958 if (iDim != shape.size() - 1) {
1959 fGC += ", ";
1960 }
1961 }
1962 fGC += "};\n";
1963 }
1964 fGC += "\nconstexpr std::array<TensorDims, " + std::to_string(fOutputTensorNames.size()) + "> outputTensorDims{\n";
1965 for (std::size_t iOutput = 0; iOutput < fOutputTensorNames.size(); ++iOutput) {
1966 auto const &name = fOutputTensorNames[iOutput];
1967 fGC += SP + "makeDims(dim_" + name + ")";
1968 if (iOutput == fOutputTensorNames.size() - 1) {
1969 fGC += "\n";
1970 } else {
1971 fGC += ",\n";
1972 }
1973 }
1974 fGC += "};\n";
1975 fGC +=
1976 "\nconstexpr bool hasDynamicOutputTensors{" + std::string{hasDynamicOutputTensors ? "true" : "false"} + "};\n\n";
1977}
1978
1980 std::cout << "Model requires following inputs:\n";
1981 for (auto& inputInfo: fInputTensorInfos) {
1982 std::cout << "Parametrised Tensor name: " << inputInfo.first << "\t";
1983 std::cout << "type: " << ConvertTypeToString(inputInfo.second.type) << "\t";
1984 std::cout << "shape: [";
1985 for (size_t i = 0; i < inputInfo.second.shape.size(); i++) {
1986 if (inputInfo.second.shape[i].isParam) {
1987 std::cout << inputInfo.second.shape[i].param;
1988 } else {
1989 std::cout << inputInfo.second.shape[i].dim ;
1990 }
1991 if (i < inputInfo.second.shape.size() - 1) std::cout << ",";
1992 }
1993 std::cout << "]" << std::endl;
1994 }
1995
1996 for (auto& inputInfo: fReadyInputTensorInfos) {
1997 std::cout << "Fully Specified Tensor name: " << inputInfo.first << "\t";
1998 std::cout << "type: " << ConvertTypeToString(inputInfo.second.type) << "\t";
1999 std::cout << "shape: [";
2000 for (size_t i = 0; i < inputInfo.second.shape.size(); i++) {
2001 std::cout << inputInfo.second.shape[i];
2002 if (i < inputInfo.second.shape.size() - 1) std::cout << ",";
2003 }
2004 std::cout << "]" << std::endl;
2005 }
2006 std::cout << "\n";
2007}
2008
2010 std::cout << "Model initialized the following tensors:\n";
2011 for (auto& it: fInitializedTensors) {
2012 std::cout << "Tensor name: \"" << it.first << "\"\t";
2013 std::cout << "type: " << ConvertTypeToString(it.second.type()) << "\t";
2014 std::cout << "shape: [";
2015 for (size_t i = 0; i < it.second.shape().size(); i++) {
2016 std::cout << it.second.shape()[i];
2017 if (i < it.second.shape().size() - 1) std::cout << ",";
2018 }
2019 std::cout << "]";
2020 if (it.second.IsConstantTensor()) std::cout << " (Constant)";
2021 if (it.second.IsNotWritable()) std::cout << " (Not Writable)";
2022 std::cout << std::endl;
2023 }
2024 std::cout << "\n";
2025}
2026
2028 std::cout << "Model specify the following intermediate tensors:\n";
2029 for (auto& it: fIntermediateTensorInfos) {
2030 std::cout << "Tensor name: \"" << it.first << "\"\t";
2031 std::cout << "type: " << ConvertTypeToString(it.second.type) << "\t";
2032 std::cout << "shape: [";
2033 for (size_t i = 0; i < it.second.shape.size(); i++) {
2034 std::cout << it.second.shape[i];
2035 if (i < it.second.shape.size() - 1) std::cout << ",";
2036 }
2037 std::cout << "]" << std::endl;
2038 }
2039 std::cout << "\n";
2040}
2041
2043 std::cout << "Model specify the following dynamic tensors:\n";
2044 for (auto& it: fDynamicTensorInfos) {
2045 std::cout << "Tensor name: \"" << it.first << "\"\t";
2046 std::cout << "type: " << ConvertTypeToString(it.second.type) << "\t";
2047 std::cout << "shape: [";
2048 for (size_t i = 0; i < it.second.shape.size(); i++) {
2049 std::cout << it.second.shape[i].GetVal();
2050 if (i < it.second.shape.size() - 1) std::cout << ",";
2051 }
2052 std::cout << "]" << std::endl;
2053 }
2054 std::cout << "\n";
2055}
2056
2058 std::cout << "Model specify the following output tensors:\n";
2059 for (auto& it: fOutputTensorNames) {
2060 std::cout << "Tensor name: \"" << it << "\"\t";
2061 try {
2062 auto shape = GetDimTensorShape(it);
2063 std::cout << "with shape: " << ConvertDimShapeToString(shape) << std::endl;
2064 } catch (...) {
2065 std::cout << "with shape not yet defined" << std::endl;
2066 }
2067 }
2068 std::cout << "\n";
2069}
2070
2071void RModel::HeadInitializedTensors(std::string name, int n_print) {
2072 auto it = fInitializedTensors.find(name);
2073 if (it == fInitializedTensors.end()) {
2074 std::cout << "Tensor " << name << " not found in model's initialized tensor list" << std::endl;
2075 return;
2076 }
2077
2078 std::cout << "Tensor name: " << it->first << "\t";
2079 std::cout << "type: " << ConvertTypeToString(it->second.type()) << "\t";
2080 int length =1;
2081 std::cout << "shape: [";
2082 for (size_t i = 0; i < it->second.shape().size(); i++) {
2083 std::cout << it->second.shape()[i];
2084 length *= it->second.shape()[i];
2085 if (i < it->second.shape().size() - 1) std::cout << ",";
2086 }
2087 std::cout << "]" << std::endl;
2088 bool ellipsis = true;
2089 if (n_print > length) {
2090 n_print = length;
2091 ellipsis = false;
2092 }
2093
2094 std::cout << "data: [" << std::endl;
2095 if (it->second.type() == ETensorType::FLOAT) {
2096 auto converted_data = it->second.data<float>();
2097 for (int i =0; i < n_print; i++) {
2098 std::cout << converted_data[i];
2099 if (i < n_print - 1) std::cout << " ,";
2100 }
2101 }
2102 if (ellipsis) std::cout << ", ...";
2103 std::cout << "]" << std::endl;
2104
2105}
2106
2107void RModel::GenerateHeaderInfo(std::string& hgname) {
2108 fGC += ("//Code generated automatically by TMVA for Inference of Model file [" + fFileName + "] at [" + fParseTime.substr(0, fParseTime.length()-1) +"] \n");
2109 // add header guards
2110 hgname = fName;
2111 std::transform(hgname.begin(), hgname.end(), hgname.begin(), [](unsigned char c) {
2112 return std::toupper(c);
2113 } );
2114 hgname = "ROOT_TMVA_SOFIE_" + hgname;
2115 fGC += "\n#ifndef " + hgname + "\n";
2116 fGC += "#define " + hgname + "\n\n";
2117 // Standard library headers the generated code relies on. Listed explicitly
2118 // now that they are no longer pulled in transitively via SOFIE_common.hxx.
2119 for (const char *h : {"cstdint", "cstring", "string", "vector", "map", "memory", "sstream", "iostream", "iomanip",
2120 "limits", "stdexcept", "algorithm", "cmath", "cassert"}) {
2121 fNeededStdLib.insert(h);
2122 }
2123 for (auto& i: fNeededStdLib) {
2124 fGC += "#include <" + i + ">\n";
2125 }
2126 for (auto& i: fCustomOpHeaders) {
2127 fGC += "#include \"" + i + "\"\n";
2128 }
2129 // Placeholder for the #include directives needed by the embedded helper
2130 // functions (filled in by EmitHelperFunctionsCode).
2132 if (fUseWeightFile)
2133 fGC += "#include <fstream>\n";
2134 // the safetensors file constructor buffers the payload with
2135 // std::istreambuf_iterator
2137 fGC += "#include <iterator>\n";
2138
2139 fGC += "\nnamespace TMVA_SOFIE_" + fName + "{\n";
2140 if (!fNeededBlasRoutines.empty()) {
2141 fGC += ("namespace BLAS{\n");
2142 for (auto &routine : fNeededBlasRoutines) {
2143 if (routine == "Gemm") {
2144 fGC += ("\textern \"C\" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k,\n"
2145 "\t const float * alpha, const float * A, const int * lda, const float * B, const int * ldb,\n"
2146 "\t const float * beta, float * C, const int * ldc);\n");
2147 // sgemm_ now declared; the standalone Gemm_Call helper will skip its copy.
2148 fBlasSgemmDeclared = true;
2149 } else if (routine == "Gemv") {
2150 fGC += ("\textern \"C\" void sgemv_(const char * trans, const int * m, const int * n, const float * alpha, const float * A,\n"
2151 "\t const int * lda, const float * X, const int * incx, const float * beta, const float * Y, const int * incy);\n");
2152 } else if (routine == "Axpy") {
2153 fGC += ("\textern \"C\" void saxpy_(const int * n, const float * alpha, const float * x,\n"
2154 "\t const int * incx, float * y, const int * incy);\n");
2155 } else if (routine == "Copy") {
2156 fGC += ("\textern \"C\" void scopy_(const int *n, const float* x, const int *incx, float* y, const int* incy);\n");
2157 }
2158 }
2159 fGC += ("}//BLAS\n");
2160 }
2161 // Placeholder for the standalone definitions of the inference helper
2162 // functions used by this model (filled in by EmitHelperFunctionsCode). It
2163 // sits inside the generated model namespace, right before the session code.
2165}
2166
2168{
2169 HelperFunctionsCode code =
2171
2172 auto replaceMarker = [this](const std::string &marker, const std::string &replacement) {
2173 auto pos = fGC.find(marker);
2174 if (pos != std::string::npos) {
2175 fGC.replace(pos, marker.size(), replacement);
2176 }
2177 };
2178
2179 replaceMarker(kHelperIncludesMarker, code.includes);
2180 replaceMarker(kHelperFunctionsMarker, code.definitions);
2181
2182 // Clad derivatives live at file scope and reference the model's helpers, so
2183 // insert them after the whole model namespace, just before the include guard.
2184 if (!code.cladDefinitions.empty()) {
2185 auto pos = fGC.rfind("#endif");
2186 if (pos != std::string::npos) {
2187 fGC.insert(pos, code.cladDefinitions + "\n");
2188 } else {
2189 fGC += code.cladDefinitions;
2190 }
2191 }
2192}
2193
2194void RModel::OutputGenerated(std::string filename, bool append) {
2195
2196 // the model can be appended only if a file name is provided
2197 if (filename.empty()) {
2198 // if a file is pr
2199 filename = fName + ".hxx";
2200 append = false;
2201 }
2202 std::ofstream f;
2203 if (append)
2204 f.open(filename, std::ios_base::app);
2205 else
2206 f.open(filename);
2207 if (!f.is_open()) {
2208 throw std::runtime_error("tmva-sofie failed to open file for output generated inference code");
2209 }
2210 f << fGC;
2211 f.close();
2212
2213 // write weights in a separate weight file
2214 if (fUseWeightFile) {
2215 const std::string extension = fWeightFile == WeightFileType::Safetensors ? ".safetensors" : ".dat";
2216 if (!filename.empty()) {
2217 size_t pos = filename.find(".hxx");
2218 if (pos != std::string::npos)
2219 filename.replace(pos, 4, extension);
2220 else
2221 filename += extension;
2222 } else {
2223 filename = fName + extension;
2224 }
2226 }
2227}
2228
2229} // namespace SOFIE::Experimental::TMVA
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define a(i)
Definition RSha256.hxx:99
#define h(i)
Definition RSha256.hxx:106
#define e(i)
Definition RSha256.hxx:103
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize id
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:142
std::string WriteInitializedTensorsToBuffer()
Definition RModel.cxx:1876
void AddShapeParam(const std::string &name, size_t def_value=0)
Definition RModel.cxx:416
void AddNeededHelperFunction(std::string name)
Definition RModel.hxx:327
std::vector< size_t > GetTensorShape(const std::string &name) const
Definition RModel.cxx:108
std::set< std::string > fNeededHelperFunctions
Definition RModel.hxx:70
std::unordered_set< std::string > fComputedShapeParams
! shape parameters computed at run time by an operator
Definition RModel.hxx:101
std::vector< Dim > GetDimTensorShape(const std::string &name) const
Definition RModel.cxx:144
std::unordered_map< std::string, DynamicTensorInfo > fDynamicTensorInfos
Definition RModel.hxx:98
bool IsDynamicTensor(const std::string &name) const
Definition RModel.cxx:367
void AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector< Dim > dim_shape)
Definition RModel.cxx:382
bool AddAliasTensor(const std::string &tensor_name, const std::string &orig_tensor_name)
Definition RModel.cxx:306
std::string GenerateInferSignature(bool isdecl=true)
Definition RModel.cxx:1188
std::unordered_set< std::string > fNeededBlasRoutines
Definition RModel.hxx:58
bool CheckIfTensorAlreadyExist(std::string tensor_name)
Definition RModel.cxx:201
void GenerateHeaderInfo(std::string &hgname)
Definition RModel.cxx:2107
void GenerateRequiredInputTensorInfo()
To emit the dimensions of the input tensors as a data member of a session, which is helpful when vali...
Definition RModel.cxx:1900
void OutputGenerated(std::string filename="", bool append=false)
Definition RModel.cxx:2194
std::unordered_map< std::string, std::string > fAliasTensors
Definition RModel.hxx:102
void AddInputTensorInfo(std::string input_name, ETensorType type, std::vector< Dim > shape)
Definition RModel.cxx:212
std::unordered_map< std::string, TensorInfo > fIntermediateTensorInfos
Definition RModel.hxx:97
void AddOutputTensorNameList(std::vector< std::string > output_tensor_names)
Definition RModel.cxx:436
std::unordered_map< std::string, TensorInfo > fReadyInputTensorInfos
Definition RModel.hxx:95
void AddConstantTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:288
void AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector< Dim > shape)
Definition RModel.cxx:399
std::vector< std::string > fDimShapeNames
Definition RModel.hxx:103
void AddInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:268
std::unordered_map< std::string_view, size_t > fIntermediateTensorFrequencyLookup
! lookup table for intermediate tensor frequency (transient)
Definition RModel.hxx:122
void AddBlasRoutines(std::vector< std::string > routines)
Definition RModel.hxx:310
void AddInputTensorName(std::string name)
Definition RModel.cxx:231
std::vector< std::string > fOutputTensorNames
Definition RModel.hxx:104
void AddNeededStdLib(std::string libname)
Definition RModel.hxx:316
bool IsDimInputTensor(const std::string &name) const
Definition RModel.cxx:372
bool IsShapeTensor(const std::string &name) const
check if a tensor is a shape tensor
Definition RModel.cxx:341
static constexpr const char * kHelperIncludesMarker
Definition RModel.hxx:80
bool IsInitializedTensor(const std::string &name) const
Definition RModel.cxx:354
std::unordered_set< std::string > fNeededStdLib
Definition RModel.hxx:64
bool IsAliasTensor(const std::string &name) const
check if a tensor is a alias tensor
Definition RModel.cxx:345
static constexpr const char * kHelperFunctionsMarker
Definition RModel.hxx:81
void CheckAndFlushIntermediateMemory(std::span< const std::string_view > op_output_tensors, const size_t &op_idx)
Definition RModel.cxx:581
void AddOperator(std::unique_ptr< ROperator > op, int order_execution=-1)
Definition RModel.cxx:235
RModel()=default
Default constructor.
void HeadInitializedTensors(std::string name, int n_print=50)
Definition RModel.cxx:2071
bool IsConstantTensor(const std::string &name) const
Definition RModel.cxx:358
void WriteInitializedTensorsToStream(std::ostream &os)
Definition RModel.cxx:1821
void Initialize(int batchSize=-1, bool verbose=false)
Definition RModel.cxx:662
long WriteInitializedTensorsToFile(std::string filename="")
Definition RModel.cxx:1746
OptimizationLevel fOptimizationLevel
Definition RModel.hxx:92
std::vector< std::string > CollectTensorMemberNames(const std::string &input)
Collects all identifiers starting with "tensor_" in the input code, provided that the occurrence is n...
Definition RModel.cxx:1130
std::vector< Dim > GetDynamicTensorShape(const std::string &name) const
Definition RModel.cxx:155
std::unordered_map< std::string, InputTensorInfo > fInputTensorInfos
Definition RModel.hxx:94
std::shared_ptr< void > GetInitializedTensorData(std::string tensor_name)
Definition RModel.cxx:459
void AddComputedShapeParam(const std::string &name)
Declare a shape parameter as computed at run time by an operator (e.g.
Definition RModel.cxx:427
MemoryPoolInfo fIntermediateMemoryInfo
! intermediate memory info (transient)
Definition RModel.hxx:121
std::string AllocateIntermediateMemory(std::span< const std::string_view > op_output_tensors)
Definition RModel.cxx:476
std::unordered_map< std::string, std::pair< std::vector< Dim >, bool > > fShapeTensors
Definition RModel.hxx:99
std::vector< std::unique_ptr< ROperator, ROperatorDeleter > > fOperators
Definition RModel.hxx:115
void InitializeSubGraph(std::shared_ptr< RModel > graph)
Definition RModel.cxx:825
std::unordered_map< std::string, std::string > fShapeParams
Definition RModel.hxx:100
void SetNotWritableInitializedTensor(const std::string &tensor_name)
Definition RModel.cxx:468
const std::string & GetName() const
Definition RModel.hxx:308
ETensorType GetTensorType(std::string name) const
Definition RModel.cxx:169
std::vector< std::string > fInputTensorNames
Definition RModel.hxx:105
std::unordered_map< std::string, InitializedTensor > fInitializedTensors
Definition RModel.hxx:96
void UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:450
void Generate(std::underlying_type_t< Options > options, int batchSize=-1, bool verbose=false)
Definition RModel.cxx:1639
const std::vector< Dim > & GetShapeTensorValues(const std::string &tensor_name) const
Definition RModel.cxx:349
std::vector< std::shared_ptr< RModel > > fSubGraphs
! sub-graph models (transient)
Definition RModel.hxx:117
bool IsReadyInputTensor(const std::string &name) const
Definition RModel.cxx:376
void UpdateOutputTensorList(std::vector< std::string > curr_output_tensor, std::vector< std::string > modify_output_tensor)
Definition RModel.cxx:443
void AddShapeTensor(const std::string &name, const std::vector< Dim > &shapeValues, bool scalar=false)
Definition RModel.cxx:298
std::unordered_set< std::string > fCustomOpHeaders
Definition RModel.hxx:65
bool IsInputTensorShapeParam(std::string const &name) const
Check if a given parameter is used for the shape of an input tensor.
Definition RModel.cxx:1112
const Int_t n
Definition legend1.C:16
std::string Clean_name(std::string input_tensor_name)
std::string ConvertDimShapeToString(const std::vector< Dim > &shape)
std::size_t ConvertShapeToLength(const std::vector< size_t > &shape)
std::string ConvertValuesToString(size_t n, const T *data, size_t maxprint=-1)
std::vector< Dim > ConvertShapeToDim(const std::vector< size_t > &shape)
Convert shape from integer format to dynamic one (based on Dim)
constexpr size_t GetTypeSize(ETensorType type)
std::string GenerateConstantTensorCode(const std::pair< std::string, InitializedTensor > &t)
Definition RModel.cxx:860
std::vector< size_t > ConvertShapeToInt(const std::vector< Dim > &shape)
Convert shape based on Dim to integer format.
std::string ConvertTypeToString(ETensorType type)
HelperFunctionsCode GenerateHelperFunctionsCode(const std::set< std::string > &neededHelpers, const std::string &modelNamespace, bool sgemmAlreadyDeclared=false)
Return the standalone C++ source of the inference helper functions requested in neededHelpers (see RM...
std::underlying_type_t< Options > operator|(Options opA, Options opB)
Definition RModel.cxx:101
std::string ConvertDimShapeToLength(const std::vector< Dim > &shape)
std::string ConvertShapeToString(const std::vector< size_t > &shape)
std::string ConvertValToString(T value)
if(fPos !=fText.size()) Fail("unexpected trailing content")
Source code of the inference helper functions to embed in generated code so that it is standalone and...
std::string definitions
function/type definitions to place inside the generated model namespace
std::string cladDefinitions
Clad custom-derivative definitions to place at file scope (outside the model namespace) so that Clad ...
std::string includes
#include directives to place in the header preamble
std::map< size_t, TensorMemoryInfo > total_stack
std::map< size_t, size_t > available_stack