18 std::default_delete<ROperator>()(ptr);
27const std::string
SP =
" ";
29void ReplaceAll(std::string &str,
const std::string &from,
const std::string &to)
32 while ((pos = str.find(from, pos)) != std::string::npos) {
33 str.replace(pos, from.length(), to);
38bool IsIdentifierChar(
char c)
40 return std::isalnum(
static_cast<unsigned char>(
c)) ||
c ==
'_';
46bool IsIdentifier(
const std::string &s)
48 if (s.empty() || std::isdigit(
static_cast<unsigned char>(s[0])))
51 if (!IsIdentifierChar(
c))
57std::string TensorMember(std::string
const &
name)
59 return "tensor_" +
name;
64std::string SafetensorsDType(ETensorType
type)
72 " cannot be written to a safetensors file");
78std::string JsonEscape(
const std::string &s)
81 out.reserve(s.size() + 2);
82 for (
const char c : s) {
84 case '"': out +=
"\\\"";
break;
85 case '\\': out +=
"\\\\";
break;
87 if (
static_cast<unsigned char>(
c) < 0x20) {
89 std::snprintf(buf,
sizeof(buf),
"\\u%04x",
c);
102 return static_cast<std::underlying_type_t<Options>
>(opA) |
static_cast<std::underlying_type_t<Options>
>(opB);
104std::underlying_type_t<Options>
operator|(std::underlying_type_t<Options> opA,
Options opB) {
105 return opA |
static_cast<std::underlying_type_t<Options>
>(opB);
111 return f->second.shape;
115 return f2->second.shape();
119 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] is an input tensor with unspecified dimension parameter");
123 return f4->second.shape;
129 if (f5->second.second)
130 return std::vector<size_t>{};
132 return std::vector<size_t>{f5->second.first.size()};
136 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] is a dynamic tensor. Use GetDynamicTensorShape instead of GetTensorShape");
141 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] for which the shape is requested is not found");
146 return f->second.shape;
149 return f->second.shape;
157 return f->second.shape;
160 return f->second.shape;
164 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] for which the shape is requested is not dynamic");
166 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] for which the shape is requested is not found");
172 return f->second.type;
176 return f2->second.type();
180 return f3->second.type;
184 return f4->second.type;
188 return f5->second.type;
198 throw std::runtime_error(
"TMVA SOFIE tensor [" +
name +
"] for which the type is requested is not found, model name: " +
fName);
215 throw std::runtime_error(
"TMVA-SOFIE: input tensor with name " + input_name +
" already exists \n");
225 throw std::runtime_error(
"TMVA-SOFIE: input tensor with name " + input_name +
" already exists \n");
238 auto libs = op->GetStdLibs();
239 auto op_input_tensors = op->GetOpInputTensors();
240 for (
auto &stdlib : libs) {
244 std::unique_ptr<ROperator, ROperatorDeleter> opStored(op.release());
245 if (order_execution >= 0) {
262 std::cout <<
"adding order execution for " << op_input_tensors[
index] <<
" order " << order_execution
272 throw std::runtime_error(
"TMVA-SOFIE: initialized tensor with name " + tensor_name +
" already exists \n");
279 const std::vector<std::size_t> &shape,
void *raw_data)
283 std::shared_ptr<void>
data(malloc(
size * itemsize), free);
284 std::memcpy(
data.get(), raw_data,
size * itemsize);
292 throw std::runtime_error(
"TMVA-SOFIE: constant tensor with name " + tensor_name +
" already exists \n");
301 throw std::runtime_error(
"TMVA-SOFIE: shape tensor with name " + tensor_name +
" already exists \n");
303 fShapeTensors[tensor_name] = std::make_pair(shape_values, scalar);
315 throw std::runtime_error(
"TMVA-SOFIE: alias tensor with name " + tensor_name +
" already exists \n");
319 origin_name = it->second;
363 return itr->second.IsConstantTensor();
384 if (!int_shape.empty())
393 throw std::runtime_error(
"TMVA-SOFIE: intermediate tensor with name " + tensor_name +
" already exists \n");
402 throw std::runtime_error(
"TMVA-SOFIE: intermediate tensor with name " + tensor_name +
" already exists \n");
407 for (
auto &
d : shape) {
409 if (
d.dim !=
size_t(-1)) {
438 for(
auto& it : outputtensornames) {
444 for(
auto& it:curr_output_tensors) {
453 throw std::runtime_error(
"TMVA-SOFIE: tensor " + tensor_name +
" not found when trying to update it");
462 throw std::runtime_error(
"TMVA-SOFIE: tensor " + tensor_name +
" not found when trying to get its data");
464 return f->second.sharedptr();
471 throw std::runtime_error(
"TMVA-SOFIE: initialized tensor " + tensor_name +
" not found when trying to get its info");
473 t->second.SetNotWritable();
478 std::stringstream code;
481 std::cout <<
"Total chunks allocated\n";
483 std::cout <<
"..... chunk " << chunk->first <<
" size " << chunk->second.tensor_size <<
" " << chunk->second.tensor_name << std::endl;
487 auto declareIntermediateTensor = [
this, &code](std::string
const &
name,
size_t size,
size_t location) {
489 code <<
"\n // Allocating memory for intermediate tensor " <<
name <<
" with size " <<
size <<
" bytes";
491 << typeName <<
"* " << TensorMember(
name) <<
" = reinterpret_cast<" << typeName
492 <<
"*>(fIntermediateMemoryPool.data() + " << location <<
");\n";
495 if (
fVerbose) std::cout <<
"*** AllocateIntermediateMemory: Loop on op output tensors\n";
497 std::vector<TensorMemoryInfo> ordered_output_tensors;
499 for (
auto &it : op_output_tensors) {
500 auto name = std::string(it);
513 ordered_output_tensors.push_back(tmi);
515 std::sort(ordered_output_tensors.begin(), ordered_output_tensors.end(),
518 for (
auto &it : ordered_output_tensors) {
519 bool allocated =
false;
520 std::string
name = std::string{it.tensor_name};
521 size_t tensor_size = it.tensor_size;
523 std::cout <<
"output tensor " <<
name <<
" size " << tensor_size << std::endl;
528 if (
fVerbose) std::cout <<
".. available chunk " << chunk->first <<
" with size = " << chunk->second;
530 if (chunk->second >= tensor_size) {
534 auto new_chunk_location = chunk->first + chunk->second - tensor_size;
537 declareIntermediateTensor(
name, tensor_size, new_chunk_location);
538 chunk->second -= tensor_size;
542 if (
fVerbose) std::cout <<
" is re-used and split in a new of size " << new_chunk.tensor_size <<
" at " << new_chunk_location;
544 if (chunk->second == 0) {
545 if (
fVerbose) std::cout <<
" and deleted since size matches";
548 if (
fVerbose) std::cout << std::endl;
554 declareIntermediateTensor(
name, tensor_size, chunk->first);
557 if (
fVerbose) std::cout <<
" is extended with a bigger one of size " << tensor_size << std::endl;
561 if (
fVerbose) std::cout << std::endl;
572 declareIntermediateTensor(
name, tensor_size, chunk_idx);
574 if (
fVerbose) std::cout <<
"no chunk available - add in total stack a new chunk with size of tensor and idx : " << chunk_idx
582 if (
fVerbose) std::cout <<
"*** CheckAndFlushIntermediateMemory: Loop on input tensors for op " << op_idx <<
"\n";
584 if (
fVerbose) std::cout <<
"available chunks before freeing them : \n";
587 if (
fVerbose) std::cout <<
"-- free chunk " << chunk->first <<
" size = " << chunk->second << std::endl;
589 for (
auto &iv : op_input_tensors) {
591 if (
fVerbose) std::cout <<
".. input tensors : " << iv;
600 if (
fVerbose) std::cout <<
" flash condition is met - looping on chunks to find matching one \n";
603 if (
fVerbose) std::cout <<
"--- chunk " << chunk->first <<
" , " << chunk->second.tensor_name <<
" size " << chunk->second.tensor_size;
604 if (chunk->second.tensor_name == it) {
605 if (
fVerbose) std::cout <<
" -- Found chunk corresponding to input tensor: " << chunk->first;
611 : std::prev(first_greater);
616 last_smaller->first + last_smaller->second == chunk->first) {
618 last_smaller->second += chunk->second.tensor_size;
620 if (
fVerbose) std::cout <<
" is adjacent in memory with previous one - merge ";
622 last_smaller->first + last_smaller->second == first_greater->first) {
624 last_smaller->second += first_greater->second;
630 if (
fVerbose) std::cout <<
" merge also with following that is free ";
633 if (
fVerbose) std::cout << std::endl;
636 chunk->first + chunk->second.tensor_size == first_greater->first) {
638 if (
fVerbose) std::cout <<
" is adjacent in memory with following one - merge \n";
640 size_t new_size = chunk->second.tensor_size + first_greater->second;
641 size_t first_greater_idx = first_greater->first;
650 if (
fVerbose) std::cout <<
" insert in the available stack the chunk with size " << chunk->second.tensor_size << std::endl;
652 chunk->second.tensor_name =
"free";
657 if (
fVerbose) std::cout << std::endl;
663 std::map<std::string, size_t> inputParams;
665 inputParams[
"input_size"] = batchSize;
666 inputParams[
"batch_size"] = batchSize;
667 inputParams[
"bs"] = batchSize;
678 std::cout <<
"Model is already initialized - skip initialization " << std::endl;
693 auto found = originalInputTensorInfos.find(inputName);
694 if (found == originalInputTensorInfos.end())
696 auto &
input = *found;
697 if (verbose) std::cout <<
"looking at the tensor " <<
input.first << std::endl;
699 if (!inputParams.empty()) {
700 for (
auto &
d :
input.second.shape) {
702 std::string pname =
d.param;
703 if (pname ==
input.first +
"_size") pname =
"input_size";
704 auto itr = inputParams.find(pname);
705 if (itr != inputParams.end() ) {
706 d =
Dim{ itr->second };
708 std::cout <<
"Tensor: " <<
input.first <<
" - fix parametric shape " << itr->first <<
" to " << itr->second << std::endl;
718 if (!shape.empty()) {
730 for (
auto &
d :
input.second.shape) {
747 std::vector<size_t> temp_available_stack;
750 std::unordered_set<std::string> runtimeInitializedInputs;
751 for(
size_t op_idx = 0; op_idx <
fOperators.size(); ++op_idx){
754 std::cout <<
"Initializing operator " << i <<
" " <<
typeid(
r).
name() << std::endl;
757 for(
auto &it:
fOperators[op_idx]->GetOpOutputTensors()){
758 std::string
name = std::string{it};
768 if (!
fOperators[op_idx]->IsOutputConstant()) {
769 for (
auto &it :
fOperators[op_idx]->GetOpInputTensors()) {
770 std::string
name = std::string{it};
772 runtimeInitializedInputs.insert(
name);
785 if (it.second.IsNotWritable() && runtimeInitializedInputs.find(it.first) != runtimeInitializedInputs.end()) {
786 it.second.SetWritable();
788 std::cout <<
"Initialized tensor " << it.first <<
" is flagged as not writable but is used by non constant operators, set it as writable \n";
794 it.second.SetConstant();
800 bool modelHasWeights =
false;
802 if (it.second.IsWeightTensor()) {
803 modelHasWeights =
true;
807 if (!modelHasWeights)
829 graph->fParentGraph =
this;
830 graph->fIsSubGraph =
true;
837 std::vector<std::string> blasRoutines;
838 for (
auto &
e : graph->fNeededBlasRoutines)
839 blasRoutines.push_back(
e);
841 for (
auto e : graph->fNeededStdLib)
845 for (
auto const &
h : graph->GetNeededHelperFunctions())
850 graph->fInputTensorNames.emplace_back(
name);
862 std::stringstream strs;
867 bool allocateOnStack = (
length > 100 || t.second.IsWeightTensor()) ?
false :
true;
869 const T *
data = t.second.data<T>();
872 bool sameData =
false;
875 if (!allocateOnStack &&
length > 1) {
878 sameData = (
data[idx] ==
data[idx - 1]);
880 }
while (sameData && idx <
length);
882 if (allocateOnStack) {
884 strs <<
type <<
" * " << TensorMember(t.first) <<
" = fTensor_" + t.first +
";\n";
886 strs <<
"std::vector<" <<
type <<
"> fTensor_" << t.first <<
" = ";
892 strs <<
type <<
" * " << TensorMember(t.first) <<
" = fTensor_" + t.first +
".data();\n";
900 fGC +=
"// initialized (weights and constant) tensors\n";
904 if (i.second.IsNotWritable())
continue;
909 bool hasInfOrNaN =
false;
910 const float *
data = i.second.data<
float>();
911 for (
size_t idx = 0; idx <
length; idx++) {
912 if (std::is_floating_point<float>::value) {
913 if (std::isinf(
data[idx]) || std::isnan(
data[idx])) {
921 fGC += GenerateConstantTensorCode<float>(i);
924 fGC += GenerateConstantTensorCode<int64_t>(i);
927 fGC += GenerateConstantTensorCode<int32_t>(i);
930 fGC += GenerateConstantTensorCode<uint8_t>(i);
938 fGC +=
"std::vector<float> fTensor_" + i.first +
" = std::vector<float>(" + std::to_string(
length) +
");\n";
939 fGC +=
"float * " + TensorMember(i.first) +
" = fTensor_" + i.first +
".data();\n";
948 fGC +=
"\n//--- Allocating session memory pool to be used for allocating intermediate tensors\n";
953 const size_t memPoolSize = totalStack.rbegin()->first + totalStack.rbegin()->second.tensor_size;
954 fGC +=
"std::vector<char> fIntermediateMemoryPool = std::vector<char>(" + std::to_string(memPoolSize) +
");\n\n";
959 std::string tensor_declaration_block =
"";
966 tensor_declaration_block +=
"std::vector<std::uint8_t> fTensor_" + i.first +
967 " = std::vector<std::uint8_t>(" +
969 tensor_declaration_block +=
970 "std::uint8_t * " + TensorMember(i.first) +
" = fTensor_" + i.first +
".data();\n";
974 bool not_in_freq_map =
976 bool not_in_output_names =
979 if ((not_in_freq_map && not_in_output_names) || (!not_in_freq_map && !is_extended && not_in_output_names)) {
983 tensor_declaration_block +=
"std::vector<float> fTensor_" + i.first +
" = std::vector<float>(" + std::to_string(
length) +
");\n";
984 tensor_declaration_block +=
"float * " + TensorMember(i.first) +
" = fTensor_" + i.first +
".data();\n";
988 tensor_declaration_block +=
"std::vector<double> fTensor_" + i.first +
" = std::vector<double>(" + std::to_string(
length) +
");\n";
989 tensor_declaration_block +=
"double * " + TensorMember(i.first) +
" = fTensor_" + i.first +
".data();\n";
993 tensor_declaration_block +=
"std::vector<int64_t> fTensor_" + i.first +
" = std::vector<int64_t>(" + std::to_string(
length) +
");\n";
994 tensor_declaration_block +=
"int64_t * " + TensorMember(i.first) +
" = fTensor_" + i.first +
".data();\n";
1000 if (tensor_declaration_block.length()) {
1001 fGC +=
"\n//--- declare and allocate the intermediate tensors\n" + tensor_declaration_block;
1006 fGC +=
"//--- declare the dynamic tensors\n";
1012 fGC +=
"//--- dynamic tensors pool\n";
1013 fGC +=
"std::vector<char> fDynamicMemoryPool;\n";
1019 std::string strcode;
1021 strcode += op->GenerateDeclCode();
1023 if (strcode.empty())
return;
1024 fGC +=
"\n//---- operator declarations \n";
1036 std::cout <<
"generating code for dynamic tensor management" << std::endl;
1043 std::stringstream out;
1044 out <<
"// dynamic tensor memory management\n";
1045 out <<
SP <<
"std::vector<TensorLifeInfo> dynamicTensorInfos;\n";
1050 std::vector<std::pair<std::string, ETensorType>> tensors;
1054 for (
auto &it : op->GetOpOutputTensors()) {
1056 auto op_ptr = op.get();
1057 std::cout <<
"Looping on operator " << op_index <<
" " <<
typeid(*op_ptr).name() << std::endl;
1060 std::string
name = std::string(it);
1066 int begin = op_index;
1071 end = it_lookup->second + 1;
1076 std::cout <<
"op " << op_index <<
"tensor_" <<
name <<
" begin " << begin <<
" " <<
" end " << end << std::endl;
1077 throw std::runtime_error(
"TMVA-SOFIE: RModel::GenerateDynamicTensorInfo: tensor_" +
name +
" has end before begin");
1081 out <<
SP <<
"dynamicTensorInfos.push_back( {" << begin <<
", " << end <<
", " << type_size <<
"* (" << tensor_size <<
") });"
1082 <<
" // tensor_" <<
name << std::endl;
1088 out <<
"\n" <<
SP <<
"auto memory_result = OrganizeMemory(dynamicTensorInfos);\n\n";
1089 out <<
"// allocating now the memory\n";
1090 out <<
SP <<
"fDynamicMemoryPool = std::vector<char>(memory_result.total_bytes);\n";
1091 out <<
SP <<
"int idx = 0;\n";
1092 for (
auto & it : tensors) {
1093 out <<
SP <<
"tensor_" << it.first <<
" = reinterpret_cast<" <<
ConvertTypeToString(it.second) <<
" *>(fDynamicMemoryPool.data() + memory_result.offsets[idx++]);\n";
1096 bool missingTensor =
false;
1100 if (std::find(tensors.begin(), tensors.end(), std::pair<std::string,ETensorType>{i.first, i.second.type}) == tensors.end()) {
1101 std::cout <<
"Dynamic tensors " << i.first <<
" is not in list of operator input/output " << std::endl;
1102 missingTensor =
true;
1106 throw std::runtime_error(
"TMVA-SOFIE: RModel::GenerateDynamicTensorInfo - some tensors are not in input/output list");
1117 for (
auto &
d : shape) {
1118 if (
d.param == paramName)
1132 const std::string
target =
"tensor_";
1134 std::vector<std::string>
result;
1136 for (
size_t i = 0; i <
input.size();) {
1138 bool doCollect =
false;
1141 (i == 0 || !IsIdentifierChar(
input[i - 1]))) {
1145 std::size_t j = i +
target.size();
1148 while (j <
input.size() && IsIdentifierChar(
input[j]))
1151 std::string fullName =
input.substr(i, j - i);
1172 result.push_back(fullName);
1192 std::unordered_map<std::string, int> inputParams;
1198 for (
auto &
d : shape) {
1199 std::string pName =
d.param;
1201 if (
d.isParam && inputParams.count(pName) == 0) {
1202 if (isdecl) rGC +=
"size_t ";
1203 rGC +=
d.param +
",";
1204 inputParams[pName] = i_input;
1211 if (
type.rfind(
"other", 0) == 0)
1212 throw std::runtime_error(
"TMVA-SOFIE: input tensor " +
name +
1213 " is of a data type which is not yet supported.");
1214 rGC +=
type +
" const* ";
1216 rGC +=
"tensor_" +
name +
",";
1233std::string memberNameForDimShape(std::string
name)
1235 if (!
name.empty()) {
1236 name[0] = std::toupper(
static_cast<unsigned char>(
name[0]));
1249 bool sameOutputTypes =
true;
1250 std::string inferReturnType;
1253 if (outputSize == 1) {
1254 fGC +=
"std::vector<" + typeForOutput(eFirstOutputType) +
">";
1259 sameOutputTypes =
false;
1261 if (sameOutputTypes)
1262 fGC +=
"std::vector<std::vector<" + typeForOutput(eFirstOutputType) +
">>";
1264 inferReturnType =
"std::tuple<";
1265 for (
size_t i = 0; i < outputSize; i++) {
1267 if (i < outputSize - 1)
1268 inferReturnType +=
",";
1270 inferReturnType +=
">";
1271 fGC += inferReturnType;
1278 if (!doInferArgs.empty())
1281 std::unordered_set<std::string> emittedShapeParams;
1293 bool hasRuntimeParam =
false;
1296 hasRuntimeParam =
true;
1298 n = hasRuntimeParam ? memberNameForDimShape(dimLen) : dimLen;
1300 std::string outputName =
"output_tensor_" +
name;
1302 doInferArgs +=
" " + outputName +
".data(),";
1306 emittedShapeParams.insert(dim.param).second) {
1307 fGC +=
SP +
"size_t " + dim.param +
" = 0;\n";
1308 doInferArgs +=
" " + dim.param +
",";
1313 if (!doInferArgs.empty())
1314 doInferArgs.back() =
' ';
1317 std::unordered_set<std::string> input_params_checked;
1318 std::string dynamic_parameters_check =
"";
1322 for (
auto &
d : shape) {
1323 std::string pName =
d.param;
1324 if (
d.isParam && input_params_checked.count(pName) == 0) {
1325 std::string memberName = memberNameForDimShape(
d.param);
1326 dynamic_parameters_check +=
d.param +
" > " + memberName +
" || ";
1327 input_params_checked.insert(pName);
1328 fGC +=
SP +
"if (" +
d.param +
" > " + memberName +
") {\n";
1329 fGC +=
SP +
SP +
"throw std::runtime_error(\"TMVA-SOFIE: dynamic input tensor shape parameter " +
1330 d.param +
" exceeds the initialized maximum allowed shape.\");\n";
1337 fGC +=
SP +
"doInfer(*this, " + doInferArgs +
");\n";
1343 std::string outputName =
"output_tensor_" +
name;
1345 fGC +=
SP + outputName +
".resize(" + tensor_size +
");\n";
1349 fGC +=
SP +
"return {";
1364 fGC +=
"struct " + sessionName +
";\n";
1368 if (!doInferSignature.empty())
1369 doInferSignature +=
", ";
1371 std::unordered_set<std::string> signatureShapeParams;
1378 signatureShapeParams.insert(dim.param).second)
1379 doInferSignature +=
" size_t &" + dim.param +
"_output,";
1383 doInferSignature.back() =
' ';
1385 doInferSignature = sessionName +
" const &session, " + doInferSignature;
1387 doInferSignature =
"inline void doInfer(" + doInferSignature +
")";
1390 fGC += doInferSignature +
";\n";
1396 std::string sessionCode = R
"(struct @SESSION@ {
1398 ~@SESSION@() = default;
1400 @SESSION@(@SESSION@ const&) = delete;
1401 @SESSION@(@SESSION@ &&) = default;
1403 @SESSION@& operator=(@SESSION@ const&) = delete;
1404 @SESSION@& operator=(@SESSION@ &&) = default;
1407 ReplaceAll(sessionCode, "@SESSION@", sessionName);
1415 std::string intermediate_memory_alloc_string =
"";
1416 intermediate_memory_alloc_string +=
"\n// --- Positioning intermediate tensor memory --";
1417 for (
size_t op_idx = 0; op_idx <
fOperators.size(); ++op_idx) {
1420 std::cout <<
"\n******************\n analyzing input/output operator " << op_idx <<
" "
1421 <<
typeid(*op).name() << std::endl;
1436 fGC += intermediate_memory_alloc_string;
1446 fGC +=
"\n// dynamic shape parameters\n";
1448 std::sort(dimShapeNames.begin(), dimShapeNames.end());
1449 for (
const auto &
p : dimShapeNames) {
1450 fGC +=
"size_t " + memberNameForDimShape(
p) +
";\n";
1457 fGC +=
"Session_" + graph->fName +
" fSession_" + graph->fName +
";\n";
1463 for (
size_t id = 0;
id <
fOperators.size();
id++) {
1464 std::string opName = std::to_string(
id);
1470 std::string dynParamDecls;
1471 std::string dynParamArgs;
1476 dynParamArgs +=
", " +
p;
1487 fGC +=
" static std::string LoadWeightsFromFile(const std::string &filename) {\n";
1488 fGC +=
" std::ifstream f(filename, std::ios::binary);\n";
1489 fGC +=
" if (!f.is_open()) {\n";
1490 fGC +=
" throw std::runtime_error(\"tmva-sofie failed to open file \" + filename + \" for input "
1493 fGC +=
" return std::string(std::istreambuf_iterator<char>(f), std::istreambuf_iterator<char>());\n";
1495 fGC += sessionName +
"(std::string filename =\"" +
fName +
".safetensors\"" + dynParamDecls +
")\n";
1496 fGC +=
" : " + sessionName +
"(SafetensorsBlob{LoadWeightsFromFile(filename)}" + dynParamArgs +
") {}\n\n";
1497 fGC += sessionName +
"(SafetensorsBlob weights_blob" + dynParamDecls +
") {\n";
1499 std::string fileName =
fName +
".dat";
1500 fGC += sessionName +
"(std::string filename =\"" + fileName +
"\"" + dynParamDecls +
") {\n";
1504 fGC += sessionName +
"(std::string = \"\"" + dynParamDecls +
") {\n";
1512 std::sort(dimShapeNames.begin(), dimShapeNames.end());
1513 for (
const auto &
p : dimShapeNames) {
1514 fGC +=
" " + memberNameForDimShape(
p) +
" = " +
p +
";\n";
1521 fGC +=
"\n//--- reading weights from file\n";
1531 for (
size_t id = 0;
id <
fOperators.size();
id++) {
1543 fGC += sessionName +
"(SafetensorsBlob" + dynParamDecls +
")\n";
1544 fGC +=
" : " + sessionName +
"(std::string{}" + dynParamArgs +
") {}\n\n";
1550 fGC +=
"// Set all weight and constant tensors to zero. This is useful to create\n"
1551 "// the tangent Session objects needed to differentiate the generated code\n"
1553 "void SetWeightsToZero() {\n";
1557 if (i.second.IsNotWritable() ||
1560 fGC +=
" for (auto &v : fTensor_" + i.first +
") v = 0;\n";
1563 fGC +=
" fSession_" + graph->fName +
".SetWeightsToZero();\n";
1571 fGC +=
"}; // end of Session\n\n";
1575 fGC += doInferSignature +
" {\n";
1580 std::cout <<
"Generating main inference code for " <<
fName << std::endl;
1583 throw std::runtime_error(
"TMVA-SOFIE: output size=0 are not supported");
1585 std::string allOperatorCode;
1587 for (
size_t op_idx = 0; op_idx <
fOperators.size(); ++op_idx) {
1589 std::cout <<
"Generating code for operator .... " << op_idx << std::endl;
1590 std::string operatorCode =
fOperators[op_idx]->Generate(std::to_string(op_idx));
1591 allOperatorCode += operatorCode;
1596 ReplaceAll(allOperatorCode,
"this->",
"session.");
1600 const std::string prefix =
"tensor_";
1601 for (
auto const&
name: tensorMemberNames) {
1605 fGC +=
" auto &" +
name +
" = session." +
name +
";\n";
1615 std::string t =
"session.tensor_" +
name;
1617 fGC +=
" std::copy(" + t +
", " + t +
" + " + std::to_string(
length) +
", tensor_" +
name +
");\n";
1621 fGC += allOperatorCode;
1623 std::unordered_set<std::string> assignedShapeParams;
1629 assignedShapeParams.insert(dim.param).second)
1630 fGC +=
" " + dim.param +
"_output = " + dim.param +
";\n";
1666 std::cout <<
"generate session code for subgraph " << graph->fName << std::endl;
1667 graph->GenerateSessionCode();
1672 std::cout <<
"generate Main session code - model " <<
fName << std::endl;
1678 fGC += (
"} //TMVA_SOFIE_" +
fName +
"\n");
1679 fGC +=
"\n#endif // " + hgname +
"\n";
1693 fGC +=
" std::ifstream f;\n";
1694 fGC +=
" f.open(filename);\n";
1695 fGC +=
" if (!f.is_open()) {\n";
1696 fGC +=
" throw std::runtime_error(\"tmva-sofie failed to open file \" + filename + \" for input weights\");\n";
1705 if (!i.second.IsWeightTensor())
continue;
1706 std::string tensor_name =
"tensor_" + i.first;
1709 fGC +=
" ReadTensorFromStream(f, " + tensor_name +
", \"" + tensor_name +
"\", " +
length +
");\n";
1711 throw std::runtime_error(
"tmva-sofie tensor " + tensor_name +
" with type " +
ConvertTypeToString(i.second.type()) +
" cannot be read from a file");
1714 fGC +=
" f.close();\n";
1724 fGC +=
" SafetensorsReader sofie_weights_reader(weights_blob);\n";
1728 if (!i.second.IsWeightTensor())
1730 std::string tensor_name =
"tensor_" + i.first;
1733 dtype = SafetensorsDType(i.second.type());
1734 }
catch (
const std::runtime_error &) {
1735 throw std::runtime_error(
"tmva-sofie tensor " + tensor_name +
" with type " +
1737 " cannot be read from a safetensors payload");
1740 fGC +=
" sofie_weights_reader.Read(\"" + tensor_name +
"\", fTensor_" + i.first +
", " +
length +
", \"" +
1748 std::string fileExtension;
1751 fileExtension =
".dat";
1754 fileExtension =
".safetensors";
1757 fileExtension =
".dat";
1768 std::ofstream
f(
filename, std::ios::binary);
1770 throw std::runtime_error(
"tmva-sofie failed to open file " +
filename +
" for tensor weight data");
1774 long curr_pos =
f.tellp();
1782 std::runtime_error(
"tmva-sofie failed to open file " +
filename +
" for tensor weight data");
1785 if (!i.second.IsWeightTensor()) {
1789 std::string tensor_name =
"tensor_" + i.first;
1790 f << tensor_name <<
" " <<
length <<
"\n";
1792 const float *
data = i.second.data<
float>();
1793 for (
size_t idx = 0; idx <
length; idx++) {
1796 if (
value != 0. && std::abs(
value) < std::numeric_limits<float>::min() )
value = 0;
1798 if (std::isinf(
value))
1799 f << (
value > 0 ?
"inf" :
"-inf");
1800 else if (std::isnan(
value))
1803 f << std::setprecision(std::numeric_limits<float>::max_digits10) <<
value;
1804 f << ( (idx <
length-1) ?
" " :
"\n" );
1808 throw std::runtime_error(
"tmva-sofie tensor " + tensor_name +
" with type " +
ConvertTypeToString(i.second.type()) +
" cannot be written to a file");
1811 throw std::runtime_error(
"tmva-sofie failed to write tensor data to file for " + tensor_name);
1813 long curr_pos =
f.tellp();
1825 const std::uint16_t one = 1;
1826 if (!*
reinterpret_cast<const std::uint8_t *
>(&one))
1827 throw std::runtime_error(
"tmva-sofie: safetensors weights can only be written on a little-endian host");
1834 std::uint64_t
offset = 0;
1836 std::vector<std::string> names;
1839 if (item.second.IsWeightTensor())
1840 names.push_back(item.first);
1842 std::sort(names.begin(), names.end());
1843 std::string headerStr =
"{";
1844 for (
size_t idx = 0; idx < names.size(); ++idx) {
1849 headerStr +=
"\"" + JsonEscape(
"tensor_" + names[idx]) +
"\":{\"dtype\":\"" + SafetensorsDType(tensor.type()) +
1851 for (
size_t i = 0; i < tensor.shape().
size(); ++i) {
1854 headerStr += std::to_string(tensor.shape()[i]);
1856 headerStr +=
"],\"data_offsets\":[" + std::to_string(
offset) +
"," + std::to_string(
offset + nbytes) +
"]}";
1861 const std::uint64_t headerSize = headerStr.size();
1863 for (
int i = 0; i < 8; ++i)
1864 sizestr[i] =
static_cast<char>((headerSize >> (8 * i)) & 0xff);
1865 f.write(sizestr, 8);
1866 f.write(headerStr.data(), headerStr.size());
1867 for (
const auto &
name : names) {
1870 f.write(
reinterpret_cast<const char *
>(tensor.data<
void>()), nbytes);
1873 throw std::runtime_error(
"tmva-sofie failed to write safetensors payload");
1878 std::ostringstream buffer;
1880 return buffer.str();
1884 std::cout <<
"Summary of model " <<
GetName() << std::endl;
1885 for(
size_t op_idx = 0; op_idx <
fOperators.size(); ++op_idx){
1887 std::string raw_name =
typeid(
r).
name();
1889 std::string
name = raw_name.substr(raw_name.find(
"ROperator_")+10, raw_name.size());
1890 std::cout << op_idx <<
" " <<
name <<
" : ";
1891 for (
auto & t_in :
r.GetOpInputTensors()) std::cout << t_in <<
" ";
1892 std::cout <<
" ----> ";
1893 for (
auto & t_out :
r.GetOpOutputTensors()) std::cout << t_out <<
" ";
1894 std::cout << std::endl;
1902 fGC +=
"\n// Input tensor dimensions\n";
1905 bool hasDynamicInputTensors =
false;
1910 hasDynamicInputTensors =
true;
1913 fGC +=
"constexpr std::array<SingleDim, " + std::to_string(shape.size()) +
"> dim_" +
name +
"{";
1914 for (std::size_t iDim = 0; iDim < shape.size(); ++iDim) {
1915 auto const &dim = shape[iDim];
1917 fGC +=
"SingleDim{\"" + dim.GetVal() +
"\"}";
1919 fGC +=
"SingleDim{" + dim.GetVal() +
"}";
1921 if (iDim != shape.size() - 1) {
1927 fGC +=
"\nconstexpr std::array<TensorDims, " + std::to_string(
fInputTensorNames.size()) +
"> inputTensorDims{\n";
1930 fGC +=
SP +
"makeDims(dim_" +
name +
")";
1940 "\nconstexpr bool hasDynamicInputTensors{" + std::string{hasDynamicInputTensors ?
"true" :
"false"} +
"};\n\n";
1942 fGC +=
"\n// Output tensor dimensions\n";
1943 bool hasDynamicOutputTensors =
false;
1947 hasDynamicOutputTensors =
true;
1950 fGC +=
"constexpr std::array<SingleDim, " + std::to_string(shape.size()) +
"> dim_" +
name +
"{";
1951 for (std::size_t iDim = 0; iDim < shape.size(); ++iDim) {
1952 auto const &dim = shape[iDim];
1954 fGC +=
"SingleDim{\"" + dim.GetVal() +
"\"}";
1956 fGC +=
"SingleDim{" + dim.GetVal() +
"}";
1958 if (iDim != shape.size() - 1) {
1964 fGC +=
"\nconstexpr std::array<TensorDims, " + std::to_string(
fOutputTensorNames.size()) +
"> outputTensorDims{\n";
1967 fGC +=
SP +
"makeDims(dim_" +
name +
")";
1976 "\nconstexpr bool hasDynamicOutputTensors{" + std::string{hasDynamicOutputTensors ?
"true" :
"false"} +
"};\n\n";
1980 std::cout <<
"Model requires following inputs:\n";
1982 std::cout <<
"Parametrised Tensor name: " << inputInfo.first <<
"\t";
1984 std::cout <<
"shape: [";
1985 for (
size_t i = 0; i < inputInfo.second.shape.size(); i++) {
1986 if (inputInfo.second.shape[i].isParam) {
1987 std::cout << inputInfo.second.shape[i].param;
1989 std::cout << inputInfo.second.shape[i].dim ;
1991 if (i < inputInfo.second.shape.size() - 1) std::cout <<
",";
1993 std::cout <<
"]" << std::endl;
1997 std::cout <<
"Fully Specified Tensor name: " << inputInfo.first <<
"\t";
1999 std::cout <<
"shape: [";
2000 for (
size_t i = 0; i < inputInfo.second.shape.size(); i++) {
2001 std::cout << inputInfo.second.shape[i];
2002 if (i < inputInfo.second.shape.size() - 1) std::cout <<
",";
2004 std::cout <<
"]" << std::endl;
2010 std::cout <<
"Model initialized the following tensors:\n";
2012 std::cout <<
"Tensor name: \"" << it.first <<
"\"\t";
2014 std::cout <<
"shape: [";
2015 for (
size_t i = 0; i < it.second.shape().
size(); i++) {
2016 std::cout << it.second.shape()[i];
2017 if (i < it.second.shape().size() - 1) std::cout <<
",";
2020 if (it.second.IsConstantTensor()) std::cout <<
" (Constant)";
2021 if (it.second.IsNotWritable()) std::cout <<
" (Not Writable)";
2022 std::cout << std::endl;
2028 std::cout <<
"Model specify the following intermediate tensors:\n";
2030 std::cout <<
"Tensor name: \"" << it.first <<
"\"\t";
2032 std::cout <<
"shape: [";
2033 for (
size_t i = 0; i < it.second.shape.size(); i++) {
2034 std::cout << it.second.shape[i];
2035 if (i < it.second.shape.size() - 1) std::cout <<
",";
2037 std::cout <<
"]" << std::endl;
2043 std::cout <<
"Model specify the following dynamic tensors:\n";
2045 std::cout <<
"Tensor name: \"" << it.first <<
"\"\t";
2047 std::cout <<
"shape: [";
2048 for (
size_t i = 0; i < it.second.shape.size(); i++) {
2049 std::cout << it.second.shape[i].GetVal();
2050 if (i < it.second.shape.size() - 1) std::cout <<
",";
2052 std::cout <<
"]" << std::endl;
2058 std::cout <<
"Model specify the following output tensors:\n";
2060 std::cout <<
"Tensor name: \"" << it <<
"\"\t";
2065 std::cout <<
"with shape not yet defined" << std::endl;
2074 std::cout <<
"Tensor " <<
name <<
" not found in model's initialized tensor list" << std::endl;
2078 std::cout <<
"Tensor name: " << it->first <<
"\t";
2081 std::cout <<
"shape: [";
2082 for (
size_t i = 0; i < it->second.shape().
size(); i++) {
2083 std::cout << it->second.shape()[i];
2084 length *= it->second.shape()[i];
2085 if (i < it->second.shape().size() - 1) std::cout <<
",";
2087 std::cout <<
"]" << std::endl;
2088 bool ellipsis =
true;
2094 std::cout <<
"data: [" << std::endl;
2096 auto converted_data = it->second.data<
float>();
2097 for (
int i =0; i < n_print; i++) {
2098 std::cout << converted_data[i];
2099 if (i < n_print - 1) std::cout <<
" ,";
2102 if (ellipsis) std::cout <<
", ...";
2103 std::cout <<
"]" << std::endl;
2108 fGC += (
"//Code generated automatically by TMVA for Inference of Model file [" +
fFileName +
"] at [" +
fParseTime.substr(0,
fParseTime.length()-1) +
"] \n");
2111 std::transform(hgname.begin(), hgname.end(), hgname.begin(), [](
unsigned char c) {
2112 return std::toupper(c);
2114 hgname =
"ROOT_TMVA_SOFIE_" + hgname;
2115 fGC +=
"\n#ifndef " + hgname +
"\n";
2116 fGC +=
"#define " + hgname +
"\n\n";
2119 for (
const char *
h : {
"cstdint",
"cstring",
"string",
"vector",
"map",
"memory",
"sstream",
"iostream",
"iomanip",
2120 "limits",
"stdexcept",
"algorithm",
"cmath",
"cassert"}) {
2124 fGC +=
"#include <" + i +
">\n";
2127 fGC +=
"#include \"" + i +
"\"\n";
2133 fGC +=
"#include <fstream>\n";
2137 fGC +=
"#include <iterator>\n";
2139 fGC +=
"\nnamespace TMVA_SOFIE_" +
fName +
"{\n";
2141 fGC += (
"namespace BLAS{\n");
2143 if (routine ==
"Gemm") {
2144 fGC += (
"\textern \"C\" void sgemm_(const char * transa, const char * transb, const int * m, const int * n, const int * k,\n"
2145 "\t const float * alpha, const float * A, const int * lda, const float * B, const int * ldb,\n"
2146 "\t const float * beta, float * C, const int * ldc);\n");
2149 }
else if (routine ==
"Gemv") {
2150 fGC += (
"\textern \"C\" void sgemv_(const char * trans, const int * m, const int * n, const float * alpha, const float * A,\n"
2151 "\t const int * lda, const float * X, const int * incx, const float * beta, const float * Y, const int * incy);\n");
2152 }
else if (routine ==
"Axpy") {
2153 fGC += (
"\textern \"C\" void saxpy_(const int * n, const float * alpha, const float * x,\n"
2154 "\t const int * incx, float * y, const int * incy);\n");
2155 }
else if (routine ==
"Copy") {
2156 fGC += (
"\textern \"C\" void scopy_(const int *n, const float* x, const int *incx, float* y, const int* incy);\n");
2159 fGC += (
"}//BLAS\n");
2172 auto replaceMarker = [
this](
const std::string &marker,
const std::string &replacement) {
2173 auto pos =
fGC.find(marker);
2174 if (pos != std::string::npos) {
2175 fGC.replace(pos, marker.size(), replacement);
2185 auto pos =
fGC.rfind(
"#endif");
2186 if (pos != std::string::npos) {
2208 throw std::runtime_error(
"tmva-sofie failed to open file for output generated inference code");
2217 size_t pos =
filename.find(
".hxx");
2218 if (pos != std::string::npos)
2219 filename.replace(pos, 4, extension);
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize id
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
std::string WriteInitializedTensorsToBuffer()
void AddShapeParam(const std::string &name, size_t def_value=0)
void AddNeededHelperFunction(std::string name)
std::vector< size_t > GetTensorShape(const std::string &name) const
void ReadInitializedTensorsFromFile()
std::set< std::string > fNeededHelperFunctions
void PrintIntermediateTensors() const
std::unordered_set< std::string > fComputedShapeParams
! shape parameters computed at run time by an operator
std::vector< Dim > GetDimTensorShape(const std::string &name) const
std::unordered_map< std::string, DynamicTensorInfo > fDynamicTensorInfos
bool IsDynamicTensor(const std::string &name) const
WeightFileType fWeightFile
void AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector< Dim > dim_shape)
void GenerateIntermediateTensorInfo()
bool AddAliasTensor(const std::string &tensor_name, const std::string &orig_tensor_name)
std::string GenerateInferSignature(bool isdecl=true)
std::string fExtraCodeForDimShapes
void GenerateOperatorDeclarations()
size_t fWeightsTensorSize
std::unordered_set< std::string > fNeededBlasRoutines
bool CheckIfTensorAlreadyExist(std::string tensor_name)
void GenerateHeaderInfo(std::string &hgname)
void GenerateRequiredInputTensorInfo()
To emit the dimensions of the input tensors as a data member of a session, which is helpful when vali...
void OutputGenerated(std::string filename="", bool append=false)
std::unordered_map< std::string, std::string > fAliasTensors
void AddInputTensorInfo(std::string input_name, ETensorType type, std::vector< Dim > shape)
std::unordered_map< std::string, TensorInfo > fIntermediateTensorInfos
void AddOutputTensorNameList(std::vector< std::string > output_tensor_names)
void EmitHelperFunctionsCode()
std::unordered_map< std::string, TensorInfo > fReadyInputTensorInfos
void AddConstantTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
void AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector< Dim > shape)
std::vector< std::string > fDimShapeNames
void AddInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
std::unordered_map< std::string_view, size_t > fIntermediateTensorFrequencyLookup
! lookup table for intermediate tensor frequency (transient)
void AddBlasRoutines(std::vector< std::string > routines)
void AddInputTensorName(std::string name)
std::vector< std::string > fOutputTensorNames
void PrintRequiredInputTensors() const
void GenerateSessionCode()
void AddNeededStdLib(std::string libname)
bool IsDimInputTensor(const std::string &name) const
void GenerateDynamicTensorInfo()
void PrintDynamicTensors() const
bool IsShapeTensor(const std::string &name) const
check if a tensor is a shape tensor
static constexpr const char * kHelperIncludesMarker
bool IsInitializedTensor(const std::string &name) const
std::unordered_set< std::string > fNeededStdLib
bool IsAliasTensor(const std::string &name) const
check if a tensor is a alias tensor
static constexpr const char * kHelperFunctionsMarker
size_t fConstantTensorSize
void CheckAndFlushIntermediateMemory(std::span< const std::string_view > op_output_tensors, const size_t &op_idx)
void AddOperator(std::unique_ptr< ROperator > op, int order_execution=-1)
void PrintOutputTensors() const
RModel()=default
Default constructor.
void HeadInitializedTensors(std::string name, int n_print=50)
bool IsConstantTensor(const std::string &name) const
void WriteInitializedTensorsToStream(std::ostream &os)
void Initialize(int batchSize=-1, bool verbose=false)
long WriteInitializedTensorsToFile(std::string filename="")
OptimizationLevel fOptimizationLevel
void PrintSummary() const
std::vector< std::string > CollectTensorMemberNames(const std::string &input)
Collects all identifiers starting with "tensor_" in the input code, provided that the occurrence is n...
std::vector< Dim > GetDynamicTensorShape(const std::string &name) const
RModel & operator=(RModel &&)
void PrintInitializedTensors() const
std::unordered_map< std::string, InputTensorInfo > fInputTensorInfos
std::shared_ptr< void > GetInitializedTensorData(std::string tensor_name)
void AddComputedShapeParam(const std::string &name)
Declare a shape parameter as computed at run time by an operator (e.g.
MemoryPoolInfo fIntermediateMemoryInfo
! intermediate memory info (transient)
void GenerateIntermediateMemoryPool()
std::string AllocateIntermediateMemory(std::span< const std::string_view > op_output_tensors)
std::unordered_map< std::string, std::pair< std::vector< Dim >, bool > > fShapeTensors
std::vector< std::unique_ptr< ROperator, ROperatorDeleter > > fOperators
void InitializeSubGraph(std::shared_ptr< RModel > graph)
std::unordered_map< std::string, std::string > fShapeParams
void SetNotWritableInitializedTensor(const std::string &tensor_name)
const std::string & GetName() const
ETensorType GetTensorType(std::string name) const
void GenerateInitializedTensorInfo()
std::vector< std::string > fInputTensorNames
std::unordered_map< std::string, InitializedTensor > fInitializedTensors
void UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
void Generate(std::underlying_type_t< Options > options, int batchSize=-1, bool verbose=false)
const std::vector< Dim > & GetShapeTensorValues(const std::string &tensor_name) const
std::vector< std::shared_ptr< RModel > > fSubGraphs
! sub-graph models (transient)
bool IsReadyInputTensor(const std::string &name) const
void UpdateOutputTensorList(std::vector< std::string > curr_output_tensor, std::vector< std::string > modify_output_tensor)
void AddShapeTensor(const std::string &name, const std::vector< Dim > &shapeValues, bool scalar=false)
std::unordered_set< std::string > fCustomOpHeaders
bool IsInputTensorShapeParam(std::string const &name) const
Check if a given parameter is used for the shape of an input tensor.
std::string Clean_name(std::string input_tensor_name)
std::string ConvertDimShapeToString(const std::vector< Dim > &shape)
std::size_t ConvertShapeToLength(const std::vector< size_t > &shape)
std::string ConvertValuesToString(size_t n, const T *data, size_t maxprint=-1)
std::vector< Dim > ConvertShapeToDim(const std::vector< size_t > &shape)
Convert shape from integer format to dynamic one (based on Dim)
constexpr size_t GetTypeSize(ETensorType type)
std::string GenerateConstantTensorCode(const std::pair< std::string, InitializedTensor > &t)
std::vector< size_t > ConvertShapeToInt(const std::vector< Dim > &shape)
Convert shape based on Dim to integer format.
std::string ConvertTypeToString(ETensorType type)
HelperFunctionsCode GenerateHelperFunctionsCode(const std::set< std::string > &neededHelpers, const std::string &modelNamespace, bool sgemmAlreadyDeclared=false)
Return the standalone C++ source of the inference helper functions requested in neededHelpers (see RM...
std::underlying_type_t< Options > operator|(Options opA, Options opB)
std::string ConvertDimShapeToLength(const std::vector< Dim > &shape)
std::string ConvertShapeToString(const std::vector< size_t > &shape)
std::string ConvertValToString(T value)
if(fPos !=fText.size()) Fail("unexpected trailing content")
Source code of the inference helper functions to embed in generated code so that it is standalone and...
std::string definitions
function/type definitions to place inside the generated model namespace
std::string cladDefinitions
Clad custom-derivative definitions to place at file scope (outside the model namespace) so that Clad ...
std::string includes
#include directives to place in the header preamble
std::map< size_t, TensorMemoryInfo > total_stack
std::map< size_t, size_t > available_stack
void operator()(ROperator *ptr) const