Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RSofieReader.hxx
Go to the documentation of this file.
1/**********************************************************************************
2 * Project: ROOT - a Root-integrated toolkit for multivariate data analysis *
3 * Package: TMVA * *
4 * *
5 * Description: *
6 * *
7 * Authors: *
8 * Lorenzo Moneta *
9 * *
10 * Copyright (c) 2022: *
11 * CERN, Switzerland *
12 * *
13 **********************************************************************************/
14
15
16#ifndef TMVA_RSOFIEREADER
17#define TMVA_RSOFIEREADER
18
19
20#include <string>
21#include <vector>
22#include <memory> // std::unique_ptr
23#include <sstream> // std::stringstream
24#include <iostream>
25#include "TROOT.h"
26#include "TSystem.h"
27#include "TError.h"
28#include "TInterpreter.h"
29#include "TUUID.h"
30#include "TMVA/RTensor.hxx"
31#include "Math/Util.h"
32
33namespace TMVA {
34namespace Experimental {
35
36
37
38
39/// TMVA::RSofieReader class for reading external Machine Learning models
40/// in ONNX files, Keras .h5 files or PyTorch .pt files
41/// and performing the inference using SOFIE
42/// It is reccomended to use ONNX if possible since there is a larger support for
43/// model operators.
44
46
47
48public:
49 /// Dummy constructor which needs model loading afterwards
51 /// Create TMVA model from ONNX file
52 /// print level can be 0 (minimal) 1 with info , 2 with all ONNX parsing info
53 RSofieReader(const std::string &path, std::vector<std::vector<size_t>> inputShapes = {}, int verbose = 0)
54 {
55 Load(path, inputShapes, verbose);
56 }
57
58 void Load(const std::string &path, std::vector<std::vector<size_t>> inputShapes = {}, int verbose = 0)
59 {
60
61 enum EModelType {kONNX, kKeras, kPt, kROOT, kNotDef}; // type of model
62 EModelType type = kNotDef;
63
64 auto pos1 = path.rfind("/");
65 auto pos2 = path.find(".onnx");
66 if (pos2 != std::string::npos) {
67 type = kONNX;
68 } else {
69 pos2 = path.find(".h5");
70 if (pos2 != std::string::npos) {
71 type = kKeras;
72 } else {
73 pos2 = path.find(".pt");
74 if (pos2 != std::string::npos) {
75 type = kPt;
76 }
77 else {
78 pos2 = path.find(".root");
79 if (pos2 != std::string::npos) {
80 type = kROOT;
81 }
82 }
83 }
84 }
85 if (type == kNotDef) {
86 throw std::runtime_error("Input file is not an ONNX or Keras or PyTorch file");
87 }
88 if (pos1 == std::string::npos)
89 pos1 = 0;
90 else
91 pos1 += 1;
92 std::string modelName = path.substr(pos1,pos2-pos1);
93 std::string fileType = path.substr(pos2+1, path.length()-pos2-1);
94 if (verbose) std::cout << "Parsing SOFIE model " << modelName << " of type " << fileType << std::endl;
95
96 // create code for parsing model and generate C++ code for inference
97 // make it in a separate scope to avoid polluting global interpreter space
98 std::string parserCode;
99 if (type == kONNX) {
100 // check first if we can load the SOFIE parser library
101 if (gSystem->Load("libROOTTMVASofieParser") < 0) {
102 throw std::runtime_error("RSofieReader: cannot use SOFIE with ONNX since libROOTTMVASofieParser is missing");
103 }
104 gInterpreter->Declare("#include \"TMVA/RModelParser_ONNX.hxx\"");
105 parserCode += "{\nTMVA::Experimental::SOFIE::RModelParser_ONNX parser ; \n";
106 if (verbose == 2)
107 parserCode += "TMVA::Experimental::SOFIE::RModel model = parser.Parse(\"" + path + "\",true); \n";
108 else
109 parserCode += "TMVA::Experimental::SOFIE::RModel model = parser.Parse(\"" + path + "\"); \n";
110 }
111 else if (type == kKeras) {
112 // use Keras direct parser
113 if (gSystem->Load("libPyMVA") < 0) {
114 throw std::runtime_error("RSofieReader: cannot use SOFIE with Keras since libPyMVA is missing");
115 }
116 parserCode += "{\nTMVA::Experimental::SOFIE::RModel model = TMVA::Experimental::SOFIE::PyKeras::Parse(\"" + path + "\"); \n";
117 }
118 else if (type == kPt) {
119 // use PyTorch direct parser
120 if (gSystem->Load("libPyMVA") < 0) {
121 throw std::runtime_error("RSofieReader: cannot use SOFIE with PyTorch since libPyMVA is missing");
122 }
123 if (inputShapes.size() == 0) {
124 throw std::runtime_error("RSofieReader: cannot use SOFIE with PyTorch since the input tensor shape is missing and is needed by the PyTorch parser");
125 }
126 std::string inputShapesStr = "{";
127 for (unsigned int i = 0; i < inputShapes.size(); i++) {
128 inputShapesStr += "{ ";
129 for (unsigned int j = 0; j < inputShapes[i].size(); j++) {
130 inputShapesStr += ROOT::Math::Util::ToString(inputShapes[i][j]);
131 if (j < inputShapes[i].size()-1) inputShapesStr += ", ";
132 }
133 inputShapesStr += "}";
134 if (i < inputShapes.size()-1) inputShapesStr += ", ";
135 }
136 inputShapesStr += "}";
137 parserCode += "{\nTMVA::Experimental::SOFIE::RModel model = TMVA::Experimental::SOFIE::PyTorch::Parse(\"" + path + "\", "
138 + inputShapesStr + "); \n";
139 }
140 else if (type == kROOT) {
141 // use parser from ROOT
142 parserCode += "{\nauto fileRead = TFile::Open(\"" + path + "\",\"READ\");\n";
143 parserCode += "TMVA::Experimental::SOFIE::RModel * modelPtr;\n";
144 parserCode += "auto keyList = fileRead->GetListOfKeys(); TString name;\n";
145 parserCode += "for (const auto&& k : *keyList) { \n";
146 parserCode += " TString cname = ((TKey*)k)->GetClassName(); if (cname==\"TMVA::Experimental::SOFIE::RModel\") name = k->GetName(); }\n";
147 parserCode += "fileRead->GetObject(name,modelPtr); fileRead->Close(); delete fileRead;\n";
148 parserCode += "TMVA::Experimental::SOFIE::RModel & model = *modelPtr;\n";
149 }
150
151 int batchSize = 1;
152 if (inputShapes.size() > 0 && inputShapes[0].size() > 0) {
153 batchSize = inputShapes[0][0];
154 if (batchSize < 1) batchSize = 1;
155 }
156 if (verbose) std::cout << "generating the code with batch size = " << batchSize << " ...\n";
157
158 parserCode += "model.Generate(TMVA::Experimental::SOFIE::Options::kDefault,"
159 + ROOT::Math::Util::ToString(batchSize) + "); \n";
160
161 // add custom operators if needed
162 if (fCustomOperators.size() > 0) {
163 if (verbose) {
164 parserCode += "model.PrintRequiredInputTensors();\n";
165 parserCode += "model.PrintIntermediateTensors();\n";
166 parserCode += "model.PrintOutputTensors();\n";
167 }
168 for (auto & op : fCustomOperators) {
169 parserCode += "{ auto p = new TMVA::Experimental::SOFIE::ROperator_Custom<float>(\""
170 + op.fOpName + "\"," + op.fInputNames + "," + op.fOutputNames + "," + op.fOutputShapes + ",\"" + op.fFileName + "\");\n";
171 parserCode += "std::unique_ptr<TMVA::Experimental::SOFIE::ROperator> op(p);\n";
172 parserCode += "model.AddOperator(std::move(op));\n}\n";
173 }
174 parserCode += "model.Generate(TMVA::Experimental::SOFIE::Options::kDefault,"
175 + ROOT::Math::Util::ToString(batchSize) + "); \n";
176 }
177 if (verbose > 1)
178 parserCode += "model.PrintGenerated(); \n";
179 parserCode += "model.OutputGenerated();\n";
180
181 parserCode += "int nInputs = model.GetInputTensorNames().size();\n";
182
183 // need information on number of inputs (assume output is 1)
184
185 //end of parsing code, close the scope and return 1 to indicate a success
186 parserCode += "return nInputs;\n}\n";
187
188 if (verbose) std::cout << "//ParserCode being executed:\n" << parserCode << std::endl;
189
190 auto iret = gROOT->ProcessLine(parserCode.c_str());
191 if (iret <= 0) {
192 std::string msg = "RSofieReader: error processing the parser code: \n" + parserCode;
193 throw std::runtime_error(msg);
194 }
195 fNInputs = iret;
196 if (fNInputs > 3) {
197 throw std::runtime_error("RSofieReader does not yet support model with > 3 inputs");
198 }
199
200 // compile now the generated code and create Session class
201 std::string modelHeader = modelName + ".hxx";
202 if (verbose) std::cout << "compile generated code from file " <<modelHeader << std::endl;
203 if (gSystem->AccessPathName(modelHeader.c_str())) {
204 std::string msg = "RSofieReader: input header file " + modelHeader + " is not existing";
205 throw std::runtime_error(msg);
206 }
207 if (verbose) std::cout << "Creating Inference function for model " << modelName << std::endl;
208 std::string declCode;
209 declCode += "#pragma cling optimize(2)\n";
210 declCode += "#include \"" + modelHeader + "\"\n";
211 // create global session instance: use UUID to have an unique name
212 std::string sessionClassName = "TMVA_SOFIE_" + modelName + "::Session";
213 TUUID uuid;
214 std::string uidName = uuid.AsString();
215 uidName.erase(std::remove_if(uidName.begin(), uidName.end(),
216 []( char const& c ) -> bool { return !std::isalnum(c); } ), uidName.end());
217
218 std::string sessionName = "session_" + uidName;
219 declCode += sessionClassName + " " + sessionName + ";";
220
221 if (verbose) std::cout << "//global session declaration\n" << declCode << std::endl;
222
223 bool ret = gInterpreter->Declare(declCode.c_str());
224 if (!ret) {
225 std::string msg = "RSofieReader: error compiling inference code and creating session class\n" + declCode;
226 throw std::runtime_error(msg);
227 }
228
229 fSessionPtr = (void *) gInterpreter->Calc(sessionName.c_str());
230
231 // define a function to be called for inference
232 std::stringstream ifuncCode;
233 std::string funcName = "SofieInference_" + uidName;
234 ifuncCode << "std::vector<float> " + funcName + "( void * ptr";
235 for (int i = 0; i < fNInputs; i++)
236 ifuncCode << ", float * data" << i;
237 ifuncCode << ") {\n";
238 ifuncCode << " " << sessionClassName << " * s = " << "(" << sessionClassName << "*) (ptr);\n";
239 ifuncCode << " return s->infer(";
240 for (int i = 0; i < fNInputs; i++) {
241 if (i>0) ifuncCode << ",";
242 ifuncCode << "data" << i;
243 }
244 ifuncCode << ");\n";
245 ifuncCode << "}\n";
246
247 if (verbose) std::cout << "//Inference function code using global session instance\n"
248 << ifuncCode.str() << std::endl;
249
250 ret = gInterpreter->Declare(ifuncCode.str().c_str());
251 if (!ret) {
252 std::string msg = "RSofieReader: error compiling inference function\n" + ifuncCode.str();
253 throw std::runtime_error(msg);
254 }
255 fFuncPtr = (void *) gInterpreter->Calc(funcName.c_str());
256 //fFuncPtr = reinterpret_cast<std::vector<float> (*)(void *, const float *)>(fptr);
257 fInitialized = true;
258 }
259
260 // Add custum operator
261 void AddCustomOperator(const std::string &opName, const std::string &inputNames, const std::string & outputNames,
262 const std::string & outputShapes, const std::string & fileName) {
263 if (fInitialized) std::cout << "WARNING: Model is already loaded and initialised. It must be done after adding the custom operators" << std::endl;
264 fCustomOperators.push_back( {fileName, opName,inputNames, outputNames,outputShapes});
265 }
266
267 // implementations for different outputs
268 std::vector<float> DoCompute(const std::vector<float> & x1) {
269 if (fNInputs != 1) {
270 std::string msg = "Wrong number of inputs - model requires " + std::to_string(fNInputs);
271 throw std::runtime_error(msg);
272 }
273 auto fptr = reinterpret_cast<std::vector<float> (*)(void *, const float *)>(fFuncPtr);
274 return fptr(fSessionPtr, x1.data());
275 }
276 std::vector<float> DoCompute(const std::vector<float> & x1, const std::vector<float> & x2) {
277 if (fNInputs != 2) {
278 std::string msg = "Wrong number of inputs - model requires " + std::to_string(fNInputs);
279 throw std::runtime_error(msg);
280 }
281 auto fptr = reinterpret_cast<std::vector<float> (*)(void *, const float *, const float *)>(fFuncPtr);
282 return fptr(fSessionPtr, x1.data(),x2.data());
283 }
284 std::vector<float> DoCompute(const std::vector<float> & x1, const std::vector<float> & x2, const std::vector<float> & x3) {
285 if (fNInputs != 3) {
286 std::string msg = "Wrong number of inputs - model requires " + std::to_string(fNInputs);
287 throw std::runtime_error(msg);
288 }
289 auto fptr = reinterpret_cast<std::vector<float> (*)(void *, const float *, const float *, const float *)>(fFuncPtr);
290 return fptr(fSessionPtr, x1.data(),x2.data(),x3.data());
291 }
292
293 /// Compute model prediction on vector
294 template<typename... T>
295 std::vector<float> Compute(T... x)
296 {
297 if(!fInitialized) {
298 return std::vector<float>();
299 }
300
301 // Take lock to protect model evaluation
303
304 // Evaluate TMVA model (need to add support for multiple outputs)
305 return DoCompute(x...);
306
307 }
308 std::vector<float> Compute(const std::vector<float> &x) {
309 if(!fInitialized) {
310 return std::vector<float>();
311 }
312
313 // Take lock to protect model evaluation
315
316 // Evaluate TMVA model (need to add support for multiple outputs)
317 return DoCompute(x);
318 }
319 /// Compute model prediction on input RTensor
320 /// The shape of the input tensor should be {nevents, nfeatures}
321 /// and the return shape will be {nevents, noutputs}
322 /// support for now only a single input
324 {
325 if(!fInitialized) {
326 return RTensor<float>({0});
327 }
328 const auto nrows = x.GetShape()[0];
329 const auto rowsize = x.GetStrides()[0];
330 auto fptr = reinterpret_cast<std::vector<float> (*)(void *, const float *)>(fFuncPtr);
331 auto result = fptr(fSessionPtr, x.GetData());
332
333 RTensor<float> y({nrows, result.size()}, MemoryLayout::ColumnMajor);
334 std::copy(result.begin(),result.end(), y.GetData());
335 //const bool layout = x.GetMemoryLayout() == MemoryLayout::ColumnMajor ? false : true;
336 // assume column major layout
337 for (size_t i = 1; i < nrows; i++) {
338 result = fptr(fSessionPtr, x.GetData() + i*rowsize);
339 std::copy(result.begin(),result.end(), y.GetData() + i*result.size());
340 }
341 return y;
342 }
343
344private:
345
346 bool fInitialized = false;
347 int fNInputs = 0;
348 void * fSessionPtr = nullptr;
349 void * fFuncPtr = nullptr;
350
351 // data to insert custom operators
353 std::string fFileName; // code implementing the custom operator
354 std::string fOpName; // operator name
355 std::string fInputNames; // input tensor names (convert as string as {"n1", "n2"})
356 std::string fOutputNames; // output tensor names converted as trind
357 std::string fOutputShapes; // output shapes
358 };
359 std::vector<CustomOperatorData> fCustomOperators;
360
361};
362
363} // namespace Experimental
364} // namespace TMVA
365
366#endif // TMVA_RREADER
#define c(i)
Definition RSha256.hxx:101
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char x2
Option_t Option_t TPoint TPoint const char x1
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
#define gInterpreter
#define gROOT
Definition TROOT.h:407
R__EXTERN TSystem * gSystem
Definition TSystem.h:555
#define R__WRITE_LOCKGUARD(mutex)
TMVA::RSofieReader class for reading external Machine Learning models in ONNX files,...
RSofieReader(const std::string &path, std::vector< std::vector< size_t > > inputShapes={}, int verbose=0)
Create TMVA model from ONNX file print level can be 0 (minimal) 1 with info , 2 with all ONNX parsing...
RTensor< float > Compute(RTensor< float > &x)
Compute model prediction on input RTensor The shape of the input tensor should be {nevents,...
std::vector< float > Compute(const std::vector< float > &x)
std::vector< float > Compute(T... x)
Compute model prediction on vector.
void Load(const std::string &path, std::vector< std::vector< size_t > > inputShapes={}, int verbose=0)
std::vector< float > DoCompute(const std::vector< float > &x1, const std::vector< float > &x2, const std::vector< float > &x3)
std::vector< CustomOperatorData > fCustomOperators
std::vector< float > DoCompute(const std::vector< float > &x1)
void AddCustomOperator(const std::string &opName, const std::string &inputNames, const std::string &outputNames, const std::string &outputShapes, const std::string &fileName)
std::vector< float > DoCompute(const std::vector< float > &x1, const std::vector< float > &x2)
RSofieReader()
Dummy constructor which needs model loading afterwards.
RTensor is a container with contiguous memory and shape information.
Definition RTensor.hxx:162
const Shape_t & GetShape() const
Definition RTensor.hxx:242
virtual int Load(const char *module, const char *entry="", Bool_t system=kFALSE)
Load a shared library.
Definition TSystem.cxx:1857
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
Returns FALSE if one can access a file using the specified access mode.
Definition TSystem.cxx:1296
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition TUUID.h:42
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
Definition TUUID.cxx:571
Double_t y[n]
Definition legend1.C:17
Double_t x[n]
Definition legend1.C:17
std::string ToString(const T &val)
Utility function for conversion to strings.
Definition Util.h:50
R__EXTERN TVirtualRWMutex * gCoreMutex
create variable transformations