Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
ROperator_Gemm.hxx
Go to the documentation of this file.
1#ifndef TMVA_SOFIE_ROPERATOR_GEMM
2#define TMVA_SOFIE_ROPERATOR_GEMM
3
4
6#include "TMVA/ROperator.hxx"
7#include "TMVA/RModel.hxx"
8
9#include <sstream>
10#include <algorithm>
11#include <iterator>
12#include <iomanip>
13#include <limits>
14#include <cassert>
15
16namespace TMVA{
17namespace Experimental{
18namespace SOFIE{
19
20
21 template <typename T>
22 class ROperator_Gemm final : public ROperator
23 {
24
25 private:
26 bool fIsDynamic = false;
27
28 float fAttrAlpha = 1.0;
29 float fAttrBeta = 1.0;
32
33 std::string fNA;
34 std::string fNB;
35 std::string fNC = "";
36 std::string fNC2; // bias tensor name after broadcasting
37 std::string fNY;
38 std::string fType;
39 std::vector<Dim> fShapeA;
40 std::vector<Dim> fShapeB;
41 std::vector<size_t> fShapeC;
42 std::vector<Dim> fShapeY;
43
44 public:
45
47 ROperator_Gemm(float alpha, float beta, int_t transA, int_t transB, std::string nameA, std::string nameB, std::string nameY):
48 fAttrAlpha(alpha), fAttrBeta(beta), fAttrTransA(transA), fAttrTransB(transB), fNA(UTILITY::Clean_name(nameA)),
49 fNB(UTILITY::Clean_name(nameB)), fNY(UTILITY::Clean_name(nameY))
50 {
51 fType = "float";
52 static_assert(std::is_same_v<T, float>,
53 "TMVA::SOFIE - Unsupported type parsing a Gemm operator");
54 }
55
56 ROperator_Gemm(float alpha, float beta, int_t transA, int_t transB, std::string nameA, std::string nameB, std::string nameC, std::string nameY):
57 fAttrAlpha(alpha), fAttrBeta(beta), fAttrTransA(transA), fAttrTransB(transB), fNA(UTILITY::Clean_name(nameA)),
58 fNB(UTILITY::Clean_name(nameB)), fNC(UTILITY::Clean_name(nameC)), fNY(UTILITY::Clean_name(nameY))
59 {
60 fType = "float";
61 static_assert(std::is_same_v<T, float>,
62 "TMVA::SOFIE - Unsupported type parsing a Gemm operator");
63 }
64
65 std::vector<ETensorType> TypeInference(std::vector<ETensorType> input){
66 ETensorType out = input[0];
67 return {out};
68 }
69
70 template <typename U>
71 std::vector<std::vector<U>> DoShapeInference(const std::vector<std::vector<U>> & input){
72 if (input.size() > 3) throw std::runtime_error("TMVA SOFIE Gemm Op Shape Inference only need 2 or 3 input tensor");
73 for (auto& i: input){
74 if (i.size() > 2){
75 throw std::runtime_error("TMVA SOFIE Gemm Op Shape Inference only accept input tensor with 2 dimensions");
76 }
77 }
78 std::vector<std::vector<U>> ret;
79 if (input.size() == 3){
80 ret.push_back(input[2]); //shape of C is shape of Y
81 return ret;
82 }
83 std::vector<U> s_a(input[0]);
84 std::vector<U> s_b(input[1]);
85 if (fAttrTransA){
86 std::reverse(s_a.begin(), s_a.end());
87 }
88 if (fAttrTransB){
89 std::reverse(s_b.begin(), s_b.end());
90 }
91 std::vector<U> s_y(2);
92 s_y[0] = s_a[0];
93 s_y[1] = s_b[1];
94 ret.push_back(s_y);
95 return ret;
96 }
97
98 std::vector<std::vector<size_t>> ShapeInference(std::vector<std::vector<size_t>> input){
99 return DoShapeInference<size_t>(input);
100 }
101 std::vector<std::vector<Dim>> DynamicShapeInference(const std::vector<std::vector<Dim>> & input){
102 return DoShapeInference<Dim>(input);
103 }
104
105
106
107 void Initialize(RModel& model){
108 //TODO: propagate A or B as specified by ONNX standard
109
110 if ((model.CheckIfTensorAlreadyExist(fNA) == false) || (model.CheckIfTensorAlreadyExist(fNB) == false) ){ //input must be a graph input, or already initialized intermediate tensor
111 throw std::runtime_error("TMVA SOFIE Gemm Op Input Tensor " + fNA + " or " + fNB + " is not found in model");
112 }
113 if (fNC != ""){
114 if (model.CheckIfTensorAlreadyExist(fNC) == false){ //input must be a graph input, or already initialized intermediate tensor
115 throw std::runtime_error("TMVA SOFIE Gemm Op Input Tensor" + fNC + " is not found in model");
116 }
117 }
118 if (model.IsDynamicTensor(fNA) || model.IsInputTensor(fNA) ) {
120 fIsDynamic = true;
121 }
122 else {
123 auto shapeA_int = model.GetTensorShape(fNA);
124 // don't think this is needed?
125 if (shapeA_int.size() == 1)
126 shapeA_int = {1,shapeA_int[0]};
127 fShapeA = ConvertShapeToDim(shapeA_int);
128 }
129
130 if (model.IsDynamicTensor(fNB) || model.IsInputTensor(fNB)) {
132 fIsDynamic = true;
133 }
134 else {
135 auto shapeB_int = model.GetTensorShape(fNB);
136 fShapeB = ConvertShapeToDim(shapeB_int);
137 }
138 if (fShapeA.size() != 2)
139 throw std::runtime_error("TMVA SOFIE Gemm Op Input Tensor" + fNA +
140 " is not of 2 dimensions: A " + ConvertDynamicShapeToString(fShapeA));
141 if (fShapeB.size() != 2)
142 throw std::runtime_error("TMVA SOFIE Gemm Op Input Tensor" + fNB +
143 " is not of 2 dimensions: B " + ConvertDynamicShapeToString(fShapeB));
144
146 std::vector<size_t> shapeY;
147 if (!fIsDynamic) {
148 shapeY = ConvertShapeToInt(fShapeY);
149 if (shapeY.empty()) {
150 throw std::runtime_error("TMVA SOFIE Gemm Op " + fNY + " has invalid shape" + ConvertDynamicShapeToString(fShapeY));
151 }
152 }
153
154 // bias is normally not dynamic (not support it for time being)
155 if (fNC != ""){
156 // normally bias is fixed and not dynamic
157 if (model.IsDynamicTensor(fNC)) {
158 throw std::runtime_error("TMVA SOFIE Gemm Op Input Tensor" + fNC + " is dynamic and is not supported");
159 }
160 fShapeC = model.GetTensorShape(fNC);
161 fNC2 = fNC;
162 bool broadcast_needed = !UTILITY::AreSameShape(fShapeC, fShapeY);
163
164 // For Gemm broadcasting is not needed if fShapeY[0] == 1 i.e. C and Y have same length
165 //if (fShapeY[0] == 1 && ConvertShapeToLength(fShapeC) != ConvertShapeToLength(fShapeY)) {
166 // broadcast_needed = false;
167 //}
168
169 // std::cout << "doing broadcast " << broadcast_needed << " use session " << model.UseSession() <<
170 // " shape C " << ConvertShapeToString(fShapeC) << " shape Y " << ConvertShapeToString(fShapeY)
171 // << std::endl;
172
173 if (broadcast_needed) {
174 if (!model.UseSession()) {
175 // without session dynamic tensors not supported in Gemm
176 if (fIsDynamic) {
177 throw std::runtime_error("TMVA SOFIE Gemm Op: dynamic tensors not supported without a session");
178 }
179 auto original_data = model.GetInitializedTensorData(fNC);
180 auto targetShape = UTILITY::UnidirectionalBroadcastShape(fShapeC, shapeY);
181 if (fType == "float") {
182 std::shared_ptr<void> new_data_ptr(UTILITY::UnidirectionalBroadcast<float>(
183 static_cast<float *>(original_data.get()), fShapeC, targetShape),
184 std::default_delete<float[]>());
185
186 model.UpdateInitializedTensor(fNC, model.GetTensorType(fNC), shapeY, new_data_ptr);
187 fShapeC = shapeY;
188 }
189 } else {
190 // In case of session add broadcasting code in Session constructor and in GenerateInitCode
191 // we need to add a new intermediate tensor for broadcasted bias tensor
192 fNC2 = fNC + "bcast";
193 if (!fIsDynamic) {
194 model.AddIntermediateTensor(fNC2, model.GetTensorType(fNC), shapeY);
195 }
196 else
198 }
199 }
200 }
201
202 if (!fIsDynamic)
203 model.AddIntermediateTensor(fNY, model.GetTensorType(fNA), shapeY);
204 else
206
207 model.AddNeededStdLib("algorithm");
208
209 }
210
211 std::string GenerateInitCode()
212 {
213 std::stringstream out;
214 // generate initialization code for broadcasting of bias tensor
215 if (fShapeC.size() != fShapeY.size() && fNC != fNC2) {
216 // we broadcast here always C in Y output, so target shape is the one of Y
217 // no need to call UTILITY::UnidirectionalBroadcastShape.
218 // here in case of parametric shape we need to assume that the parameters will be defined in the initialization code.
219 auto targetShape = fShapeY;
220 // include a separate scope to avoid defining unique operator temp variables
221 out << "//--- broadcast bias tensor " << fNC << "for Gemm op\n";
222 out << SP << "{\n";
223 out << " float * data = TMVA::Experimental::SOFIE::UTILITY::UnidirectionalBroadcast<float>(tensor_"
224 << fNC << "," << ConvertShapeToString(fShapeC) << ", " << ConvertDynamicShapeToString(fShapeY) << ");\n";
226 out << SP << SP << "std::copy(data, data + " << length << ", tensor_" << fNC2 << ");\n";
227 out << SP << SP << "delete [] data;\n";
228 out << SP << "}\n";
229 }
230 return out.str();
231 }
232
233 std::string Generate(std::string OpName){
234 OpName = "op_" + OpName;
235
236 if (fShapeA.empty() || fShapeB.empty() || fShapeY.empty() || (fNC != "" && fShapeC.empty())) {
237 throw std::runtime_error("TMVA SOFIE Gemm Op called to Generate without being initialized first");
238 }
239 std::stringstream out;
240 out << "\n//--------- Gemm\n";
241 out << SP << "char " << OpName << "_transA = " << (fAttrTransA ? "\'t\'" : "\'n\'") << ";\n";
242 out << SP << "char " << OpName << "_transB = " << (fAttrTransB ? "\'t\'" : "\'n\'") << ";\n";
243
244 auto m = (fAttrTransA ? fShapeA[1].GetVal() : fShapeA[0].GetVal());
245 auto n = (fAttrTransB ? fShapeB[0].GetVal() : fShapeB[1].GetVal());
246 auto k = (fAttrTransA ? fShapeA[0].GetVal() : fShapeA[1].GetVal());
248
249 out << SP << "int " << OpName << "_m = " << m << ";\n";
250 out << SP << "int " << OpName << "_n = " << n << ";\n";
251 out << SP << "int " << OpName << "_k = " << k << ";\n";
252 out << SP << "float " << OpName << "_alpha = " << std::setprecision(std::numeric_limits<float>::max_digits10) << fAttrAlpha << ";\n";
253 out << SP << "float " << OpName << "_beta = " << std::setprecision(std::numeric_limits<float>::max_digits10) << fAttrBeta << ";\n";
254 out << SP << "int " << OpName << "_lda = " << (fAttrTransA ? m : k) << ";\n";
255 out << SP << "int " << OpName << "_ldb = " << (fAttrTransB ? k : n) << ";\n";
256 // case bias is present
257 if (!fNC.empty()){
258 if (fNC2 == fNC) {
259 // add a check in case broadcasting was not needed or done outside of session
260 if (!fIsDynamic) {
261 if (std::stoi(ConvertDynamicShapeToLength(fShapeY)) != static_cast<int>(ConvertShapeToLength(fShapeC)))
262 throw std::runtime_error("TMVA SOFIE Gemm Op " + OpName + " Bias tensor has not correct size "
263 + ConvertShapeToString(fShapeC) + " output length " + length);
264 } else {
265 // add a dynamic check
266 out << SP << "assert(" << length << " != " << ConvertShapeToLength(fShapeC) << ");\n";
267 }
268 }
269 out << SP << "std::copy(" << "tensor_" << fNC2 << ", " << "tensor_" << fNC2 << " + " << length << ", " << "tensor_" << fNY << ");\n";
270 } else {
271 //in this case fAttrBeta needs to be equal to zero otherwise second time we run we will use
272 // the previous result
273 if (fAttrBeta != 0) {
274 throw std::runtime_error("TMVA SOFIE Gemm Op " + OpName + " Bias tensor is not present but beta value in Gemm is not zero");
275 }
276 }
277 if (fType == "float"){
278 out << SP << "BLAS::sgemm_(&" << OpName << "_transB, &" << OpName << "_transA, &" << OpName
279 << "_n, &" << OpName << "_m, &" << OpName << "_k, &" << OpName << "_alpha, " << "tensor_" << fNB
280 << ", &" << OpName << "_ldb, " << "tensor_" << fNA << ", &" << OpName << "_lda, &" << OpName << "_beta, " << "tensor_" << fNY << ", &"
281 << OpName << "_n);\n";
282 }
283
284 return out.str();
285
286 }
287
288 std::vector<std::string> GetBlasRoutines() { return { std::string("Gemm"), std::string("Gemv") }; }
289
290 };
291
292
293}//SOFIE
294}//Experimental
295}//TMVA
296
297
298#endif //TMVA_SOFIE_ROPERATOR_GEMM
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void input
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
void AddNeededStdLib(std::string libname)
const ETensorType & GetTensorType(std::string name)
Definition RModel.cxx:91
bool IsDynamicTensor(const std::string &name) const
Definition RModel.cxx:177
void AddIntermediateTensor(std::string tensor_name, ETensorType type, std::vector< Dim > dim_shape)
Definition RModel.cxx:187
std::vector< Dim > GetDynamicTensorShape(std::string name)
Definition RModel.cxx:79
bool CheckIfTensorAlreadyExist(std::string tensor_name)
Definition RModel.cxx:116
void AddDynamicTensor(std::string tensor_name, ETensorType type, std::vector< Dim > shape)
Definition RModel.cxx:204
const std::vector< size_t > & GetTensorShape(std::string name)
Definition RModel.cxx:56
bool IsInputTensor(const std::string &name) const
Definition RModel.cxx:181
std::shared_ptr< void > GetInitializedTensorData(std::string tensor_name)
Definition RModel.cxx:248
void UpdateInitializedTensor(std::string tensor_name, ETensorType type, std::vector< std::size_t > shape, std::shared_ptr< void > data)
Definition RModel.cxx:239
ROperator_Gemm(float alpha, float beta, int_t transA, int_t transB, std::string nameA, std::string nameB, std::string nameC, std::string nameY)
std::vector< std::vector< U > > DoShapeInference(const std::vector< std::vector< U > > &input)
std::string Generate(std::string OpName)
std::vector< std::string > GetBlasRoutines()
std::vector< std::vector< Dim > > DynamicShapeInference(const std::vector< std::vector< Dim > > &input)
std::vector< std::vector< size_t > > ShapeInference(std::vector< std::vector< size_t > > input)
ROperator_Gemm(float alpha, float beta, int_t transA, int_t transB, std::string nameA, std::string nameB, std::string nameY)
std::vector< ETensorType > TypeInference(std::vector< ETensorType > input)
const std::string SP
space used to correctly indent the generated C++ code
Definition ROperator.hxx:41
const Int_t n
Definition legend1.C:16
bool AreSameShape(const std::vector< size_t > &, const std::vector< size_t > &)
std::vector< size_t > UnidirectionalBroadcastShape(std::vector< size_t >, std::vector< size_t >)
std::vector< Dim > ConvertShapeToDim(std::vector< size_t > shape)
Convert shape from integer format to dynamic one (based on Dim)
std::string ConvertDynamicShapeToLength(std::vector< Dim > shape)
std::string ConvertShapeToString(std::vector< size_t > shape)
std::string ConvertDynamicShapeToString(std::vector< Dim > shape)
std::vector< size_t > ConvertShapeToInt(std::vector< Dim > shape)
Convert shape based on Dim to integer format.
std::size_t ConvertShapeToLength(std::vector< size_t > shape)
create variable transformations
TMarker m
Definition textangle.C:8