Logo ROOT  
Reference Guide
RDFUtils.cxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 03/2017
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#include "RConfigure.h" // R__USE_IMT
12#include "ROOT/RDataSource.hxx"
15#include "ROOT/RDF/Utils.hxx"
16#include "ROOT/RLogger.hxx"
17#include "RtypesCore.h"
18#include "TBranch.h"
19#include "TBranchElement.h"
20#include "TClass.h"
21#include "TClassEdit.h"
22#include "TClassRef.h"
23#include "TError.h" // Info
24#include "TInterpreter.h"
25#include "TLeaf.h"
26#include "TROOT.h" // IsImplicitMTEnabled, GetThreadPoolSize
27#include "TTree.h"
28
29#include <stdexcept>
30#include <string>
31#include <cstring>
32#include <typeinfo>
33
34using namespace ROOT::Detail::RDF;
35using namespace ROOT::RDF;
36
38{
39 static ROOT::Experimental::RLogChannel c("ROOT.RDF");
40 return c;
41}
42
43namespace ROOT {
44namespace Internal {
45namespace RDF {
46
47/// Return the type_info associated to a name. If the association fails, an
48/// exception is thrown.
49/// References and pointers are not supported since those cannot be stored in
50/// columns.
51const std::type_info &TypeName2TypeID(const std::string &name)
52{
53 if (auto c = TClass::GetClass(name.c_str())) {
54 if (!c->GetTypeInfo()) {
55 std::string msg("Cannot extract type_info of type ");
56 msg += name.c_str();
57 msg += ".";
58 throw std::runtime_error(msg);
59 }
60 return *c->GetTypeInfo();
61 } else if (name == "char" || name == "Char_t")
62 return typeid(char);
63 else if (name == "unsigned char" || name == "UChar_t")
64 return typeid(unsigned char);
65 else if (name == "int" || name == "Int_t")
66 return typeid(int);
67 else if (name == "unsigned int" || name == "UInt_t")
68 return typeid(unsigned int);
69 else if (name == "short" || name == "Short_t")
70 return typeid(short);
71 else if (name == "unsigned short" || name == "UShort_t")
72 return typeid(unsigned short);
73 else if (name == "long" || name == "Long_t")
74 return typeid(long);
75 else if (name == "unsigned long" || name == "ULong_t")
76 return typeid(unsigned long);
77 else if (name == "double" || name == "Double_t")
78 return typeid(double);
79 else if (name == "float" || name == "Float_t")
80 return typeid(float);
81 else if (name == "long long" || name == "long long int" || name == "Long64_t")
82 return typeid(Long64_t);
83 else if (name == "unsigned long long" || name == "unsigned long long int" || name == "ULong64_t")
84 return typeid(ULong64_t);
85 else if (name == "bool" || name == "Bool_t")
86 return typeid(bool);
87 else {
88 std::string msg("Cannot extract type_info of type ");
89 msg += name.c_str();
90 msg += ".";
91 throw std::runtime_error(msg);
92 }
93}
94
95/// Returns the name of a type starting from its type_info
96/// An empty string is returned in case of failure
97/// References and pointers are not supported since those cannot be stored in
98/// columns.
99std::string TypeID2TypeName(const std::type_info &id)
100{
101 if (auto c = TClass::GetClass(id)) {
102 return c->GetName();
103 } else if (id == typeid(char))
104 return "char";
105 else if (id == typeid(unsigned char))
106 return "unsigned char";
107 else if (id == typeid(int))
108 return "int";
109 else if (id == typeid(unsigned int))
110 return "unsigned int";
111 else if (id == typeid(short))
112 return "short";
113 else if (id == typeid(unsigned short))
114 return "unsigned short";
115 else if (id == typeid(long))
116 return "long";
117 else if (id == typeid(unsigned long))
118 return "unsigned long";
119 else if (id == typeid(double))
120 return "double";
121 else if (id == typeid(float))
122 return "float";
123 else if (id == typeid(Long64_t))
124 return "Long64_t";
125 else if (id == typeid(ULong64_t))
126 return "ULong64_t";
127 else if (id == typeid(bool))
128 return "bool";
129 else
130 return "";
131}
132
133std::string ComposeRVecTypeName(const std::string &valueType)
134{
135 return "ROOT::VecOps::RVec<" + valueType + ">";
136}
137
138std::string GetLeafTypeName(TLeaf *leaf, const std::string &colName)
139{
140 const char *colTypeCStr = leaf->GetTypeName();
141 std::string colType = colTypeCStr == nullptr ? "" : colTypeCStr;
142 if (colType.empty())
143 throw std::runtime_error("Could not deduce type of leaf " + colName);
144 if (leaf->GetLeafCount() != nullptr && leaf->GetLenStatic() == 1) {
145 // this is a variable-sized array
146 colType = ComposeRVecTypeName(colType);
147 } else if (leaf->GetLeafCount() == nullptr && leaf->GetLenStatic() > 1) {
148 // this is a fixed-sized array (we do not differentiate between variable- and fixed-sized arrays)
149 colType = ComposeRVecTypeName(colType);
150 } else if (leaf->GetLeafCount() != nullptr && leaf->GetLenStatic() > 1) {
151 // we do not know how to deal with this branch
152 throw std::runtime_error("TTree leaf " + colName +
153 " has both a leaf count and a static length. This is not supported.");
154 }
155
156 return colType;
157}
158
159/// Return the typename of object colName stored in t, if any. Return an empty string if colName is not in t.
160/// Supported cases:
161/// - leaves corresponding to single values, variable- and fixed-length arrays, with following syntax:
162/// - "leafname", as long as TTree::GetLeaf resolves it
163/// - "b1.b2...leafname", as long as TTree::GetLeaf("b1.b2....", "leafname") resolves it
164/// - TBranchElements, as long as TTree::GetBranch resolves their names
165std::string GetBranchOrLeafTypeName(TTree &t, const std::string &colName)
166{
167 // look for TLeaf either with GetLeaf(colName) or with GetLeaf(branchName, leafName) (splitting on last dot)
168 auto *leaf = t.GetLeaf(colName.c_str());
169 if (!leaf)
170 leaf = t.FindLeaf(colName.c_str()); // try harder
171 if (!leaf) {
172 // try splitting branchname and leafname
173 const auto dotPos = colName.find_last_of('.');
174 const auto hasDot = dotPos != std::string::npos;
175 if (hasDot) {
176 const auto branchName = colName.substr(0, dotPos);
177 const auto leafName = colName.substr(dotPos + 1);
178 leaf = t.GetLeaf(branchName.c_str(), leafName.c_str());
179 }
180 }
181 if (leaf)
182 return GetLeafTypeName(leaf, std::string(leaf->GetFullName()));
183
184 // we could not find a leaf named colName, so we look for a branch called like this
185 auto branch = t.GetBranch(colName.c_str());
186 if (!branch)
187 branch = t.FindBranch(colName.c_str()); // try harder
188 if (branch) {
189 static const TClassRef tbranchelement("TBranchElement");
190 if (branch->InheritsFrom(tbranchelement)) {
191 auto be = static_cast<TBranchElement *>(branch);
192 if (auto currentClass = be->GetCurrentClass())
193 return currentClass->GetName();
194 else {
195 // Here we have a special case for getting right the type of data members
196 // of classes sorted in TClonesArrays: ROOT-9674
197 auto mother = be->GetMother();
198 if (mother && mother->InheritsFrom(tbranchelement) && mother != be) {
199 auto beMom = static_cast<TBranchElement *>(mother);
200 auto beMomClass = beMom->GetClass();
201 if (beMomClass && 0 == std::strcmp("TClonesArray", beMomClass->GetName()))
202 return be->GetTypeName();
203 }
204 return be->GetClassName();
205 }
206 } else if (branch->IsA() == TBranch::Class() && branch->GetListOfLeaves()->GetEntriesUnsafe() == 1) {
207 // normal branch (not a TBranchElement): if it has only one leaf, we pick the type of the leaf:
208 // RDF and TTreeReader allow referring to branch.leaf as just branch if branch has only one leaf
209 leaf = static_cast<TLeaf *>(branch->GetListOfLeaves()->UncheckedAt(0));
210 return GetLeafTypeName(leaf, std::string(leaf->GetFullName()));
211 }
212 }
213
214 // we could not find a branch or a leaf called colName
215 return std::string();
216}
217
218/// Return a string containing the type of the given branch. Works both with real TTree branches and with temporary
219/// column created by Define. Throws if type name deduction fails.
220/// Note that for fixed- or variable-sized c-style arrays the returned type name will be RVec<T>.
221/// vector2rvec specifies whether typename 'std::vector<T>' should be converted to 'RVec<T>' or returned as is
222/// customColID is only used if isDefine is true, and must correspond to the custom column's unique identifier
223/// returned by its `GetID()` method.
224std::string ColumnName2ColumnTypeName(const std::string &colName, TTree *tree, RDataSource *ds, RDefineBase *define,
225 bool vector2rvec)
226{
227 std::string colType;
228
229 // must check defines first: we want Redefines to have precedence over everything else
230 if (colType.empty() && define) {
231 colType = define->GetTypeName();
232 }
233
234 if (ds && ds->HasColumn(colName))
235 colType = ds->GetTypeName(colName);
236
237 if (colType.empty() && tree) {
238 colType = GetBranchOrLeafTypeName(*tree, colName);
239 if (vector2rvec && TClassEdit::IsSTLCont(colType) == ROOT::ESTLType::kSTLvector) {
240 std::vector<std::string> split;
241 int dummy;
242 TClassEdit::GetSplit(colType.c_str(), split, dummy);
243 auto &valueType = split[1];
244 colType = ComposeRVecTypeName(valueType);
245 }
246 }
247
248 if (colType.empty())
249 throw std::runtime_error("Column \"" + colName +
250 "\" is not in a dataset and is not a custom column been defined.");
251
252 return colType;
253}
254
255/// Convert type name (e.g. "Float_t") to ROOT type code (e.g. 'F') -- see TBranch documentation.
256/// Return a space ' ' in case no match was found.
257char TypeName2ROOTTypeName(const std::string &b)
258{
259 if (b == "Char_t" || b == "char")
260 return 'B';
261 if (b == "UChar_t" || b == "unsigned char")
262 return 'b';
263 if (b == "Short_t" || b == "short" || b == "short int")
264 return 'S';
265 if (b == "UShort_t" || b == "unsigned short" || b == "unsigned short int")
266 return 's';
267 if (b == "Int_t" || b == "int")
268 return 'I';
269 if (b == "UInt_t" || b == "unsigned" || b == "unsigned int")
270 return 'i';
271 if (b == "Float_t" || b == "float")
272 return 'F';
273 if (b == "Double_t" || b == "double")
274 return 'D';
275 if (b == "Long64_t" || b == "long long" || b == "long long int")
276 return 'L';
277 if (b == "ULong64_t" || b == "unsigned long long" || b == "unsigned long long int")
278 return 'l';
279 if (b == "Long_t" || b == "long" || b == "long int")
280 return 'G';
281 if (b == "ULong_t" || b == "unsigned long" || b == "unsigned long int")
282 return 'g';
283 if (b == "Bool_t" || b == "bool")
284 return 'O';
285 return ' ';
286}
287
288unsigned int GetNSlots()
289{
290 unsigned int nSlots = 1;
291#ifdef R__USE_IMT
293 nSlots = ROOT::GetThreadPoolSize();
294#endif // R__USE_IMT
295 return nSlots;
296}
297
298/// Replace occurrences of '.' with '_' in each string passed as argument.
299/// An Info message is printed when this happens. Dots at the end of the string are not replaced.
300/// An exception is thrown in case the resulting set of strings would contain duplicates.
301std::vector<std::string> ReplaceDotWithUnderscore(const std::vector<std::string> &columnNames)
302{
303 auto newColNames = columnNames;
304 for (auto &col : newColNames) {
305 const auto dotPos = col.find('.');
306 if (dotPos != std::string::npos && dotPos != col.size() - 1 && dotPos != 0u) {
307 auto oldName = col;
308 std::replace(col.begin(), col.end(), '.', '_');
309 if (std::find(columnNames.begin(), columnNames.end(), col) != columnNames.end())
310 throw std::runtime_error("Column " + oldName + " would be written as " + col +
311 " but this column already exists. Please use Alias to select a new name for " +
312 oldName);
313 Info("Snapshot", "Column %s will be saved as %s", oldName.c_str(), col.c_str());
314 }
315 }
316
317 return newColNames;
318}
319
320void InterpreterDeclare(const std::string &code)
321{
322 R__LOG_DEBUG(10, RDFLogChannel()) << "Declaring the following code to cling:\n\n" << code << '\n';
323
324 if (!gInterpreter->Declare(code.c_str())) {
325 const auto msg =
326 "\nRDataFrame: An error occurred during just-in-time compilation. The lines above might indicate the cause of "
327 "the crash\n All RDF objects that have not run an event loop yet should be considered in an invalid state.\n";
328 throw std::runtime_error(msg);
329 }
330}
331
332Long64_t InterpreterCalc(const std::string &code, const std::string &context)
333{
334 R__LOG_DEBUG(10, RDFLogChannel()) << "Jitting and executing the following code:\n\n" << code << '\n';
335
336 TInterpreter::EErrorCode errorCode(TInterpreter::kNoError); // storage for cling errors
337
338 auto callCalc = [&errorCode, &context](const std::string &codeSlice) {
339 gInterpreter->Calc(codeSlice.c_str(), &errorCode);
340 if (errorCode != TInterpreter::EErrorCode::kNoError) {
341 std::string msg = "\nAn error occurred during just-in-time compilation";
342 if (!context.empty())
343 msg += " in " + context;
344 msg +=
345 ". The lines above might indicate the cause of the crash\nAll RDF objects that have not run their event "
346 "loop yet should be considered in an invalid state.\n";
347 throw std::runtime_error(msg);
348 }
349 };
350
351 // Call Calc every 1000 newlines in order to avoid jitting a very large function body, which is slow:
352 // see https://github.com/root-project/root/issues/9312 and https://github.com/root-project/root/issues/7604
353 std::size_t substr_start = 0;
354 std::size_t substr_end = 0;
355 while (substr_end != std::string::npos && substr_start != code.size() - 1) {
356 for (std::size_t i = 0u; i < 1000u && substr_end != std::string::npos; ++i) {
357 substr_end = code.find('\n', substr_end + 1);
358 }
359 const std::string subs = code.substr(substr_start, substr_end - substr_start);
360 substr_start = substr_end;
361
362 callCalc(subs);
363 }
364
365 return 0; // we used to forward the return value of Calc, but that's not possible anymore.
366}
367
369{
370 const auto str = colName.data();
371 const auto goodPrefix = colName.size() > 3 && // has at least more characters than {r,t}df
372 ('r' == str[0] || 't' == str[0]) && // starts with r or t
373 0 == strncmp("df", str + 1, 2); // 2nd and 3rd letters are df
374 return goodPrefix && '_' == colName.back(); // also ends with '_'
375}
376
377unsigned int GetColumnWidth(const std::vector<std::string>& names, const unsigned int minColumnSpace)
378{
379 auto columnWidth = 0u;
380 for (const auto& name : names) {
381 const auto length = name.length();
382 if (length > columnWidth)
383 columnWidth = length;
384 }
385 columnWidth = (columnWidth / minColumnSpace + 1) * minColumnSpace;
386 return columnWidth;
387}
388
389void CheckDefineType(RDefineBase &define, const std::type_info &tid)
390{
391 const auto &colTId = define.GetTypeId();
392
393 // Here we compare names and not typeinfos since they may come from two different contexts: a compiled
394 // and a jitted one.
395 const auto diffTypes = (0 != std::strcmp(colTId.name(), tid.name()));
396 auto inheritedType = [&]() {
397 auto colTClass = TClass::GetClass(colTId);
398 return colTClass && colTClass->InheritsFrom(TClass::GetClass(tid));
399 };
400
401 if (diffTypes && !inheritedType()) {
402 const auto tName = TypeID2TypeName(tid);
403 const auto colTypeName = TypeID2TypeName(colTId);
404 std::string errMsg = "RDefineReader: column \"" + define.GetName() + "\" is being used as ";
405 if (tName.empty()) {
406 errMsg += tid.name();
407 errMsg += " (extracted from type info)";
408 } else {
409 errMsg += tName;
410 }
411 errMsg += " but defined column has type ";
412 if (colTypeName.empty()) {
413 auto &id = colTId;
414 errMsg += id.name();
415 errMsg += " (extracted from type info)";
416 } else {
417 errMsg += colTypeName;
418 }
419 throw std::runtime_error(errMsg);
420 }
421}
422
423} // end NS RDF
424} // end NS Internal
425} // end NS ROOT
void Class()
Definition: Class.C:29
double
Definition: Converters.cxx:939
l unsigned short
Definition: Converters.cxx:880
long
Definition: Converters.cxx:876
#define R__LOG_DEBUG(DEBUGLEVEL,...)
Definition: RLogger.hxx:365
#define b(i)
Definition: RSha256.hxx:100
#define c(i)
Definition: RSha256.hxx:101
long long Long64_t
Definition: RtypesCore.h:80
unsigned long long ULong64_t
Definition: RtypesCore.h:81
void Info(const char *location, const char *msgfmt,...)
Use this function for informational messages.
Definition: TError.cxx:220
char name[80]
Definition: TGX11.cxx:110
#define gInterpreter
Definition: TInterpreter.h:562
std::string GetTypeName() const
Definition: RDefineBase.cxx:44
std::string GetName() const
Definition: RDefineBase.cxx:39
virtual const std::type_info & GetTypeId() const =0
A log configuration for a channel, e.g.
Definition: RLogger.hxx:101
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual bool HasColumn(std::string_view colName) const =0
Checks if the dataset has a certain column.
virtual std::string GetTypeName(std::string_view colName) const =0
Type of a column as a string, e.g.
A Branch for the case of an object.
virtual TClass * GetClass() const
TClassRef is used to implement a permanent reference to a TClass object.
Definition: TClassRef.h:28
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2955
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition: TLeaf.h:57
virtual const char * GetTypeName() const
Definition: TLeaf.h:139
virtual TLeaf * GetLeafCount() const
If this leaf stores a variable-sized array or a multi-dimensional array whose last dimension has vari...
Definition: TLeaf.h:121
virtual Int_t GetLenStatic() const
Return the fixed length of this leaf.
Definition: TLeaf.h:132
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
A TTree represents a columnar dataset.
Definition: TTree.h:79
virtual TBranch * FindBranch(const char *name)
Return the branch that correspond to the path 'branchname', which can include the name of the tree or...
Definition: TTree.cxx:4810
virtual TBranch * GetBranch(const char *name)
Return pointer to the branch with the given name in this tree or its friends.
Definition: TTree.cxx:5257
virtual TLeaf * GetLeaf(const char *branchname, const char *leafname)
Return pointer to the 1st Leaf named name in any Branch of this Tree or any branch in the list of fri...
Definition: TTree.cxx:6163
virtual TLeaf * FindLeaf(const char *name)
Find leaf..
Definition: TTree.cxx:4882
basic_string_view< char > string_view
ROOT::Experimental::RLogChannel & RDFLogChannel()
Definition: RDFUtils.cxx:37
std::vector< std::string > ReplaceDotWithUnderscore(const std::vector< std::string > &columnNames)
Replace occurrences of '.
Definition: RDFUtils.cxx:301
const std::type_info & TypeName2TypeID(const std::string &name)
Return the type_info associated to a name.
Definition: RDFUtils.cxx:51
unsigned int GetNSlots()
Definition: RDFUtils.cxx:288
std::string ComposeRVecTypeName(const std::string &valueType)
Definition: RDFUtils.cxx:133
std::string ColumnName2ColumnTypeName(const std::string &colName, TTree *, RDataSource *, RDefineBase *, bool vector2rvec=true)
Return a string containing the type of the given branch.
Definition: RDFUtils.cxx:224
std::string GetLeafTypeName(TLeaf *leaf, const std::string &colName)
Definition: RDFUtils.cxx:138
char TypeName2ROOTTypeName(const std::string &b)
Convert type name (e.g.
Definition: RDFUtils.cxx:257
std::string TypeID2TypeName(const std::type_info &id)
Returns the name of a type starting from its type_info An empty string is returned in case of failure...
Definition: RDFUtils.cxx:99
unsigned int GetColumnWidth(const std::vector< std::string > &names, const unsigned int minColumnSpace=8u)
Get optimal column width for printing a table given the names and the desired minimal space between c...
Definition: RDFUtils.cxx:377
std::string GetBranchOrLeafTypeName(TTree &t, const std::string &colName)
Return the typename of object colName stored in t, if any.
Definition: RDFUtils.cxx:165
void CheckDefineType(RDefineBase &define, const std::type_info &tid)
Definition: RDFUtils.cxx:389
Long64_t InterpreterCalc(const std::string &code, const std::string &context="")
Jit code in the interpreter with TInterpreter::Calc, throw in case of errors.
Definition: RDFUtils.cxx:332
bool IsInternalColumn(std::string_view colName)
Whether custom column with name colName is an "internal" column such as rdfentry_ or rdfslot_.
Definition: RDFUtils.cxx:368
void InterpreterDeclare(const std::string &code)
Declare code in the interpreter via the TInterpreter::Declare method, throw in case of errors.
Definition: RDFUtils.cxx:320
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition: TROOT.cxx:558
UInt_t GetThreadPoolSize()
Returns the size of ROOT's thread pool.
Definition: TROOT.cxx:565
@ kSTLvector
Definition: ESTLType.h:30
ROOT::ESTLType IsSTLCont(std::string_view type)
type : type name: vector<list<classA,allocator>,allocator> result: 0 : not stl container code of cont...
int GetSplit(const char *type, std::vector< std::string > &output, int &nestedLoc, EModType mode=TClassEdit::kNone)
Stores in output (after emptying it) the split type.
Definition: tree.py:1