Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
InterfaceUtils.hxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 02/2018
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#ifndef ROOT_RDF_TINTERFACE_UTILS
12#define ROOT_RDF_TINTERFACE_UTILS
13
14#include "RColumnRegister.hxx"
15#include <ROOT/RDF/RAction.hxx>
16#include <ROOT/RDF/ActionHelpers.hxx> // for BuildAction
18#include <ROOT/RDF/RDefine.hxx>
20#include <ROOT/RDF/RFilter.hxx>
21#include <ROOT/RDF/Utils.hxx>
27#include <string_view>
29#include <ROOT/TypeTraits.hxx>
30#include <TError.h> // gErrorIgnoreLevel
31#include <TH1.h>
32#include <TROOT.h> // IsImplicitMTEnabled
33
34#include <deque>
35#include <functional>
36#include <map>
37#include <memory>
38#include <string>
39#include <type_traits>
40#include <typeinfo>
41#include <vector>
42#include <unordered_map>
43
44class TObjArray;
45class TTree;
46namespace ROOT {
47namespace Detail {
48namespace RDF {
49class RNodeBase;
50}
51}
52namespace RDF {
53template <typename T>
54class RResultPtr;
55template<typename T, typename V>
56class RInterface;
58class RDataSource;
59} // namespace RDF
60
61} // namespace ROOT
62
63/// \cond HIDDEN_SYMBOLS
64
65namespace ROOT {
66namespace Internal {
67namespace RDF {
68using namespace ROOT::Detail::RDF;
69using namespace ROOT::RDF;
70namespace TTraits = ROOT::TypeTraits;
71
72std::string DemangleTypeIdName(const std::type_info &typeInfo);
73
75ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName);
76
77/// An helper object that sets and resets gErrorIgnoreLevel via RAII.
78class RIgnoreErrorLevelRAII {
79private:
80 int fCurIgnoreErrorLevel = gErrorIgnoreLevel;
81
82public:
83 RIgnoreErrorLevelRAII(int errorIgnoreLevel) { gErrorIgnoreLevel = errorIgnoreLevel; }
84 ~RIgnoreErrorLevelRAII() { gErrorIgnoreLevel = fCurIgnoreErrorLevel; }
85};
86
87/****** BuildAction overloads *******/
88
89// clang-format off
90/// This namespace defines types to be used for tag dispatching in RInterface.
91namespace ActionTags {
92struct Histo1D{};
93struct Histo2D{};
94struct Histo3D{};
95struct HistoND{};
96struct Graph{};
97struct GraphAsymmErrors{};
98struct Profile1D{};
99struct Profile2D{};
100struct Min{};
101struct Max{};
102struct Sum{};
103struct Mean{};
104struct Fill{};
105struct StdDev{};
106struct Display{};
107struct Snapshot{};
108struct Book{};
109}
110// clang-format on
111
112template <typename T, bool ISV6HISTO = std::is_base_of<TH1, std::decay_t<T>>::value>
113struct HistoUtils {
114 static void SetCanExtendAllAxes(T &h) { h.SetCanExtend(::TH1::kAllAxes); }
115 static bool HasAxisLimits(T &h)
116 {
117 auto xaxis = h.GetXaxis();
118 return !(xaxis->GetXmin() == 0. && xaxis->GetXmax() == 0.);
119 }
120};
121
122template <typename T>
123struct HistoUtils<T, false> {
124 static void SetCanExtendAllAxes(T &) {}
125 static bool HasAxisLimits(T &) { return true; }
126};
127
128// Generic filling (covers Histo2D, Histo3D, HistoND, Profile1D and Profile2D actions, with and without weights)
129template <typename... ColTypes, typename ActionTag, typename ActionResultType, typename PrevNodeType>
130std::unique_ptr<RActionBase>
131BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &h, const unsigned int nSlots,
132 std::shared_ptr<PrevNodeType> prevNode, ActionTag, const RColumnRegister &colRegister)
133{
134 using Helper_t = FillHelper<ActionResultType>;
135 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
136 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
137}
138
139// Histo1D filling (must handle the special case of distinguishing FillHelper and BufferedFillHelper
140template <typename... ColTypes, typename PrevNodeType>
141std::unique_ptr<RActionBase>
142BuildAction(const ColumnNames_t &bl, const std::shared_ptr<::TH1D> &h, const unsigned int nSlots,
143 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Histo1D, const RColumnRegister &colRegister)
144{
145 auto hasAxisLimits = HistoUtils<::TH1D>::HasAxisLimits(*h);
146
147 if (hasAxisLimits || !IsImplicitMTEnabled()) {
148 using Helper_t = FillHelper<::TH1D>;
149 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
150 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
151 } else {
152 using Helper_t = BufferedFillHelper;
153 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
154 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
155 }
156}
157
158template <typename... ColTypes, typename PrevNodeType>
159std::unique_ptr<RActionBase>
160BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraph> &g, const unsigned int nSlots,
161 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Graph, const RColumnRegister &colRegister)
162{
163 using Helper_t = FillTGraphHelper;
164 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
165 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
166}
167
168template <typename... ColTypes, typename PrevNodeType>
169std::unique_ptr<RActionBase>
170BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraphAsymmErrors> &g, const unsigned int nSlots,
171 std::shared_ptr<PrevNodeType> prevNode, ActionTags::GraphAsymmErrors, const RColumnRegister &colRegister)
172{
173 using Helper_t = FillTGraphAsymmErrorsHelper;
174 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
175 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
176}
177
178// Min action
179template <typename ColType, typename PrevNodeType, typename ActionResultType>
180std::unique_ptr<RActionBase>
181BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &minV, const unsigned int nSlots,
182 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Min, const RColumnRegister &colRegister)
183{
184 using Helper_t = MinHelper<ActionResultType>;
185 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
186 return std::make_unique<Action_t>(Helper_t(minV, nSlots), bl, std::move(prevNode), colRegister);
187}
188
189// Max action
190template <typename ColType, typename PrevNodeType, typename ActionResultType>
191std::unique_ptr<RActionBase>
192BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &maxV, const unsigned int nSlots,
193 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Max, const RColumnRegister &colRegister)
194{
195 using Helper_t = MaxHelper<ActionResultType>;
196 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
197 return std::make_unique<Action_t>(Helper_t(maxV, nSlots), bl, std::move(prevNode), colRegister);
198}
199
200// Sum action
201template <typename ColType, typename PrevNodeType, typename ActionResultType>
202std::unique_ptr<RActionBase>
203BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &sumV, const unsigned int nSlots,
204 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Sum, const RColumnRegister &colRegister)
205{
206 using Helper_t = SumHelper<ActionResultType>;
207 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
208 return std::make_unique<Action_t>(Helper_t(sumV, nSlots), bl, std::move(prevNode), colRegister);
209}
210
211// Mean action
212template <typename ColType, typename PrevNodeType>
213std::unique_ptr<RActionBase>
214BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &meanV, const unsigned int nSlots,
215 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Mean, const RColumnRegister &colRegister)
216{
217 using Helper_t = MeanHelper;
218 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
219 return std::make_unique<Action_t>(Helper_t(meanV, nSlots), bl, std::move(prevNode), colRegister);
220}
221
222// Standard Deviation action
223template <typename ColType, typename PrevNodeType>
224std::unique_ptr<RActionBase>
225BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &stdDeviationV, const unsigned int nSlots,
226 std::shared_ptr<PrevNodeType> prevNode, ActionTags::StdDev, const RColumnRegister &colRegister)
227{
228 using Helper_t = StdDevHelper;
229 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
230 return std::make_unique<Action_t>(Helper_t(stdDeviationV, nSlots), bl, prevNode, colRegister);
231}
232
233using displayHelperArgs_t = std::pair<size_t, std::shared_ptr<ROOT::RDF::RDisplay>>;
234
235// Display action
236template <typename... ColTypes, typename PrevNodeType>
237std::unique_ptr<RActionBase>
238BuildAction(const ColumnNames_t &bl, const std::shared_ptr<displayHelperArgs_t> &helperArgs, const unsigned int,
239 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Display, const RColumnRegister &colRegister)
240{
241 using Helper_t = DisplayHelper<PrevNodeType>;
242 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
243 return std::make_unique<Action_t>(Helper_t(helperArgs->first, helperArgs->second, prevNode), bl, prevNode,
244 colRegister);
245}
246
247struct SnapshotHelperArgs {
248 std::string fFileName;
249 std::string fDirName;
250 std::string fTreeName;
251 std::vector<std::string> fOutputColNames;
253};
254
255// Snapshot action
256template <typename... ColTypes, typename PrevNodeType>
257std::unique_ptr<RActionBase>
258BuildAction(const ColumnNames_t &colNames, const std::shared_ptr<SnapshotHelperArgs> &snapHelperArgs,
259 const unsigned int nSlots, std::shared_ptr<PrevNodeType> prevNode, ActionTags::Snapshot,
260 const RColumnRegister &colRegister)
261{
262 const auto &filename = snapHelperArgs->fFileName;
263 const auto &dirname = snapHelperArgs->fDirName;
264 const auto &treename = snapHelperArgs->fTreeName;
265 const auto &outputColNames = snapHelperArgs->fOutputColNames;
266 const auto &options = snapHelperArgs->fOptions;
267
268 auto sz = sizeof...(ColTypes);
269 std::vector<bool> isDefine(sz);
270 for (auto i = 0u; i < sz; ++i)
271 isDefine[i] = colRegister.IsDefineOrAlias(colNames[i]);
272
273 std::unique_ptr<RActionBase> actionPtr;
275 // single-thread snapshot
276 using Helper_t = SnapshotHelper<ColTypes...>;
277 using Action_t = RAction<Helper_t, PrevNodeType>;
278 actionPtr.reset(
279 new Action_t(Helper_t(filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
280 colNames, prevNode, colRegister));
281 } else {
282 // multi-thread snapshot
283 using Helper_t = SnapshotHelperMT<ColTypes...>;
284 using Action_t = RAction<Helper_t, PrevNodeType>;
285 actionPtr.reset(new Action_t(
286 Helper_t(nSlots, filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
287 colNames, prevNode, colRegister));
288 }
289 return actionPtr;
290}
291
292// Book with custom helper type
293template <typename... ColTypes, typename PrevNodeType, typename Helper_t>
294std::unique_ptr<RActionBase>
295BuildAction(const ColumnNames_t &bl, const std::shared_ptr<Helper_t> &h, const unsigned int /*nSlots*/,
296 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Book, const RColumnRegister &colRegister)
297{
298 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
299 return std::make_unique<Action_t>(Helper_t(std::move(*h)), bl, std::move(prevNode), colRegister);
300}
301
302/****** end BuildAndBook ******/
303
304template <typename Filter>
305void CheckFilter(Filter &)
306{
307 using FilterRet_t = typename RDF::CallableTraits<Filter>::ret_type;
308 static_assert(std::is_convertible<FilterRet_t, bool>::value,
309 "filter expression returns a type that is not convertible to bool");
310}
311
312ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action);
313
314void CheckValidCppVarName(std::string_view var, const std::string &where);
315
316void CheckForRedefinition(const std::string &where, std::string_view definedCol, const RColumnRegister &colRegister,
317 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
318
319void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister,
320 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
321
322void CheckForNoVariations(const std::string &where, std::string_view definedColView,
323 const RColumnRegister &colRegister);
324
325std::string PrettyPrintAddr(const void *const addr);
326
327std::shared_ptr<RJittedFilter> BookFilterJit(std::shared_ptr<RNodeBase> *prevNodeOnHeap, std::string_view name,
328 std::string_view expression, const ColumnNames_t &branches,
329 const RColumnRegister &colRegister, TTree *tree, RDataSource *ds);
330
331std::shared_ptr<RJittedDefine> BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm,
332 RDataSource *ds, const RColumnRegister &colRegister,
333 const ColumnNames_t &branches, std::shared_ptr<RNodeBase> *prevNodeOnHeap);
334
335std::shared_ptr<RJittedDefine> BookDefinePerSampleJit(std::string_view name, std::string_view expression,
336 RLoopManager &lm, const RColumnRegister &colRegister,
337 std::shared_ptr<RNodeBase> *upcastNodeOnHeap);
338
339std::shared_ptr<RJittedVariation>
340BookVariationJit(const std::vector<std::string> &colNames, std::string_view variationName,
341 const std::vector<std::string> &variationTags, std::string_view expression, RLoopManager &lm,
342 RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches,
343 std::shared_ptr<RNodeBase> *upcastNodeOnHeap, bool isSingleColumn);
344
345std::string JitBuildAction(const ColumnNames_t &bl, std::shared_ptr<RDFDetail::RNodeBase> *prevNode,
346 const std::type_info &art, const std::type_info &at, void *rOnHeap, TTree *tree,
347 const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds,
348 std::weak_ptr<RJittedAction> *jittedActionOnHeap);
349
350// Allocate a weak_ptr on the heap, return a pointer to it. The user is responsible for deleting this weak_ptr.
351// This function is meant to be used by RInterface's methods that book code for jitting.
352// The problem it solves is that we generate code to be lazily jitted with the addresses of certain objects in them,
353// and we need to check those objects are still alive when the generated code is finally jitted and executed.
354// So we pass addresses to weak_ptrs allocated on the heap to the jitted code, which is then responsible for
355// the deletion of the weak_ptr object.
356template <typename T>
357std::weak_ptr<T> *MakeWeakOnHeap(const std::shared_ptr<T> &shPtr)
358{
359 return new std::weak_ptr<T>(shPtr);
360}
361
362// Same as MakeWeakOnHeap, but create a shared_ptr that makes sure the object is definitely kept alive.
363template <typename T>
364std::shared_ptr<T> *MakeSharedOnHeap(const std::shared_ptr<T> &shPtr)
365{
366 return new std::shared_ptr<T>(shPtr);
367}
368
369bool AtLeastOneEmptyString(const std::vector<std::string_view> strings);
370
371/// Take a shared_ptr<AnyNodeType> and return a shared_ptr<RNodeBase>.
372/// This works for RLoopManager nodes as well as filters and ranges.
373std::shared_ptr<RNodeBase> UpcastNode(std::shared_ptr<RNodeBase> ptr);
374
375ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns,
376 const RColumnRegister &validDefines, RDataSource *ds);
377
378std::vector<std::string> GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister,
379 TTree *tree, RDataSource *ds, const std::string &context,
380 bool vector2rvec);
381
382std::vector<bool> FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedDSCols);
383
384template <typename T>
385void AddDSColumnsHelper(const std::string &colName, RLoopManager &lm, RDataSource &ds, RColumnRegister &colRegister)
386{
387 if (colRegister.IsDefineOrAlias(colName) || !ds.HasColumn(colName) ||
388 lm.HasDataSourceColumnReaders(colName, typeid(T)))
389 return;
390
391 const auto nSlots = lm.GetNSlots();
392 std::vector<std::unique_ptr<RColumnReaderBase>> colReaders;
393 colReaders.reserve(nSlots);
394
395 const auto valuePtrs = ds.GetColumnReaders<T>(colName);
396 if (!valuePtrs.empty()) { // we are using the old GetColumnReaders mechanism in this RDataSource
397 for (auto *ptr : valuePtrs)
398 colReaders.emplace_back(new RDSColumnReader<T>(ptr));
399
400 } else { // using the new GetColumnReaders mechanism
401 // TODO consider changing the interface so we return all of these for all slots in one go
402 for (auto slot = 0u; slot < lm.GetNSlots(); ++slot)
403 colReaders.emplace_back(ds.GetColumnReaders(slot, colName, typeid(T)));
404 }
405
406 lm.AddDataSourceColumnReaders(colName, std::move(colReaders), typeid(T));
407}
408
409/// Take list of column names that must be defined, current map of custom columns, current list of defined column names,
410/// and return a new map of custom columns (with the new datasource columns added to it)
411template <typename... ColumnTypes>
412void AddDSColumns(const std::vector<std::string> &requiredCols, RLoopManager &lm, RDataSource &ds,
413 TTraits::TypeList<ColumnTypes...>, RColumnRegister &colRegister)
414{
415 // hack to expand a template parameter pack without c++17 fold expressions.
416 using expander = int[];
417 int i = 0;
418 (void)expander{(AddDSColumnsHelper<ColumnTypes>(requiredCols[i], lm, ds, colRegister), ++i)..., 0};
419}
420
421// this function is meant to be called by the jitted code generated by BookFilterJit
422template <typename F, typename PrevNode>
423void JitFilterHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name,
424 std::weak_ptr<RJittedFilter> *wkJittedFilter, std::shared_ptr<PrevNode> *prevNodeOnHeap,
425 RColumnRegister *colRegister) noexcept
426{
427 if (wkJittedFilter->expired()) {
428 // The branch of the computation graph that needed this jitted code went out of scope between the type
429 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
430 delete wkJittedFilter;
431 delete colRegister;
432 delete prevNodeOnHeap;
433 return;
434 }
435
436 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
437 delete[] colsPtr;
438
439 const auto jittedFilter = wkJittedFilter->lock();
440
441 // mock Filter logic -- validity checks and Define-ition of RDataSource columns
442 using Callable_t = std::decay_t<F>;
444 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
445 constexpr auto nColumns = ColTypes_t::list_size;
446 CheckFilter(f);
447
448 auto &lm = *jittedFilter->GetLoopManagerUnchecked(); // RLoopManager must exist at this time
449 auto ds = lm.GetDataSource();
450
451 if (ds != nullptr)
452 AddDSColumns(cols, lm, *ds, ColTypes_t(), *colRegister);
453
454 jittedFilter->SetFilter(
455 std::unique_ptr<RFilterBase>(new F_t(std::forward<F>(f), cols, *prevNodeOnHeap, *colRegister, name)));
456 // colRegister points to the columns structure in the heap, created before the jitted call so that the jitter can
457 // share data after it has lazily compiled the code. Here the data has been used and the memory can be freed.
458 delete colRegister;
459 delete prevNodeOnHeap;
460 delete wkJittedFilter;
461}
462
463namespace DefineTypes {
464struct RDefineTag {};
465struct RDefinePerSampleTag {};
466}
467
468template <typename F>
469auto MakeDefineNode(DefineTypes::RDefineTag, std::string_view name, std::string_view dummyType, F &&f,
470 const ColumnNames_t &cols, RColumnRegister &colRegister, RLoopManager &lm)
471{
472 return std::unique_ptr<RDefineBase>(new RDefine<std::decay_t<F>, ExtraArgsForDefine::None>(
473 name, dummyType, std::forward<F>(f), cols, colRegister, lm));
474}
475
476template <typename F>
477auto MakeDefineNode(DefineTypes::RDefinePerSampleTag, std::string_view name, std::string_view dummyType, F &&f,
478 const ColumnNames_t &, RColumnRegister &, RLoopManager &lm)
479{
480 return std::unique_ptr<RDefineBase>(
481 new RDefinePerSample<std::decay_t<F>>(name, dummyType, std::forward<F>(f), lm));
482}
483
484// Build a RDefine or a RDefinePerSample object and attach it to an existing RJittedDefine
485// This function is meant to be called by jitted code right before starting the event loop.
486// If colsPtr is null, build a RDefinePerSample (it has no input columns), otherwise a RDefine.
487template <typename RDefineTypeTag, typename F>
488void JitDefineHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name, RLoopManager *lm,
489 std::weak_ptr<RJittedDefine> *wkJittedDefine, RColumnRegister *colRegister,
490 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
491{
492 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
493 auto doDeletes = [&] {
494 delete wkJittedDefine;
495 delete colRegister;
496 delete prevNodeOnHeap;
497 delete[] colsPtr;
498 };
499
500 if (wkJittedDefine->expired()) {
501 // The branch of the computation graph that needed this jitted code went out of scope between the type
502 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
503 doDeletes();
504 return;
505 }
506
507 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
508
509 auto jittedDefine = wkJittedDefine->lock();
510
511 using Callable_t = std::decay_t<F>;
512 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
513
514 auto ds = lm->GetDataSource();
515 if (ds != nullptr)
516 AddDSColumns(cols, *lm, *ds, ColTypes_t(), *colRegister);
517
518 // will never actually be used (trumped by jittedDefine->GetTypeName()), but we set it to something meaningful
519 // to help devs debugging
520 const auto dummyType = "jittedCol_t";
521 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
522 std::unique_ptr<RDefineBase> newCol{
523 MakeDefineNode(RDefineTypeTag{}, name, dummyType, std::forward<F>(f), cols, *colRegister, *lm)};
524 jittedDefine->SetDefine(std::move(newCol));
525
526 doDeletes();
527}
528
529template <bool IsSingleColumn, typename F>
530void JitVariationHelper(F &&f, const char **colsPtr, std::size_t colsSize, const char **variedCols,
531 std::size_t variedColsSize, const char **variationTags, std::size_t variationTagsSize,
532 std::string_view variationName, RLoopManager *lm,
533 std::weak_ptr<RJittedVariation> *wkJittedVariation, RColumnRegister *colRegister,
534 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
535{
536 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
537 auto doDeletes = [&] {
538 delete[] colsPtr;
539 delete[] variedCols;
540 delete[] variationTags;
541
542 delete wkJittedVariation;
543 delete colRegister;
544 delete prevNodeOnHeap;
545 };
546
547 if (wkJittedVariation->expired()) {
548 // The branch of the computation graph that needed this jitted variation went out of scope between the type
549 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
550 doDeletes();
551 return;
552 }
553
554 const ColumnNames_t inputColNames(colsPtr, colsPtr + colsSize);
555 std::vector<std::string> variedColNames(variedCols, variedCols + variedColsSize);
556 std::vector<std::string> tags(variationTags, variationTags + variationTagsSize);
557
558 auto jittedVariation = wkJittedVariation->lock();
559
560 using Callable_t = std::decay_t<F>;
561 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
562
563 auto ds = lm->GetDataSource();
564 if (ds != nullptr)
565 AddDSColumns(inputColNames, *lm, *ds, ColTypes_t(), *colRegister);
566
567 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
568 std::unique_ptr<RVariationBase> newVariation{new RVariation<std::decay_t<F>, IsSingleColumn>(
569 std::move(variedColNames), variationName, std::forward<F>(f), std::move(tags), jittedVariation->GetTypeName(),
570 *colRegister, *lm, inputColNames)};
571 jittedVariation->SetVariation(std::move(newVariation));
572
573 doDeletes();
574}
575
576/// Convenience function invoked by jitted code to build action nodes at runtime
577template <typename ActionTag, typename... ColTypes, typename PrevNodeType, typename HelperArgType>
578void CallBuildAction(std::shared_ptr<PrevNodeType> *prevNodeOnHeap, const char **colsPtr, std::size_t colsSize,
579 const unsigned int nSlots, std::shared_ptr<HelperArgType> *helperArgOnHeap,
580 std::weak_ptr<RJittedAction> *wkJittedActionOnHeap, RColumnRegister *colRegister) noexcept
581{
582 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
583 auto doDeletes = [&] {
584 delete[] colsPtr;
585 delete helperArgOnHeap;
586 delete wkJittedActionOnHeap;
587 // colRegister must be deleted before prevNodeOnHeap because their dtor needs the RLoopManager to be alive
588 // and prevNodeOnHeap is what keeps it alive if the rest of the computation graph is already out of scope
589 delete colRegister;
590 delete prevNodeOnHeap;
591 };
592
593 if (wkJittedActionOnHeap->expired()) {
594 // The branch of the computation graph that needed this jitted variation went out of scope between the type
595 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
596 doDeletes();
597 return;
598 }
599
600 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
601
602 auto jittedActionOnHeap = wkJittedActionOnHeap->lock();
603
604 // if we are here it means we are jitting, if we are jitting the loop manager must be alive
605 auto &prevNodePtr = *prevNodeOnHeap;
606 auto &loopManager = *prevNodePtr->GetLoopManagerUnchecked();
607 using ColTypes_t = TypeList<ColTypes...>;
608 constexpr auto nColumns = ColTypes_t::list_size;
609 auto ds = loopManager.GetDataSource();
610 if (ds != nullptr)
611 AddDSColumns(cols, loopManager, *ds, ColTypes_t(), *colRegister);
612
613 auto actionPtr = BuildAction<ColTypes...>(cols, std::move(*helperArgOnHeap), nSlots, std::move(prevNodePtr),
614 ActionTag{}, *colRegister);
615 jittedActionOnHeap->SetAction(std::move(actionPtr));
616
617 doDeletes();
618}
619
620/// The contained `type` alias is `double` if `T == RInferredType`, `U` if `T == std::container<U>`, `T` otherwise.
621template <typename T, bool Container = IsDataContainer<T>::value && !std::is_same<T, std::string>::value>
622struct RMinReturnType {
623 using type = T;
624};
625
626template <>
627struct RMinReturnType<RInferredType, false> {
628 using type = double;
629};
630
631template <typename T>
632struct RMinReturnType<T, true> {
633 using type = TTraits::TakeFirstParameter_t<T>;
634};
635
636// return wrapper around f that prepends an `unsigned int slot` parameter
637template <typename R, typename F, typename... Args>
638std::function<R(unsigned int, Args...)> AddSlotParameter(F &f, TypeList<Args...>)
639{
640 return [f](unsigned int, Args... a) mutable -> R { return f(a...); };
641}
642
643template <typename ColType, typename... Rest>
644struct RNeedJittingHelper {
645 static constexpr bool value = RNeedJittingHelper<Rest...>::value;
646};
647
648template <typename... Rest>
649struct RNeedJittingHelper<RInferredType, Rest...> {
650 static constexpr bool value = true;
651};
652
653template <typename T>
654struct RNeedJittingHelper<T> {
655 static constexpr bool value = false;
656};
657
658template <>
659struct RNeedJittingHelper<RInferredType> {
660 static constexpr bool value = true;
661};
662
663template <typename ...ColTypes>
664struct RNeedJitting {
665 static constexpr bool value = RNeedJittingHelper<ColTypes...>::value;
666};
667
668template <>
669struct RNeedJitting<> {
670 static constexpr bool value = false;
671};
672
673///////////////////////////////////////////////////////////////////////////////
674/// Check preconditions for RInterface::Aggregate:
675/// - the aggregator callable must have signature `U(U,T)` or `void(U&,T)`.
676/// - the merge callable must have signature `U(U,U)` or `void(std::vector<U>&)`
677template <typename R, typename Merge, typename U, typename T, typename decayedU = std::decay_t<U>,
678 typename mergeArgsNoDecay_t = typename CallableTraits<Merge>::arg_types_nodecay,
679 typename mergeArgs_t = typename CallableTraits<Merge>::arg_types,
680 typename mergeRet_t = typename CallableTraits<Merge>::ret_type>
681void CheckAggregate(TypeList<U, T>)
682{
683 constexpr bool isAggregatorOk =
684 (std::is_same<R, decayedU>::value) || (std::is_same<R, void>::value && std::is_lvalue_reference<U>::value);
685 static_assert(isAggregatorOk, "aggregator function must have signature `U(U,T)` or `void(U&,T)`");
686 constexpr bool isMergeOk =
687 (std::is_same<TypeList<decayedU, decayedU>, mergeArgs_t>::value && std::is_same<decayedU, mergeRet_t>::value) ||
688 (std::is_same<TypeList<std::vector<decayedU> &>, mergeArgsNoDecay_t>::value &&
689 std::is_same<void, mergeRet_t>::value);
690 static_assert(isMergeOk, "merge function must have signature `U(U,U)` or `void(std::vector<U>&)`");
691}
692
693///////////////////////////////////////////////////////////////////////////////
694/// This overload of CheckAggregate is called when the aggregator takes more than two arguments
695template <typename R, typename T>
696void CheckAggregate(T)
697{
698 static_assert(sizeof(T) == 0, "aggregator function must take exactly two arguments");
699}
700
701///////////////////////////////////////////////////////////////////////////////
702/// Check as many template parameters were passed as the number of column names, throw if this is not the case.
703void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames);
704
705/// Return local BranchNames or default BranchNames according to which one should be used
706const ColumnNames_t SelectColumns(unsigned int nArgs, const ColumnNames_t &bl, const ColumnNames_t &defBl);
707
708/// Check whether column names refer to a valid branch of a TTree or have been `Define`d. Return invalid column names.
709ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns,
710 const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns);
711
712/// Returns the list of Filters defined in the whole graph
713std::vector<std::string> GetFilterNames(const std::shared_ptr<RLoopManager> &loopManager);
714
715/// Returns the list of Filters defined in the branch
716template <typename NodeType>
717std::vector<std::string> GetFilterNames(const std::shared_ptr<NodeType> &node)
718{
719 std::vector<std::string> filterNames;
720 node->AddFilterName(filterNames);
721 return filterNames;
722}
723
724struct ParsedTreePath {
725 std::string fTreeName;
726 std::string fDirName;
727};
728
729ParsedTreePath ParseTreePath(std::string_view fullTreeName);
730
731// Check if a condition is true for all types
732template <bool...>
733struct TBoolPack;
734
735template <bool... bs>
736using IsTrueForAllImpl_t = typename std::is_same<TBoolPack<bs..., true>, TBoolPack<true, bs...>>;
737
738template <bool... Conditions>
739struct TEvalAnd {
740 static constexpr bool value = IsTrueForAllImpl_t<Conditions...>::value;
741};
742
743// Check if a class is a specialisation of stl containers templates
744// clang-format off
745
746template <typename>
747struct IsList_t : std::false_type {};
748
749template <typename T>
750struct IsList_t<std::list<T>> : std::true_type {};
751
752template <typename>
753struct IsDeque_t : std::false_type {};
754
755template <typename T>
756struct IsDeque_t<std::deque<T>> : std::true_type {};
757// clang-format on
758
760
761template <typename T>
762struct InnerValueType {
763 using type = T; // fallback for when T is not a nested RVec
764};
765
766template <typename Elem>
767struct InnerValueType<ROOT::VecOps::RVec<ROOT::VecOps::RVec<Elem>>> {
768 using type = Elem;
769};
770
771template <typename T>
772using InnerValueType_t = typename InnerValueType<T>::type;
773
774std::pair<std::vector<std::string>, std::vector<std::string>>
775AddSizeBranches(const std::vector<std::string> &branches, TTree *tree, std::vector<std::string> &&colsWithoutAliases,
776 std::vector<std::string> &&colsWithAliases);
777
778void RemoveDuplicates(ColumnNames_t &columnNames);
779
780} // namespace RDF
781} // namespace Internal
782
783namespace Detail {
784namespace RDF {
785
786/// The aliased type is `double` if `T == RInferredType`, `U` if `T == container<U>`, `T` otherwise.
787template <typename T>
788using MinReturnType_t = typename RDFInternal::RMinReturnType<T>::type;
789
790template <typename T>
791using MaxReturnType_t = MinReturnType_t<T>;
792
793template <typename T>
794using SumReturnType_t = MinReturnType_t<T>;
795
796} // namespace RDF
797} // namespace Detail
798} // namespace ROOT
799
800/// \endcond
801
802#endif
#define f(i)
Definition RSha256.hxx:104
#define g(i)
Definition RSha256.hxx:105
#define a(i)
Definition RSha256.hxx:99
#define h(i)
Definition RSha256.hxx:106
Int_t gErrorIgnoreLevel
Error handling routines.
Definition TError.cxx:31
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
The head node of a RDF computation graph.
void AddDataSourceColumnReaders(const std::string &col, std::vector< std::unique_ptr< RColumnReaderBase > > &&readers, const std::type_info &ti)
RDataSource * GetDataSource() const
RLoopManager * GetLoopManagerUnchecked() final
bool HasDataSourceColumnReaders(const std::string &col, const std::type_info &ti) const
Return true if AddDataSourceColumnReaders was called for column name col.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual bool HasColumn(std::string_view colName) const =0
Checks if the dataset has a certain column.
std::vector< T ** > GetColumnReaders(std::string_view columnName)
Called at most once per column by RDF.
The public interface to the RDataFrame federation of classes.
A "std::vector"-like collection of values implementing handy operation to analyse them.
Definition RVec.hxx:1529
@ kAllAxes
Definition TH1.h:76
An array of TObjects.
Definition TObjArray.h:31
A TTree represents a columnar dataset.
Definition TTree.h:79
T Sum(const RVec< T > &v, const T zero=T(0))
Sum elements of an RVec.
Definition RVec.hxx:1954
#define F(x, y, z)
const ColumnNames_t SelectColumns(unsigned int nRequiredNames, const ColumnNames_t &names, const ColumnNames_t &defaultNames)
Choose between local column names or default column names, throw in case of errors.
void CheckForNoVariations(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister)
Throw if the column has systematic variations attached.
ParsedTreePath ParseTreePath(std::string_view fullTreeName)
void CheckForRedefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is already there.
void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is not already there.
std::shared_ptr< RJittedDefine > BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a Define call.
void CheckValidCppVarName(std::string_view var, const std::string &where)
void RemoveDuplicates(ColumnNames_t &columnNames)
ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns, const RColumnRegister &colRegister, RDataSource *ds)
Given the desired number of columns and the user-provided list of columns:
std::shared_ptr< RNodeBase > UpcastNode(std::shared_ptr< RNodeBase > ptr)
std::vector< std::string > GetFilterNames(const std::shared_ptr< RLoopManager > &loopManager)
std::string PrettyPrintAddr(const void *const addr)
void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames)
std::string DemangleTypeIdName(const std::type_info &typeInfo)
bool AtLeastOneEmptyString(const std::vector< std::string_view > strings)
std::shared_ptr< RDFDetail::RJittedFilter > BookFilterJit(std::shared_ptr< RDFDetail::RNodeBase > *prevNodeOnHeap, std::string_view name, std::string_view expression, const ColumnNames_t &branches, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds)
Book the jitting of a Filter call.
std::string JitBuildAction(const ColumnNames_t &cols, std::shared_ptr< RDFDetail::RNodeBase > *prevNode, const std::type_info &helperArgType, const std::type_info &at, void *helperArgOnHeap, TTree *tree, const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds, std::weak_ptr< RJittedAction > *jittedActionOnHeap)
ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action)
Take a list of column names, return that list with entries starting by '#' filtered out.
std::shared_ptr< RJittedVariation > BookVariationJit(const std::vector< std::string > &colNames, std::string_view variationName, const std::vector< std::string > &variationTags, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap, bool isSingleColumn)
Book the jitting of a Vary call.
void CheckForDuplicateSnapshotColumns(const ColumnNames_t &cols)
ColumnNames_t ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName)
std::pair< std::vector< std::string >, std::vector< std::string > > AddSizeBranches(const std::vector< std::string > &branches, TTree *tree, std::vector< std::string > &&colsWithoutAliases, std::vector< std::string > &&colsWithAliases)
Return copies of colsWithoutAliases and colsWithAliases with size branches for variable-sized array b...
std::vector< bool > FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedCols)
Return a bitset each element of which indicates whether the corresponding element in selectedColumns ...
std::shared_ptr< RJittedDefine > BookDefinePerSampleJit(std::string_view name, std::string_view expression, RLoopManager &lm, const RColumnRegister &colRegister, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a DefinePerSample call.
std::vector< std::string > GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds, const std::string &context, bool vector2rvec)
ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns, const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns)
void(off) SmallVectorTemplateBase< T
double T(double x)
std::vector< std::string > ColumnNames_t
ROOT type_traits extensions.
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition TROOT.cxx:570
Short_t Max(Short_t a, Short_t b)
Returns the largest of a and b.
Definition TMathBase.h:250
Short_t Min(Short_t a, Short_t b)
Returns the smallest of a and b.
Definition TMathBase.h:198
Double_t Mean(Long64_t n, const T *a, const Double_t *w=nullptr)
Returns the weighted mean of an array a with length n.
Definition TMath.h:1089
Double_t StdDev(Long64_t n, const T *a, const Double_t *w=nullptr)
Definition TMath.h:527
A collection of options to steer the creation of the dataset on file.
Lightweight storage for a collection of types.