Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
InterfaceUtils.hxx
Go to the documentation of this file.
1// Author: Enrico Guiraud, Danilo Piparo CERN 02/2018
2
3/*************************************************************************
4 * Copyright (C) 1995-2018, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11#ifndef ROOT_RDF_TINTERFACE_UTILS
12#define ROOT_RDF_TINTERFACE_UTILS
13
14#include "RColumnRegister.hxx"
15#include <ROOT/RDF/RAction.hxx>
16#include <ROOT/RDF/ActionHelpers.hxx> // for BuildAction
18#include <ROOT/RDF/RDefine.hxx>
20#include <ROOT/RDF/RFilter.hxx>
21#include <ROOT/RDF/Utils.hxx>
27#include <string_view>
29#include <ROOT/TypeTraits.hxx>
30#include <TError.h> // gErrorIgnoreLevel
31#include <TH1.h>
32#include <TROOT.h> // IsImplicitMTEnabled
33
34#include <deque>
35#include <functional>
36#include <map>
37#include <memory>
38#include <string>
39#include <type_traits>
40#include <typeinfo>
41#include <vector>
42#include <unordered_map>
43
44class TObjArray;
45class TTree;
46namespace ROOT {
47namespace Detail {
48namespace RDF {
49class RNodeBase;
50}
51}
52namespace RDF {
53template <typename T>
54class RResultPtr;
55template<typename T, typename V>
56class RInterface;
58class RDataSource;
59} // namespace RDF
60
61} // namespace ROOT
62
63/// \cond HIDDEN_SYMBOLS
64
65namespace ROOT {
66namespace Internal {
67namespace RDF {
68using namespace ROOT::Detail::RDF;
69using namespace ROOT::RDF;
70namespace TTraits = ROOT::TypeTraits;
71
72std::string DemangleTypeIdName(const std::type_info &typeInfo);
73
75ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName);
76
77/// An helper object that sets and resets gErrorIgnoreLevel via RAII.
78class RIgnoreErrorLevelRAII {
79private:
80 int fCurIgnoreErrorLevel = gErrorIgnoreLevel;
81
82public:
83 RIgnoreErrorLevelRAII(int errorIgnoreLevel) { gErrorIgnoreLevel = errorIgnoreLevel; }
84 ~RIgnoreErrorLevelRAII() { gErrorIgnoreLevel = fCurIgnoreErrorLevel; }
85};
86
87/****** BuildAction overloads *******/
88
89// clang-format off
90/// This namespace defines types to be used for tag dispatching in RInterface.
91namespace ActionTags {
92struct Histo1D{};
93struct Histo2D{};
94struct Histo3D{};
95struct HistoND{};
96struct Graph{};
97struct GraphAsymmErrors{};
98struct Profile1D{};
99struct Profile2D{};
100struct Min{};
101struct Max{};
102struct Sum{};
103struct Mean{};
104struct Fill{};
105struct StdDev{};
106struct Display{};
107struct Snapshot{};
108struct Book{};
109}
110// clang-format on
111
112template <typename T, bool ISV6HISTO = std::is_base_of<TH1, std::decay_t<T>>::value>
113struct HistoUtils {
114 static void SetCanExtendAllAxes(T &h) { h.SetCanExtend(::TH1::kAllAxes); }
115 static bool HasAxisLimits(T &h)
116 {
117 auto xaxis = h.GetXaxis();
118 return !(xaxis->GetXmin() == 0. && xaxis->GetXmax() == 0.);
119 }
120};
121
122template <typename T>
123struct HistoUtils<T, false> {
124 static void SetCanExtendAllAxes(T &) {}
125 static bool HasAxisLimits(T &) { return true; }
126};
127
128// Generic filling (covers Histo2D, Histo3D, HistoND, Profile1D and Profile2D actions, with and without weights)
129template <typename... ColTypes, typename ActionTag, typename ActionResultType, typename PrevNodeType>
130std::unique_ptr<RActionBase>
131BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &h, const unsigned int nSlots,
132 std::shared_ptr<PrevNodeType> prevNode, ActionTag, const RColumnRegister &colRegister)
133{
134 using Helper_t = FillHelper<ActionResultType>;
135 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
136 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
137}
138
139// Histo1D filling (must handle the special case of distinguishing FillHelper and BufferedFillHelper
140template <typename... ColTypes, typename PrevNodeType>
141std::unique_ptr<RActionBase>
142BuildAction(const ColumnNames_t &bl, const std::shared_ptr<::TH1D> &h, const unsigned int nSlots,
143 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Histo1D, const RColumnRegister &colRegister)
144{
145 auto hasAxisLimits = HistoUtils<::TH1D>::HasAxisLimits(*h);
146
147 if (hasAxisLimits || !IsImplicitMTEnabled()) {
148 using Helper_t = FillHelper<::TH1D>;
149 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
150 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
151 } else {
152 using Helper_t = BufferedFillHelper;
153 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
154 return std::make_unique<Action_t>(Helper_t(h, nSlots), bl, std::move(prevNode), colRegister);
155 }
156}
157
158template <typename... ColTypes, typename PrevNodeType>
159std::unique_ptr<RActionBase>
160BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraph> &g, const unsigned int nSlots,
161 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Graph, const RColumnRegister &colRegister)
162{
163 using Helper_t = FillTGraphHelper;
164 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
165 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
166}
167
168template <typename... ColTypes, typename PrevNodeType>
169std::unique_ptr<RActionBase>
170BuildAction(const ColumnNames_t &bl, const std::shared_ptr<TGraphAsymmErrors> &g, const unsigned int nSlots,
171 std::shared_ptr<PrevNodeType> prevNode, ActionTags::GraphAsymmErrors, const RColumnRegister &colRegister)
172{
173 using Helper_t = FillTGraphAsymmErrorsHelper;
174 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
175 return std::make_unique<Action_t>(Helper_t(g, nSlots), bl, std::move(prevNode), colRegister);
176}
177
178// Min action
179template <typename ColType, typename PrevNodeType, typename ActionResultType>
180std::unique_ptr<RActionBase>
181BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &minV, const unsigned int nSlots,
182 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Min, const RColumnRegister &colRegister)
183{
184 using Helper_t = MinHelper<ActionResultType>;
185 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
186 return std::make_unique<Action_t>(Helper_t(minV, nSlots), bl, std::move(prevNode), colRegister);
187}
188
189// Max action
190template <typename ColType, typename PrevNodeType, typename ActionResultType>
191std::unique_ptr<RActionBase>
192BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &maxV, const unsigned int nSlots,
193 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Max, const RColumnRegister &colRegister)
194{
195 using Helper_t = MaxHelper<ActionResultType>;
196 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
197 return std::make_unique<Action_t>(Helper_t(maxV, nSlots), bl, std::move(prevNode), colRegister);
198}
199
200// Sum action
201template <typename ColType, typename PrevNodeType, typename ActionResultType>
202std::unique_ptr<RActionBase>
203BuildAction(const ColumnNames_t &bl, const std::shared_ptr<ActionResultType> &sumV, const unsigned int nSlots,
204 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Sum, const RColumnRegister &colRegister)
205{
206 using Helper_t = SumHelper<ActionResultType>;
207 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
208 return std::make_unique<Action_t>(Helper_t(sumV, nSlots), bl, std::move(prevNode), colRegister);
209}
210
211// Mean action
212template <typename ColType, typename PrevNodeType>
213std::unique_ptr<RActionBase>
214BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &meanV, const unsigned int nSlots,
215 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Mean, const RColumnRegister &colRegister)
216{
217 using Helper_t = MeanHelper;
218 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
219 return std::make_unique<Action_t>(Helper_t(meanV, nSlots), bl, std::move(prevNode), colRegister);
220}
221
222// Standard Deviation action
223template <typename ColType, typename PrevNodeType>
224std::unique_ptr<RActionBase>
225BuildAction(const ColumnNames_t &bl, const std::shared_ptr<double> &stdDeviationV, const unsigned int nSlots,
226 std::shared_ptr<PrevNodeType> prevNode, ActionTags::StdDev, const RColumnRegister &colRegister)
227{
228 using Helper_t = StdDevHelper;
229 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColType>>;
230 return std::make_unique<Action_t>(Helper_t(stdDeviationV, nSlots), bl, prevNode, colRegister);
231}
232
233using displayHelperArgs_t = std::pair<size_t, std::shared_ptr<ROOT::RDF::RDisplay>>;
234
235// Display action
236template <typename... ColTypes, typename PrevNodeType>
237std::unique_ptr<RActionBase>
238BuildAction(const ColumnNames_t &bl, const std::shared_ptr<displayHelperArgs_t> &helperArgs, const unsigned int,
239 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Display, const RColumnRegister &colRegister)
240{
241 using Helper_t = DisplayHelper<PrevNodeType>;
242 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
243 return std::make_unique<Action_t>(Helper_t(helperArgs->first, helperArgs->second, prevNode), bl, prevNode,
244 colRegister);
245}
246
247struct SnapshotHelperArgs {
248 std::string fFileName;
249 std::string fDirName;
250 std::string fTreeName;
251 std::vector<std::string> fOutputColNames;
253};
254
255// Snapshot action
256template <typename... ColTypes, typename PrevNodeType>
257std::unique_ptr<RActionBase>
258BuildAction(const ColumnNames_t &colNames, const std::shared_ptr<SnapshotHelperArgs> &snapHelperArgs,
259 const unsigned int nSlots, std::shared_ptr<PrevNodeType> prevNode, ActionTags::Snapshot,
260 const RColumnRegister &colRegister)
261{
262 const auto &filename = snapHelperArgs->fFileName;
263 const auto &dirname = snapHelperArgs->fDirName;
264 const auto &treename = snapHelperArgs->fTreeName;
265 const auto &outputColNames = snapHelperArgs->fOutputColNames;
266 const auto &options = snapHelperArgs->fOptions;
267
268 auto makeIsDefine = [&] {
269 std::vector<bool> isDef;
270 isDef.reserve(sizeof...(ColTypes));
271 for (auto i = 0u; i < sizeof...(ColTypes); ++i)
272 isDef.push_back(colRegister.IsDefineOrAlias(colNames[i]));
273 return isDef;
274 };
275 std::vector<bool> isDefine = makeIsDefine();
276
277 std::unique_ptr<RActionBase> actionPtr;
279 // single-thread snapshot
280 using Helper_t = SnapshotHelper<ColTypes...>;
281 using Action_t = RAction<Helper_t, PrevNodeType>;
282 actionPtr.reset(
283 new Action_t(Helper_t(filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
284 colNames, prevNode, colRegister));
285 } else {
286 // multi-thread snapshot
287 using Helper_t = SnapshotHelperMT<ColTypes...>;
288 using Action_t = RAction<Helper_t, PrevNodeType>;
289 actionPtr.reset(new Action_t(
290 Helper_t(nSlots, filename, dirname, treename, colNames, outputColNames, options, std::move(isDefine)),
291 colNames, prevNode, colRegister));
292 }
293 return actionPtr;
294}
295
296// Book with custom helper type
297template <typename... ColTypes, typename PrevNodeType, typename Helper_t>
298std::unique_ptr<RActionBase>
299BuildAction(const ColumnNames_t &bl, const std::shared_ptr<Helper_t> &h, const unsigned int /*nSlots*/,
300 std::shared_ptr<PrevNodeType> prevNode, ActionTags::Book, const RColumnRegister &colRegister)
301{
302 using Action_t = RAction<Helper_t, PrevNodeType, TTraits::TypeList<ColTypes...>>;
303 return std::make_unique<Action_t>(Helper_t(std::move(*h)), bl, std::move(prevNode), colRegister);
304}
305
306/****** end BuildAndBook ******/
307
308template <typename Filter>
309void CheckFilter(Filter &)
310{
311 using FilterRet_t = typename RDF::CallableTraits<Filter>::ret_type;
312 static_assert(std::is_convertible<FilterRet_t, bool>::value,
313 "filter expression returns a type that is not convertible to bool");
314}
315
316ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action);
317
318void CheckValidCppVarName(std::string_view var, const std::string &where);
319
320void CheckForRedefinition(const std::string &where, std::string_view definedCol, const RColumnRegister &colRegister,
321 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
322
323void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister,
324 const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns);
325
326void CheckForNoVariations(const std::string &where, std::string_view definedColView,
327 const RColumnRegister &colRegister);
328
329std::string PrettyPrintAddr(const void *const addr);
330
331std::shared_ptr<RJittedFilter> BookFilterJit(std::shared_ptr<RNodeBase> *prevNodeOnHeap, std::string_view name,
332 std::string_view expression, const ColumnNames_t &branches,
333 const RColumnRegister &colRegister, TTree *tree, RDataSource *ds);
334
335std::shared_ptr<RJittedDefine> BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm,
336 RDataSource *ds, const RColumnRegister &colRegister,
337 const ColumnNames_t &branches, std::shared_ptr<RNodeBase> *prevNodeOnHeap);
338
339std::shared_ptr<RJittedDefine> BookDefinePerSampleJit(std::string_view name, std::string_view expression,
340 RLoopManager &lm, const RColumnRegister &colRegister,
341 std::shared_ptr<RNodeBase> *upcastNodeOnHeap);
342
343std::shared_ptr<RJittedVariation>
344BookVariationJit(const std::vector<std::string> &colNames, std::string_view variationName,
345 const std::vector<std::string> &variationTags, std::string_view expression, RLoopManager &lm,
346 RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches,
347 std::shared_ptr<RNodeBase> *upcastNodeOnHeap, bool isSingleColumn);
348
349std::string JitBuildAction(const ColumnNames_t &bl, std::shared_ptr<RDFDetail::RNodeBase> *prevNode,
350 const std::type_info &art, const std::type_info &at, void *rOnHeap, TTree *tree,
351 const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds,
352 std::weak_ptr<RJittedAction> *jittedActionOnHeap);
353
354// Allocate a weak_ptr on the heap, return a pointer to it. The user is responsible for deleting this weak_ptr.
355// This function is meant to be used by RInterface's methods that book code for jitting.
356// The problem it solves is that we generate code to be lazily jitted with the addresses of certain objects in them,
357// and we need to check those objects are still alive when the generated code is finally jitted and executed.
358// So we pass addresses to weak_ptrs allocated on the heap to the jitted code, which is then responsible for
359// the deletion of the weak_ptr object.
360template <typename T>
361std::weak_ptr<T> *MakeWeakOnHeap(const std::shared_ptr<T> &shPtr)
362{
363 return new std::weak_ptr<T>(shPtr);
364}
365
366// Same as MakeWeakOnHeap, but create a shared_ptr that makes sure the object is definitely kept alive.
367template <typename T>
368std::shared_ptr<T> *MakeSharedOnHeap(const std::shared_ptr<T> &shPtr)
369{
370 return new std::shared_ptr<T>(shPtr);
371}
372
373bool AtLeastOneEmptyString(const std::vector<std::string_view> strings);
374
375/// Take a shared_ptr<AnyNodeType> and return a shared_ptr<RNodeBase>.
376/// This works for RLoopManager nodes as well as filters and ranges.
377std::shared_ptr<RNodeBase> UpcastNode(std::shared_ptr<RNodeBase> ptr);
378
379ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns,
380 const RColumnRegister &validDefines, RDataSource *ds);
381
382std::vector<std::string> GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister,
383 TTree *tree, RDataSource *ds, const std::string &context,
384 bool vector2rvec);
385
386std::vector<bool> FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedDSCols);
387
388template <typename T>
389void AddDSColumnsHelper(const std::string &colName, RLoopManager &lm, RDataSource &ds, RColumnRegister &colRegister)
390{
391 if (colRegister.IsDefineOrAlias(colName) || !ds.HasColumn(colName) ||
392 lm.HasDataSourceColumnReaders(colName, typeid(T)))
393 return;
394
395 const auto nSlots = lm.GetNSlots();
396 std::vector<std::unique_ptr<RColumnReaderBase>> colReaders;
397 colReaders.reserve(nSlots);
398
399 const auto valuePtrs = ds.GetColumnReaders<T>(colName);
400 if (!valuePtrs.empty()) { // we are using the old GetColumnReaders mechanism in this RDataSource
401 for (auto *ptr : valuePtrs)
402 colReaders.emplace_back(new RDSColumnReader<T>(ptr));
403
404 } else { // using the new GetColumnReaders mechanism
405 // TODO consider changing the interface so we return all of these for all slots in one go
406 for (auto slot = 0u; slot < lm.GetNSlots(); ++slot)
407 colReaders.emplace_back(ds.GetColumnReaders(slot, colName, typeid(T)));
408 }
409
410 lm.AddDataSourceColumnReaders(colName, std::move(colReaders), typeid(T));
411}
412
413/// Take list of column names that must be defined, current map of custom columns, current list of defined column names,
414/// and return a new map of custom columns (with the new datasource columns added to it)
415template <typename... ColumnTypes>
416void AddDSColumns(const std::vector<std::string> &requiredCols, RLoopManager &lm, RDataSource &ds,
417 TTraits::TypeList<ColumnTypes...>, RColumnRegister &colRegister)
418{
419 // hack to expand a template parameter pack without c++17 fold expressions.
420 using expander = int[];
421 int i = 0;
422 (void)expander{(AddDSColumnsHelper<ColumnTypes>(requiredCols[i], lm, ds, colRegister), ++i)..., 0};
423}
424
425// this function is meant to be called by the jitted code generated by BookFilterJit
426template <typename F, typename PrevNode>
427void JitFilterHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name,
428 std::weak_ptr<RJittedFilter> *wkJittedFilter, std::shared_ptr<PrevNode> *prevNodeOnHeap,
429 RColumnRegister *colRegister) noexcept
430{
431 if (wkJittedFilter->expired()) {
432 // The branch of the computation graph that needed this jitted code went out of scope between the type
433 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
434 delete wkJittedFilter;
435 delete colRegister;
436 delete prevNodeOnHeap;
437 return;
438 }
439
440 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
441 delete[] colsPtr;
442
443 const auto jittedFilter = wkJittedFilter->lock();
444
445 // mock Filter logic -- validity checks and Define-ition of RDataSource columns
446 using Callable_t = std::decay_t<F>;
448 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
449 constexpr auto nColumns = ColTypes_t::list_size;
450 CheckFilter(f);
451
452 auto &lm = *jittedFilter->GetLoopManagerUnchecked(); // RLoopManager must exist at this time
453 auto ds = lm.GetDataSource();
454
455 if (ds != nullptr)
456 AddDSColumns(cols, lm, *ds, ColTypes_t(), *colRegister);
457
458 jittedFilter->SetFilter(
459 std::unique_ptr<RFilterBase>(new F_t(std::forward<F>(f), cols, *prevNodeOnHeap, *colRegister, name)));
460 // colRegister points to the columns structure in the heap, created before the jitted call so that the jitter can
461 // share data after it has lazily compiled the code. Here the data has been used and the memory can be freed.
462 delete colRegister;
463 delete prevNodeOnHeap;
464 delete wkJittedFilter;
465}
466
467namespace DefineTypes {
468struct RDefineTag {};
469struct RDefinePerSampleTag {};
470}
471
472template <typename F>
473auto MakeDefineNode(DefineTypes::RDefineTag, std::string_view name, std::string_view dummyType, F &&f,
474 const ColumnNames_t &cols, RColumnRegister &colRegister, RLoopManager &lm)
475{
476 return std::unique_ptr<RDefineBase>(new RDefine<std::decay_t<F>, ExtraArgsForDefine::None>(
477 name, dummyType, std::forward<F>(f), cols, colRegister, lm));
478}
479
480template <typename F>
481auto MakeDefineNode(DefineTypes::RDefinePerSampleTag, std::string_view name, std::string_view dummyType, F &&f,
482 const ColumnNames_t &, RColumnRegister &, RLoopManager &lm)
483{
484 return std::unique_ptr<RDefineBase>(
485 new RDefinePerSample<std::decay_t<F>>(name, dummyType, std::forward<F>(f), lm));
486}
487
488// Build a RDefine or a RDefinePerSample object and attach it to an existing RJittedDefine
489// This function is meant to be called by jitted code right before starting the event loop.
490// If colsPtr is null, build a RDefinePerSample (it has no input columns), otherwise a RDefine.
491template <typename RDefineTypeTag, typename F>
492void JitDefineHelper(F &&f, const char **colsPtr, std::size_t colsSize, std::string_view name, RLoopManager *lm,
493 std::weak_ptr<RJittedDefine> *wkJittedDefine, RColumnRegister *colRegister,
494 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
495{
496 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
497 auto doDeletes = [&] {
498 delete wkJittedDefine;
499 delete colRegister;
500 delete prevNodeOnHeap;
501 delete[] colsPtr;
502 };
503
504 if (wkJittedDefine->expired()) {
505 // The branch of the computation graph that needed this jitted code went out of scope between the type
506 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
507 doDeletes();
508 return;
509 }
510
511 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
512
513 auto jittedDefine = wkJittedDefine->lock();
514
515 using Callable_t = std::decay_t<F>;
516 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
517
518 auto ds = lm->GetDataSource();
519 if (ds != nullptr)
520 AddDSColumns(cols, *lm, *ds, ColTypes_t(), *colRegister);
521
522 // will never actually be used (trumped by jittedDefine->GetTypeName()), but we set it to something meaningful
523 // to help devs debugging
524 const auto dummyType = "jittedCol_t";
525 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
526 std::unique_ptr<RDefineBase> newCol{
527 MakeDefineNode(RDefineTypeTag{}, name, dummyType, std::forward<F>(f), cols, *colRegister, *lm)};
528 jittedDefine->SetDefine(std::move(newCol));
529
530 doDeletes();
531}
532
533template <bool IsSingleColumn, typename F>
534void JitVariationHelper(F &&f, const char **colsPtr, std::size_t colsSize, const char **variedCols,
535 std::size_t variedColsSize, const char **variationTags, std::size_t variationTagsSize,
536 std::string_view variationName, RLoopManager *lm,
537 std::weak_ptr<RJittedVariation> *wkJittedVariation, RColumnRegister *colRegister,
538 std::shared_ptr<RNodeBase> *prevNodeOnHeap) noexcept
539{
540 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
541 auto doDeletes = [&] {
542 delete[] colsPtr;
543 delete[] variedCols;
544 delete[] variationTags;
545
546 delete wkJittedVariation;
547 delete colRegister;
548 delete prevNodeOnHeap;
549 };
550
551 if (wkJittedVariation->expired()) {
552 // The branch of the computation graph that needed this jitted variation went out of scope between the type
553 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
554 doDeletes();
555 return;
556 }
557
558 const ColumnNames_t inputColNames(colsPtr, colsPtr + colsSize);
559 std::vector<std::string> variedColNames(variedCols, variedCols + variedColsSize);
560 std::vector<std::string> tags(variationTags, variationTags + variationTagsSize);
561
562 auto jittedVariation = wkJittedVariation->lock();
563
564 using Callable_t = std::decay_t<F>;
565 using ColTypes_t = typename TTraits::CallableTraits<Callable_t>::arg_types;
566
567 auto ds = lm->GetDataSource();
568 if (ds != nullptr)
569 AddDSColumns(inputColNames, *lm, *ds, ColTypes_t(), *colRegister);
570
571 // use unique_ptr<RDefineBase> instead of make_unique<NewCol_t> to reduce jit/compile-times
572 std::unique_ptr<RVariationBase> newVariation{new RVariation<std::decay_t<F>, IsSingleColumn>(
573 std::move(variedColNames), variationName, std::forward<F>(f), std::move(tags), jittedVariation->GetTypeName(),
574 *colRegister, *lm, inputColNames)};
575 jittedVariation->SetVariation(std::move(newVariation));
576
577 doDeletes();
578}
579
580/// Convenience function invoked by jitted code to build action nodes at runtime
581template <typename ActionTag, typename... ColTypes, typename PrevNodeType, typename HelperArgType>
582void CallBuildAction(std::shared_ptr<PrevNodeType> *prevNodeOnHeap, const char **colsPtr, std::size_t colsSize,
583 const unsigned int nSlots, std::shared_ptr<HelperArgType> *helperArgOnHeap,
584 std::weak_ptr<RJittedAction> *wkJittedActionOnHeap, RColumnRegister *colRegister) noexcept
585{
586 // a helper to delete objects allocated before jitting, so that the jitter can share data with lazily jitted code
587 auto doDeletes = [&] {
588 delete[] colsPtr;
589 delete helperArgOnHeap;
590 delete wkJittedActionOnHeap;
591 // colRegister must be deleted before prevNodeOnHeap because their dtor needs the RLoopManager to be alive
592 // and prevNodeOnHeap is what keeps it alive if the rest of the computation graph is already out of scope
593 delete colRegister;
594 delete prevNodeOnHeap;
595 };
596
597 if (wkJittedActionOnHeap->expired()) {
598 // The branch of the computation graph that needed this jitted variation went out of scope between the type
599 // jitting was booked and the time jitting actually happened. Nothing to do other than cleaning up.
600 doDeletes();
601 return;
602 }
603
604 const ColumnNames_t cols(colsPtr, colsPtr + colsSize);
605
606 auto jittedActionOnHeap = wkJittedActionOnHeap->lock();
607
608 // if we are here it means we are jitting, if we are jitting the loop manager must be alive
609 auto &prevNodePtr = *prevNodeOnHeap;
610 auto &loopManager = *prevNodePtr->GetLoopManagerUnchecked();
611 using ColTypes_t = TypeList<ColTypes...>;
612 constexpr auto nColumns = ColTypes_t::list_size;
613 auto ds = loopManager.GetDataSource();
614 if (ds != nullptr)
615 AddDSColumns(cols, loopManager, *ds, ColTypes_t(), *colRegister);
616
617 auto actionPtr = BuildAction<ColTypes...>(cols, std::move(*helperArgOnHeap), nSlots, std::move(prevNodePtr),
618 ActionTag{}, *colRegister);
619 jittedActionOnHeap->SetAction(std::move(actionPtr));
620
621 doDeletes();
622}
623
624/// The contained `type` alias is `double` if `T == RInferredType`, `U` if `T == std::container<U>`, `T` otherwise.
625template <typename T, bool Container = IsDataContainer<T>::value && !std::is_same<T, std::string>::value>
626struct RMinReturnType {
627 using type = T;
628};
629
630template <>
631struct RMinReturnType<RInferredType, false> {
632 using type = double;
633};
634
635template <typename T>
636struct RMinReturnType<T, true> {
637 using type = TTraits::TakeFirstParameter_t<T>;
638};
639
640// return wrapper around f that prepends an `unsigned int slot` parameter
641template <typename R, typename F, typename... Args>
642std::function<R(unsigned int, Args...)> AddSlotParameter(F &f, TypeList<Args...>)
643{
644 return [f](unsigned int, Args... a) mutable -> R { return f(a...); };
645}
646
647template <typename ColType, typename... Rest>
648struct RNeedJittingHelper {
649 static constexpr bool value = RNeedJittingHelper<Rest...>::value;
650};
651
652template <typename... Rest>
653struct RNeedJittingHelper<RInferredType, Rest...> {
654 static constexpr bool value = true;
655};
656
657template <typename T>
658struct RNeedJittingHelper<T> {
659 static constexpr bool value = false;
660};
661
662template <>
663struct RNeedJittingHelper<RInferredType> {
664 static constexpr bool value = true;
665};
666
667template <typename ...ColTypes>
668struct RNeedJitting {
669 static constexpr bool value = RNeedJittingHelper<ColTypes...>::value;
670};
671
672template <>
673struct RNeedJitting<> {
674 static constexpr bool value = false;
675};
676
677///////////////////////////////////////////////////////////////////////////////
678/// Check preconditions for RInterface::Aggregate:
679/// - the aggregator callable must have signature `U(U,T)` or `void(U&,T)`.
680/// - the merge callable must have signature `U(U,U)` or `void(std::vector<U>&)`
681template <typename R, typename Merge, typename U, typename T, typename decayedU = std::decay_t<U>,
682 typename mergeArgsNoDecay_t = typename CallableTraits<Merge>::arg_types_nodecay,
683 typename mergeArgs_t = typename CallableTraits<Merge>::arg_types,
684 typename mergeRet_t = typename CallableTraits<Merge>::ret_type>
685void CheckAggregate(TypeList<U, T>)
686{
687 constexpr bool isAggregatorOk =
688 (std::is_same<R, decayedU>::value) || (std::is_same<R, void>::value && std::is_lvalue_reference<U>::value);
689 static_assert(isAggregatorOk, "aggregator function must have signature `U(U,T)` or `void(U&,T)`");
690 constexpr bool isMergeOk =
691 (std::is_same<TypeList<decayedU, decayedU>, mergeArgs_t>::value && std::is_same<decayedU, mergeRet_t>::value) ||
692 (std::is_same<TypeList<std::vector<decayedU> &>, mergeArgsNoDecay_t>::value &&
693 std::is_same<void, mergeRet_t>::value);
694 static_assert(isMergeOk, "merge function must have signature `U(U,U)` or `void(std::vector<U>&)`");
695}
696
697///////////////////////////////////////////////////////////////////////////////
698/// This overload of CheckAggregate is called when the aggregator takes more than two arguments
699template <typename R, typename T>
700void CheckAggregate(T)
701{
702 static_assert(sizeof(T) == 0, "aggregator function must take exactly two arguments");
703}
704
705///////////////////////////////////////////////////////////////////////////////
706/// Check as many template parameters were passed as the number of column names, throw if this is not the case.
707void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames);
708
709/// Return local BranchNames or default BranchNames according to which one should be used
710const ColumnNames_t SelectColumns(unsigned int nArgs, const ColumnNames_t &bl, const ColumnNames_t &defBl);
711
712/// Check whether column names refer to a valid branch of a TTree or have been `Define`d. Return invalid column names.
713ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns,
714 const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns);
715
716/// Returns the list of Filters defined in the whole graph
717std::vector<std::string> GetFilterNames(const std::shared_ptr<RLoopManager> &loopManager);
718
719/// Returns the list of Filters defined in the branch
720template <typename NodeType>
721std::vector<std::string> GetFilterNames(const std::shared_ptr<NodeType> &node)
722{
723 std::vector<std::string> filterNames;
724 node->AddFilterName(filterNames);
725 return filterNames;
726}
727
728struct ParsedTreePath {
729 std::string fTreeName;
730 std::string fDirName;
731};
732
733ParsedTreePath ParseTreePath(std::string_view fullTreeName);
734
735// Check if a condition is true for all types
736template <bool...>
737struct TBoolPack;
738
739template <bool... bs>
740using IsTrueForAllImpl_t = typename std::is_same<TBoolPack<bs..., true>, TBoolPack<true, bs...>>;
741
742template <bool... Conditions>
743struct TEvalAnd {
744 static constexpr bool value = IsTrueForAllImpl_t<Conditions...>::value;
745};
746
747// Check if a class is a specialisation of stl containers templates
748// clang-format off
749
750template <typename>
751struct IsList_t : std::false_type {};
752
753template <typename T>
754struct IsList_t<std::list<T>> : std::true_type {};
755
756template <typename>
757struct IsDeque_t : std::false_type {};
758
759template <typename T>
760struct IsDeque_t<std::deque<T>> : std::true_type {};
761// clang-format on
762
764
765template <typename T>
766struct InnerValueType {
767 using type = T; // fallback for when T is not a nested RVec
768};
769
770template <typename Elem>
771struct InnerValueType<ROOT::VecOps::RVec<ROOT::VecOps::RVec<Elem>>> {
772 using type = Elem;
773};
774
775template <typename T>
776using InnerValueType_t = typename InnerValueType<T>::type;
777
778std::pair<std::vector<std::string>, std::vector<std::string>>
779AddSizeBranches(const std::vector<std::string> &branches, TTree *tree, std::vector<std::string> &&colsWithoutAliases,
780 std::vector<std::string> &&colsWithAliases);
781
782void RemoveDuplicates(ColumnNames_t &columnNames);
783
784} // namespace RDF
785} // namespace Internal
786
787namespace Detail {
788namespace RDF {
789
790/// The aliased type is `double` if `T == RInferredType`, `U` if `T == container<U>`, `T` otherwise.
791template <typename T>
792using MinReturnType_t = typename RDFInternal::RMinReturnType<T>::type;
793
794template <typename T>
795using MaxReturnType_t = MinReturnType_t<T>;
796
797template <typename T>
798using SumReturnType_t = MinReturnType_t<T>;
799
800} // namespace RDF
801} // namespace Detail
802} // namespace ROOT
803
804/// \endcond
805
806#endif
#define f(i)
Definition RSha256.hxx:104
#define g(i)
Definition RSha256.hxx:105
#define a(i)
Definition RSha256.hxx:99
#define h(i)
Definition RSha256.hxx:106
Int_t gErrorIgnoreLevel
Error handling routines.
Definition TError.cxx:31
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char filename
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
The head node of a RDF computation graph.
void AddDataSourceColumnReaders(const std::string &col, std::vector< std::unique_ptr< RColumnReaderBase > > &&readers, const std::type_info &ti)
RDataSource * GetDataSource() const
RLoopManager * GetLoopManagerUnchecked() final
bool HasDataSourceColumnReaders(const std::string &col, const std::type_info &ti) const
Return true if AddDataSourceColumnReaders was called for column name col.
RDataSource defines an API that RDataFrame can use to read arbitrary data formats.
virtual bool HasColumn(std::string_view colName) const =0
Checks if the dataset has a certain column.
std::vector< T ** > GetColumnReaders(std::string_view columnName)
Called at most once per column by RDF.
The public interface to the RDataFrame federation of classes.
A "std::vector"-like collection of values implementing handy operation to analyse them.
Definition RVec.hxx:1492
@ kAllAxes
Definition TH1.h:75
An array of TObjects.
Definition TObjArray.h:31
A TTree represents a columnar dataset.
Definition TTree.h:79
T Sum(const RVec< T > &v, const T zero=T(0))
Sum elements of an RVec.
Definition RVec.hxx:1917
#define F(x, y, z)
const ColumnNames_t SelectColumns(unsigned int nRequiredNames, const ColumnNames_t &names, const ColumnNames_t &defaultNames)
Choose between local column names or default column names, throw in case of errors.
void CheckForNoVariations(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister)
Throw if the column has systematic variations attached.
ParsedTreePath ParseTreePath(std::string_view fullTreeName)
void CheckForRedefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is already there.
void CheckForDefinition(const std::string &where, std::string_view definedColView, const RColumnRegister &colRegister, const ColumnNames_t &treeColumns, const ColumnNames_t &dataSourceColumns)
Throw if column definedColView is not already there.
std::shared_ptr< RJittedDefine > BookDefineJit(std::string_view name, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a Define call.
void CheckValidCppVarName(std::string_view var, const std::string &where)
void RemoveDuplicates(ColumnNames_t &columnNames)
ColumnNames_t GetValidatedColumnNames(RLoopManager &lm, const unsigned int nColumns, const ColumnNames_t &columns, const RColumnRegister &colRegister, RDataSource *ds)
Given the desired number of columns and the user-provided list of columns:
std::shared_ptr< RNodeBase > UpcastNode(std::shared_ptr< RNodeBase > ptr)
std::vector< std::string > GetFilterNames(const std::shared_ptr< RLoopManager > &loopManager)
std::string PrettyPrintAddr(const void *const addr)
void CheckTypesAndPars(unsigned int nTemplateParams, unsigned int nColumnNames)
std::string DemangleTypeIdName(const std::type_info &typeInfo)
bool AtLeastOneEmptyString(const std::vector< std::string_view > strings)
std::shared_ptr< RDFDetail::RJittedFilter > BookFilterJit(std::shared_ptr< RDFDetail::RNodeBase > *prevNodeOnHeap, std::string_view name, std::string_view expression, const ColumnNames_t &branches, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds)
Book the jitting of a Filter call.
std::string JitBuildAction(const ColumnNames_t &cols, std::shared_ptr< RDFDetail::RNodeBase > *prevNode, const std::type_info &helperArgType, const std::type_info &at, void *helperArgOnHeap, TTree *tree, const unsigned int nSlots, const RColumnRegister &colRegister, RDataSource *ds, std::weak_ptr< RJittedAction > *jittedActionOnHeap)
ColumnNames_t FilterArraySizeColNames(const ColumnNames_t &columnNames, const std::string &action)
Take a list of column names, return that list with entries starting by '#' filtered out.
std::shared_ptr< RJittedVariation > BookVariationJit(const std::vector< std::string > &colNames, std::string_view variationName, const std::vector< std::string > &variationTags, std::string_view expression, RLoopManager &lm, RDataSource *ds, const RColumnRegister &colRegister, const ColumnNames_t &branches, std::shared_ptr< RNodeBase > *upcastNodeOnHeap, bool isSingleColumn)
Book the jitting of a Vary call.
void CheckForDuplicateSnapshotColumns(const ColumnNames_t &cols)
ColumnNames_t ConvertRegexToColumns(const ColumnNames_t &colNames, std::string_view columnNameRegexp, std::string_view callerName)
std::pair< std::vector< std::string >, std::vector< std::string > > AddSizeBranches(const std::vector< std::string > &branches, TTree *tree, std::vector< std::string > &&colsWithoutAliases, std::vector< std::string > &&colsWithAliases)
Return copies of colsWithoutAliases and colsWithAliases with size branches for variable-sized array b...
std::vector< bool > FindUndefinedDSColumns(const ColumnNames_t &requestedCols, const ColumnNames_t &definedCols)
Return a bitset each element of which indicates whether the corresponding element in selectedColumns ...
std::shared_ptr< RJittedDefine > BookDefinePerSampleJit(std::string_view name, std::string_view expression, RLoopManager &lm, const RColumnRegister &colRegister, std::shared_ptr< RNodeBase > *upcastNodeOnHeap)
Book the jitting of a DefinePerSample call.
std::vector< std::string > GetValidatedArgTypes(const ColumnNames_t &colNames, const RColumnRegister &colRegister, TTree *tree, RDataSource *ds, const std::string &context, bool vector2rvec)
ColumnNames_t FindUnknownColumns(const ColumnNames_t &requiredCols, const ColumnNames_t &datasetColumns, const RColumnRegister &definedCols, const ColumnNames_t &dataSourceColumns)
void(off) SmallVectorTemplateBase< T
double T(double x)
std::vector< std::string > ColumnNames_t
ROOT type_traits extensions.
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
Bool_t IsImplicitMTEnabled()
Returns true if the implicit multi-threading in ROOT is enabled.
Definition TROOT.cxx:568
Short_t Max(Short_t a, Short_t b)
Returns the largest of a and b.
Definition TMathBase.h:250
Short_t Min(Short_t a, Short_t b)
Returns the smallest of a and b.
Definition TMathBase.h:198
Double_t Mean(Long64_t n, const T *a, const Double_t *w=nullptr)
Returns the weighted mean of an array a with length n.
Definition TMath.h:1089
Double_t StdDev(Long64_t n, const T *a, const Double_t *w=nullptr)
Definition TMath.h:527
A collection of options to steer the creation of the dataset on file.
Lightweight storage for a collection of types.