49#include <unordered_set>
57void assignSpan(std::span<T> &to, std::span<T>
const &from)
62void logArchitectureInfo(
bool useGPU)
74 static std::pair<bool, bool> lastUseGPU;
75 if (lastUseGPU.second && lastUseGPU.first == useGPU)
77 lastUseGPU = {useGPU,
true};
80 auto log = [](std::string_view message) {
85 log(
"using generic CPU library compiled with no vectorizations");
90 log(
"using CUDA computation library");
100void logMissingCudaSupport(
RooAbsArg const &arg)
105 static std::set<std::string> warnedClasses;
106 static std::mutex warnedClassesMutex;
107 std::scoped_lock guard{warnedClassesMutex};
108 if (warnedClasses.insert(arg.
ClassName()).second) {
110 <<
" could not be evaluated on the GPU because it doesn't support it."
111 <<
" Consider requesting or implementing it to benefit from a speed up."
127 std::shared_ptr<RooBatchCompute::AbsBuffer>
buffer;
153 : _topNode{const_cast<
RooAbsReal &>(absReal)}, _useGPU{useGPU}
157 throw std::runtime_error(
"Can't create Evaluator in CUDA mode because RooBatchCompute CUDA could not be loaded!");
173 std::map<RooFit::Detail::DataKey, NodeInfo *> nodeInfos;
177 std::size_t iNode = 0;
181 auto &nodeInfo =
_nodes.back();
184 nodeInfo.absArg = arg;
185 nodeInfo.originalOperMode = arg->
operMode();
186 nodeInfo.iNode = iNode;
187 nodeInfos[arg] = &nodeInfo;
190 nodeInfo.isVariable =
true;
195 nodeInfo.isCategory =
true;
202 info.serverInfos.reserve(info.absArg->servers().size());
203 for (
RooAbsArg *server : info.absArg->servers()) {
204 if (server->isValueServer(*info.absArg)) {
205 auto *serverInfo = nodeInfos.at(server);
206 info.serverInfos.emplace_back(serverInfo);
207 serverInfo->clientInfos.emplace_back(&info);
213 _nodes.back().isValueServer =
true;
214 for (
auto iter =
_nodes.rbegin(); iter !=
_nodes.rend(); ++iter) {
215 if (!iter->isValueServer)
217 for (
auto &serverInfo : iter->serverInfos) {
218 serverInfo->isValueServer =
true;
232 for (
auto &info :
_nodes) {
244 std::size_t iValueServer = 0;
245 for (
RooAbsArg *server : info.absArg->servers()) {
246 if (server->isValueServer(*info.absArg)) {
247 auto *knownServer = info.serverInfos[iValueServer]->absArg;
248 if (knownServer->hasDataToken()) {
249 server->setDataToken(knownServer->dataToken());
260 throw std::runtime_error(
"Evaluator can only take device array as input in CUDA mode!");
277 static std::atomic<std::size_t> nextInputGeneration{1};
278 const std::size_t gen = ++nextInputGeneration;
309 info.
buffer->assignFromDevice(gpuSpan);
315 info.
buffer->assignFromHost(cpuSpan);
322 std::map<RooFit::Detail::DataKey, std::size_t> sizeMap;
323 for (
auto &info :
_nodes) {
324 if (info.fromArrayInput) {
325 sizeMap[info.absArg] = info.outputSize;
334 auto found = sizeMap.find(key);
335 return found != sizeMap.end() ? found->second : -1;
338 for (
auto &info :
_nodes) {
339 info.outputSize = outputSizeMap.at(info.absArg);
352 for (
auto &info :
_nodes) {
354 if (!info.isVariable && info.absArg) {
355 info.absArg->resetDataToken();
370 if (found !=
_nodesMap.end() && found->second->absArg == &arg) {
371 found->second->absArg =
nullptr;
377 using namespace Detail;
381 double *buffer =
nullptr;
391 logMissingCudaSupport(*info.
absArg);
398 buffer = info.
buffer->hostWritePtr();
408 buffer[0] = nodeAbsCategory->getCurrentIndex();
410 throw std::runtime_error(
"RooFit::Evaluator - non-scalar category values are not supported!");
413 auto nodeAbsReal =
static_cast<RooAbsReal const *
>(node);
431 auto *var =
static_cast<RooRealVar const *
>(node);
435 clientInfo->isDirty =
true;
448 if (nodeInfo.
lastCatVal != cat->getCurrentIndex()) {
451 clientInfo->isDirty =
true;
463 clientInfo->isDirty =
true;
484 for (
auto &nodeInfo :
_nodes) {
485 if (!nodeInfo.fromArrayInput) {
486 if (nodeInfo.isVariable) {
488 }
else if (nodeInfo.isCategory) {
491 if (nodeInfo.isDirty) {
494 nodeInfo.isDirty =
false;
535 for (
auto &info :
_nodes) {
536 if (info.fromArrayInput) {
539 if (info.isVariable) {
541 }
else if (info.isCategory) {
543 }
else if (info.isDirty) {
545 if (info.computeInGPU) {
550 info.isDirty =
false;
583 for (
auto &action : ctx->_deferredActions) {
586 ctx->_deferredActions.clear();
596 using namespace Detail;
602 double *buffer =
nullptr;
611 buffer = info.
buffer->deviceWritePtr();
628 for (
auto &info :
_nodes) {
629 info.computeInGPU =
false;
630 if (!info.absArg->canComputeBatchWithCuda()) {
633 for (
NodeInfo const *serverInfo : info.serverInfos) {
634 if (serverInfo->outputSize > 1) {
635 info.computeInGPU =
true;
642 for (
auto &info :
_nodes) {
643 info.copyAfterEvaluation =
false;
645 if (!info.isScalar()) {
646 for (
auto *clientInfo : info.clientInfos) {
647 if (info.computeInGPU != clientInfo->computeInGPU) {
648 info.copyAfterEvaluation =
true;
665 for (
auto &info :
_nodes) {
666 if (info.isVariable) {
698 auto out = std::make_unique<ChangeOperModeRAII>();
699 std::unordered_set<RooAbsArg *> visited;
701 std::vector<RooAbsArg *> queue;
702 queue.reserve(
_nodes.size());
703 for (
auto &info :
_nodes) {
704 queue.push_back(info.absArg);
707 while (!queue.empty()) {
710 if (!visited.insert(node).second)
713 out->change(node, opMode);
719 queue.push_back(client);
728 std::cout <<
"--- RooFit BatchMode evaluation ---\n";
730 std::vector<int> widths{9, 37, 20, 9, 10, 20};
732 auto printElement = [&](
int iCol,
auto const &t) {
733 const char separator =
' ';
734 os << separator << std::left << std::setw(widths[iCol]) << std::setfill(separator) << t;
738 auto printHorizontalRow = [&]() {
740 for (
int w : widths) {
743 for (
int i = 0; i <
n; i++) {
749 printHorizontalRow();
752 printElement(0,
"Index");
753 printElement(1,
"Name");
754 printElement(2,
"Class");
755 printElement(3,
"Size");
756 printElement(4,
"From Data");
757 printElement(5,
"1st value");
760 printHorizontalRow();
762 for (std::size_t iNode = 0; iNode <
_nodes.size(); ++iNode) {
763 auto &nodeInfo =
_nodes[iNode];
769 printElement(0, iNode);
770 printElement(1, node->
GetName());
772 printElement(3, nodeInfo.outputSize);
773 printElement(4, nodeInfo.fromArrayInput);
774 printElement(5, span[0]);
779 printHorizontalRow();
791 for (
auto &nodeInfo :
_nodes) {
792 if (nodeInfo.isValueServer && nodeInfo.absArg->isFundamental()) {
793 parameters.
add(*nodeInfo.absArg);
818 for (
auto &nodeInfo :
_nodes) {
819 if (nodeInfo.absArg->isReducerNode()) {
820 nodeInfo.isDirty =
true;
Option_t Option_t TPoint TPoint const char mode
Common abstract base class for objects that represent a value and a "shape" in RooFit.
const TNamed * namePtr() const
De-duplicated pointer to this object's name.
const RefCountList_t & valueClients() const
List of all value clients of this object. Value clients receive value updates.
void setDataToken(std::size_t index, RooFit::Evaluator *evaluator=nullptr)
Sets the token for retrieving results in the BatchMode.
OperMode operMode() const
Query the operation mode of this node.
A space to attach TBranches.
virtual bool add(const RooAbsArg &var, bool silent=false)
Add the specified argument to list.
Storage_t::size_type size() const
void sort(bool reverse=false)
Sort collection using std::sort and name comparison.
Abstract base class for objects that represent a real value and implements functionality common to al...
virtual void doEval(RooFit::EvalContext &) const
Base function for computing multiple values of a RooAbsReal.
RooArgSet is a container object that can hold multiple RooAbsArg objects.
Minimal configuration struct to steer the evaluation of a single node with the RooBatchCompute librar...
void setCudaStream(CudaInterface::CudaStream *cudaStream)
void setNThreads(int nThreads)
Number of threads to use for CPU batch computations and reductions.
virtual void synchronizeCudaStream(CudaInterface::CudaStream *) const =0
Wait until all work that was enqueued on the stream has completed.
virtual std::unique_ptr< AbsBufferManager > createBufferManager() const =0
virtual CudaInterface::CudaStream * newCudaStream() const =0
virtual void deleteCudaStream(CudaInterface::CudaStream *) const =0
std::size_t _inputGeneration
std::vector< std::function< void()> > _deferredActions
void set(RooAbsArg const *arg, std::span< const double > const &span)
std::span< const double > at(RooAbsArg const *arg, RooAbsArg const *caller=nullptr)
void resetVectorBuffers()
void enableVectorBuffers(bool enable)
RooBatchCompute::Config config(RooAbsArg const *arg) const
void setConfig(RooAbsArg const *arg, RooBatchCompute::Config const &config)
std::span< double > _currentOutput
void resize(std::size_t n)
void print(std::ostream &os)
void setClientsDirty(NodeInfo &nodeInfo)
Flags all the clients of a given node dirty.
std::unique_ptr< ChangeOperModeRAII > setOperModes(RooAbsArg::OperMode opMode)
RooArgSet getParameters() const
Gets all the parameters of the RooAbsReal.
void setOffsetMode(RooFit::EvalContext::OffsetMode)
Sets the offset mode for evaluation.
void unregisterNode(RooAbsArg const &arg)
Called from RooAbsArg::~RooAbsArg() when a node of the computation graph is deleted while this Evalua...
void syncDataTokens()
If there are servers with the same name that got de-duplicated in the _nodes list,...
std::unordered_map< TNamed const *, NodeInfo * > _nodesMap
std::unique_ptr< ChangeOperModeRAII > _operModeChanges
std::vector< NodeInfo > _nodes
bool _needToUpdateOutputSizes
std::span< const double > getValHeterogeneous()
Returns the value of the top node in the computation graph.
std::span< const double > run()
Returns the value of the top node in the computation graph.
Evaluator(const RooAbsReal &absReal, bool useGPU=false)
Construct a new Evaluator.
void setNThreads(int nThreads)
Sets the number of threads to use for the evaluation of a single node.
void processVariable(NodeInfo &nodeInfo)
Process a variable in the computation graph.
void processCategory(NodeInfo &nodeInfo)
Process a category in the computation graph.
RooBatchCompute::CudaInterface::CudaStream * _cudaStream
std::unique_ptr< RooBatchCompute::AbsBufferManager > _bufferManager
void markGPUNodes()
Decides which nodes are assigned to the GPU in a CUDA fit.
void assignToGPU(NodeInfo &info)
Enqueue the computation of a node on the GPU.
void setInput(std::string const &name, std::span< const double > inputArray, bool isOnDevice)
RooFit::EvalContext _evalContextCUDA
RooFit::EvalContext _evalContextCPU
void computeCPUNode(const RooAbsArg *node, NodeInfo &info)
void setOperMode(RooAbsArg *arg, RooAbsArg::OperMode opMode)
Temporarily change the operation mode of a RooAbsArg until the Evaluator gets deleted.
static RooMsgService & instance()
Return reference to singleton instance.
static const TNamed * ptr(const char *stringPtr)
Return a unique TNamed pointer for given C++ string.
Variable that can be changed from the outside.
const char * GetName() const override
Returns name of object.
virtual const char * ClassName() const
Returns name of class to which the object belongs.
RVec< PromoteType< T > > log(const RVec< T > &v)
R__EXTERN RooBatchComputeInterface * dispatchCUDA
std::string cpuArchitectureName()
R__EXTERN RooBatchComputeInterface * dispatchCPU
This dispatch pointer points to an implementation of the compute library, provided one has been loade...
Architecture cpuArchitecture()
int initCPU()
Inspect hardware capabilities, and load the optimal library for RooFit computations.
The namespace RooFit contains mostly switches that change the behaviour of functions of PDFs (or othe...
void getSortedComputationGraph(RooAbsArg const &func, RooArgSet &out)
A struct used by the Evaluator to store information on the RooAbsArgs in the computation graph.
std::size_t lastSetValCount
std::vector< NodeInfo * > serverInfos
RooAbsArg::OperMode originalOperMode
std::vector< NodeInfo * > clientInfos
std::shared_ptr< RooBatchCompute::AbsBuffer > buffer