60 std::unique_ptr<ROOT::RFieldBase>
CloneImpl(std::string_view )
const final
62 return std::make_unique<RRDFCardinalityField>();
97 *
static_cast<std::size_t *
>(to) =
size;
106 *
static_cast<std::size_t *
>(to) =
size;
120 std::unique_ptr<ROOT::RFieldBase>
CloneImpl(std::string_view)
const final
160 std::unique_ptr<RFieldBase::RValue>
fValue;
195 auto onDiskType =
source.GetSharedDescriptorGuard()->GetFieldDescriptor(
fField->GetOnDiskId()).GetTypeName();
196 std::string
msg =
"RNTupleDS: invalid type \"" +
fField->GetTypeName() +
"\" for column \"" +
199 throw std::runtime_error(
msg);
208 fValue = std::make_unique<RFieldBase::RValue>(
fField->CreateValue());
228 return fValue->GetPtr<
void>().get();
284 fieldDesc.GetTypeName().substr(0, 12) ==
"std::vector<" ||
fieldDesc.GetTypeName() ==
"");
314 f.SetOnDiskId(desc.
FindFieldId(
f.GetFieldName(),
f.GetParent()->GetOnDiskId()));
320 if (
info.fNRepetitions > 0) {
323 cardinalityField = std::make_unique<ROOT::Internal::RDF::RRDFCardinalityField>();
378 fPrincipalDescriptor =
pageSource->GetSharedDescriptorGuard()->Clone();
379 fStagingArea.emplace_back(std::move(
pageSource));
381 AddField(fPrincipalDescriptor,
"", fPrincipalDescriptor.GetFieldZeroId(),
382 std::vector<ROOT::RDF::RNTupleDS::RFieldInfo>());
393 static std::once_flag
flag;
394 std::call_once(
flag, []() {
405std::unique_ptr<ROOT::Internal::RPageSource> CreatePageSource(std::string_view
ntupleName, std::string_view fileName)
435std::unique_ptr<ROOT::Detail::RDF::RColumnReaderBase>
439 const auto index = std::distance(fColumnNames.begin(), std::find(fColumnNames.begin(), fColumnNames.end(),
name));
450 return fld->GetTypeName() == requestedType;
457 throw std::runtime_error(
"RNTupleDS: Could not create field with type \"" +
requestedType +
458 "\" for column \"" + std::string(
name));
471 fFieldId2QualifiedName[
field->GetOnDiskId()] = fPrincipalDescriptor.GetQualifiedFieldName(
field->GetOnDiskId());
472 for (
const auto &s : *
field) {
473 fFieldId2QualifiedName[s.GetOnDiskId()] = fPrincipalDescriptor.GetQualifiedFieldName(s.GetOnDiskId());
476 auto reader = std::make_unique<ROOT::Internal::RDF::RNTupleColumnReader>(
this,
field);
477 fActiveColumnReaders[
slot].emplace_back(
reader.get());
485 std::unique_lock lock(fMutexStaging);
486 fCvStaging.wait(lock, [
this] {
return fIsReadyForStaging || fStagingThreadShouldTerminate; });
487 if (fStagingThreadShouldTerminate)
492 fHasNextSources =
true;
493 fIsReadyForStaging =
false;
496 fCvStaging.notify_one();
502 const auto nFiles = fFileNames.empty() ? 1 : fFileNames.size();
503 for (
auto i = fNextFileIndex; (i <
nFiles) && ((i - fNextFileIndex) < fNSlots); ++i) {
504 if (fStagingThreadShouldTerminate)
507 if (fStagingArea[i]) {
511 fStagingArea[i] = CreatePageSource(fNTupleName, fFileNames[i]);
512 fStagingArea[i]->LoadStructure();
519 assert(fNextRanges.empty());
520 auto nFiles = fFileNames.empty() ? 1 : fFileNames.size();
527 while ((fNextRanges.size() < fNSlots) && (fNextFileIndex <
nFiles)) {
530 std::swap(fStagingArea[fNextFileIndex],
range.fSource);
532 if (!
range.fSource) {
535 range.fSource = CreatePageSource(fNTupleName, fFileNames[fNextFileIndex]);
537 range.fSource->Attach();
545 fNextRanges.emplace_back(std::move(
range));
554 for (std::size_t i = 0; (fNextRanges.size() < fNSlots) && (fNextFileIndex <
nFiles); ++i) {
555 std::unique_ptr<ROOT::Internal::RPageSource>
source;
556 std::swap(fStagingArea[fNextFileIndex],
source);
559 source = CreatePageSource(fNTupleName, fFileNames[fNextFileIndex]);
589 unsigned int iSlot = 0;
604 range.fSource->SetEntryRange({start, end - start});
605 range.fFirstEntry = start;
606 range.fLastEntry = end;
607 fNextRanges.emplace_back(std::move(
range));
614 std::vector<std::pair<ULong64_t, ULong64_t>> ranges;
623 for (
auto r : fActiveColumnReaders[0]) {
624 r->Disconnect(
true );
630 if (fCurrentRanges.empty() || (fSeenEntries > 0)) {
634 std::unique_lock lock(fMutexStaging);
635 fCvStaging.wait(lock, [
this] {
return fHasNextSources; });
638 if (fNextRanges.empty()) {
643 assert(fNextRanges.size() <= fNSlots);
645 fCurrentRanges.clear();
646 std::swap(fCurrentRanges, fNextRanges);
651 std::lock_guard
_(fMutexStaging);
652 fIsReadyForStaging =
true;
653 fHasNextSources =
false;
655 fCvStaging.notify_one();
662 fFirstEntry2RangeIdx.clear();
664 for (std::size_t i = 0; i < fCurrentRanges.size(); ++i) {
667 if (fCurrentRanges[i].fFirstEntry == 0) {
672 auto start = fCurrentRanges[i].fFirstEntry + fSeenEntries;
673 auto end = fCurrentRanges[i].fLastEntry + fSeenEntries;
676 fFirstEntry2RangeIdx[start] = i;
677 ranges.emplace_back(start, end);
681 if ((fNSlots == 1) && (fCurrentRanges[0].fSource)) {
682 for (
auto r : fActiveColumnReaders[0]) {
683 r->Connect(*fCurrentRanges[0].fSource, ranges[0].first);
696 for (
auto r : fActiveColumnReaders[
slot]) {
706 for (
auto r : fActiveColumnReaders[
slot]) {
707 r->Disconnect(
true );
716 auto msg = std::string(
"RNTupleDS: There is no column with name \"") + std::string(
colName) +
"\"";
717 throw std::runtime_error(
msg);
721 return fColumnTypes[
index];
726 return std::find(fColumnNames.begin(), fColumnNames.end(),
colName) != fColumnNames.end();
733 fIsReadyForStaging = fHasNextSources = fStagingThreadShouldTerminate =
false;
734 fThreadStaging = std::thread(&RNTupleDS::ExecStaging,
this);
735 assert(fNextRanges.empty());
737 if (fCurrentRanges.empty() || (fFileNames.size() > fNSlots)) {
740 std::lock_guard
_(fMutexStaging);
741 fIsReadyForStaging =
true;
743 fCvStaging.notify_one();
747 fNextFileIndex = std::max(fFileNames.size(), std::size_t(1));
753 for (
unsigned int i = 0; i < fNSlots; ++i) {
754 for (
auto r : fActiveColumnReaders[i]) {
755 r->Disconnect(
false );
759 std::lock_guard
_(fMutexStaging);
760 fStagingThreadShouldTerminate =
true;
762 fCvStaging.notify_one();
763 fThreadStaging.join();
766 if (fFileNames.size() > fNSlots) {
767 fCurrentRanges.clear();
769 fStagingArea.clear();
770 fStagingArea.resize(fFileNames.size());
779 fActiveColumnReaders.resize(fNSlots);
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
unsigned long long ULong64_t
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
R__EXTERN TSystem * gSystem
void GetCollectionInfo(const ROOT::NTupleSize_t globalIndex, RNTupleLocalIndex *collectionStart, ROOT::NTupleSize_t *collectionSize)
For offset columns only, look at the two adjacent values that define a collection's coordinates.
An artificial field that provides the size of a fixed-size array.
void GenerateColumns(const ROOT::RNTupleDescriptor &) final
Implementations in derived classes should create the backing columns corresponding to the field type ...
std::unique_ptr< ROOT::RFieldBase > CloneImpl(std::string_view) const final
Called by Clone(), which additionally copies the on-disk ID.
void ReadGlobalImpl(ROOT::NTupleSize_t, void *to) final
RArraySizeField(const RArraySizeField &other)=delete
RArraySizeField(RArraySizeField &&other)=default
RArraySizeField & operator=(RArraySizeField &&other)=default
void ReadInClusterImpl(RNTupleLocalIndex, void *to) final
std::size_t GetValueSize() const final
The number of bytes taken by a value of the appropriate type.
RArraySizeField(std::size_t arrayLength)
void ConstructValue(void *where) const final
Constructs value in a given location of size at least GetValueSize(). Called by the base class' Creat...
~RArraySizeField() final=default
RArraySizeField & operator=(const RArraySizeField &other)=delete
std::size_t GetAlignment() const final
As a rule of thumb, the alignment is equal to the size of the type.
void GenerateColumns() final
Implementations in derived classes should create the backing columns corresponding to the field type ...
Every RDF column is represented by exactly one RNTuple field.
void * GetImpl(Long64_t entry) final
void Connect(RPageSource &source, Long64_t entryOffset)
Connect the field and its subfields to the page source.
RNTupleColumnReader(RNTupleDS *ds, RFieldBase *protoField)
std::unique_ptr< RFieldBase::RValue > fValue
The memory location used to read from fField.
std::unique_ptr< RFieldBase > fField
The field backing the RDF column.
Long64_t fEntryOffset
For chains, the logical entry and the physical entry in any particular file can be different.
std::shared_ptr< void > fValuePtr
Used to reuse the object created by fValue when reconnecting sources.
RNTupleDS * fDataSource
The data source that owns this column reader.
~RNTupleColumnReader() override=default
RFieldBase * fProtoField
The prototype field from which fField is cloned.
Long64_t fLastEntry
Last entry number that was read.
void Disconnect(bool keepValue)
An artificial field that transforms an RNTuple column that contains the offset of collections into co...
RRDFCardinalityField(RRDFCardinalityField &&other)=default
size_t GetAlignment() const final
As a rule of thumb, the alignment is equal to the size of the type.
void GenerateColumns() final
Implementations in derived classes should create the backing columns corresponding to the field type ...
void ReadGlobalImpl(ROOT::NTupleSize_t globalIndex, void *to) final
Get the number of elements of the collection identified by globalIndex.
RRDFCardinalityField & operator=(RRDFCardinalityField &&other)=default
void GenerateColumns(const ROOT::RNTupleDescriptor &desc) final
Implementations in derived classes should create the backing columns corresponding to the field type ...
~RRDFCardinalityField() override=default
const RColumnRepresentations & GetColumnRepresentations() const final
Implementations in derived classes should return a static RColumnRepresentations object.
size_t GetValueSize() const final
The number of bytes taken by a value of the appropriate type.
void ReadInClusterImpl(ROOT::RNTupleLocalIndex localIndex, void *to) final
Get the number of elements of the collection identified by clusterIndex.
std::unique_ptr< ROOT::RFieldBase > CloneImpl(std::string_view) const final
Called by Clone(), which additionally copies the on-disk ID.
void ConstructValue(void *where) const final
Constructs value in a given location of size at least GetValueSize(). Called by the base class' Creat...
static void SetClusterBunchSize(RNTupleReadOptions &options, unsigned int val)
Storage provider that reads ntuple pages from a file.
Abstract interface to read data from an ntuple.
static std::unique_ptr< RPageSource > Create(std::string_view ntupleName, std::string_view location, const ROOT::RNTupleReadOptions &options=ROOT::RNTupleReadOptions())
Guess the concrete derived page source from the file name (location)
std::vector< void * > Record_t
The RDataSource implementation for RNTuple.
void AddField(const ROOT::RNTupleDescriptor &desc, std::string_view colName, ROOT::DescriptorId_t fieldId, std::vector< RFieldInfo > fieldInfos, bool convertToRVec=true)
Provides the RDF column "colName" given the field identified by fieldID.
std::vector< std::pair< ULong64_t, ULong64_t > > GetEntryRanges() final
Return ranges of entries to distribute to tasks.
void ExecStaging()
The main function of the fThreadStaging background thread.
std::vector< std::unique_ptr< ROOT::Internal::RPageSource > > fStagingArea
The staging area is relevant for chains of files, i.e.
std::unique_ptr< ROOT::Detail::RDF::RColumnReaderBase > GetColumnReaders(unsigned int, std::string_view, const std::type_info &) final
If the other GetColumnReaders overload returns an empty vector, this overload will be called instead.
std::vector< std::unique_ptr< ROOT::RFieldBase > > fProtoFields
We prepare a prototype field for every column.
void SetNSlots(unsigned int nSlots) final
Inform RDataSource of the number of processing slots (i.e.
std::vector< std::string > fFileNames
void InitSlot(unsigned int slot, ULong64_t firstEntry) final
Convenience method called at the start of the data processing associated to a slot.
RNTupleDS(std::unique_ptr< ROOT::Internal::RPageSource > pageSource)
std::string GetTypeName(std::string_view colName) const final
Type of a column as a string, e.g.
std::unordered_map< ROOT::DescriptorId_t, std::string > fFieldId2QualifiedName
Connects the IDs of active proto fields and their subfields to their fully qualified name (a....
std::string fNTupleName
The data source may be constructed with an ntuple name and a list of files.
void PrepareNextRanges()
Populates fNextRanges with the next set of entry ranges.
void StageNextSources()
Starting from fNextFileIndex, opens the next fNSlots files.
void Finalize() final
Convenience method called after concluding an event-loop.
std::vector< std::string > fColumnTypes
void Initialize() final
Convenience method called before starting an event-loop.
std::vector< std::string > fColumnNames
bool HasColumn(std::string_view colName) const final
Checks if the dataset has a certain column.
Record_t GetColumnReadersImpl(std::string_view name, const std::type_info &) final
type-erased vector of pointers to pointers to column values - one per slot
void FinalizeSlot(unsigned int slot) final
Convenience method called at the end of the data processing associated to a slot.
ROOT's RDataFrame offers a modern, high-level interface for analysis of data stored in TTree ,...
Base class for all ROOT issued exceptions.
The list of column representations a field can have.
A field translates read and write calls from/to underlying columns to/from tree values.
ROOT::Internal::RColumn * fPrincipalColumn
All fields that have columns have a distinct main column.
RConstSchemaIterator cbegin() const
const std::string & GetFieldName() const
static RResult< std::unique_ptr< RFieldBase > > Create(const std::string &fieldName, const std::string &typeName, const ROOT::RCreateFieldOptions &options, const ROOT::RNTupleDescriptor *desc, ROOT::DescriptorId_t fieldId)
Factory method to resurrect a field from the stored on-disk type information.
ROOT::DescriptorId_t GetOnDiskId() const
std::unique_ptr< RFieldBase > Clone(std::string_view newName) const
Copies the field and its subfields using a possibly new name and a new, unconnected set of columns.
The on-storage metadata of an RNTuple.
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
const RFieldDescriptor & GetFieldDescriptor(ROOT::DescriptorId_t fieldId) const
ROOT::DescriptorId_t FindFieldId(std::string_view fieldName, ROOT::DescriptorId_t parentId) const
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Common user-tunable settings for reading RNTuples.
Representation of an RNTuple data set in a ROOT file.
const_iterator begin() const
const_iterator end() const
virtual const char * Getenv(const char *env)
Get environment variable.
std::string TypeID2TypeName(const std::type_info &id)
Returns the name of a type starting from its type_info An empty string is returned in case of failure...
void CallConnectPageSourceOnField(RFieldBase &, ROOT::Internal::RPageSource &)
std::string GetRenormalizedTypeName(const std::string &metaNormalizedName)
Given a type name normalized by ROOT meta, renormalize it for RNTuple. E.g., insert std::prefix.
RDataFrame FromRNTuple(std::string_view ntupleName, std::string_view fileName)
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
constexpr DescriptorId_t kInvalidDescriptorId
ENTupleStructure
The fields in the ntuple model tree can carry different structural information about the type system.
The PrepareNextRanges() method populates the fNextRanges list with REntryRangeDS records.