Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleDescriptor.cxx
Go to the documentation of this file.
1/// \file RNTupleDescriptor.cxx
2/// \author Jakob Blomer <jblomer@cern.ch>
3/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
4/// \date 2018-10-04
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#include <ROOT/RError.hxx>
15#include <ROOT/RFieldBase.hxx>
16#include <ROOT/RNTuple.hxx>
18#include <ROOT/RNTupleModel.hxx>
19#include <ROOT/RNTupleTypes.hxx>
20#include <ROOT/RNTupleUtils.hxx>
21#include <ROOT/RPage.hxx>
22#include <string_view>
23
24#include <RZip.h>
25#include <TError.h>
26
27#include <algorithm>
28#include <cstdint>
29#include <deque>
30#include <functional>
31#include <iostream>
32#include <set>
33#include <utility>
34
36{
37 static std::string gEmpty;
38 return gEmpty;
39}
40
42{
43 return fFieldId == other.fFieldId && fFieldVersion == other.fFieldVersion && fTypeVersion == other.fTypeVersion &&
44 GetFieldName() == other.GetFieldName() && GetFieldDescription() == other.GetFieldDescription() &&
45 GetTypeName() == other.GetTypeName() && GetTypeAlias() == other.GetTypeAlias() &&
46 fNRepetitions == other.fNRepetitions && fStructure == other.fStructure && fParentId == other.fParentId &&
47 fProjectionSourceId == other.fProjectionSourceId && fLinkIds == other.fLinkIds &&
48 fLogicalColumnIds == other.fLogicalColumnIds && fTypeChecksum == other.fTypeChecksum &&
49 fIsSoACollection == other.fIsSoACollection;
50}
51
53{
54 fFieldId = source.fFieldId;
55 fFieldVersion = source.fFieldVersion;
56 fTypeVersion = source.fTypeVersion;
57 fNRepetitions = source.fNRepetitions;
58 fStructure = source.fStructure;
59 fParentId = source.fParentId;
60 fProjectionSourceId = source.fProjectionSourceId;
61 fLinkIds = source.fLinkIds;
62 fColumnCardinality = source.fColumnCardinality;
63 fLogicalColumnIds = source.fLogicalColumnIds;
64 fTypeChecksum = source.fTypeChecksum;
65 fIsSoACollection = source.fIsSoACollection;
66
67 fFieldName = stringPool.Intern(source.GetFieldName());
68 fFieldDescription = stringPool.Intern(source.GetFieldDescription());
69 fTypeName = stringPool.Intern(source.GetTypeName());
70 fTypeAlias = stringPool.Intern(source.GetTypeAlias());
71}
72
74{
75 RFieldDescriptor clone;
76 clone.fStringPool = std::make_unique<Internal::RStringPool>();
77 clone.InitFrom(*this, *clone.fStringPool);
78 return clone;
79}
80
81std::unique_ptr<ROOT::RFieldBase>
83{
84 if (GetStructure() == ROOT::ENTupleStructure::kStreamer) {
85 auto streamerField = std::make_unique<ROOT::RStreamerField>(GetFieldName(), GetTypeName());
86 if ((streamerField->GetTraits() & RFieldBase::kTraitEmulatedField) && !options.GetEmulateUnknownTypes()) {
87 throw RException(
88 R__FAIL("streamer field " + GetFieldName() + " has an emulated class but emulation is turned off"));
89 }
90 streamerField->SetOnDiskId(fFieldId);
91 return streamerField;
92 }
93
94 // The structure may be unknown if the descriptor comes from a deserialized field with an unknown structural role.
95 // For forward compatibility, we allow this case and return an InvalidField.
96 if (GetStructure() == ROOT::ENTupleStructure::kUnknown) {
97 if (options.GetReturnInvalidOnError()) {
98 auto invalidField = std::make_unique<ROOT::RInvalidField>(GetFieldName(), GetTypeName(), "",
100 invalidField->SetOnDiskId(fFieldId);
101 return invalidField;
102 } else {
103 throw RException(R__FAIL("unexpected on-disk field structure value for field \"" + GetFieldName() + "\""));
104 }
105 }
106
107 // Untyped records and collections
108 if (GetTypeName().empty()) {
109 switch (GetStructure()) {
111 std::vector<std::unique_ptr<ROOT::RFieldBase>> memberFields;
112 memberFields.reserve(fLinkIds.size());
113 for (auto id : fLinkIds) {
114 const auto &memberDesc = ntplDesc.GetFieldDescriptor(id);
115 auto field = memberDesc.CreateField(ntplDesc, options);
116 if (field->GetTraits() & ROOT::RFieldBase::kTraitInvalidField)
117 return field;
118 memberFields.emplace_back(std::move(field));
119 }
120 auto recordField = std::make_unique<ROOT::RRecordField>(GetFieldName(), std::move(memberFields));
121 recordField->SetOnDiskId(fFieldId);
122 return recordField;
123 }
125 if (fLinkIds.size() != 1) {
126 throw RException(R__FAIL("unsupported untyped collection for field \"" + GetFieldName() + "\""));
127 }
128 auto itemField = ntplDesc.GetFieldDescriptor(fLinkIds[0]).CreateField(ntplDesc, options);
129 if (itemField->GetTraits() & ROOT::RFieldBase::kTraitInvalidField)
130 return itemField;
131 auto collectionField = ROOT::RVectorField::CreateUntyped(GetFieldName(), std::move(itemField));
132 collectionField->SetOnDiskId(fFieldId);
133 return collectionField;
134 }
135 default: throw RException(R__FAIL("unsupported untyped field structure for field \"" + GetFieldName() + "\""));
136 }
137 }
138
139 try {
140 const auto &fieldName = GetFieldName();
141 const auto &typeName = GetTypeAlias().empty() ? GetTypeName() : GetTypeAlias();
142 // NOTE: Unwrap() here may throw an exception, hence the try block.
143 // If options.fReturnInvalidOnError is false we just rethrow it, otherwise we return an InvalidField wrapping the
144 // error.
145 auto field = ROOT::Internal::CallFieldBaseCreate(fieldName, typeName, options, &ntplDesc, fFieldId).Unwrap();
146 field->SetOnDiskId(fFieldId);
147
148 for (auto &subfield : *field) {
149 const auto subfieldId = ntplDesc.FindFieldId(subfield.GetFieldName(), subfield.GetParent()->GetOnDiskId());
150 subfield.SetOnDiskId(subfieldId);
151 if (subfield.GetTraits() & ROOT::RFieldBase::kTraitInvalidField) {
152 auto &invalidField = static_cast<ROOT::RInvalidField &>(subfield);
153 // A subfield being invalid "infects" its entire ancestry.
154 return invalidField.Clone(fieldName);
155 }
156 }
157
158 return field;
159 } catch (const RException &ex) {
160 if (options.GetReturnInvalidOnError())
161 return std::make_unique<ROOT::RInvalidField>(GetFieldName(), GetTypeName(), ex.what(),
163 else
164 throw ex;
165 }
166}
167
168////////////////////////////////////////////////////////////////////////////////
169
171{
172 return fLogicalColumnId == other.fLogicalColumnId && fPhysicalColumnId == other.fPhysicalColumnId &&
173 fBitsOnStorage == other.fBitsOnStorage && fType == other.fType && fFieldId == other.fFieldId &&
174 fIndex == other.fIndex && fRepresentationIndex == other.fRepresentationIndex &&
175 fValueRange == other.fValueRange;
176}
177
179{
180 RColumnDescriptor clone;
181 clone.fLogicalColumnId = fLogicalColumnId;
182 clone.fPhysicalColumnId = fPhysicalColumnId;
183 clone.fBitsOnStorage = fBitsOnStorage;
184 clone.fType = fType;
185 clone.fFieldId = fFieldId;
186 clone.fIndex = fIndex;
187 clone.fFirstElementIndex = fFirstElementIndex;
188 clone.fRepresentationIndex = fRepresentationIndex;
189 if (fValueRange)
190 clone.fValueRange = std::make_unique<RValueRange>(*fValueRange);
191 return clone;
192}
193
194////////////////////////////////////////////////////////////////////////////////
195
198{
199 if (!fCumulativeNElements) {
200 // Small range, just iterate through fPageInfos
201 NTupleSize_t pageNumber = 0;
202 NTupleSize_t firstInPage = 0;
203 for (const auto &pi : fPageInfos) {
204 if (firstInPage + pi.GetNElements() > idxInCluster) {
205 return RPageInfoExtended{pi, firstInPage, pageNumber};
206 }
207 pageNumber++;
208 firstInPage += pi.GetNElements();
209 }
210 R__ASSERT(false);
211 }
212
213 const auto N = fCumulativeNElements->size();
214 R__ASSERT(N > 0);
215 R__ASSERT(N == fPageInfos.size());
216
217 std::size_t left = 0;
218 std::size_t right = N - 1;
219 std::size_t midpoint = N;
220 while (left <= right) {
221 midpoint = (left + right) / 2;
222 if ((*fCumulativeNElements)[midpoint] <= idxInCluster) {
223 left = midpoint + 1;
224 continue;
225 }
226
227 if ((midpoint == 0) || ((*fCumulativeNElements)[midpoint - 1] <= idxInCluster))
228 break;
229
230 right = midpoint - 1;
231 }
232 R__ASSERT(midpoint < N);
233
234 auto pageInfo = fPageInfos[midpoint];
235 decltype(idxInCluster) firstInPage = (midpoint == 0) ? 0 : (*fCumulativeNElements)[midpoint - 1];
236 R__ASSERT(firstInPage <= idxInCluster);
237 R__ASSERT((firstInPage + pageInfo.GetNElements()) > idxInCluster);
238 return RPageInfoExtended{pageInfo, firstInPage, midpoint};
239}
240
241std::size_t
244 std::size_t pageSize)
245{
246 R__ASSERT(fPhysicalColumnId == columnRange.GetPhysicalColumnId());
247 R__ASSERT(!columnRange.IsSuppressed());
248
249 const auto nElements =
250 std::accumulate(fPageInfos.begin(), fPageInfos.end(), 0U,
251 [](std::size_t n, const auto &pageInfo) { return n + pageInfo.GetNElements(); });
252 const auto nElementsRequired = static_cast<std::uint64_t>(columnRange.GetNElements());
253
254 if (nElementsRequired == nElements)
255 return 0U;
256 R__ASSERT((nElementsRequired > nElements) && "invalid attempt to shrink RPageRange");
257
258 std::vector<RPageInfo> pageInfos;
259 // Synthesize new `RPageInfo`s as needed
260 const std::uint64_t nElementsPerPage = pageSize / element.GetSize();
261 R__ASSERT(nElementsPerPage > 0);
262 for (auto nRemainingElements = nElementsRequired - nElements; nRemainingElements > 0;) {
263 RPageInfo pageInfo;
264 pageInfo.SetNElements(std::min(nElementsPerPage, nRemainingElements));
265 RNTupleLocator locator;
267 locator.SetNBytesOnStorage(element.GetPackedSize(pageInfo.GetNElements()));
268 pageInfo.SetLocator(locator);
269 pageInfos.emplace_back(pageInfo);
270 nRemainingElements -= pageInfo.GetNElements();
271 }
272
273 pageInfos.insert(pageInfos.end(), std::make_move_iterator(fPageInfos.begin()),
274 std::make_move_iterator(fPageInfos.end()));
275 std::swap(fPageInfos, pageInfos);
276 return nElementsRequired - nElements;
277}
278
280{
281 return fClusterId == other.fClusterId && fFirstEntryIndex == other.fFirstEntryIndex &&
282 fNEntries == other.fNEntries && fColumnRanges == other.fColumnRanges && fPageRanges == other.fPageRanges;
283}
284
286{
287 std::uint64_t nbytes = 0;
288 for (const auto &pr : fPageRanges) {
289 for (const auto &pi : pr.second.GetPageInfos()) {
290 nbytes += pi.GetLocator().GetNBytesOnStorage();
291 }
292 }
293 return nbytes;
294}
295
297{
298 RClusterDescriptor clone;
299 clone.fClusterId = fClusterId;
300 clone.fFirstEntryIndex = fFirstEntryIndex;
301 clone.fNEntries = fNEntries;
302 clone.fColumnRanges = fColumnRanges;
303 for (const auto &d : fPageRanges)
304 clone.fPageRanges.emplace(d.first, d.second.Clone());
305 return clone;
306}
307
308////////////////////////////////////////////////////////////////////////////////
309
311{
312 return fContentId == other.fContentId && fTypeName == other.fTypeName && fTypeVersion == other.fTypeVersion;
313}
314
316{
318 clone.fContentId = fContentId;
319 clone.fTypeVersion = fTypeVersion;
320 clone.fTypeName = fTypeName;
321 clone.fContent = fContent;
322 return clone;
323}
324
325////////////////////////////////////////////////////////////////////////////////
326
327ROOT::RNTupleDescriptor::RNTupleDescriptor() : fStringPool(std::make_shared<Internal::RStringPool>())
328{
329 // We need to make sure that for cloning fields, the empty string that is not explicitly entered for
330 // default constructed field descriptors is available.
331 fStringPool->Intern("");
332}
333
335{
336 // clang-format off
337 return fName == other.fName &&
338 fDescription == other.fDescription &&
339 fNEntries == other.fNEntries &&
340 fGeneration == other.fGeneration &&
341 fFieldZeroId == other.fFieldZeroId &&
342 fFieldDescriptors == other.fFieldDescriptors &&
343 fColumnDescriptors == other.fColumnDescriptors &&
344 fClusterGroupDescriptors == other.fClusterGroupDescriptors &&
345 fClusterDescriptors == other.fClusterDescriptors;
346 // clang-format on
347}
348
349ROOT::NTupleSize_t ROOT::RNTupleDescriptor::GetNElements(ROOT::DescriptorId_t physicalColumnId) const
350{
352 for (const auto &cd : fClusterDescriptors) {
353 if (!cd.second.ContainsColumn(physicalColumnId))
354 continue;
355 auto columnRange = cd.second.GetColumnRange(physicalColumnId);
356 result = std::max(result, columnRange.GetFirstElementIndex() + columnRange.GetNElements());
357 }
358 return result;
359}
360
361////////////////////////////////////////////////////////////////////////////////
362/// Return the cluster boundaries for each cluster in this RNTuple.
363std::vector<ROOT::Internal::RNTupleClusterBoundaries>
365{
366 std::vector<Internal::RNTupleClusterBoundaries> boundaries;
367 boundaries.reserve(desc.GetNClusters());
368 R__ASSERT(desc.GetNClusters() == desc.GetNActiveClusters());
369 for (const auto &clusterDesc : desc.GetActiveClusterIterable()) {
370 R__ASSERT(clusterDesc.GetNEntries() > 0);
371 boundaries.emplace_back(ROOT::Internal::RNTupleClusterBoundaries{
372 clusterDesc.GetFirstEntryIndex(), clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries()});
373 }
374 return boundaries;
375}
376
378ROOT::RNTupleDescriptor::FindFieldId(std::string_view fieldName, ROOT::DescriptorId_t parentId) const
379{
380 std::string leafName(fieldName);
381 auto posDot = leafName.find_last_of('.');
382 if (posDot != std::string::npos) {
383 auto parentName = leafName.substr(0, posDot);
384 leafName = leafName.substr(posDot + 1);
385 parentId = FindFieldId(parentName, parentId);
386 }
387 auto itrFieldDesc = fFieldDescriptors.find(parentId);
388 if (itrFieldDesc == fFieldDescriptors.end())
390 for (const auto linkId : itrFieldDesc->second.GetLinkIds()) {
391 if (fFieldDescriptors.at(linkId).GetFieldName() == leafName)
392 return linkId;
393 }
395}
396
398{
399 if (fieldId == ROOT::kInvalidDescriptorId)
400 return "";
401
402 const auto &fieldDescriptor = fFieldDescriptors.at(fieldId);
403 auto prefix = GetQualifiedFieldName(fieldDescriptor.GetParentId());
404 if (prefix.empty())
405 return fieldDescriptor.GetFieldName();
406 return prefix + "." + fieldDescriptor.GetFieldName();
407}
408
410{
411 R__ASSERT(fVersionEpoch == 1);
412 return fVersionMajor == 0 && fVersionMinor == 0 && fVersionPatch < 1;
413}
414
416{
417 std::string typeName = fieldDesc.GetTypeName();
418
419 if (FieldTypeNamesMayNeedFixup()) {
420 typeName = ROOT::Internal::GetRenormalizedTypeName(typeName);
421 }
422
423 return typeName;
424}
425
427{
428 return FindFieldId(fieldName, GetFieldZeroId());
429}
430
432 std::uint32_t columnIndex,
433 std::uint16_t representationIndex) const
434{
435 auto itr = fFieldDescriptors.find(fieldId);
436 if (itr == fFieldDescriptors.cend())
438 if (columnIndex >= itr->second.GetColumnCardinality())
440 const auto idx = representationIndex * itr->second.GetColumnCardinality() + columnIndex;
441 if (itr->second.GetLogicalColumnIds().size() <= idx)
443 return itr->second.GetLogicalColumnIds()[idx];
444}
445
447 std::uint32_t columnIndex,
448 std::uint16_t representationIndex) const
449{
450 auto logicalId = FindLogicalColumnId(fieldId, columnIndex, representationIndex);
451 if (logicalId == ROOT::kInvalidDescriptorId)
453 return GetColumnDescriptor(logicalId).GetPhysicalId();
454}
455
458{
459 if (GetNClusterGroups() == 0)
461
462 // Binary search in the cluster group list, followed by a binary search in the clusters of that cluster group
463
464 std::size_t cgLeft = 0;
465 std::size_t cgRight = GetNClusterGroups() - 1;
466 while (cgLeft <= cgRight) {
467 const std::size_t cgMidpoint = (cgLeft + cgRight) / 2;
468 const auto &clusterIds = GetClusterGroupDescriptor(fSortedClusterGroupIds[cgMidpoint]).GetClusterIds();
469 R__ASSERT(!clusterIds.empty());
470
471 const auto &clusterDesc = GetClusterDescriptor(clusterIds.front());
472 // this may happen if the RNTuple has an empty schema
473 if (!clusterDesc.ContainsColumn(physicalColumnId))
475
476 const auto firstElementInGroup = clusterDesc.GetColumnRange(physicalColumnId).GetFirstElementIndex();
477 if (firstElementInGroup > index) {
478 // Look into the lower half of cluster groups
479 R__ASSERT(cgMidpoint > 0);
480 cgRight = cgMidpoint - 1;
481 continue;
482 }
483
484 const auto &lastColumnRange = GetClusterDescriptor(clusterIds.back()).GetColumnRange(physicalColumnId);
485 if ((lastColumnRange.GetFirstElementIndex() + lastColumnRange.GetNElements()) <= index) {
486 // Look into the upper half of cluster groups
487 cgLeft = cgMidpoint + 1;
488 continue;
489 }
490
491 // Binary search in the current cluster group; since we already checked the element range boundaries,
492 // the element must be in that cluster group.
493 std::size_t clusterLeft = 0;
494 std::size_t clusterRight = clusterIds.size() - 1;
495 while (clusterLeft <= clusterRight) {
496 const std::size_t clusterMidpoint = (clusterLeft + clusterRight) / 2;
497 const auto clusterId = clusterIds[clusterMidpoint];
498 const auto &columnRange = GetClusterDescriptor(clusterId).GetColumnRange(physicalColumnId);
499
500 if (columnRange.Contains(index))
501 return clusterId;
502
503 if (columnRange.GetFirstElementIndex() > index) {
504 R__ASSERT(clusterMidpoint > 0);
505 clusterRight = clusterMidpoint - 1;
506 continue;
507 }
508
509 if (columnRange.GetFirstElementIndex() + columnRange.GetNElements() <= index) {
510 clusterLeft = clusterMidpoint + 1;
511 continue;
512 }
513 }
514 R__ASSERT(false);
515 }
517}
518
520{
521 if (GetNClusterGroups() == 0)
523
524 // Binary search in the cluster group list, followed by a binary search in the clusters of that cluster group
525
526 std::size_t cgLeft = 0;
527 std::size_t cgRight = GetNClusterGroups() - 1;
528 while (cgLeft <= cgRight) {
529 const std::size_t cgMidpoint = (cgLeft + cgRight) / 2;
530 const auto &cgDesc = GetClusterGroupDescriptor(fSortedClusterGroupIds[cgMidpoint]);
531
532 if (cgDesc.GetMinEntry() > entryIdx) {
533 R__ASSERT(cgMidpoint > 0);
534 cgRight = cgMidpoint - 1;
535 continue;
536 }
537
538 if (cgDesc.GetMinEntry() + cgDesc.GetEntrySpan() <= entryIdx) {
539 cgLeft = cgMidpoint + 1;
540 continue;
541 }
542
543 // Binary search in the current cluster group; since we already checked the element range boundaries,
544 // the element must be in that cluster group.
545 const auto &clusterIds = cgDesc.GetClusterIds();
546 R__ASSERT(!clusterIds.empty());
547 std::size_t clusterLeft = 0;
548 std::size_t clusterRight = clusterIds.size() - 1;
549 while (clusterLeft <= clusterRight) {
550 const std::size_t clusterMidpoint = (clusterLeft + clusterRight) / 2;
551 const auto &clusterDesc = GetClusterDescriptor(clusterIds[clusterMidpoint]);
552
553 if (clusterDesc.GetFirstEntryIndex() > entryIdx) {
554 R__ASSERT(clusterMidpoint > 0);
555 clusterRight = clusterMidpoint - 1;
556 continue;
557 }
558
559 if (clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries() <= entryIdx) {
560 clusterLeft = clusterMidpoint + 1;
561 continue;
562 }
563
564 return clusterIds[clusterMidpoint];
565 }
566 R__ASSERT(false);
567 }
569}
570
571ROOT::DescriptorId_t ROOT::RNTupleDescriptor::FindNextClusterId(ROOT::DescriptorId_t clusterId) const
572{
573 // TODO(jblomer): we may want to shortcut the common case and check if clusterId + 1 contains
574 // firstEntryInNextCluster. This shortcut would currently always trigger. We do not want, however, to depend
575 // on the linearity of the descriptor IDs, so we should only enable the shortcut if we can ensure that the
576 // binary search code path remains tested.
577 const auto &clusterDesc = GetClusterDescriptor(clusterId);
578 const auto firstEntryInNextCluster = clusterDesc.GetFirstEntryIndex() + clusterDesc.GetNEntries();
579 return FindClusterId(firstEntryInNextCluster);
580}
581
582ROOT::DescriptorId_t ROOT::RNTupleDescriptor::FindPrevClusterId(ROOT::DescriptorId_t clusterId) const
583{
584 // TODO(jblomer): we may want to shortcut the common case and check if clusterId - 1 contains
585 // firstEntryInNextCluster. This shortcut would currently always trigger. We do not want, however, to depend
586 // on the linearity of the descriptor IDs, so we should only enable the shortcut if we can ensure that the
587 // binary search code path remains tested.
588 const auto &clusterDesc = GetClusterDescriptor(clusterId);
589 if (clusterDesc.GetFirstEntryIndex() == 0)
591 return FindClusterId(clusterDesc.GetFirstEntryIndex() - 1);
592}
593
594std::vector<ROOT::DescriptorId_t>
596{
597 std::vector<ROOT::DescriptorId_t> fields;
598 for (const auto fieldId : fFieldIdsOrder) {
599 if (fFieldIdsLookup.count(desc.GetFieldDescriptor(fieldId).GetParentId()) == 0)
600 fields.emplace_back(fieldId);
601 }
602 return fields;
603}
604
606 const RFieldDescriptor &field)
607 : fNTuple(ntuple), fColumns(field.GetLogicalColumnIds())
608{
609}
610
612 : fNTuple(ntuple)
613{
614 std::deque<ROOT::DescriptorId_t> fieldIdQueue{ntuple.GetFieldZeroId()};
615
616 while (!fieldIdQueue.empty()) {
617 auto currFieldId = fieldIdQueue.front();
618 fieldIdQueue.pop_front();
619
620 const auto &columns = ntuple.GetFieldDescriptor(currFieldId).GetLogicalColumnIds();
621 fColumns.insert(fColumns.end(), columns.begin(), columns.end());
622
623 for (const auto &field : ntuple.GetFieldIterable(currFieldId)) {
624 auto fieldId = field.GetId();
625 fieldIdQueue.push_back(fieldId);
626 }
627 }
628}
629
630std::vector<std::uint64_t> ROOT::RNTupleDescriptor::GetFeatureFlags() const
631{
632 std::vector<std::uint64_t> result;
633 unsigned int base = 0;
634 std::uint64_t flags = 0;
635 for (auto f : fFeatureFlags) {
636 if ((f > 0) && ((f % 64) == 0))
637 throw RException(R__FAIL("invalid feature flag: " + std::to_string(f)));
638 while (f > base + 64) {
639 result.emplace_back(flags);
640 flags = 0;
641 base += 64;
642 }
643 // Note that in the following iterations of the outer loop over fFeatureFlags, we can never have the situation
644 // where base is larger than the feature flag, because they are stored ordered in the std::set.
645 assert(f >= base);
646 f -= base;
647 flags |= std::uint64_t(1) << f;
648 }
649 result.emplace_back(flags);
650 return result;
651}
652
654 std::vector<RClusterDescriptor> &clusterDescs)
655{
656 auto iter = fClusterGroupDescriptors.find(clusterGroupId);
657 if (iter == fClusterGroupDescriptors.end())
658 return R__FAIL("invalid attempt to add details of unknown cluster group");
659 if (iter->second.HasClusterDetails())
660 return R__FAIL("invalid attempt to re-populate cluster group details");
661 if (iter->second.GetNClusters() != clusterDescs.size())
662 return R__FAIL("mismatch of number of clusters");
663
664 std::vector<ROOT::DescriptorId_t> clusterIds;
665 for (unsigned i = 0; i < clusterDescs.size(); ++i) {
666 clusterIds.emplace_back(clusterDescs[i].GetId());
667 auto [_, success] = fClusterDescriptors.emplace(clusterIds.back(), std::move(clusterDescs[i]));
668 if (!success) {
669 return R__FAIL("invalid attempt to re-populate existing cluster");
670 }
671 }
672 std::sort(clusterIds.begin(), clusterIds.end(), [this](ROOT::DescriptorId_t a, ROOT::DescriptorId_t b) {
673 return fClusterDescriptors[a].GetFirstEntryIndex() < fClusterDescriptors[b].GetFirstEntryIndex();
674 });
675 auto cgBuilder = Internal::RClusterGroupDescriptorBuilder::FromSummary(iter->second);
676 cgBuilder.AddSortedClusters(clusterIds);
677 iter->second = cgBuilder.MoveDescriptor().Unwrap();
678 return RResult<void>::Success();
679}
680
682{
683 auto iter = fClusterGroupDescriptors.find(clusterGroupId);
684 if (iter == fClusterGroupDescriptors.end())
685 return R__FAIL("invalid attempt to drop cluster details of unknown cluster group");
686 if (!iter->second.HasClusterDetails())
687 return R__FAIL("invalid attempt to drop details of cluster group summary");
688
689 for (auto clusterId : iter->second.GetClusterIds())
690 fClusterDescriptors.erase(clusterId);
691 iter->second = iter->second.CloneSummary();
692 return RResult<void>::Success();
693}
694
695std::unique_ptr<ROOT::RNTupleModel> ROOT::RNTupleDescriptor::CreateModel(const RCreateModelOptions &options) const
696{
697 // Collect all top-level fields that have invalid columns (recursively): by default if we find any we throw an
698 // exception; if we are in ForwardCompatible mode, we proceed but skip of all those top-level fields.
699 std::unordered_set<ROOT::DescriptorId_t> invalidFields;
700 for (const auto &colDesc : GetColumnIterable()) {
701 if (colDesc.GetType() == ROOT::ENTupleColumnType::kUnknown) {
702 auto fieldId = colDesc.GetFieldId();
703 while (1) {
704 const auto &field = GetFieldDescriptor(fieldId);
705 if (field.GetParentId() == GetFieldZeroId())
706 break;
707 fieldId = field.GetParentId();
708 }
709 invalidFields.insert(fieldId);
710
711 // No need to look for all invalid fields if we're gonna error out anyway
712 if (!options.GetForwardCompatible())
713 break;
714 }
715 }
716
717 if (!options.GetForwardCompatible() && !invalidFields.empty())
719 "cannot create Model: descriptor contains unknown column types. Use 'SetForwardCompatible(true)' on the "
720 "RCreateModelOptions to create a partial model containing only the fields made up by known columns."));
721
722 auto fieldZero = std::make_unique<ROOT::RFieldZero>();
723 fieldZero->SetOnDiskId(GetFieldZeroId());
724 auto model = options.GetCreateBare() ? RNTupleModel::CreateBare(std::move(fieldZero))
725 : RNTupleModel::Create(std::move(fieldZero));
726 ROOT::RCreateFieldOptions createFieldOpts;
727 createFieldOpts.SetReturnInvalidOnError(options.GetForwardCompatible());
728 createFieldOpts.SetEmulateUnknownTypes(options.GetEmulateUnknownTypes());
729 for (const auto &topDesc : GetTopLevelFields()) {
730 if (invalidFields.count(topDesc.GetId()) > 0) {
731 // Field contains invalid columns: skip it
732 continue;
733 }
734
735 auto field = topDesc.CreateField(*this, createFieldOpts);
736
737 // If we got an InvalidField here, figure out if it's a hard error or if the field must simply be skipped.
738 // The only case where it's not a hard error is if the field has an unknown structure, as that case is
739 // covered by the ForwardCompatible flag (note that if the flag is off we would not get here
740 // in the first place, so we don't need to check for that flag again).
741 if (field->GetTraits() & ROOT::RFieldBase::kTraitInvalidField) {
742 const auto &invalid = static_cast<const RInvalidField &>(*field);
743 const auto cat = invalid.GetCategory();
744 bool mustThrow = cat != RInvalidField::ECategory::kUnknownStructure;
745 if (mustThrow)
746 throw RException(R__FAIL(invalid.GetError()));
747
748 // Not a hard error: skip the field and go on.
749 continue;
750 }
751
752 if (options.GetReconstructProjections() && topDesc.IsProjectedField()) {
753 model->AddProjectedField(std::move(field), [this](const std::string &targetName) -> std::string {
754 return GetQualifiedFieldName(GetFieldDescriptor(FindFieldId(targetName)).GetProjectionSourceId());
755 });
756 } else {
757 model->AddField(std::move(field));
758 }
759 }
760 model->Freeze();
761 return model;
762}
763
765{
766 RNTupleDescriptor clone;
767 clone.fName = fName;
772 // OnDiskHeaderSize, OnDiskHeaderXxHash3 not copied because they may come from a merged header + extension header
773 // and therefore not represent the actual sources's header.
774 // OnDiskFooterSize not copied because it contains information beyond the schema, for example the clustering.
775
777 shareStringPool = false;
778
779 if (shareStringPool)
780 clone.fStringPool = fStringPool;
781
783 // In case we are copying the schema from a pre-1.0.0.1 RNTuple we need to patch all field type names
784 // to use the proper normalization. In this case, the clone will get a new string pool because we may need
785 // to insert.
786 for (const auto &d : fFieldDescriptors) {
787 Internal::RFieldDescriptorBuilder fieldDescBuilder(d.second.Clone(), *clone.fStringPool);
788 fieldDescBuilder.TypeName(ROOT::Internal::GetRenormalizedTypeName(d.second.GetTypeName()));
789 clone.fFieldDescriptors.emplace(d.first, fieldDescBuilder.MoveDescriptor().Unwrap());
790 }
791 } else {
792 for (const auto &d : fFieldDescriptors) {
793 clone.fFieldDescriptors.emplace(
795 }
796 }
797
798 for (const auto &d : fColumnDescriptors)
799 clone.fColumnDescriptors.emplace(d.first, d.second.Clone());
800
801 for (const auto &d : fExtraTypeInfoDescriptors)
802 clone.fExtraTypeInfoDescriptors.emplace_back(d.Clone());
804 clone.fHeaderExtension = std::make_unique<RHeaderExtension>(*fHeaderExtension);
805
806 return clone;
807}
808
810{
811 RNTupleDescriptor clone = CloneSchema(true /* shareStringPool */);
812 clone.fStringPool->Freeze();
813
818
822 clone.fNEntries = fNEntries;
823 clone.fNClusters = fNClusters;
824 clone.fGeneration = fGeneration;
825 for (const auto &d : fClusterGroupDescriptors)
826 clone.fClusterGroupDescriptors.emplace(d.first, d.second.Clone());
828 for (const auto &d : fClusterDescriptors)
829 clone.fClusterDescriptors.emplace(d.first, d.second.Clone());
830 for (const auto &d : fAttributeSets)
831 clone.fAttributeSets.emplace_back(d.Clone());
832 return clone;
833}
834
835////////////////////////////////////////////////////////////////////////////////
836
838{
839 return fClusterGroupId == other.fClusterGroupId && fClusterIds == other.fClusterIds &&
840 fMinEntry == other.fMinEntry && fEntrySpan == other.fEntrySpan && fNClusters == other.fNClusters;
841}
842
844{
846 clone.fClusterGroupId = fClusterGroupId;
847 clone.fPageListLocator = fPageListLocator;
848 clone.fPageListLength = fPageListLength;
849 clone.fMinEntry = fMinEntry;
850 clone.fEntrySpan = fEntrySpan;
851 clone.fNClusters = fNClusters;
852 return clone;
853}
854
856{
857 RClusterGroupDescriptor clone = CloneSummary();
858 clone.fClusterIds = fClusterIds;
859 return clone;
860}
861
862////////////////////////////////////////////////////////////////////////////////
863
866 std::uint64_t firstElementIndex,
867 std::uint32_t compressionSettings,
868 const RClusterDescriptor::RPageRange &pageRange)
869{
870 if (physicalId != pageRange.fPhysicalColumnId)
871 return R__FAIL("column ID mismatch");
872 if (fCluster.fColumnRanges.count(physicalId) > 0)
873 return R__FAIL("column ID conflict");
874 RClusterDescriptor::RColumnRange columnRange{physicalId, firstElementIndex, 0, compressionSettings};
875 for (const auto &pi : pageRange.fPageInfos) {
876 columnRange.IncrementNElements(pi.GetNElements());
877 }
878 fCluster.fPageRanges[physicalId] = pageRange.Clone();
879 fCluster.fColumnRanges[physicalId] = columnRange;
880 return RResult<void>::Success();
881}
882
885{
886 if (fCluster.fColumnRanges.count(physicalId) > 0)
887 return R__FAIL("column ID conflict");
888
890 columnRange.SetPhysicalColumnId(physicalId);
891 columnRange.SetIsSuppressed(true);
892 fCluster.fColumnRanges[physicalId] = columnRange;
893 return RResult<void>::Success();
894}
895
898{
899 for (auto &[_, columnRange] : fCluster.fColumnRanges) {
900 if (!columnRange.IsSuppressed())
901 continue;
902 R__ASSERT(columnRange.GetFirstElementIndex() == ROOT::kInvalidNTupleIndex);
903
904 const auto &columnDesc = desc.GetColumnDescriptor(columnRange.GetPhysicalColumnId());
905 const auto &fieldDesc = desc.GetFieldDescriptor(columnDesc.GetFieldId());
906 // We expect only few columns and column representations per field, so we do a linear search
907 for (const auto otherColumnLogicalId : fieldDesc.GetLogicalColumnIds()) {
908 const auto &otherColumnDesc = desc.GetColumnDescriptor(otherColumnLogicalId);
909 if (otherColumnDesc.GetRepresentationIndex() == columnDesc.GetRepresentationIndex())
910 continue;
911 if (otherColumnDesc.GetIndex() != columnDesc.GetIndex())
912 continue;
913
914 // Found corresponding column of a different column representation
915 const auto &otherColumnRange = fCluster.GetColumnRange(otherColumnDesc.GetPhysicalId());
916 if (otherColumnRange.IsSuppressed())
917 continue;
918
919 columnRange.SetFirstElementIndex(otherColumnRange.GetFirstElementIndex());
920 columnRange.SetNElements(otherColumnRange.GetNElements());
921 break;
922 }
923
924 if (columnRange.GetFirstElementIndex() == ROOT::kInvalidNTupleIndex) {
925 return R__FAIL(std::string("cannot find non-suppressed column for column ID ") +
926 std::to_string(columnRange.GetPhysicalColumnId()) +
927 ", cluster ID: " + std::to_string(fCluster.GetId()));
928 }
929 }
930 return RResult<void>::Success();
931}
932
935{
936 /// Carries out a depth-first traversal of a field subtree rooted at `rootFieldId`. For each field, `visitField` is
937 /// called passing the field ID and the number of overall repetitions, taking into account the repetitions of each
938 /// parent field in the hierarchy.
939 auto fnTraverseSubtree = [&](ROOT::DescriptorId_t rootFieldId, std::uint64_t nRepetitionsAtThisLevel,
940 const auto &visitField, const auto &enterSubtree) -> void {
941 visitField(rootFieldId, nRepetitionsAtThisLevel);
942 for (const auto &f : desc.GetFieldIterable(rootFieldId)) {
943 const std::uint64_t nRepetitions = std::max(f.GetNRepetitions(), std::uint64_t{1U}) * nRepetitionsAtThisLevel;
944 enterSubtree(f.GetId(), nRepetitions, visitField, enterSubtree);
945 }
946 };
947
948 // Extended columns can only be part of the header extension
949 if (!desc.GetHeaderExtension())
950 return *this;
951
952 // Ensure that all columns in the header extension have their associated `R(Column|Page)Range`
953 // Extended columns can be attached both to fields of the regular header and to fields of the extension header
954 for (const auto &topLevelField : desc.GetTopLevelFields()) {
955 fnTraverseSubtree(
956 topLevelField.GetId(), std::max(topLevelField.GetNRepetitions(), std::uint64_t{1U}),
957 [&](ROOT::DescriptorId_t fieldId, std::uint64_t nRepetitions) {
958 for (const auto &c : desc.GetColumnIterable(fieldId)) {
959 const ROOT::DescriptorId_t physicalId = c.GetPhysicalId();
960 auto &columnRange = fCluster.fColumnRanges[physicalId];
961
962 // Initialize a RColumnRange for `physicalId` if it was not there. Columns that were created during model
963 // extension won't have on-disk metadata for the clusters that were already committed before the model
964 // was extended. Therefore, these need to be synthetically initialized upon reading.
965 if (columnRange.GetPhysicalColumnId() == ROOT::kInvalidDescriptorId) {
966 columnRange.SetPhysicalColumnId(physicalId);
967 columnRange.SetFirstElementIndex(0);
968 columnRange.SetNElements(0);
969 columnRange.SetIsSuppressed(c.IsSuppressedDeferredColumn());
970 }
971 // Fixup the RColumnRange and RPageRange in deferred columns. We know what the first element index and
972 // number of elements should have been if the column was not deferred; fix those and let
973 // `ExtendToFitColumnRange()` synthesize RPageInfos accordingly.
974 if (c.IsDeferredColumn()) {
975 if (c.GetRepresentationIndex() == 0) {
976 // Note that a deferred column (i.e, whose first element index is > 0) for the 0th representation
977 // index already met the criteria of `ROOT::RFieldBase::EntryToColumnElementIndex()`, i.e. it is a
978 // principal column reachable from the field zero excluding subfields of collection and variant
979 // fields.
980 columnRange.SetFirstElementIndex(fCluster.GetFirstEntryIndex() * nRepetitions);
981 columnRange.SetNElements(fCluster.GetNEntries() * nRepetitions);
982 } else {
983 // Deferred representations which are not the first cannot count on the number of elements being
984 // equal to Entries * nRepetitions because they might have been added in a later cluster. But they
985 // can rely on the first representation having the correct FirstElement/NElements (by definition
986 // the first representation cannot be an "extended" one), therefore they can just copy the value
987 // from it.
988 const auto &field = desc.GetFieldDescriptor(fieldId);
989 const auto firstReprColumnId = field.GetLogicalColumnIds()[c.GetIndex()];
990 const auto &firstReprColumnRange = fCluster.fColumnRanges[firstReprColumnId];
991 columnRange.SetFirstElementIndex(firstReprColumnRange.GetFirstElementIndex());
992 columnRange.SetNElements(firstReprColumnRange.GetNElements());
993 }
994 if (!columnRange.IsSuppressed()) {
995 auto &pageRange = fCluster.fPageRanges[physicalId];
996 pageRange.fPhysicalColumnId = physicalId;
997 const auto element = ROOT::Internal::RColumnElementBase::Generate<void>(c.GetType());
998 pageRange.ExtendToFitColumnRange(columnRange, *element, ROOT::Internal::RPage::kPageZeroSize);
999 }
1000 } else if (!columnRange.IsSuppressed()) {
1001 fCluster.fPageRanges[physicalId].fPhysicalColumnId = physicalId;
1002 }
1003 }
1004 },
1005 fnTraverseSubtree);
1006 }
1007 return *this;
1008}
1009
1011{
1012 if (fCluster.fClusterId == ROOT::kInvalidDescriptorId)
1013 return R__FAIL("unset cluster ID");
1014 if (fCluster.fNEntries == 0)
1015 return R__FAIL("empty cluster");
1016 for (auto &pr : fCluster.fPageRanges) {
1017 if (fCluster.fColumnRanges.count(pr.first) == 0) {
1018 return R__FAIL("missing column range");
1019 }
1020 pr.second.fCumulativeNElements.reset();
1021 const auto nPages = pr.second.fPageInfos.size();
1023 pr.second.fCumulativeNElements = std::make_unique<std::vector<NTupleSize_t>>();
1024 pr.second.fCumulativeNElements->reserve(nPages);
1026 for (const auto &pi : pr.second.fPageInfos) {
1027 sum += pi.GetNElements();
1028 pr.second.fCumulativeNElements->emplace_back(sum);
1029 }
1030 }
1031 }
1033 std::swap(result, fCluster);
1034 return result;
1035}
1036
1037////////////////////////////////////////////////////////////////////////////////
1038
1041{
1043 builder.ClusterGroupId(clusterGroupDesc.GetId())
1044 .PageListLocator(clusterGroupDesc.GetPageListLocator())
1045 .PageListLength(clusterGroupDesc.GetPageListLength())
1046 .MinEntry(clusterGroupDesc.GetMinEntry())
1047 .EntrySpan(clusterGroupDesc.GetEntrySpan())
1048 .NClusters(clusterGroupDesc.GetNClusters());
1049 return builder;
1050}
1051
1053{
1054 if (fClusterGroup.fClusterGroupId == ROOT::kInvalidDescriptorId)
1055 return R__FAIL("unset cluster group ID");
1057 std::swap(result, fClusterGroup);
1058 return result;
1059}
1060
1061////////////////////////////////////////////////////////////////////////////////
1062
1064{
1065 if (fExtraTypeInfo.fContentId == EExtraTypeInfoIds::kInvalid)
1066 throw RException(R__FAIL("invalid extra type info content id"));
1068 std::swap(result, fExtraTypeInfo);
1069 return result;
1070}
1071
1072////////////////////////////////////////////////////////////////////////////////
1073
1075{
1076 if (fDescriptor.fFieldDescriptors.count(fieldId) == 0)
1077 return R__FAIL("field with id '" + std::to_string(fieldId) + "' doesn't exist");
1078 return RResult<void>::Success();
1079}
1080
1082{
1083 if (fDescriptor.fVersionEpoch != RNTuple::kVersionEpoch) {
1084 return R__FAIL("unset or unsupported RNTuple epoch version");
1085 }
1086
1087 // Reuse field name validity check
1088 auto validName = ROOT::Internal::EnsureValidNameForRNTuple(fDescriptor.GetName(), "Field");
1089 if (!validName) {
1090 return R__FORWARD_ERROR(validName);
1091 }
1092
1093 for (const auto &[fieldId, fieldDesc] : fDescriptor.fFieldDescriptors) {
1094 // parent not properly set?
1095 if (fieldId != fDescriptor.GetFieldZeroId() && fieldDesc.GetParentId() == ROOT::kInvalidDescriptorId) {
1096 return R__FAIL("field with id '" + std::to_string(fieldId) + "' has an invalid parent id");
1097 }
1098
1099 // Same number of columns in every column representation?
1100 const auto columnCardinality = fieldDesc.GetColumnCardinality();
1101 if (columnCardinality == 0)
1102 continue;
1103
1104 // In AddColumn, we already checked that all but the last representation are complete.
1105 // Check that the last column representation is complete, i.e. has all columns.
1106 const auto &logicalColumnIds = fieldDesc.GetLogicalColumnIds();
1107 const auto nColumns = logicalColumnIds.size();
1108 // If we have only a single column representation, the following condition is true by construction
1109 if ((nColumns + 1) == columnCardinality)
1110 continue;
1111
1112 const auto &lastColumn = fDescriptor.GetColumnDescriptor(logicalColumnIds.back());
1113 if (lastColumn.GetIndex() + 1 != columnCardinality)
1114 return R__FAIL("field with id '" + std::to_string(fieldId) + "' has incomplete column representations");
1115 }
1116
1117 return RResult<void>::Success();
1118}
1119
1121{
1122 EnsureValidDescriptor().ThrowOnError();
1123 fDescriptor.fSortedClusterGroupIds.reserve(fDescriptor.fClusterGroupDescriptors.size());
1124 for (const auto &[id, _] : fDescriptor.fClusterGroupDescriptors)
1125 fDescriptor.fSortedClusterGroupIds.emplace_back(id);
1126 std::sort(fDescriptor.fSortedClusterGroupIds.begin(), fDescriptor.fSortedClusterGroupIds.end(),
1128 return fDescriptor.fClusterGroupDescriptors[a].GetMinEntry() <
1129 fDescriptor.fClusterGroupDescriptors[b].GetMinEntry();
1130 });
1131 fDescriptor.fStringPool->Freeze();
1133 std::swap(result, fDescriptor);
1134 return result;
1135}
1136
1137void ROOT::Internal::RNTupleDescriptorBuilder::SetVersion(std::uint16_t versionEpoch, std::uint16_t versionMajor,
1138 std::uint16_t versionMinor, std::uint16_t versionPatch)
1139{
1140 if (versionEpoch != RNTuple::kVersionEpoch) {
1141 throw RException(R__FAIL("unsupported RNTuple epoch version: " + std::to_string(versionEpoch)));
1142 }
1143 fDescriptor.fVersionEpoch = versionEpoch;
1144 fDescriptor.fVersionMajor = versionMajor;
1145 fDescriptor.fVersionMinor = versionMinor;
1146 fDescriptor.fVersionPatch = versionPatch;
1147}
1148
1150{
1151 fDescriptor.fVersionEpoch = RNTuple::kVersionEpoch;
1152 fDescriptor.fVersionMajor = RNTuple::kVersionMajor;
1153 fDescriptor.fVersionMinor = RNTuple::kVersionMinor;
1154 fDescriptor.fVersionPatch = RNTuple::kVersionPatch;
1155}
1156
1157void ROOT::Internal::RNTupleDescriptorBuilder::SetNTuple(std::string_view name, std::string_view description)
1158{
1159 fDescriptor.fName = std::string(name);
1160 fDescriptor.fDescription = std::string(description);
1161}
1162
1164{
1165 if (flag > 0 && flag % 64 == 0)
1166 throw RException(R__FAIL("invalid feature flag: " + std::to_string(flag)));
1167 fDescriptor.fFeatureFlags.insert(flag);
1168}
1169
1172{
1173 if (fDesc.fName.empty())
1174 return R__FAIL("attribute set name cannot be empty");
1175 if (fDesc.fAnchorLength == 0)
1176 return R__FAIL("invalid anchor length");
1177 if (fDesc.fAnchorLocator.GetType() == RNTupleLocator::kTypeUnknown)
1178 return R__FAIL("invalid locator type");
1179
1180 return std::move(fDesc);
1181}
1182
1184{
1185 if (fColumn.GetLogicalId() == ROOT::kInvalidDescriptorId)
1186 return R__FAIL("invalid logical column id");
1187 if (fColumn.GetPhysicalId() == ROOT::kInvalidDescriptorId)
1188 return R__FAIL("invalid physical column id");
1189 if (fColumn.GetFieldId() == ROOT::kInvalidDescriptorId)
1190 return R__FAIL("invalid field id, dangling column");
1191
1192 // NOTE: if the column type is unknown we don't want to fail, as we might be reading an RNTuple
1193 // created with a future version of ROOT. In this case we just skip the valid bit range check,
1194 // as we have no idea what the valid range is.
1195 // In general, reading the metadata of an unknown column is fine, it becomes an error only when
1196 // we try to read the actual data contained in it.
1197 if (fColumn.GetType() != ENTupleColumnType::kUnknown) {
1198 const auto [minBits, maxBits] = ROOT::Internal::RColumnElementBase::GetValidBitRange(fColumn.GetType());
1199 if (fColumn.GetBitsOnStorage() < minBits || fColumn.GetBitsOnStorage() > maxBits)
1200 return R__FAIL("invalid column bit width");
1201 }
1202
1204 std::swap(result, fColumn);
1205 return result;
1206}
1207
1210{
1211 RFieldDescriptor clone;
1212 clone.InitFrom(source, stringPool);
1213 return clone;
1214}
1215
1217{
1218 if (fField.GetId() == ROOT::kInvalidDescriptorId) {
1219 return R__FAIL("invalid field id");
1220 }
1221 if (fField.GetStructure() == ROOT::ENTupleStructure::kInvalid) {
1222 return R__FAIL("invalid field structure");
1223 }
1224 if (fField.IsSoACollection() && (fField.GetStructure() != ROOT::ENTupleStructure::kCollection)) {
1225 return R__FAIL("invalid SoA flag on non-collection field");
1226 }
1227 // FieldZero is usually named "" and would be a false positive here
1228 if (fField.GetParentId() != ROOT::kInvalidDescriptorId) {
1229 auto validName = ROOT::Internal::EnsureValidNameForRNTuple(fField.GetFieldName(), "Field");
1230 if (!validName) {
1231 return R__FORWARD_ERROR(validName);
1232 }
1233 if (fField.GetFieldName().empty()) {
1234 return R__FAIL("name cannot be empty string \"\"");
1235 }
1236 }
1237
1239 std::swap(result, fField);
1240 return result;
1241}
1242
1244{
1245 RFieldDescriptorBuilder fieldDesc(GetStringPool());
1246 fieldDesc.FieldId(fieldId)
1248 .TypeVersion(field.GetTypeVersion())
1249 .FieldName(field.GetFieldName())
1251 .TypeName(field.GetTypeName())
1252 .TypeAlias(field.GetTypeAlias())
1253 .Structure(field.GetStructure())
1254 .NRepetitions(field.GetNRepetitions());
1256 fieldDesc.TypeChecksum(field.GetTypeChecksum());
1259 fieldDesc.IsSoACollection(true);
1260 }
1261 AddField(fieldDesc.MoveDescriptor().Unwrap());
1262}
1263
1265{
1266 const auto id = fieldDesc.GetId();
1267 if (fDescriptor.fHeaderExtension)
1268 fDescriptor.fHeaderExtension->MarkExtendedField(fieldDesc);
1269 if (fieldDesc.GetFieldName().empty() && fieldDesc.GetParentId() == ROOT::kInvalidDescriptorId) {
1270 fDescriptor.fFieldZeroId = id;
1271 }
1272
1273 fDescriptor.fFieldDescriptors.emplace(id, std::move(fieldDesc));
1274}
1275
1278{
1279 auto fieldExists = RResult<void>::Success();
1280 if (!(fieldExists = EnsureFieldExists(fieldId)))
1281 return R__FORWARD_ERROR(fieldExists);
1282 if (!(fieldExists = EnsureFieldExists(linkId)))
1283 return R__FAIL("child field with id '" + std::to_string(linkId) + "' doesn't exist in NTuple");
1284
1285 if (linkId == fDescriptor.GetFieldZeroId()) {
1286 return R__FAIL("cannot make FieldZero a child field");
1287 }
1288 // fail if field already has another valid parent
1289 auto parentId = fDescriptor.fFieldDescriptors.at(linkId).GetParentId();
1290 if ((parentId != ROOT::kInvalidDescriptorId) && (parentId != fieldId)) {
1291 return R__FAIL("field '" + std::to_string(linkId) + "' already has a parent ('" + std::to_string(parentId) + ")");
1292 }
1293 if (fieldId == linkId) {
1294 return R__FAIL("cannot make field '" + std::to_string(fieldId) + "' a child of itself");
1295 }
1296 fDescriptor.fFieldDescriptors.at(linkId).fParentId = fieldId;
1297 fDescriptor.fFieldDescriptors.at(fieldId).fLinkIds.push_back(linkId);
1298 return RResult<void>::Success();
1299}
1300
1302 ROOT::DescriptorId_t targetId)
1303{
1304 auto fieldExists = RResult<void>::Success();
1305 if (!(fieldExists = EnsureFieldExists(sourceId)))
1306 return R__FORWARD_ERROR(fieldExists);
1307 if (!(fieldExists = EnsureFieldExists(targetId)))
1308 return R__FAIL("projected field with id '" + std::to_string(targetId) + "' doesn't exist in NTuple");
1309
1310 if (targetId == fDescriptor.GetFieldZeroId()) {
1311 return R__FAIL("cannot make FieldZero a projected field");
1312 }
1313 if (sourceId == targetId) {
1314 return R__FAIL("cannot make field '" + std::to_string(targetId) + "' a projection of itself");
1315 }
1316 if (fDescriptor.fFieldDescriptors.at(sourceId).IsProjectedField()) {
1317 return R__FAIL("cannot make field '" + std::to_string(targetId) + "' a projection of an already projected field");
1318 }
1319 // fail if target field already has another valid projection source
1320 auto &targetDesc = fDescriptor.fFieldDescriptors.at(targetId);
1321 if (targetDesc.IsProjectedField() && targetDesc.GetProjectionSourceId() != sourceId) {
1322 return R__FAIL("field '" + std::to_string(targetId) + "' has already a projection source ('" +
1323 std::to_string(targetDesc.GetProjectionSourceId()) + ")");
1324 }
1325 fDescriptor.fFieldDescriptors.at(targetId).fProjectionSourceId = sourceId;
1326 return RResult<void>::Success();
1327}
1328
1330{
1331 const auto fieldId = columnDesc.GetFieldId();
1332 const auto columnIndex = columnDesc.GetIndex();
1333 const auto representationIndex = columnDesc.GetRepresentationIndex();
1334
1335 auto fieldExists = EnsureFieldExists(fieldId);
1336 if (!fieldExists) {
1337 return R__FORWARD_ERROR(fieldExists);
1338 }
1339 auto &fieldDesc = fDescriptor.fFieldDescriptors.find(fieldId)->second;
1340
1341 if (columnDesc.IsAliasColumn()) {
1342 if (columnDesc.GetType() != fDescriptor.GetColumnDescriptor(columnDesc.GetPhysicalId()).GetType())
1343 return R__FAIL("alias column type mismatch");
1344 }
1345 if (fDescriptor.FindLogicalColumnId(fieldId, columnIndex, representationIndex) != ROOT::kInvalidDescriptorId) {
1346 return R__FAIL("column index clash");
1347 }
1348 if (columnIndex > 0) {
1349 if (fDescriptor.FindLogicalColumnId(fieldId, columnIndex - 1, representationIndex) == ROOT::kInvalidDescriptorId)
1350 return R__FAIL("out of bounds column index");
1351 }
1352 if (representationIndex > 0) {
1353 if (fDescriptor.FindLogicalColumnId(fieldId, 0, representationIndex - 1) == ROOT::kInvalidDescriptorId) {
1354 return R__FAIL("out of bounds representation index");
1355 }
1356 if (columnIndex == 0) {
1357 assert(fieldDesc.fColumnCardinality > 0);
1358 if (fDescriptor.FindLogicalColumnId(fieldId, fieldDesc.fColumnCardinality - 1, representationIndex - 1) ==
1360 return R__FAIL("incomplete column representations");
1361 }
1362 } else {
1363 if (columnIndex >= fieldDesc.fColumnCardinality)
1364 return R__FAIL("irregular column representations");
1365 }
1366 } else {
1367 // This will set the column cardinality to the number of columns of the first representation
1368 fieldDesc.fColumnCardinality = columnIndex + 1;
1369 }
1370
1371 const auto logicalId = columnDesc.GetLogicalId();
1372 fieldDesc.fLogicalColumnIds.emplace_back(logicalId);
1373
1374 if (!columnDesc.IsAliasColumn())
1375 fDescriptor.fNPhysicalColumns++;
1376 if (fDescriptor.fHeaderExtension)
1377 fDescriptor.fHeaderExtension->MarkExtendedColumn(columnDesc);
1378 fDescriptor.fColumnDescriptors.emplace(logicalId, std::move(columnDesc));
1379
1380 return RResult<void>::Success();
1381}
1382
1384{
1385 const auto id = clusterGroup.GetId();
1386 if (fDescriptor.fClusterGroupDescriptors.count(id) > 0)
1387 return R__FAIL("cluster group id clash");
1388 fDescriptor.fNEntries = std::max(fDescriptor.fNEntries, clusterGroup.GetMinEntry() + clusterGroup.GetEntrySpan());
1389 fDescriptor.fNClusters += clusterGroup.GetNClusters();
1390 fDescriptor.fClusterGroupDescriptors.emplace(id, std::move(clusterGroup));
1391 return RResult<void>::Success();
1392}
1393
1395{
1396 // We expect the resulting descriptor to be ammended, so use a dedicated string pool
1397 fDescriptor = descriptor.CloneSchema(false /* shareStringPool */);
1398}
1399
1401{
1402 if (!fDescriptor.fHeaderExtension)
1403 fDescriptor.fHeaderExtension = std::make_unique<RNTupleDescriptor::RHeaderExtension>();
1404}
1405
1407{
1408 if (fDescriptor.GetNLogicalColumns() == 0)
1409 return;
1410 R__ASSERT(fDescriptor.GetNPhysicalColumns() > 0);
1411
1412 for (ROOT::DescriptorId_t id = fDescriptor.GetNLogicalColumns() - 1; id >= fDescriptor.GetNPhysicalColumns(); --id) {
1413 auto c = fDescriptor.fColumnDescriptors[id].Clone();
1414 R__ASSERT(c.IsAliasColumn());
1415 R__ASSERT(id == c.GetLogicalId());
1416 fDescriptor.fColumnDescriptors.erase(id);
1417 for (auto &link : fDescriptor.fFieldDescriptors[c.fFieldId].fLogicalColumnIds) {
1418 if (link == c.fLogicalColumnId) {
1419 link += offset;
1420 break;
1421 }
1422 }
1423 c.fLogicalColumnId += offset;
1424 R__ASSERT(fDescriptor.fColumnDescriptors.count(c.fLogicalColumnId) == 0);
1425 fDescriptor.fColumnDescriptors.emplace(c.fLogicalColumnId, std::move(c));
1426 }
1427
1428 // Patch up column ids in the header extension
1429 if (auto &xHeader = fDescriptor.fHeaderExtension) {
1430 for (auto &columnId : xHeader->fExtendedColumnRepresentations) {
1431 if (columnId >= fDescriptor.GetNPhysicalColumns())
1432 columnId += offset;
1433 }
1434 }
1435}
1436
1438{
1439 auto clusterId = clusterDesc.GetId();
1440 if (fDescriptor.fClusterDescriptors.count(clusterId) > 0)
1441 return R__FAIL("cluster id clash");
1442 fDescriptor.fClusterDescriptors.emplace(clusterId, std::move(clusterDesc));
1443 return RResult<void>::Success();
1444}
1445
1448{
1449 // Make sure we have no duplicates
1450 if (std::find(fDescriptor.fExtraTypeInfoDescriptors.begin(), fDescriptor.fExtraTypeInfoDescriptors.end(),
1451 extraTypeInfoDesc) != fDescriptor.fExtraTypeInfoDescriptors.end()) {
1452 return R__FAIL("extra type info duplicates");
1453 }
1454 fDescriptor.fExtraTypeInfoDescriptors.emplace_back(std::move(extraTypeInfoDesc));
1455 return RResult<void>::Success();
1456}
1457
1459{
1460 auto it = std::find(fDescriptor.fExtraTypeInfoDescriptors.begin(), fDescriptor.fExtraTypeInfoDescriptors.end(),
1461 extraTypeInfoDesc);
1462 if (it != fDescriptor.fExtraTypeInfoDescriptors.end())
1463 *it = std::move(extraTypeInfoDesc);
1464 else
1465 fDescriptor.fExtraTypeInfoDescriptors.emplace_back(std::move(extraTypeInfoDesc));
1466}
1467
1470{
1471 auto &attrSets = fDescriptor.fAttributeSets;
1472 if (std::find_if(attrSets.begin(), attrSets.end(), [&name = attrSetDesc.GetName()](const auto &desc) {
1473 return desc.GetName() == name;
1474 }) != attrSets.end()) {
1475 return R__FAIL("attribute sets with duplicate names");
1476 }
1477 attrSets.push_back(std::move(attrSetDesc));
1478 return RResult<void>::Success();
1479}
1480
1485
1488{
1489 return RFieldDescriptorIterable(*this, fieldDesc);
1490}
1491
1493 const RFieldDescriptor &fieldDesc,
1494 const std::function<bool(ROOT::DescriptorId_t, ROOT::DescriptorId_t)> &comparator) const
1495{
1496 return RFieldDescriptorIterable(*this, fieldDesc, comparator);
1497}
1498
1504
1511
1516
1522
1527
1530{
1531 return RColumnDescriptorIterable(*this, fieldDesc);
1532}
1533
1539
1544
1545ROOT::RNTupleDescriptor::RClusterDescriptorIterable ROOT::RNTupleDescriptor::GetClusterIterable() const
1546{
1547 return GetActiveClusterIterable();
1548}
1549
1554
1559
1564
1566{
1567 return fAnchorLength == other.fAnchorLength && fSchemaVersionMajor == other.fSchemaVersionMajor &&
1568 fSchemaVersionMinor == other.fSchemaVersionMinor && fAnchorLocator == other.fAnchorLocator &&
1569 fName == other.fName;
1570};
1571
1573{
1575 desc.fAnchorLength = fAnchorLength;
1576 desc.fSchemaVersionMajor = fSchemaVersionMajor;
1577 desc.fSchemaVersionMinor = fSchemaVersionMinor;
1578 desc.fAnchorLocator = fAnchorLocator;
1579 desc.fName = fName;
1580 return desc;
1581}
1582
1584{
1586 return false;
1587 if (fieldDesc.GetTypeName().rfind("std::", 0) == 0)
1588 return false;
1589
1590 auto subFieldId = desc.FindFieldId("_0", fieldDesc.GetId());
1591 if (subFieldId == kInvalidDescriptorId)
1592 return false;
1593
1594 static const std::string gIntTypeNames[] = {"bool", "char", "std::int8_t", "std::uint8_t",
1595 "std::int16_t", "std::uint16_t", "std::int32_t", "std::uint32_t",
1596 "std::int64_t", "std::uint64_t"};
1597 return std::find(std::begin(gIntTypeNames), std::end(gIntTypeNames),
1598 desc.GetFieldDescriptor(subFieldId).GetTypeName()) != std::end(gIntTypeNames);
1599}
1600
1602{
1604 return false;
1605 return (fieldDesc.GetTypeName().rfind("std::atomic<", 0) == 0);
1606}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:326
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:322
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define a(i)
Definition RSha256.hxx:99
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:130
#define N
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize id
char name[80]
Definition TGX11.cxx:142
#define _(A, B)
Definition cfortran.h:108
RResult< ROOT::Experimental::RNTupleAttrSetDescriptor > MoveDescriptor()
Attempt to make an AttributeSet descriptor.
Used to loop over all the Attribute Sets linked to an RNTuple.
Metadata stored for every Attribute Set linked to an RNTuple.
bool operator==(const RNTupleAttrSetDescriptor &other) const
std::uint32_t fAnchorLength
uncompressed size of the linked anchor
A helper class for piece-wise construction of an RClusterDescriptor.
RResult< void > MarkSuppressedColumnRange(ROOT::DescriptorId_t physicalId)
Books the given column ID as being suppressed in this cluster.
RResult< void > CommitColumnRange(ROOT::DescriptorId_t physicalId, std::uint64_t firstElementIndex, std::uint32_t compressionSettings, const RClusterDescriptor::RPageRange &pageRange)
RClusterDescriptorBuilder & AddExtendedColumnRanges(const RNTupleDescriptor &desc)
Add column and page ranges for columns created during late model extension missing in this cluster.
RResult< void > CommitSuppressedColumnRanges(const RNTupleDescriptor &desc)
Sets the first element index and number of elements for all the suppressed column ranges.
RResult< RClusterDescriptor > MoveDescriptor()
Move out the full cluster descriptor including page locations.
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & EntrySpan(std::uint64_t entrySpan)
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
static RClusterGroupDescriptorBuilder FromSummary(const RClusterGroupDescriptor &clusterGroupDesc)
RClusterGroupDescriptorBuilder & PageListLength(std::uint64_t pageListLength)
RClusterGroupDescriptorBuilder & MinEntry(std::uint64_t minEntry)
RResult< RClusterGroupDescriptor > MoveDescriptor()
RClusterGroupDescriptorBuilder & ClusterGroupId(ROOT::DescriptorId_t clusterGroupId)
RClusterGroupDescriptorBuilder & NClusters(std::uint32_t nClusters)
RResult< RColumnDescriptor > MoveDescriptor()
Attempt to make a column descriptor.
A column element encapsulates the translation between basic C++ types and their column representation...
static std::pair< std::uint16_t, std::uint16_t > GetValidBitRange(ROOT::ENTupleColumnType type)
Most types have a fixed on-disk bit width.
std::size_t GetPackedSize(std::size_t nElements=1U) const
RResult< RExtraTypeInfoDescriptor > MoveDescriptor()
A helper class for piece-wise construction of an RFieldDescriptor.
RResult< RFieldDescriptor > MoveDescriptor()
Attempt to make a field descriptor.
RFieldDescriptorBuilder & NRepetitions(std::uint64_t nRepetitions)
RFieldDescriptorBuilder & Structure(const ROOT::ENTupleStructure &structure)
RFieldDescriptorBuilder & TypeAlias(const std::string &typeAlias)
static RFieldDescriptor CloneDescriptor(const RFieldDescriptor &source, RStringPool &stringPool)
RFieldDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
RFieldDescriptorBuilder & IsSoACollection(bool val)
RFieldDescriptorBuilder & TypeChecksum(const std::optional< std::uint32_t > typeChecksum)
RFieldDescriptorBuilder & FieldDescription(const std::string &fieldDescription)
RFieldDescriptorBuilder & FieldVersion(std::uint32_t fieldVersion)
RFieldDescriptorBuilder & FieldName(const std::string &fieldName)
RFieldDescriptorBuilder & FieldId(ROOT::DescriptorId_t fieldId)
RFieldDescriptorBuilder & TypeName(const std::string &typeName)
void SetNTuple(std::string_view name, std::string_view description)
void SetSchemaFromExisting(const RNTupleDescriptor &descriptor)
Copies the "schema" part of descriptor into the builder's descriptor.
RResult< void > AddColumn(RColumnDescriptor &&columnDesc)
void AddField(RFieldDescriptor fieldDesc)
RResult< void > AddAttributeSet(Experimental::RNTupleAttrSetDescriptor &&attrSetDesc)
RResult< void > AddFieldProjection(ROOT::DescriptorId_t sourceId, ROOT::DescriptorId_t targetId)
void ReplaceExtraTypeInfo(RExtraTypeInfoDescriptor &&extraTypeInfoDesc)
RResult< void > AddExtraTypeInfo(RExtraTypeInfoDescriptor &&extraTypeInfoDesc)
void ShiftAliasColumns(std::uint32_t offset)
Shift column IDs of alias columns by offset
void SetVersion(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch)
void BeginHeaderExtension()
Mark the beginning of the header extension; any fields and columns added after a call to this functio...
RResult< void > AddCluster(RClusterDescriptor &&clusterDesc)
RResult< void > EnsureValidDescriptor() const
Checks whether invariants hold:
RResult< void > AddFieldLink(ROOT::DescriptorId_t fieldId, ROOT::DescriptorId_t linkId)
RResult< void > AddClusterGroup(RClusterGroupDescriptor &&clusterGroup)
RResult< void > EnsureFieldExists(ROOT::DescriptorId_t fieldId) const
void SetFeature(unsigned int flag)
Sets the flag-th bit of the feature flag to 1.
Used for string interning of repeated type names, field names, etc.
const std::string * Intern(std::string_view str)
Searches for str in the pool.
The window of element indexes of a particular column in a particular cluster.
void SetPhysicalColumnId(ROOT::DescriptorId_t id)
ROOT::DescriptorId_t GetPhysicalColumnId() const
void IncrementNElements(ROOT::NTupleSize_t by)
Records the partition of data into pages for a particular column in a particular cluster.
static constexpr std::size_t kLargeRangeThreshold
Create the fCumulativeNElements only when its needed, i.e. when there are many pages to search throug...
RPageInfoExtended Find(ROOT::NTupleSize_t idxInCluster) const
Find the page in the RPageRange that contains the given element. The element must exist.
std::size_t ExtendToFitColumnRange(const RColumnRange &columnRange, const ROOT::Internal::RColumnElementBase &element, std::size_t pageSize)
Extend this RPageRange to fit the given RColumnRange.
Metadata for RNTuple clusters.
ROOT::NTupleSize_t fFirstEntryIndex
Clusters can be swapped by adjusting the entry offsets of the cluster and all ranges.
std::unordered_map< ROOT::DescriptorId_t, RColumnRange > fColumnRanges
ROOT::DescriptorId_t fClusterId
RClusterDescriptor Clone() const
bool operator==(const RClusterDescriptor &other) const
RColumnRangeIterable GetColumnRangeIterable() const
Returns an iterator over pairs { columnId, columnRange }. The iteration order is unspecified.
std::unordered_map< ROOT::DescriptorId_t, RPageRange > fPageRanges
std::uint64_t GetNBytesOnStorage() const
Clusters are bundled in cluster groups.
RNTupleLocator fPageListLocator
The page list that corresponds to the cluster group.
RClusterGroupDescriptor Clone() const
std::vector< ROOT::DescriptorId_t > fClusterIds
The cluster IDs can be empty if the corresponding page list is not loaded.
ROOT::DescriptorId_t GetId() const
std::uint64_t fMinEntry
The minimum first entry number of the clusters in the cluster group.
std::uint32_t fNClusters
Number of clusters is always known even if the cluster IDs are not (yet) populated.
std::uint64_t fPageListLength
Uncompressed size of the page list.
std::uint64_t GetPageListLength() const
RNTupleLocator GetPageListLocator() const
std::uint64_t fEntrySpan
Number of entries that are (partially for sharded clusters) covered by this cluster group.
bool operator==(const RClusterGroupDescriptor &other) const
RClusterGroupDescriptor CloneSummary() const
Creates a clone without the cluster IDs.
Metadata stored for every column of an RNTuple.
ROOT::DescriptorId_t fPhysicalColumnId
Usually identical to the logical column ID, except for alias columns where it references the shadowed...
bool operator==(const RColumnDescriptor &other) const
ROOT::DescriptorId_t fLogicalColumnId
The actual column identifier, which is the link to the corresponding field.
std::unique_ptr< RValueRange > fValueRange
Optional value range (used e.g. by quantized real fields)
ROOT::DescriptorId_t fFieldId
Every column belongs to one and only one field.
std::int64_t fFirstElementIndex
The absolute value specifies the index for the first stored element for this column.
std::uint32_t fIndex
A field can be serialized into several columns, which are numbered from zero to $n$.
std::uint16_t fBitsOnStorage
The size in bits of elements of this column.
std::uint16_t fRepresentationIndex
A field may use multiple column representations, which are numbered from zero to $m$.
ROOT::ENTupleColumnType fType
The on-disk column type.
RColumnDescriptor Clone() const
Get a copy of the descriptor.
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Field specific extra type information from the header / extenstion header.
bool operator==(const RExtraTypeInfoDescriptor &other) const
RExtraTypeInfoDescriptor Clone() const
EExtraTypeInfoIds fContentId
Specifies the meaning of the extra information.
std::string fTypeName
The type name the extra information refers to; empty for RNTuple-wide extra information.
std::string fContent
The content format depends on the content ID and may be binary.
std::uint32_t fTypeVersion
Type version the extra type information is bound to.
A field translates read and write calls from/to underlying columns to/from tree values.
ROOT::ENTupleStructure GetStructure() const
@ kTraitEmulatedField
This field is a user defined type that was missing dictionaries and was reconstructed from the on-dis...
@ kTraitSoACollection
The field represents a collection in SoA layout.
@ kTraitInvalidField
This field is an instance of RInvalidField and can be safely static_cast to it.
@ kTraitTypeChecksum
The TClass checksum is set and valid.
const std::string & GetFieldName() const
const std::string & GetTypeAlias() const
const std::string & GetDescription() const
Get the field's description.
virtual std::uint32_t GetFieldVersion() const
Indicates an evolution of the mapping scheme from C++ type to columns.
virtual std::uint32_t GetTypeChecksum() const
Return the current TClass reported checksum of this class. Only valid if kTraitTypeChecksum is set.
std::uint32_t GetTraits() const
std::size_t GetNRepetitions() const
const std::string & GetTypeName() const
std::unique_ptr< RFieldBase > Clone(std::string_view newName) const
Copies the field and its subfields using a possibly new name and a new, unconnected set of columns.
virtual std::uint32_t GetTypeVersion() const
Indicates an evolution of the C++ type itself.
Metadata stored for every field of an RNTuple.
const std::string & GetTypeAlias() const
std::unique_ptr< ROOT::RFieldBase > CreateField(const RNTupleDescriptor &ntplDesc, const ROOT::RCreateFieldOptions &options={}) const
In general, we create a field simply from the C++ type name.
std::uint32_t fFieldVersion
The version of the C++-type-to-column translation mechanics.
ROOT::DescriptorId_t fFieldId
RFieldDescriptor Clone() const
Get a copy of the descriptor.
ROOT::DescriptorId_t GetId() const
std::uint64_t fNRepetitions
The number of elements per entry for fixed-size arrays.
const std::vector< ROOT::DescriptorId_t > & GetLogicalColumnIds() const
std::uint32_t fColumnCardinality
The number of columns in the column representations of the field.
ROOT::DescriptorId_t fProjectionSourceId
For projected fields, the source field ID.
static const std::string & GetEmptyString()
ROOT::ENTupleStructure GetStructure() const
bool operator==(const RFieldDescriptor &other) const
ROOT::DescriptorId_t fParentId
Establishes sub field relationships, such as classes and collections.
ROOT::DescriptorId_t GetParentId() const
ROOT::ENTupleStructure fStructure
The structural information carried by this field in the data model tree.
std::vector< ROOT::DescriptorId_t > fLinkIds
The pointers in the other direction from parent to children.
const std::string & GetFieldDescription() const
void InitFrom(const RFieldDescriptor &source, Internal::RStringPool &stringPool)
bool fIsSoACollection
Indicates if this is a collection that should be represented in memory by a SoA layout.
std::uint32_t fTypeVersion
The version of the C++ type itself.
const std::string & GetFieldName() const
std::unique_ptr< Internal::RStringPool > fStringPool
Optional string storage for a free-standing field descriptor.
std::vector< ROOT::DescriptorId_t > fLogicalColumnIds
The ordered list of columns attached to this field: first by representation index then by column inde...
const std::string & GetTypeName() const
std::optional< std::uint32_t > fTypeChecksum
For custom classes, we store the ROOT TClass reported checksum to facilitate the use of I/O rules tha...
Used in RFieldBase::Check() to record field creation failures.
Definition RField.hxx:96
@ kGeneric
Generic unrecoverable error.
@ kUnknownStructure
The field could not be created because its descriptor had an unknown structural role.
ECategory GetCategory() const
Definition RField.hxx:128
Loop over all the clusters of an RNTuple, ordered by first entry number.
Used to loop over all the cluster groups of an RNTuple in order of entry ranges.
Used to loop over a field's associated columns.
std::vector< ROOT::DescriptorId_t > fColumns
The descriptor ids of the columns ordered by field, representation, and column index.
RColumnDescriptorIterable(const RNTupleDescriptor &ntuple, const RFieldDescriptor &fieldDesc)
Used to loop over all the extra type info record of an RNTuple (in unspecified order)
Used to loop over a field's child fields.
std::vector< ROOT::DescriptorId_t > GetTopMostFields(const RNTupleDescriptor &desc) const
Return a vector containing the IDs of the top-level fields defined in the extension header,...
The on-storage metadata of an RNTuple.
const RColumnDescriptor & GetColumnDescriptor(ROOT::DescriptorId_t columnId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
std::set< unsigned int > fFeatureFlags
std::unordered_map< ROOT::DescriptorId_t, RClusterGroupDescriptor > fClusterGroupDescriptors
const RFieldDescriptor & GetFieldDescriptor(ROOT::DescriptorId_t fieldId) const
std::uint64_t fNPhysicalColumns
Updated by the descriptor builder when columns are added.
std::vector< Experimental::RNTupleAttrSetDescriptor > fAttributeSets
List of AttributeSets linked to this RNTuple.
ROOT::DescriptorId_t fFieldZeroId
Set by the descriptor builder.
std::uint64_t fNEntries
Updated by the descriptor builder when the cluster groups are added.
RClusterDescriptorIterable GetActiveClusterIterable() const
RClusterGroupDescriptorIterable GetClusterGroupIterable() const
std::size_t GetNActiveClusters() const
RColumnDescriptorIterable GetColumnIterable() const
bool operator==(const RNTupleDescriptor &other) const
std::uint64_t fOnDiskFooterSize
Like fOnDiskHeaderSize, contains both cluster summaries and page locations.
std::uint16_t fVersionMinor
Set by the descriptor builder when deserialized.
ROOT::DescriptorId_t FindClusterId(ROOT::NTupleSize_t entryIdx) const
ROOT::NTupleSize_t R__DEPRECATED(6, 46, "This function is ill-conceived in the descriptor " "as not all cluster descriptors may be present. This interface is no longer publicly exposed.") GetNElements(ROOT ROOT::DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level RNTuple data fields.
std::vector< std::uint64_t > GetFeatureFlags() const
std::unique_ptr< ROOT::RNTupleModel > CreateModel(const RCreateModelOptions &options=RCreateModelOptions()) const
Re-create the C++ model from the stored metadata.
std::string GetTypeNameForComparison(const RFieldDescriptor &fieldDesc) const
Adjust the type name of the passed RFieldDescriptor for comparison with another renormalized type nam...
std::unordered_map< ROOT::DescriptorId_t, RClusterDescriptor > fClusterDescriptors
Potentially a subset of all the available clusters.
std::size_t GetNClusters() const
ROOT::DescriptorId_t FindPhysicalColumnId(ROOT::DescriptorId_t fieldId, std::uint32_t columnIndex, std::uint16_t representationIndex) const
RExtraTypeInfoDescriptorIterable GetExtraTypeInfoIterable() const
std::shared_ptr< Internal::RStringPool > fStringPool
Storage for all the field strings. Shared among descriptor clones.
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
std::uint64_t fNClusters
Updated by the descriptor builder when the cluster groups are added.
RNTupleDescriptor CloneSchema(bool shareStringPool) const
Creates a descriptor containing only the schema information about this RNTuple, i....
std::uint64_t fOnDiskHeaderXxHash3
Set by the descriptor builder when deserialized.
ROOT::DescriptorId_t FindFieldId(std::string_view fieldName, ROOT::DescriptorId_t parentId) const
std::string fName
The RNTuple name needs to be unique in a given storage location (file)
std::uint64_t fOnDiskHeaderSize
Set by the descriptor builder when deserialized.
RResult< void > DropClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId)
std::uint16_t fVersionMajor
Set by the descriptor builder when deserialized.
std::vector< ROOT::DescriptorId_t > fSortedClusterGroupIds
References cluster groups sorted by entry range and thus allows for binary search.
std::unordered_map< ROOT::DescriptorId_t, RColumnDescriptor > fColumnDescriptors
ROOT::DescriptorId_t FindLogicalColumnId(ROOT::DescriptorId_t fieldId, std::uint32_t columnIndex, std::uint16_t representationIndex) const
std::unordered_map< ROOT::DescriptorId_t, RFieldDescriptor > fFieldDescriptors
RResult< void > AddClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId, std::vector< RClusterDescriptor > &clusterDescs)
Methods to load and drop cluster group details (cluster IDs and page locations)
std::uint16_t fVersionPatch
Set by the descriptor builder when deserialized.
std::string fDescription
Free text from the user.
ROOT::Experimental::RNTupleAttrSetDescriptorIterable GetAttrSetIterable() const
RFieldDescriptorIterable GetTopLevelFields() const
std::uint16_t fVersionEpoch
Set by the descriptor builder when deserialized.
std::vector< RExtraTypeInfoDescriptor > fExtraTypeInfoDescriptors
RNTupleDescriptor Clone() const
ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindClusterId(ROOT ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindNextClusterId(ROOT ROOT::DescriptorId_t R__DEPRECATED(6, 46, "This function is ill-defined in the descriptor " "as not all cluster descriptors may be present. This interface is no longer exposed.") FindPrevClusterId(ROOT std::string GetQualifiedFieldName(ROOT::DescriptorId_t fieldId) const
Walks up the parents of the field ID and returns a field name of the form a.b.c.d In case of invalid ...
bool FieldTypeNamesMayNeedFixup() const
ROOT v6.34, with spec versions before 1.0.0.1, did not properly renormalize the type name.
std::uint64_t fGeneration
The generation of the descriptor.
std::unique_ptr< RHeaderExtension > fHeaderExtension
Generic information about the physical location of data.
void SetType(ELocatorType type)
void SetNBytesOnStorage(std::uint64_t nBytesOnStorage)
static std::unique_ptr< RNTupleModel > Create()
static std::unique_ptr< RNTupleModel > CreateBare()
Creates a "bare model", i.e. an RNTupleModel with no default entry.
static constexpr std::uint16_t kVersionPatch
Definition RNTuple.hxx:82
static constexpr std::uint16_t kVersionMajor
Definition RNTuple.hxx:80
static constexpr std::uint16_t kVersionEpoch
Definition RNTuple.hxx:79
static constexpr std::uint16_t kVersionMinor
Definition RNTuple.hxx:81
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:222
static std::unique_ptr< RVectorField > CreateUntyped(std::string_view fieldName, std::unique_ptr< RFieldBase > itemField)
const Int_t n
Definition legend1.C:16
Double_t ex[n]
Definition legend1.C:17
RResult< void > EnsureValidNameForRNTuple(std::string_view name, std::string_view where)
Check whether a given string is a valid name according to the RNTuple specification.
ROOT::RResult< std::unique_ptr< ROOT::RFieldBase > > CallFieldBaseCreate(const std::string &fieldName, const std::string &typeName, const ROOT::RCreateFieldOptions &options, const ROOT::RNTupleDescriptor *desc, ROOT::DescriptorId_t fieldId)
bool IsCustomEnumFieldDesc(const RNTupleDescriptor &desc, const RFieldDescriptor &fieldDesc)
Tells if the field describes a user-defined enum type.
std::vector< ROOT::Internal::RNTupleClusterBoundaries > GetClusterBoundaries(const RNTupleDescriptor &desc)
Return the cluster boundaries for each cluster in this RNTuple.
std::string GetRenormalizedTypeName(const std::string &metaNormalizedName)
Given a type name normalized by ROOT meta, renormalize it for RNTuple. E.g., insert std::prefix.
bool IsStdAtomicFieldDesc(const RFieldDescriptor &fieldDesc)
Tells if the field describes a std::atomic<T> type.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr NTupleSize_t kInvalidNTupleIndex
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
constexpr DescriptorId_t kInvalidDescriptorId
Additional information about a page in an in-memory RPageRange.
Information about a single page in the context of a cluster's page range.
void SetLocator(const RNTupleLocator &locator)
static uint64_t sum(uint64_t i)
Definition Factory.cxx:2335