Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorage.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorage.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2018-07-19
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RPageStorage
17#define ROOT7_RPageStorage
18
19#include <ROOT/RError.hxx>
20#include <ROOT/RCluster.hxx>
26#include <ROOT/RNTupleUtil.hxx>
27#include <ROOT/RPage.hxx>
28#include <ROOT/RPagePool.hxx>
29#include <ROOT/RSpan.hxx>
30#include <string_view>
31
32#include <atomic>
33#include <cassert>
34#include <cstddef>
35#include <deque>
36#include <functional>
37#include <memory>
38#include <mutex>
39#include <set>
40#include <shared_mutex>
41#include <unordered_set>
42#include <vector>
43
44namespace ROOT {
45namespace Experimental {
46
47class RNTupleModel;
48
49namespace Internal {
50class RColumn;
51class RColumnElementBase;
52class RNTupleCompressor;
53struct RNTupleModelChangeset;
54class RPageAllocator;
55
56enum class EPageStorageType {
57 kSink,
58 kSource,
59};
60
61// clang-format off
62/**
63\class ROOT::Experimental::Internal::RPageStorage
64\ingroup NTuple
65\brief Common functionality of an ntuple storage for both reading and writing
66
67The RPageStore provides access to a storage container that keeps the bits of pages and clusters comprising
68an ntuple. Concrete implementations can use a TFile, a raw file, an object store, and so on.
69*/
70// clang-format on
72public:
73 /// The page checksum is a 64bit xxhash3
74 static constexpr std::size_t kNBytesPageChecksum = sizeof(std::uint64_t);
75
76 /// The interface of a task scheduler to schedule page (de)compression tasks
78 public:
79 virtual ~RTaskScheduler() = default;
80 /// Take a callable that represents a task
81 virtual void AddTask(const std::function<void(void)> &taskFunc) = 0;
82 /// Blocks until all scheduled tasks finished
83 virtual void Wait() = 0;
84 };
85
86 /// A sealed page contains the bytes of a page as written to storage (packed & compressed). It is used
87 /// as an input to UnsealPages() as well as to transfer pages between different storage media.
88 /// RSealedPage does _not_ own the buffer it is pointing to in order to not interfere with the memory management
89 /// of concrete page sink and page source implementations.
90 struct RSealedPage {
91 private:
92 const void *fBuffer = nullptr;
93 std::size_t fBufferSize = 0; ///< Size of the page payload and the trailing checksum (if available)
94 std::uint32_t fNElements = 0;
95 bool fHasChecksum = false; ///< If set, the last 8 bytes of the buffer are the xxhash of the rest of the buffer
96
97 public:
98 RSealedPage() = default;
99 RSealedPage(const void *buffer, std::size_t bufferSize, std::uint32_t nElements, bool hasChecksum = false)
100 : fBuffer(buffer), fBufferSize(bufferSize), fNElements(nElements), fHasChecksum(hasChecksum)
101 {
102 }
103 RSealedPage(const RSealedPage &other) = default;
104 RSealedPage &operator=(const RSealedPage &other) = default;
105 RSealedPage(RSealedPage &&other) = default;
106 RSealedPage &operator=(RSealedPage &&other) = default;
107
108 const void *GetBuffer() const { return fBuffer; }
109 void SetBuffer(const void *buffer) { fBuffer = buffer; }
110
111 std::size_t GetDataSize() const
112 {
115 }
116 std::size_t GetBufferSize() const { return fBufferSize; }
117 void SetBufferSize(std::size_t bufferSize) { fBufferSize = bufferSize; }
118
119 std::uint32_t GetNElements() const { return fNElements; }
120 void SetNElements(std::uint32_t nElements) { fNElements = nElements; }
121
122 bool GetHasChecksum() const { return fHasChecksum; }
123 void SetHasChecksum(bool hasChecksum) { fHasChecksum = hasChecksum; }
124
125 void ChecksumIfEnabled();
127 /// Returns a failure if the sealed page has no checksum
129 };
130
131 using SealedPageSequence_t = std::deque<RSealedPage>;
132 /// A range of sealed pages referring to the same column that can be used for vector commit
135 SealedPageSequence_t::const_iterator fFirst;
136 SealedPageSequence_t::const_iterator fLast;
137
138 RSealedPageGroup() = default;
139 RSealedPageGroup(DescriptorId_t d, SealedPageSequence_t::const_iterator b, SealedPageSequence_t::const_iterator e)
141 {
142 }
143 };
144
145protected:
147
148 /// For the time being, we will use the heap allocator for all sources and sinks. This may change in the future.
149 std::unique_ptr<RPageAllocator> fPageAllocator;
150
151 std::string fNTupleName;
154 {
155 if (!fTaskScheduler)
156 return;
158 }
159
160public:
161 explicit RPageStorage(std::string_view name);
162 RPageStorage(const RPageStorage &other) = delete;
163 RPageStorage &operator=(const RPageStorage &other) = delete;
164 RPageStorage(RPageStorage &&other) = default;
166 virtual ~RPageStorage();
167
168 /// Whether the concrete implementation is a sink or a source
170
173 RColumn *fColumn = nullptr;
174
175 /// Returns true for a valid column handle; fColumn and fPhysicalId should always either both
176 /// be valid or both be invalid.
177 explicit operator bool() const { return fPhysicalId != kInvalidDescriptorId && fColumn; }
178 };
179 /// The column handle identifies a column with the current open page storage
181
182 /// Register a new column. When reading, the column must exist in the ntuple on disk corresponding to the meta-data.
183 /// When writing, every column can only be attached once.
184 virtual ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) = 0;
185 /// Unregisters a column. A page source decreases the reference counter for the corresponding active column.
186 /// For a page sink, dropping columns is currently a no-op.
187 virtual void DropColumn(ColumnHandle_t columnHandle) = 0;
188 ColumnId_t GetColumnId(ColumnHandle_t columnHandle) const { return columnHandle.fPhysicalId; }
189
190 /// Returns the default metrics object. Subclasses might alternatively provide their own metrics object by
191 /// overriding this.
193
194 /// Returns the NTuple name.
195 const std::string &GetNTupleName() const { return fNTupleName; }
196
197 void SetTaskScheduler(RTaskScheduler *taskScheduler) { fTaskScheduler = taskScheduler; }
198}; // class RPageStorage
199
200// clang-format off
201/**
202\class ROOT::Experimental::Internal::RWritePageMemoryManager
203\ingroup NTuple
204\brief Helper to maintain a memory budget for the write pages of a set of columns
205
206The memory manager keeps track of the sum of bytes used by the write pages of a set of columns.
207It will flush (and shrink) large pages of other columns on the attempt to expand a page.
208*/
209// clang-format on
211private:
212 struct RColumnInfo {
213 RColumn *fColumn = nullptr;
214 std::size_t fCurrentPageSize = 0;
215 std::size_t fInitialPageSize = 0;
216
217 bool operator>(const RColumnInfo &other) const;
218 };
219
220 /// Sum of all the write page sizes (their capacity) of the columns in `fColumnsSortedByPageSize`
221 std::size_t fCurrentAllocatedBytes = 0;
222 /// Maximum allowed value for `fCurrentAllocatedBytes`, set from RNTupleWriteOptions::fPageBufferBudget
223 std::size_t fMaxAllocatedBytes = 0;
224 /// All columns that called `ReservePage()` (hence `TryUpdate()`) at least once,
225 /// sorted by their current write page size from large to small
226 std::set<RColumnInfo, std::greater<RColumnInfo>> fColumnsSortedByPageSize;
227
228 /// Flush columns in order of allocated write page size until the sum of all write page allocations
229 /// leaves space for at least targetAvailableSize bytes. Only use columns with a write page size larger
230 /// than pageSizeLimit.
231 bool TryEvict(std::size_t targetAvailableSize, std::size_t pageSizeLimit);
232
233public:
234 explicit RWritePageMemoryManager(std::size_t maxAllocatedBytes) : fMaxAllocatedBytes(maxAllocatedBytes) {}
235
236 /// Try to register the new write page size for the given column. Flush large columns to make space, if necessary.
237 /// If not enough space is available after all (sum of write pages would be larger than fMaxAllocatedBytes),
238 /// return false.
239 bool TryUpdate(RColumn &column, std::size_t newWritePageSize);
240};
241
242// clang-format off
243/**
244\class ROOT::Experimental::Internal::RPageSink
245\ingroup NTuple
246\brief Abstract interface to write data into an ntuple
247
248The page sink takes the list of columns and afterwards a series of page commits and cluster commits.
249The user is responsible to commit clusters at a consistent point, i.e. when all pages corresponding to data
250up to the given entry number are committed.
251
252An object of this class may either be a wrapper (for example a RPageSinkBuf) or a "persistent" sink,
253inheriting from RPagePersistentSink.
254*/
255// clang-format on
256class RPageSink : public RPageStorage {
257public:
258 using Callback_t = std::function<void(RPageSink &)>;
259
260 /// Cluster that was staged, but not yet logically appended to the RNTuple
262 std::uint64_t fNBytesWritten = 0;
264
265 struct RColumnInfo {
268 bool fIsSuppressed = false;
269 };
270
271 std::vector<RColumnInfo> fColumnInfos;
272 };
273
274protected:
275 /// Parameters for the SealPage() method
277 const RPage *fPage = nullptr; ///< Input page to be sealed
278 const RColumnElementBase *fElement = nullptr; ///< Corresponds to the page's elements, for size calculation etc.
279 int fCompressionSetting = 0; ///< Compression algorithm and level to apply
280 /// Adds a 8 byte little-endian xxhash3 checksum to the page payload. The buffer has to be large enough to
281 /// to store the additional 8 bytes.
282 bool fWriteChecksum = true;
283 /// If false, the output buffer must not point to the input page buffer, which would otherwise be an option
284 /// if the page is mappable and should not be compressed
285 bool fAllowAlias = false;
286 /// Location for sealed output. The memory buffer has to be large enough.
287 void *fBuffer = nullptr;
288 };
289
290 std::unique_ptr<RNTupleWriteOptions> fOptions;
291
292 /// Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
293 /// There could be concrete page sinks that don't need a compressor. Therefore, and in order to stay consistent
294 /// with the page source, we leave it up to the derived class whether or not the compressor gets constructed.
295 std::unique_ptr<RNTupleCompressor> fCompressor;
296
297 /// Helper for streaming a page. This is commonly used in derived, concrete page sinks. Note that if
298 /// compressionSetting is 0 (uncompressed) and the page is mappable and not checksummed, the returned sealed page
299 /// will point directly to the input page buffer. Otherwise, the sealed page references an internal buffer
300 /// of fCompressor. Thus, the buffer pointed to by the RSealedPage should never be freed.
301 /// Usage of this method requires construction of fCompressor.
302 RSealedPage SealPage(const RPage &page, const RColumnElementBase &element);
303
304 /// Seal a page using the provided info.
305 static RSealedPage SealPage(const RSealPageConfig &config);
306
307private:
308 /// Flag if sink was initialized
309 bool fIsInitialized = false;
310 std::vector<Callback_t> fOnDatasetCommitCallbacks;
311 std::vector<unsigned char> fSealPageBuffer; ///< Used as destination buffer in the simple SealPage overload
312
313 /// Used in ReservePage to maintain the page buffer budget
315
316public:
317 RPageSink(std::string_view ntupleName, const RNTupleWriteOptions &options);
318
319 RPageSink(const RPageSink &) = delete;
320 RPageSink &operator=(const RPageSink &) = delete;
321 RPageSink(RPageSink &&) = default;
323 ~RPageSink() override;
324
326 /// Returns the sink's write options.
327 const RNTupleWriteOptions &GetWriteOptions() const { return *fOptions; }
328
329 void DropColumn(ColumnHandle_t /*columnHandle*/) final {}
330
331 bool IsInitialized() const { return fIsInitialized; }
332
333 /// Return the RNTupleDescriptor being constructed.
334 virtual const RNTupleDescriptor &GetDescriptor() const = 0;
335
336 /// Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket)
337 /// Init() associates column handles to the columns referenced by the model
338 void Init(RNTupleModel &model)
339 {
340 if (fIsInitialized) {
341 throw RException(R__FAIL("already initialized"));
342 }
343 fIsInitialized = true;
344 InitImpl(model);
345 }
346
347protected:
348 virtual void InitImpl(RNTupleModel &model) = 0;
349 virtual void CommitDatasetImpl() = 0;
350
351public:
352 /// Incorporate incremental changes to the model into the ntuple descriptor. This happens, e.g. if new fields were
353 /// added after the initial call to `RPageSink::Init(RNTupleModel &)`.
354 /// `firstEntry` specifies the global index for the first stored element in the added columns.
355 virtual void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) = 0;
356 /// Adds an extra type information record to schema. The extra type information will be written to the
357 /// extension header. The information in the record will be merged with the existing information, e.g.
358 /// duplicate streamer info records will be removed. This method is called by the "on commit dataset" callback
359 /// registered by specific fields (e.g., unsplit field) and during merging.
360 virtual void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) = 0;
361
362 /// Commits a suppressed column for the current cluster. Can be called anytime before CommitCluster().
363 /// For any given column and cluster, there must be no calls to both CommitSuppressedColumn() and page commits.
364 virtual void CommitSuppressedColumn(ColumnHandle_t columnHandle) = 0;
365 /// Write a page to the storage. The column must have been added before.
366 virtual void CommitPage(ColumnHandle_t columnHandle, const RPage &page) = 0;
367 /// Write a preprocessed page to storage. The column must have been added before.
368 virtual void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) = 0;
369 /// Write a vector of preprocessed pages to storage. The corresponding columns must have been added before.
370 virtual void CommitSealedPageV(std::span<RPageStorage::RSealedPageGroup> ranges) = 0;
371 /// Stage the current cluster and create a new one for the following data.
372 /// Returns the object that must be passed to CommitStagedClusters to logically append the staged cluster to the
373 /// ntuple descriptor.
374 virtual RStagedCluster StageCluster(NTupleSize_t nNewEntries) = 0;
375 /// Commit staged clusters, logically appending them to the ntuple descriptor.
376 virtual void CommitStagedClusters(std::span<RStagedCluster> clusters) = 0;
377 /// Finalize the current cluster and create a new one for the following data.
378 /// Returns the number of bytes written to storage (excluding meta-data).
379 virtual std::uint64_t CommitCluster(NTupleSize_t nNewEntries)
380 {
381 RStagedCluster stagedClusters[] = {StageCluster(nNewEntries)};
382 CommitStagedClusters(stagedClusters);
383 return stagedClusters[0].fNBytesWritten;
384 }
385 /// Write out the page locations (page list envelope) for all the committed clusters since the last call of
386 /// CommitClusterGroup (or the beginning of writing).
387 virtual void CommitClusterGroup() = 0;
388
389 /// The registered callback is executed at the beginning of CommitDataset();
391 /// Run the registered callbacks and finalize the current cluster and the entrire data set.
392 void CommitDataset();
393
394 /// Get a new, empty page for the given column that can be filled with up to nElements;
395 /// nElements must be larger than zero.
396 virtual RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements);
397
398 /// An RAII wrapper used to synchronize a page sink. See GetSinkGuard().
400 std::mutex *fLock;
401
402 public:
403 explicit RSinkGuard(std::mutex *lock) : fLock(lock)
404 {
405 if (fLock != nullptr) {
406 fLock->lock();
407 }
408 }
409 RSinkGuard(const RSinkGuard &) = delete;
410 RSinkGuard &operator=(const RSinkGuard &) = delete;
411 RSinkGuard(RSinkGuard &&) = delete;
414 {
415 if (fLock != nullptr) {
416 fLock->unlock();
417 }
418 }
419 };
420
422 {
423 // By default, there is no lock and the guard does nothing.
424 return RSinkGuard(nullptr);
425 }
426}; // class RPageSink
427
428// clang-format off
429/**
430\class ROOT::Experimental::Internal::RPagePersistentSink
431\ingroup NTuple
432\brief Base class for a sink with a physical storage backend
433*/
434// clang-format on
436private:
437 /// Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization
439
440 /// Remembers the starting cluster id for the next cluster group
441 std::uint64_t fNextClusterInGroup = 0;
442 /// Used to calculate the number of entries in the current cluster
444 /// Keeps track of the number of elements in the currently open cluster. Indexed by column id.
445 std::vector<RClusterDescriptor::RColumnRange> fOpenColumnRanges;
446 /// Keeps track of the written pages in the currently open cluster. Indexed by column id.
447 std::vector<RClusterDescriptor::RPageRange> fOpenPageRanges;
448
449 /// Union of the streamer info records that are sent from unsplit fields to the sink before committing the dataset.
451
452protected:
453 /// Set of optional features supported by the persistent sink
454 struct RFeatures {
455 bool fCanMergePages = false;
456 };
457
460
461 /// Default I/O performance counters that get registered in fMetrics
462 struct RCounters {
470 };
471 std::unique_ptr<RCounters> fCounters;
472
473 virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length) = 0;
474
475 virtual RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) = 0;
476 virtual RNTupleLocator
477 CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) = 0;
478 /// Vector commit of preprocessed pages. The `ranges` array specifies a range of sealed pages to be
479 /// committed for each column. The returned vector contains, in order, the RNTupleLocator for each
480 /// page on each range in `ranges`, i.e. the first N entries refer to the N pages in `ranges[0]`,
481 /// followed by M entries that refer to the M pages in `ranges[1]`, etc.
482 /// The mask allows to skip writing out certain pages. The vector has the size of all the pages.
483 /// For every `false` value in the mask, the corresponding locator is skipped (missing) in the output vector.
484 /// The default is to call `CommitSealedPageImpl` for each page; derived classes may provide an
485 /// optimized implementation though.
486 virtual std::vector<RNTupleLocator>
487 CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask);
488 /// Returns the number of bytes written to storage (excluding metadata)
489 virtual std::uint64_t StageClusterImpl() = 0;
490 /// Returns the locator of the page list envelope of the given buffer that contains the serialized page list.
491 /// Typically, the implementation takes care of compressing and writing the provided buffer.
492 virtual RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) = 0;
493 virtual void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) = 0;
494
495 /// Enables the default set of metrics provided by RPageSink. `prefix` will be used as the prefix for
496 /// the counters registered in the internal RNTupleMetrics object.
497 /// This set of counters can be extended by a subclass by calling `fMetrics.MakeCounter<...>()`.
498 ///
499 /// A subclass using the default set of metrics is always responsible for updating the counters
500 /// appropriately, e.g. `fCounters->fNPageCommited.Inc()`
501 void EnableDefaultMetrics(const std::string &prefix);
502
503public:
504 RPagePersistentSink(std::string_view ntupleName, const RNTupleWriteOptions &options);
505
510 ~RPagePersistentSink() override;
511
512 /// Guess the concrete derived page source from the location
513 static std::unique_ptr<RPageSink> Create(std::string_view ntupleName, std::string_view location,
514 const RNTupleWriteOptions &options = RNTupleWriteOptions());
515
516 ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) final;
517
519
520 /// Updates the descriptor and calls InitImpl() that handles the backend-specific details (file, DAOS, etc.)
521 void InitImpl(RNTupleModel &model) final;
522 void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) final;
523 void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) final;
524
525 /// Initialize sink based on an existing descriptor and fill into the descriptor builder.
526 void InitFromDescriptor(const RNTupleDescriptor &descriptor);
527
528 void CommitSuppressedColumn(ColumnHandle_t columnHandle) final;
529 void CommitPage(ColumnHandle_t columnHandle, const RPage &page) final;
530 void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final;
531 void CommitSealedPageV(std::span<RPageStorage::RSealedPageGroup> ranges) final;
532 RStagedCluster StageCluster(NTupleSize_t nNewEntries) final;
533 void CommitStagedClusters(std::span<RStagedCluster> clusters) final;
534 void CommitClusterGroup() final;
535 void CommitDatasetImpl() final;
536}; // class RPagePersistentSink
537
538// clang-format off
539/**
540\class ROOT::Experimental::Internal::RPageSource
541\ingroup NTuple
542\brief Abstract interface to read data from an ntuple
543
544The page source is initialized with the columns of interest. Alias columns from projected fields are mapped to the
545corresponding physical columns. Pages from the columns of interest can then be mapped into memory.
546The page source also gives access to the ntuple's meta-data.
547*/
548// clang-format on
549class RPageSource : public RPageStorage {
550public:
551 /// Used in SetEntryRange / GetEntryRange
552 struct REntryRange {
554 NTupleSize_t fNEntries = 0;
555
556 /// Returns true if the given cluster has entries within the entry range
557 bool IntersectsWith(const RClusterDescriptor &clusterDesc) const;
558 };
559
560 /// An RAII wrapper used for the read-only access to `RPageSource::fDescriptor`. See `GetExclDescriptorGuard()``.
563 std::shared_mutex &fLock;
564
565 public:
566 RSharedDescriptorGuard(const RNTupleDescriptor &desc, std::shared_mutex &lock) : fDescriptor(desc), fLock(lock)
567 {
568 fLock.lock_shared();
569 }
574 ~RSharedDescriptorGuard() { fLock.unlock_shared(); }
575 const RNTupleDescriptor *operator->() const { return &fDescriptor; }
576 const RNTupleDescriptor &GetRef() const { return fDescriptor; }
577 };
578
579 /// An RAII wrapper used for the writable access to `RPageSource::fDescriptor`. See `GetSharedDescriptorGuard()`.
582 std::shared_mutex &fLock;
583
584 public:
585 RExclDescriptorGuard(RNTupleDescriptor &desc, std::shared_mutex &lock) : fDescriptor(desc), fLock(lock)
586 {
587 fLock.lock();
588 }
594 {
595 fDescriptor.IncGeneration();
596 fLock.unlock();
597 }
598 RNTupleDescriptor *operator->() const { return &fDescriptor; }
599 void MoveIn(RNTupleDescriptor &&desc) { fDescriptor = std::move(desc); }
600 };
601
602private:
604 mutable std::shared_mutex fDescriptorLock;
605 REntryRange fEntryRange; ///< Used by the cluster pool to prevent reading beyond the given range
606 bool fHasStructure = false; ///< Set to true once `LoadStructure()` is called
607 bool fIsAttached = false; ///< Set to true once `Attach()` is called
608
609protected:
610 /// Default I/O performance counters that get registered in `fMetrics`
611 struct RCounters {
629 };
630
631 /// Keeps track of the requested physical column IDs. When using alias columns (projected fields), physical
632 /// columns may be requested multiple times.
634 private:
635 std::vector<DescriptorId_t> fIDs;
636 std::vector<std::size_t> fRefCounters;
637
638 public:
639 void Insert(DescriptorId_t physicalColumnID);
640 void Erase(DescriptorId_t physicalColumnID);
641 RCluster::ColumnSet_t ToColumnSet() const;
642 };
643
644 /// Summarizes cluster-level information that are necessary to load a certain page.
645 /// Used by LoadPageImpl().
647 DescriptorId_t fClusterId = 0;
648 /// Location of the page on disk
650 /// The first element number of the page's column in the given cluster
651 std::uint64_t fColumnOffset = 0;
652 };
653
654 std::unique_ptr<RCounters> fCounters;
655
657 /// The active columns are implicitly defined by the model fields or views
659
660 /// Pages that are unzipped with IMT are staged into the page pool
662
663 virtual void LoadStructureImpl() = 0;
664 /// `LoadStructureImpl()` has been called before `AttachImpl()` is called
666 /// Returns a new, unattached page source for the same data set
667 virtual std::unique_ptr<RPageSource> CloneImpl() const = 0;
668 // Only called if a task scheduler is set. No-op be default.
669 virtual void UnzipClusterImpl(RCluster *cluster);
670 // Returns a page from storage if not found in the page pool. Should be able to handle zero page locators.
671 virtual RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo,
672 ClusterSize_t::ValueType idxInCluster) = 0;
673
674 /// Prepare a page range read for the column set in `clusterKey`. Specifically, pages referencing the
675 /// `kTypePageZero` locator are filled in `pageZeroMap`; otherwise, `perPageFunc` is called for each page. This is
676 /// commonly used as part of `LoadClusters()` in derived classes.
677 void PrepareLoadCluster(
678 const RCluster::RKey &clusterKey, ROnDiskPageMap &pageZeroMap,
679 std::function<void(DescriptorId_t, NTupleSize_t, const RClusterDescriptor::RPageRange::RPageInfo &)> perPageFunc);
680
681 /// Enables the default set of metrics provided by RPageSource. `prefix` will be used as the prefix for
682 /// the counters registered in the internal RNTupleMetrics object.
683 /// A subclass using the default set of metrics is responsible for updating the counters
684 /// appropriately, e.g. `fCounters->fNRead.Inc()`
685 /// Alternatively, a subclass might provide its own RNTupleMetrics object by overriding the
686 /// `GetMetrics()` member function.
687 void EnableDefaultMetrics(const std::string &prefix);
688
689 /// Note that the underlying lock is not recursive. See `GetSharedDescriptorGuard()` for further information.
690 RExclDescriptorGuard GetExclDescriptorGuard() { return RExclDescriptorGuard(fDescriptor, fDescriptorLock); }
691
692public:
693 RPageSource(std::string_view ntupleName, const RNTupleReadOptions &fOptions);
694 RPageSource(const RPageSource &) = delete;
698 ~RPageSource() override;
699 /// Guess the concrete derived page source from the file name (location)
700 static std::unique_ptr<RPageSource> Create(std::string_view ntupleName, std::string_view location,
701 const RNTupleReadOptions &options = RNTupleReadOptions());
702 /// Open the same storage multiple time, e.g. for reading in multiple threads.
703 /// If the source is already attached, the clone will be attached, too. The clone will use, however,
704 /// it's own connection to the underlying storage (e.g., file descriptor, XRootD handle, etc.)
705 std::unique_ptr<RPageSource> Clone() const;
706
708 const RNTupleReadOptions &GetReadOptions() const { return fOptions; }
709
710 /// Takes the read lock for the descriptor. Multiple threads can take the lock concurrently.
711 /// The underlying `std::shared_mutex`, however, is neither read nor write recursive:
712 /// within one thread, only one lock (shared or exclusive) must be acquired at the same time. This requires special
713 /// care in sections protected by `GetSharedDescriptorGuard()` and `GetExclDescriptorGuard()` especially to avoid
714 /// that the locks are acquired indirectly (e.g. by a call to `GetNEntries()`). As a general guideline, no other
715 /// method of the page source should be called (directly or indirectly) in a guarded section.
717 {
718 return RSharedDescriptorGuard(fDescriptor, fDescriptorLock);
719 }
720
721 ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) override;
722 void DropColumn(ColumnHandle_t columnHandle) override;
723
724 /// Loads header and footer without decompressing or deserializing them. This can be used to asynchronously open
725 /// a file in the background. The method is idempotent and it is called as a first step in `Attach()`.
726 /// Pages sources may or may not make use of splitting loading and processing meta-data.
727 /// Therefore, `LoadStructure()` may do nothing and defer loading the meta-data to `Attach()`.
728 void LoadStructure();
729 /// Open the physical storage container and deserialize header and footer
730 void Attach();
731 NTupleSize_t GetNEntries();
732 NTupleSize_t GetNElements(ColumnHandle_t columnHandle);
733
734 /// Promise to only read from the given entry range. If set, prevents the cluster pool from reading-ahead beyond
735 /// the given range. The range needs to be within `[0, GetNEntries())`.
736 void SetEntryRange(const REntryRange &range);
737 REntryRange GetEntryRange() const { return fEntryRange; }
738
739 /// Allocates and fills a page that contains the index-th element. The default implementation searches
740 /// the page and calls LoadPageImpl(). Returns a default-constructed RPage for suppressed columns.
741 virtual RPageRef LoadPage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex);
742 /// Another version of `LoadPage` that allows to specify cluster-relative indexes.
743 /// Returns a default-constructed RPage for suppressed columns.
744 virtual RPageRef LoadPage(ColumnHandle_t columnHandle, RClusterIndex clusterIndex);
745
746 /// Read the packed and compressed bytes of a page into the memory buffer provided by `sealedPage`. The sealed page
747 /// can be used subsequently in a call to `RPageSink::CommitSealedPage`.
748 /// The `fSize` and `fNElements` member of the sealedPage parameters are always set. If `sealedPage.fBuffer` is
749 /// `nullptr`, no data will be copied but the returned size information can be used by the caller to allocate a large
750 /// enough buffer and call `LoadSealedPage` again.
751 virtual void
752 LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage) = 0;
753
754 /// Helper for unstreaming a page. This is commonly used in derived, concrete page sources. The implementation
755 /// currently always makes a memory copy, even if the sealed page is uncompressed and in the final memory layout.
756 /// The optimization of directly mapping pages is left to the concrete page source implementations.
758 UnsealPage(const RSealedPage &sealedPage, const RColumnElementBase &element, DescriptorId_t physicalColumnId);
759
760 /// Populates all the pages of the given cluster ids and columns; it is possible that some columns do not
761 /// contain any pages. The page source may load more columns than the minimal necessary set from `columns`.
762 /// To indicate which columns have been loaded, `LoadClusters()`` must mark them with `SetColumnAvailable()`.
763 /// That includes the ones from the `columns` that don't have pages; otherwise subsequent requests
764 /// for the cluster would assume an incomplete cluster and trigger loading again.
765 /// `LoadClusters()` is typically called from the I/O thread of a cluster pool, i.e. the method runs
766 /// concurrently to other methods of the page source.
767 virtual std::vector<std::unique_ptr<RCluster>> LoadClusters(std::span<RCluster::RKey> clusterKeys) = 0;
768
769 /// Parallel decompression and unpacking of the pages in the given cluster. The unzipped pages are supposed
770 /// to be preloaded in a page pool attached to the source. The method is triggered by the cluster pool's
771 /// unzip thread. It is an optional optimization, the method can safely do nothing. In particular, the
772 /// actual implementation will only run if a task scheduler is set. In practice, a task scheduler is set
773 /// if implicit multi-threading is turned on.
774 void UnzipCluster(RCluster *cluster);
775}; // class RPageSource
776
777} // namespace Internal
778
779} // namespace Experimental
780} // namespace ROOT
781
782#endif
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:290
#define d(i)
Definition RSha256.hxx:102
#define b(i)
Definition RSha256.hxx:100
#define e(i)
Definition RSha256.hxx:103
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
char name[80]
Definition TGX11.cxx:110
A thread-safe integral performance counter.
A metric element that computes its floating point value from other counters.
A collection of Counter objects with a name, a unit, and a description.
An either thread-safe or non thread safe counter for CPU ticks.
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:152
std::unordered_set< DescriptorId_t > ColumnSet_t
Definition RCluster.hxx:154
A column element encapsulates the translation between basic C++ types and their column representation...
A column is a storage-backed array of a simple, fixed-size type, from which pages can be mapped into ...
Definition RColumn.hxx:40
A helper class for piece-wise construction of an RNTupleDescriptor.
The serialization context is used for the piecewise serialization of a descriptor.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
A memory region that contains packed and compressed pages.
Definition RCluster.hxx:103
Base class for a sink with a physical storage backend.
RPagePersistentSink(const RPagePersistentSink &)=delete
ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column) final
Register a new column.
RPagePersistentSink(RPagePersistentSink &&)=default
RStagedCluster StageCluster(NTupleSize_t nNewEntries) final
Stage the current cluster and create a new one for the following data.
std::uint64_t fNextClusterInGroup
Remembers the starting cluster id for the next cluster group.
virtual std::uint64_t StageClusterImpl()=0
Returns the number of bytes written to storage (excluding metadata)
RPagePersistentSink & operator=(RPagePersistentSink &&)=default
virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length)=0
virtual std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask)
Vector commit of preprocessed pages.
RNTupleSerializer::RContext fSerializationContext
Used to map the IDs of the descriptor to the physical IDs issued during header/footer serialization.
virtual RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page)=0
void InitFromDescriptor(const RNTupleDescriptor &descriptor)
Initialize sink based on an existing descriptor and fill into the descriptor builder.
virtual RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length)=0
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
NTupleSize_t fPrevClusterNEntries
Used to calculate the number of entries in the current cluster.
std::vector< RClusterDescriptor::RPageRange > fOpenPageRanges
Keeps track of the written pages in the currently open cluster. Indexed by column id.
const RNTupleDescriptor & GetDescriptor() const final
Return the RNTupleDescriptor being constructed.
void CommitPage(ColumnHandle_t columnHandle, const RPage &page) final
Write a page to the storage. The column must have been added before.
virtual void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length)=0
RPagePersistentSink & operator=(const RPagePersistentSink &)=delete
static std::unique_ptr< RPageSink > Create(std::string_view ntupleName, std::string_view location, const RNTupleWriteOptions &options=RNTupleWriteOptions())
Guess the concrete derived page source from the location.
Internal::RNTupleDescriptorBuilder fDescriptorBuilder
RNTupleSerializer::StreamerInfoMap_t fStreamerInfos
Union of the streamer info records that are sent from unsplit fields to the sink before committing th...
void CommitClusterGroup() final
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
std::vector< RClusterDescriptor::RColumnRange > fOpenColumnRanges
Keeps track of the number of elements in the currently open cluster. Indexed by column id.
void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry) final
Incorporate incremental changes to the model into the ntuple descriptor.
void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
Write a preprocessed page to storage. The column must have been added before.
void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges) final
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
void CommitSuppressedColumn(ColumnHandle_t columnHandle) final
Commits a suppressed column for the current cluster.
void CommitStagedClusters(std::span< RStagedCluster > clusters) final
Commit staged clusters, logically appending them to the ntuple descriptor.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo) final
Adds an extra type information record to schema.
virtual RNTupleLocator CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0
A thread-safe cache of pages loaded from the page source.
Definition RPagePool.hxx:43
Reference to a page stored in the page pool.
Definition RPagePool.hxx:85
An RAII wrapper used to synchronize a page sink. See GetSinkGuard().
RSinkGuard & operator=(const RSinkGuard &)=delete
RSinkGuard & operator=(RSinkGuard &&)=delete
Abstract interface to write data into an ntuple.
std::vector< unsigned char > fSealPageBuffer
Used as destination buffer in the simple SealPage overload.
std::vector< Callback_t > fOnDatasetCommitCallbacks
RPageSink & operator=(RPageSink &&)=default
virtual RStagedCluster StageCluster(NTupleSize_t nNewEntries)=0
Stage the current cluster and create a new one for the following data.
bool fIsInitialized
Flag if sink was initialized.
virtual void UpdateExtraTypeInfo(const RExtraTypeInfoDescriptor &extraTypeInfo)=0
Adds an extra type information record to schema.
void CommitDataset()
Run the registered callbacks and finalize the current cluster and the entrire data set.
virtual void CommitSuppressedColumn(ColumnHandle_t columnHandle)=0
Commits a suppressed column for the current cluster.
virtual const RNTupleDescriptor & GetDescriptor() const =0
Return the RNTupleDescriptor being constructed.
void Init(RNTupleModel &model)
Physically creates the storage container to hold the ntuple (e.g., a keys a TFile or an S3 bucket) In...
virtual RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements)
Get a new, empty page for the given column that can be filled with up to nElements; nElements must be...
RWritePageMemoryManager fWritePageMemoryManager
Used in ReservePage to maintain the page buffer budget.
virtual void CommitPage(ColumnHandle_t columnHandle, const RPage &page)=0
Write a page to the storage. The column must have been added before.
const RNTupleWriteOptions & GetWriteOptions() const
Returns the sink's write options.
RPageSink & operator=(const RPageSink &)=delete
virtual void CommitClusterGroup()=0
Write out the page locations (page list envelope) for all the committed clusters since the last call ...
virtual std::uint64_t CommitCluster(NTupleSize_t nNewEntries)
Finalize the current cluster and create a new one for the following data.
void RegisterOnCommitDatasetCallback(Callback_t callback)
The registered callback is executed at the beginning of CommitDataset();.
RPageSink(const RPageSink &)=delete
std::function< void(RPageSink &)> Callback_t
virtual void CommitSealedPage(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage)=0
Write a preprocessed page to storage. The column must have been added before.
virtual void CommitStagedClusters(std::span< RStagedCluster > clusters)=0
Commit staged clusters, logically appending them to the ntuple descriptor.
virtual void InitImpl(RNTupleModel &model)=0
void DropColumn(ColumnHandle_t) final
Unregisters a column.
EPageStorageType GetType() final
Whether the concrete implementation is a sink or a source.
std::unique_ptr< RNTupleCompressor > fCompressor
Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
virtual void UpdateSchema(const RNTupleModelChangeset &changeset, NTupleSize_t firstEntry)=0
Incorporate incremental changes to the model into the ntuple descriptor.
virtual void CommitSealedPageV(std::span< RPageStorage::RSealedPageGroup > ranges)=0
Write a vector of preprocessed pages to storage. The corresponding columns must have been added befor...
std::unique_ptr< RNTupleWriteOptions > fOptions
RSealedPage SealPage(const RPage &page, const RColumnElementBase &element)
Helper for streaming a page.
Keeps track of the requested physical column IDs.
An RAII wrapper used for the writable access to RPageSource::fDescriptor. See GetSharedDescriptorGuar...
RExclDescriptorGuard(RNTupleDescriptor &desc, std::shared_mutex &lock)
RExclDescriptorGuard(const RExclDescriptorGuard &)=delete
RExclDescriptorGuard & operator=(RExclDescriptorGuard &&)=delete
RExclDescriptorGuard & operator=(const RExclDescriptorGuard &)=delete
An RAII wrapper used for the read-only access to RPageSource::fDescriptor. See GetExclDescriptorGuard...
RSharedDescriptorGuard & operator=(RSharedDescriptorGuard &&)=delete
RSharedDescriptorGuard(const RSharedDescriptorGuard &)=delete
RSharedDescriptorGuard(const RNTupleDescriptor &desc, std::shared_mutex &lock)
RSharedDescriptorGuard & operator=(const RSharedDescriptorGuard &)=delete
Abstract interface to read data from an ntuple.
virtual void LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage)=0
Read the packed and compressed bytes of a page into the memory buffer provided by sealedPage.
RPagePool fPagePool
Pages that are unzipped with IMT are staged into the page pool.
EPageStorageType GetType() final
Whether the concrete implementation is a sink or a source.
RPageSource(const RPageSource &)=delete
RPageSource & operator=(RPageSource &&)=delete
std::unique_ptr< RCounters > fCounters
RExclDescriptorGuard GetExclDescriptorGuard()
Note that the underlying lock is not recursive. See GetSharedDescriptorGuard() for further informatio...
RActivePhysicalColumns fActivePhysicalColumns
The active columns are implicitly defined by the model fields or views.
RPageSource & operator=(const RPageSource &)=delete
virtual RNTupleDescriptor AttachImpl()=0
LoadStructureImpl() has been called before AttachImpl() is called
const RNTupleReadOptions & GetReadOptions() const
virtual std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys)=0
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
REntryRange fEntryRange
Used by the cluster pool to prevent reading beyond the given range.
virtual RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster)=0
virtual std::unique_ptr< RPageSource > CloneImpl() const =0
Returns a new, unattached page source for the same data set.
const RSharedDescriptorGuard GetSharedDescriptorGuard() const
Takes the read lock for the descriptor.
The interface of a task scheduler to schedule page (de)compression tasks.
virtual void Wait()=0
Blocks until all scheduled tasks finished.
virtual void AddTask(const std::function< void(void)> &taskFunc)=0
Take a callable that represents a task.
Common functionality of an ntuple storage for both reading and writing.
std::deque< RSealedPage > SealedPageSequence_t
std::unique_ptr< RPageAllocator > fPageAllocator
For the time being, we will use the heap allocator for all sources and sinks. This may change in the ...
virtual ColumnHandle_t AddColumn(DescriptorId_t fieldId, RColumn &column)=0
Register a new column.
static constexpr std::size_t kNBytesPageChecksum
The page checksum is a 64bit xxhash3.
virtual void DropColumn(ColumnHandle_t columnHandle)=0
Unregisters a column.
virtual Detail::RNTupleMetrics & GetMetrics()
Returns the default metrics object.
const std::string & GetNTupleName() const
Returns the NTuple name.
virtual EPageStorageType GetType()=0
Whether the concrete implementation is a sink or a source.
RPageStorage & operator=(RPageStorage &&other)=default
RPageStorage & operator=(const RPageStorage &other)=delete
RPageStorage(const RPageStorage &other)=delete
RPageStorage(RPageStorage &&other)=default
void SetTaskScheduler(RTaskScheduler *taskScheduler)
RColumnHandle ColumnHandle_t
The column handle identifies a column with the current open page storage.
ColumnId_t GetColumnId(ColumnHandle_t columnHandle) const
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:46
Helper to maintain a memory budget for the write pages of a set of columns.
bool TryUpdate(RColumn &column, std::size_t newWritePageSize)
Try to register the new write page size for the given column.
std::size_t fCurrentAllocatedBytes
Sum of all the write page sizes (their capacity) of the columns in fColumnsSortedByPageSize
bool TryEvict(std::size_t targetAvailableSize, std::size_t pageSizeLimit)
Flush columns in order of allocated write page size until the sum of all write page allocations leave...
std::size_t fMaxAllocatedBytes
Maximum allowed value for fCurrentAllocatedBytes, set from RNTupleWriteOptions::fPageBufferBudget.
std::set< RColumnInfo, std::greater< RColumnInfo > > fColumnsSortedByPageSize
All columns that called ReservePage() (hence TryUpdate()) at least once, sorted by their current writ...
Records the partition of data into pages for a particular column in a particular cluster.
Meta-data for a set of ntuple clusters.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Field specific extra type information from the header / extenstion header.
The on-storage meta-data of an ntuple.
The RNTupleModel encapulates the schema of an ntuple.
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr NTupleSize_t kInvalidNTupleIndex
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
constexpr ClusterSize_t kInvalidClusterIndex(std::uint64_t(-1))
constexpr DescriptorId_t kInvalidDescriptorId
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
The identifiers that specifies the content of a (partial) cluster.
Definition RCluster.hxx:156
The incremental changes to a RNTupleModel
Default I/O performance counters that get registered in fMetrics.
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuZip
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuWrite
Set of optional features supported by the persistent sink.
const RColumnElementBase * fElement
Corresponds to the page's elements, for size calculation etc.
void * fBuffer
Location for sealed output. The memory buffer has to be large enough.
bool fAllowAlias
If false, the output buffer must not point to the input page buffer, which would otherwise be an opti...
int fCompressionSetting
Compression algorithm and level to apply.
bool fWriteChecksum
Adds a 8 byte little-endian xxhash3 checksum to the page payload.
Cluster that was staged, but not yet logically appended to the RNTuple.
Summarizes cluster-level information that are necessary to load a certain page.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
Default I/O performance counters that get registered in fMetrics
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuUnzip
Detail::RNTupleTickCounter< Detail::RNTupleAtomicCounter > & fTimeCpuRead
A range of sealed pages referring to the same column that can be used for vector commit.
RSealedPageGroup(DescriptorId_t d, SealedPageSequence_t::const_iterator b, SealedPageSequence_t::const_iterator e)
A sealed page contains the bytes of a page as written to storage (packed & compressed).
RResult< std::uint64_t > GetChecksum() const
Returns a failure if the sealed page has no checksum.
bool fHasChecksum
If set, the last 8 bytes of the buffer are the xxhash of the rest of the buffer.
std::size_t fBufferSize
Size of the page payload and the trailing checksum (if available)
RSealedPage(const void *buffer, std::size_t bufferSize, std::uint32_t nElements, bool hasChecksum=false)
RSealedPage & operator=(const RSealedPage &other)=default
RSealedPage & operator=(RSealedPage &&other)=default
We do not need to store the element size / uncompressed page size because we know to which column the...
Wrap the integer in a struct in order to avoid template specialization clash with std::uint64_t.
Generic information about the physical location of data.