Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorageDaos.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorageDaos.hxx
2/// \ingroup NTuple ROOT7
3/// \author Javier Lopez-Gomez <j.lopez@cern.ch>
4/// \date 2020-11-03
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RPageStorageDaos
17#define ROOT7_RPageStorageDaos
18
19#include <ROOT/RError.hxx>
20#include <ROOT/RPageStorage.hxx>
21#include <ROOT/RNTuple.hxx>
23#include <ROOT/RNTupleZip.hxx>
24#include <string_view>
25
26#include <array>
27#include <atomic>
28#include <cstdio>
29#include <memory>
30#include <string>
31#include <optional>
32
33namespace ROOT {
34namespace Experimental {
35
36namespace Internal {
37using ntuple_index_t = std::uint32_t;
38class RCluster;
39class RClusterPool;
40class RDaosPool;
41class RDaosContainer;
43class RPagePool;
45 // Indicates that the referenced page is "caged", i.e. it is stored in a larger blob that contains multiple pages.
46 kCagedPage = 0x01,
47};
48
49// clang-format off
50/**
51\class ROOT::Experimental::Internal::RDaosNTupleAnchor
52\ingroup NTuple
53\brief Entry point for an RNTuple in a DAOS container. It encodes essential
54information to read the ntuple; currently, it contains (un)compressed size of
55the header/footer blobs and the object class for user data OIDs.
56The length of a serialized anchor cannot be greater than the value returned by the `GetSize` function.
57*/
58// clang-format on
60 /// Allows for evolving the struct in future versions
61 std::uint64_t fVersionAnchor = 1;
62 /// Version of the binary format supported by the writer
67 /// The size of the compressed ntuple header
68 std::uint32_t fNBytesHeader = 0;
69 /// The size of the uncompressed ntuple header
70 std::uint32_t fLenHeader = 0;
71 /// The size of the compressed ntuple footer
72 std::uint32_t fNBytesFooter = 0;
73 /// The size of the uncompressed ntuple footer
74 std::uint32_t fLenFooter = 0;
75 /// The object class for user data OIDs, e.g. `SX`
76 std::string fObjClass{};
77
78 bool operator ==(const RDaosNTupleAnchor &other) const {
79 return fVersionAnchor == other.fVersionAnchor && fVersionEpoch == other.fVersionEpoch &&
82 fLenHeader == other.fLenHeader && fNBytesFooter == other.fNBytesFooter && fLenFooter == other.fLenFooter &&
83 fObjClass == other.fObjClass;
84 }
85
86 std::uint32_t Serialize(void *buffer) const;
87 RResult<std::uint32_t> Deserialize(const void *buffer, std::uint32_t bufSize);
88
89 static std::uint32_t GetSize();
90}; // struct RDaosNTupleAnchor
91
92// clang-format off
93/**
94\class ROOT::Experimental::Internal::RPageSinkDaos
95\ingroup NTuple
96\brief Storage provider that writes ntuple pages to into a DAOS container
97
98Currently, an object is allocated for ntuple metadata (anchor/header/footer).
99Objects can correspond to pages or clusters of pages depending on the RNTuple-DAOS mapping strategy.
100*/
101// clang-format on
103private:
104 std::unique_ptr<RPageAllocatorHeap> fPageAllocator;
105
106 /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
107 /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
108 /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
109 /// after (which calls `daos_pool_disconect()`).
110 std::unique_ptr<RDaosContainer> fDaosContainer;
111 /// Page identifier for the next committed page; it is automatically incremented in `CommitSealedPageImpl()`
112 std::atomic<std::uint64_t> fPageId{0};
113 /// Cluster group counter for the next committed cluster pagelist; incremented in `CommitClusterGroupImpl()`
114 std::atomic<std::uint64_t> fClusterGroupId{0};
115 /// \brief A URI to a DAOS pool of the form 'daos://pool-label/container-label'
116 std::string fURI;
117 /// Tracks the number of bytes committed to the current cluster
118 std::uint64_t fNBytesCurrentCluster{0};
119
122 uint32_t fCageSizeLimit{};
123
124protected:
125 void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final;
126 RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final;
128 CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final;
129 std::vector<RNTupleLocator> CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges) final;
130 std::uint64_t CommitClusterImpl() final;
131 RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final;
132 void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
133 void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
134 void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
135 void WriteNTupleAnchor();
136
137public:
138 RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options);
139 ~RPageSinkDaos() override;
140
141 RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final;
142 void ReleasePage(RPage &page) final;
143}; // class RPageSinkDaos
144
145// clang-format off
146/**
147\class ROOT::Experimental::Internal::RPageSourceDaos
148\ingroup NTuple
149\brief Storage provider that reads ntuple pages from a DAOS container
150*/
151// clang-format on
153private:
154 /// Summarizes cluster-level information that are necessary to populate a certain page.
155 /// Used by PopulatePageFromCluster().
157 DescriptorId_t fClusterId = 0;
158 /// Location of the page on disk
160 /// The first element number of the page's column in the given cluster
161 std::uint64_t fColumnOffset = 0;
162 };
163
165
166 /// Populated pages might be shared; the page pool might, at some point, be used by multiple page sources
167 std::shared_ptr<RPagePool> fPagePool;
168 /// The last cluster from which a page got populated. Points into fClusterPool->fPool
169 RCluster *fCurrentCluster = nullptr;
170 /// A container that stores object data (header/footer, pages, etc.)
171 std::unique_ptr<RDaosContainer> fDaosContainer;
172 /// A URI to a DAOS pool of the form 'daos://pool-label/container-label'
173 std::string fURI;
174 /// The cluster pool asynchronously preloads the next few clusters
175 std::unique_ptr<RClusterPool> fClusterPool;
176
178
179 RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo,
180 ClusterSize_t::ValueType idxInCluster);
181
182protected:
183 RNTupleDescriptor AttachImpl() final;
184 void UnzipClusterImpl(RCluster *cluster) final;
185
186public:
187 RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options);
188 /// The cloned page source creates a new connection to the pool/container.
189 /// The meta-data (header and footer) is reread and parsed by the clone.
190 std::unique_ptr<RPageSource> Clone() const final;
191 ~RPageSourceDaos() override;
192
193 RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final;
194 RPage PopulatePage(ColumnHandle_t columnHandle, RClusterIndex clusterIndex) final;
195 void ReleasePage(RPage &page) final;
196
197 void LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage) final;
198
199 std::vector<std::unique_ptr<RCluster>> LoadClusters(std::span<RCluster::RKey> clusterKeys) final;
200
201 /// Return the object class used for user data OIDs in this ntuple.
202 std::string GetObjectClass() const;
203}; // class RPageSourceDaos
204
205} // namespace Internal
206
207} // namespace Experimental
208} // namespace ROOT
209
210#endif
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Managed a set of clusters containing compressed and packed pages.
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:152
A RDaosContainer provides read/write access to objects in a given container.
Definition RDaos.hxx:157
A RDaosPool provides access to containers in a specific DAOS pool.
Definition RDaos.hxx:66
A helper class for piece-wise construction of an RNTupleDescriptor.
Uses standard C++ memory allocation for the column data pages.
Base class for a sink with a physical storage backend.
A thread-safe cache of column pages.
Definition RPagePool.hxx:47
Storage provider that writes ntuple pages to into a DAOS container.
RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final
Get a new, empty page for the given column that can be filled with up to nElements.
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges) final
Vector commit of preprocessed pages.
std::unique_ptr< RDaosContainer > fDaosContainer
Underlying DAOS container.
std::uint64_t fNBytesCurrentCluster
Tracks the number of bytes committed to the current cluster.
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final
std::atomic< std::uint64_t > fPageId
Page identifier for the next committed page; it is automatically incremented in CommitSealedPageImpl(...
std::atomic< std::uint64_t > fClusterGroupId
Cluster group counter for the next committed cluster pagelist; incremented in CommitClusterGroupImpl(...
void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final
std::uint64_t CommitClusterImpl() final
Returns the number of bytes written to storage (excluding metadata)
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
std::unique_ptr< RPageAllocatorHeap > fPageAllocator
RNTupleLocator CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
Storage provider that reads ntuple pages from a DAOS container.
std::unique_ptr< RClusterPool > fClusterPool
The cluster pool asynchronously preloads the next few clusters.
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
std::shared_ptr< RPagePool > fPagePool
Populated pages might be shared; the page pool might, at some point, be used by multiple page sources...
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
Abstract interface to read data from an ntuple.
RColumnHandle ColumnHandle_t
The column handle identifies a column with the current open page storage.
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:41
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
The on-storage meta-data of an ntuple.
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
static constexpr std::uint16_t kVersionEpoch
Definition RNTuple.hxx:67
static constexpr std::uint16_t kVersionPatch
Definition RNTuple.hxx:70
static constexpr std::uint16_t kVersionMajor
Definition RNTuple.hxx:68
static constexpr std::uint16_t kVersionMinor
Definition RNTuple.hxx:69
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint64_t fVersionAnchor
Allows for evolving the struct in future versions.
bool operator==(const RDaosNTupleAnchor &other) const
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint16_t fVersionEpoch
Version of the binary format supported by the writer.
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Summarizes cluster-level information that are necessary to populate a certain page.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
Generic information about the physical location of data.