Logo ROOT  
Reference Guide
RPageStorageDaos.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorageDaos.hxx
2/// \ingroup NTuple ROOT7
3/// \author Javier Lopez-Gomez <j.lopez@cern.ch>
4/// \date 2020-11-03
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RPageStorageDaos
17#define ROOT7_RPageStorageDaos
18
19#include <ROOT/RError.hxx>
20#include <ROOT/RPageStorage.hxx>
22#include <ROOT/RNTupleZip.hxx>
23#include <ROOT/RStringView.hxx>
24
25#include <array>
26#include <atomic>
27#include <cstdio>
28#include <memory>
29#include <string>
30
31namespace ROOT {
32
33namespace Experimental {
34namespace Detail {
35
36class RCluster;
37class RClusterPool;
38class RPageAllocatorHeap;
39class RPagePool;
40class RDaosPool;
41class RDaosContainer;
42
43// clang-format off
44/**
45\class ROOT::Experimental::Detail::RDaosNTupleAnchor
46\ingroup NTuple
47\brief Entry point for an RNTuple in a DAOS container. It encodes essential
48information to read the ntuple; currently, it contains (un)compressed size of
49the header/footer blobs and the object class for user data OIDs.
50The length of a serialized anchor cannot be greater than the value returned by the `GetSize` function.
51*/
52// clang-format on
54 /// Allows for evolving the struct in future versions
55 std::uint32_t fVersion = 0;
56 /// The size of the compressed ntuple header
57 std::uint32_t fNBytesHeader = 0;
58 /// The size of the uncompressed ntuple header
59 std::uint32_t fLenHeader = 0;
60 /// The size of the compressed ntuple footer
61 std::uint32_t fNBytesFooter = 0;
62 /// The size of the uncompressed ntuple footer
63 std::uint32_t fLenFooter = 0;
64 /// The object class for user data OIDs, e.g. `SX`
65 std::string fObjClass{};
66
67 bool operator ==(const RDaosNTupleAnchor &other) const {
68 return fVersion == other.fVersion &&
70 fLenHeader == other.fLenHeader &&
72 fLenFooter == other.fLenFooter &&
73 fObjClass == other.fObjClass;
74 }
75
76 std::uint32_t Serialize(void *buffer) const;
77 RResult<std::uint32_t> Deserialize(const void *buffer, std::uint32_t bufSize);
78
79 static std::uint32_t GetSize();
80};
81
82// clang-format off
83/**
84\class ROOT::Experimental::Detail::RPageSinkDaos
85\ingroup NTuple
86\brief Storage provider that writes ntuple pages to into a DAOS container
87
88Currently, an object is allocated for ntuple metadata (anchor/header/footer).
89Objects can correspond to pages or clusters of pages depending on the RNTuple-DAOS mapping strategy.
90*/
91// clang-format on
92class RPageSinkDaos : public RPageSink {
93private:
94 std::unique_ptr<RPageAllocatorHeap> fPageAllocator;
95
96 /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
97 /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
98 /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
99 /// after (which calls `daos_pool_disconect()`).
100 std::unique_ptr<RDaosContainer> fDaosContainer;
101 /// Page identifier for the next committed page; it is automatically incremented in `CommitSealedPageImpl()`
102 std::atomic<std::uint64_t> fPageId{0};
103 /// Cluster group counter for the next committed cluster pagelist; incremented in `CommitClusterGroupImpl()`
104 std::atomic<std::uint64_t> fClusterGroupId{0};
105 /// \brief A URI to a DAOS pool of the form 'daos://pool-label/container-label'
106 std::string fURI;
107 /// Tracks the number of bytes committed to the current cluster
108 std::uint64_t fNBytesCurrentCluster{0};
109
111
112protected:
113 void CreateImpl(const RNTupleModel &model, unsigned char *serializedHeader, std::uint32_t length) final;
114 RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final;
116 std::vector<RNTupleLocator> CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges) final;
117 std::uint64_t CommitClusterImpl(NTupleSize_t nEntries) final;
118 RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final;
119 void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
120 void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
121 void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
122 void WriteNTupleAnchor();
123
124public:
126 virtual ~RPageSinkDaos();
127
128 RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final;
129 void ReleasePage(RPage &page) final;
130};
131
132
133// clang-format off
134/**
135\class ROOT::Experimental::Detail::RPageAllocatorDaos
136\ingroup NTuple
137\brief Manages pages read from a DAOS container
138*/
139// clang-format on
141public:
142 static RPage NewPage(ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements);
143 static void DeletePage(const RPage& page);
144};
145
146
147// clang-format off
148/**
149\class ROOT::Experimental::Detail::RPageSourceDaos
150\ingroup NTuple
151\brief Storage provider that reads ntuple pages from a DAOS container
152*/
153// clang-format on
155private:
156 /// Summarizes cluster-level information that are necessary to populate a certain page.
157 /// Used by PopulatePageFromCluster().
160 /// Location of the page on disk
162 /// The first element number of the page's column in the given cluster
163 std::uint64_t fColumnOffset = 0;
164 };
165
166 /// Populated pages might be shared; the memory buffer is managed by the RPageAllocatorDaos
167 std::unique_ptr<RPageAllocatorDaos> fPageAllocator;
168 // TODO: the page pool should probably be handled by the base class.
169 /// The page pool might, at some point, be used by multiple page sources
170 std::shared_ptr<RPagePool> fPagePool;
171 /// The last cluster from which a page got populated. Points into fClusterPool->fPool
173 /// A container that stores object data (header/footer, pages, etc.)
174 std::unique_ptr<RDaosContainer> fDaosContainer;
175 /// A URI to a DAOS pool of the form 'daos://pool-label/container-label'
176 std::string fURI;
177 /// The cluster pool asynchronously preloads the next few clusters
178 std::unique_ptr<RClusterPool> fClusterPool;
179
180 RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo,
181 ClusterSize_t::ValueType idxInCluster);
182
183protected:
185 void UnzipClusterImpl(RCluster *cluster) final;
186
187public:
188 RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options);
189 /// The cloned page source creates a new connection to the pool/container.
190 /// The meta-data (header and footer) is reread and parsed by the clone.
191 std::unique_ptr<RPageSource> Clone() const final;
192 virtual ~RPageSourceDaos();
193
194 RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final;
195 RPage PopulatePage(ColumnHandle_t columnHandle, const RClusterIndex &clusterIndex) final;
196 void ReleasePage(RPage &page) final;
197
198 void LoadSealedPage(DescriptorId_t columnId, const RClusterIndex &clusterIndex,
199 RSealedPage &sealedPage) final;
200
201 std::vector<std::unique_ptr<RCluster>> LoadClusters(std::span<RCluster::RKey> clusterKeys) final;
202
203 /// Return the object class used for user data OIDs in this ntuple.
204 std::string GetObjectClass() const;
205};
206
207} // namespace Detail
208
209} // namespace Experimental
210} // namespace ROOT
211
212#endif
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
An in-memory subset of the packed and compressed pages of a cluster.
Definition: RCluster.hxx:154
Manages pages read from a DAOS container.
static RPage NewPage(ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements)
Storage provider that writes ntuple pages to into a DAOS container.
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options)
RNTupleLocator CommitSealedPageImpl(DescriptorId_t columnId, const RPageStorage::RSealedPage &sealedPage) final
std::uint64_t fNBytesCurrentCluster
Tracks the number of bytes committed to the current cluster.
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
std::uint64_t CommitClusterImpl(NTupleSize_t nEntries) final
Returns the number of bytes written to storage (excluding metadata)
void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
std::atomic< std::uint64_t > fClusterGroupId
Cluster group counter for the next committed cluster pagelist; incremented in CommitClusterGroupImpl(...
void CreateImpl(const RNTupleModel &model, unsigned char *serializedHeader, std::uint32_t length) final
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges) final
Vector commit of preprocessed pages.
RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final
Get a new, empty page for the given column that can be filled with up to nElements.
std::atomic< std::uint64_t > fPageId
Page identifier for the next committed page; it is automatically incremented in CommitSealedPageImpl(...
std::unique_ptr< RDaosContainer > fDaosContainer
Underlying DAOS container.
std::unique_ptr< RPageAllocatorHeap > fPageAllocator
Abstract interface to write data into an ntuple.
Storage provider that reads ntuple pages from a DAOS container.
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
void LoadSealedPage(DescriptorId_t columnId, const RClusterIndex &clusterIndex, RSealedPage &sealedPage) final
Read the packed and compressed bytes of a page into the memory buffer provided by selaedPage.
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
void UnzipClusterImpl(RCluster *cluster) final
std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys) final
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final
Allocates and fills a page that contains the index-th element.
RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster)
RCluster * fCurrentCluster
The last cluster from which a page got populated. Points into fClusterPool->fPool.
std::string GetObjectClass() const
Return the object class used for user data OIDs in this ntuple.
std::shared_ptr< RPagePool > fPagePool
The page pool might, at some point, be used by multiple page sources.
std::unique_ptr< RPageAllocatorDaos > fPageAllocator
Populated pages might be shared; the memory buffer is managed by the RPageAllocatorDaos.
std::unique_ptr< RClusterPool > fClusterPool
The cluster pool asynchronously preloads the next few clusters.
std::unique_ptr< RPageSource > Clone() const final
The cloned page source creates a new connection to the pool/container.
Abstract interface to read data from an ntuple.
A page is a slice of a column that is mapped into memory.
Definition: RPage.hxx:41
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Definition: RNTupleUtil.hxx:87
The on-storage meta-data of an ntuple.
The RNTupleModel encapulates the schema of an ntuple.
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition: RError.hxx:195
basic_string_view< char > string_view
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
Definition: RNTupleUtil.hxx:47
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
Definition: RNTupleUtil.hxx:83
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
Definition: RNTupleUtil.hxx:79
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint32_t fVersion
Allows for evolving the struct in future versions.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
std::uint32_t Serialize(void *buffer) const
bool operator==(const RDaosNTupleAnchor &other) const
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
Summarizes cluster-level information that are necessary to populate a certain page.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
std::uint64_t fColumnOffset
The first element number of the page's column in the given cluster.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
Generic information about the physical location of data.