Logo ROOT  
Reference Guide
RPageStorageDaos.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorageDaos.hxx
2/// \ingroup NTuple ROOT7
3/// \author Javier Lopez-Gomez <j.lopez@cern.ch>
4/// \date 2020-11-03
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RPageStorageDaos
17#define ROOT7_RPageStorageDaos
18
19#include <ROOT/RPageStorage.hxx>
20#include <ROOT/RNTupleZip.hxx>
21#include <ROOT/RStringView.hxx>
22
23#include <array>
24#include <atomic>
25#include <cstdio>
26#include <memory>
27#include <string>
28
29namespace ROOT {
30
31namespace Experimental {
32namespace Detail {
33
34class RCluster;
35class RClusterPool;
36class RPageAllocatorHeap;
37class RPagePool;
38class RDaosPool;
39class RDaosContainer;
40
41
42// clang-format off
43/**
44\class ROOT::Experimental::Detail::RDaosNTupleAnchor
45\ingroup NTuple
46\brief Entry point for an RNTuple in a DAOS container. It encodes essential
47information to read the ntuple; currently, it contains (un)compressed size of
48the header/footer blobs and the object class for user data OIDs.
49The length of a serialized anchor cannot be greater than the value returned by the `GetSize` function.
50*/
51// clang-format on
53 /// Allows for evolving the struct in future versions
54 std::uint32_t fVersion = 0;
55 /// The size of the compressed ntuple header
56 std::uint32_t fNBytesHeader = 0;
57 /// The size of the uncompressed ntuple header
58 std::uint32_t fLenHeader = 0;
59 /// The size of the compressed ntuple footer
60 std::uint32_t fNBytesFooter = 0;
61 /// The size of the uncompressed ntuple footer
62 std::uint32_t fLenFooter = 0;
63 /// The object class for user data OIDs, e.g. `SX`
64 std::string fObjClass{};
65
66 bool operator ==(const RDaosNTupleAnchor &other) const {
67 return fVersion == other.fVersion &&
69 fLenHeader == other.fLenHeader &&
71 fLenFooter == other.fLenFooter &&
72 fObjClass == other.fObjClass;
73 }
74
75 std::uint32_t Serialize(void *buffer) const;
76 std::uint32_t Deserialize(const void *buffer);
77
78 static std::uint32_t GetSize();
79};
80
81// clang-format off
82/**
83\class ROOT::Experimental::Detail::RPageSinkDaos
84\ingroup NTuple
85\brief Storage provider that writes ntuple pages to into a DAOS container
86
87Currently, an object is allocated for each page + 3 additional objects (anchor/header/footer).
88*/
89// clang-format on
90class RPageSinkDaos : public RPageSink {
91private:
92 std::unique_ptr<RPageAllocatorHeap> fPageAllocator;
93
94 /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
95 /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
96 /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
97 /// after (which calls `daos_pool_disconect()`).
98 std::unique_ptr<RDaosContainer> fDaosContainer;
99 /// OID for the next committed page; it is automatically incremented in `CommitSealedPageImpl()`
100 std::atomic<std::uint64_t> fOid{0};
101 /// \brief A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'
102 std::string fURI;
103 /// Tracks the number of bytes committed to the current cluster
104 std::uint64_t fNBytesCurrentCluster{0};
105
107
108protected:
109 void CreateImpl(const RNTupleModel &model) final;
110 RClusterDescriptor::RLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final;
112 const RPageStorage::RSealedPage &sealedPage) final;
113 std::uint64_t CommitClusterImpl(NTupleSize_t nEntries) final;
114 void CommitDatasetImpl() final;
115 void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
116 void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
117 void WriteNTupleAnchor();
118
119public:
120 RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options);
121 virtual ~RPageSinkDaos();
122
123 RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final;
124 void ReleasePage(RPage &page) final;
125};
126
127
128// clang-format off
129/**
130\class ROOT::Experimental::Detail::RPageAllocatorDaos
131\ingroup NTuple
132\brief Manages pages read from a DAOS container
133*/
134// clang-format on
136public:
137 static RPage NewPage(ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements);
138 static void DeletePage(const RPage& page);
139};
140
141
142// clang-format off
143/**
144\class ROOT::Experimental::Detail::RPageSourceDaos
145\ingroup NTuple
146\brief Storage provider that reads ntuple pages from a DAOS container
147*/
148// clang-format on
150private:
151 /// Populated pages might be shared; the memory buffer is managed by the RPageAllocatorDaos
152 std::unique_ptr<RPageAllocatorDaos> fPageAllocator;
153 // TODO: the page pool should probably be handled by the base class.
154 /// The page pool might, at some point, be used by multiple page sources
155 std::shared_ptr<RPagePool> fPagePool;
156 /// The last cluster from which a page got populated. Points into fClusterPool->fPool
157 RCluster *fCurrentCluster = nullptr;
158 /// A container that stores object data (header/footer, pages, etc.)
159 std::unique_ptr<RDaosContainer> fDaosContainer;
160 /// A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'
161 std::string fURI;
162 /// The cluster pool asynchronously preloads the next few clusters
163 std::unique_ptr<RClusterPool> fClusterPool;
164
165 RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterDescriptor &clusterDescriptor,
166 ClusterSize_t::ValueType idxInCluster);
167
168protected:
169 RNTupleDescriptor AttachImpl() final;
170 void UnzipClusterImpl(RCluster *cluster) final;
171
172public:
173 RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options);
174 /// The cloned page source creates a new connection to the pool/container.
175 /// The meta-data (header and footer) is reread and parsed by the clone.
176 std::unique_ptr<RPageSource> Clone() const final;
177 virtual ~RPageSourceDaos();
178
179 RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final;
180 RPage PopulatePage(ColumnHandle_t columnHandle, const RClusterIndex &clusterIndex) final;
181 void ReleasePage(RPage &page) final;
182
183 void LoadSealedPage(DescriptorId_t columnId, const RClusterIndex &clusterIndex,
184 RSealedPage &sealedPage) final;
185
186 std::vector<std::unique_ptr<RCluster>> LoadClusters(std::span<RCluster::RKey> clusterKeys) final;
187
188 /// Return the object class used for user data OIDs in this ntuple.
189 std::string GetObjectClass() const;
190};
191
192
193} // namespace Detail
194
195} // namespace Experimental
196} // namespace ROOT
197
198#endif
An in-memory subset of the packed and compressed pages of a cluster.
Definition: RCluster.hxx:154
Manages pages read from a DAOS container.
Storage provider that writes ntuple pages to into a DAOS container.
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
std::uint64_t fNBytesCurrentCluster
Tracks the number of bytes committed to the current cluster.
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'.
std::uint64_t CommitClusterImpl(NTupleSize_t nEntries) final
Returns the number of bytes written to storage (excluding metadata)
std::atomic< std::uint64_t > fOid
OID for the next committed page; it is automatically incremented in CommitSealedPageImpl()
void CreateImpl(const RNTupleModel &model) final
RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final
Get a new, empty page for the given column that can be filled with up to nElements.
std::unique_ptr< RDaosContainer > fDaosContainer
Underlying DAOS container.
std::unique_ptr< RPageAllocatorHeap > fPageAllocator
RClusterDescriptor::RLocator CommitSealedPageImpl(DescriptorId_t columnId, const RPageStorage::RSealedPage &sealedPage) final
RClusterDescriptor::RLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
Abstract interface to write data into an ntuple.
Storage provider that reads ntuple pages from a DAOS container.
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-uuid:svc_replicas/container-uuid'.
std::shared_ptr< RPagePool > fPagePool
The page pool might, at some point, be used by multiple page sources.
std::unique_ptr< RPageAllocatorDaos > fPageAllocator
Populated pages might be shared; the memory buffer is managed by the RPageAllocatorDaos.
std::unique_ptr< RClusterPool > fClusterPool
The cluster pool asynchronously preloads the next few clusters.
Abstract interface to read data from an ntuple.
A page is a slice of a column that is mapped into memory.
Definition: RPage.hxx:41
Meta-data for a set of ntuple clusters.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
The on-storage meta-data of an ntuple.
The RNTupleModel encapulates the schema of an ntuple.
Common user-tunable settings for reading ntuples.
Common user-tunable settings for storing ntuples.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
Definition: RNTupleUtil.hxx:77
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint32_t fVersion
Allows for evolving the struct in future versions.
std::uint32_t Deserialize(const void *buffer)
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
std::uint32_t Serialize(void *buffer) const
bool operator==(const RDaosNTupleAnchor &other) const
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
Generic information about the physical location of data.