Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorageDaos.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorageDaos.hxx
2/// \ingroup NTuple
3/// \author Javier Lopez-Gomez <j.lopez@cern.ch>
4/// \date 2020-11-03
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT_RPageStorageDaos
17#define ROOT_RPageStorageDaos
18
19#include <ROOT/RError.hxx>
20#include <ROOT/RPageStorage.hxx>
21#include <ROOT/RNTuple.hxx>
23#include <ROOT/RNTupleZip.hxx>
24#include <string_view>
25
26#include <array>
27#include <atomic>
28#include <cstdio>
29#include <memory>
30#include <string>
31#include <optional>
32
33namespace ROOT {
34
35namespace Internal {
36class RCluster;
37class RClusterPool;
38} // namespace Internal
39
40namespace Experimental {
41namespace Internal {
42using ntuple_index_t = std::uint32_t;
43class RDaosPool;
44class RDaosContainer;
47 // Indicates that the referenced page is "caged", i.e. it is stored in a larger blob that contains multiple pages.
48 kCagedPage = 0x01,
49};
50
51// clang-format off
52/**
53\class ROOT::Experimental::Internal::RDaosNTupleAnchor
54\ingroup NTuple
55\brief Entry point for an RNTuple in a DAOS container. It encodes essential
56information to read the ntuple; currently, it contains (un)compressed size of
57the header/footer blobs and the object class for user data OIDs.
58The length of a serialized anchor cannot be greater than the value returned by the `GetSize` function.
59*/
60// clang-format on
62 /// Allows for evolving the struct in future versions
63 std::uint64_t fVersionAnchor = 1;
64 /// Version of the binary format supported by the writer
69 /// The size of the compressed ntuple header
70 std::uint32_t fNBytesHeader = 0;
71 /// The size of the uncompressed ntuple header
72 std::uint32_t fLenHeader = 0;
73 /// The size of the compressed ntuple footer
74 std::uint32_t fNBytesFooter = 0;
75 /// The size of the uncompressed ntuple footer
76 std::uint32_t fLenFooter = 0;
77 /// The object class for user data OIDs, e.g. `SX`
78 std::string fObjClass{};
79
80 bool operator ==(const RDaosNTupleAnchor &other) const {
81 return fVersionAnchor == other.fVersionAnchor && fVersionEpoch == other.fVersionEpoch &&
82 fVersionMajor == other.fVersionMajor && fVersionMinor == other.fVersionMinor &&
83 fVersionPatch == other.fVersionPatch && fNBytesHeader == other.fNBytesHeader &&
84 fLenHeader == other.fLenHeader && fNBytesFooter == other.fNBytesFooter && fLenFooter == other.fLenFooter &&
85 fObjClass == other.fObjClass;
86 }
87
88 std::uint32_t Serialize(void *buffer) const;
89 RResult<std::uint32_t> Deserialize(const void *buffer, std::uint32_t bufSize);
90
91 static std::uint32_t GetSize();
92}; // struct RDaosNTupleAnchor
93
94// clang-format off
95/**
96\class ROOT::Experimental::Internal::RPageSinkDaos
97\ingroup NTuple
98\brief Storage provider that writes ntuple pages to into a DAOS container
99
100Currently, an object is allocated for ntuple metadata (anchor/header/footer).
101Objects can correspond to pages or clusters of pages depending on the RNTuple-DAOS mapping strategy.
102*/
103// clang-format on
105private:
106 /// \brief Underlying DAOS container. An internal `std::shared_ptr` keep the pool connection alive.
107 /// ISO C++ ensures the correct destruction order, i.e., `~RDaosContainer` is invoked first
108 /// (which calls `daos_cont_close()`; the destructor for the `std::shared_ptr<RDaosPool>` is invoked
109 /// after (which calls `daos_pool_disconect()`).
110 std::unique_ptr<RDaosContainer> fDaosContainer;
111 /// Page identifier for the next committed page; it is automatically incremented in `CommitSealedPageImpl()`
112 std::atomic<std::uint64_t> fPageId{0};
113 /// Cluster group counter for the next committed cluster pagelist; incremented in `CommitClusterGroupImpl()`
114 std::atomic<std::uint64_t> fClusterGroupId{0};
115 /// \brief A URI to a DAOS pool of the form 'daos://pool-label/container-label'
116 std::string fURI;
117 /// Tracks the number of bytes committed to the current cluster
118 std::uint64_t fNBytesCurrentCluster{0};
119
122 uint32_t fCageSizeLimit{};
123
124protected:
126 void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final;
130 std::vector<RNTupleLocator>
131 CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask) final;
132 std::uint64_t StageClusterImpl() final;
135 void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
136 void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
137 void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
138 void WriteNTupleAnchor();
139
140public:
141 RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const ROOT::RNTupleWriteOptions &options);
143}; // class RPageSinkDaos
144
145// clang-format off
146/**
147\class ROOT::Experimental::Internal::RPageSourceDaos
148\ingroup NTuple
149\brief Storage provider that reads ntuple pages from a DAOS container
150*/
151// clang-format on
153private:
155
156 /// The last cluster from which a page got loaded. Points into fClusterPool->fPool
157 ROOT::Internal::RCluster *fCurrentCluster = nullptr;
158 /// A container that stores object data (header/footer, pages, etc.)
159 std::unique_ptr<RDaosContainer> fDaosContainer;
160 /// A URI to a DAOS pool of the form 'daos://pool-label/container-label'
161 std::string fURI;
162 /// The cluster pool asynchronously preloads the next few clusters
163 std::unique_ptr<ROOT::Internal::RClusterPool> fClusterPool;
164
166
169
170protected:
173 /// The cloned page source creates a new connection to the pool/container.
174 std::unique_ptr<RPageSource> CloneImpl() const final;
175
176public:
177 RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const ROOT::RNTupleReadOptions &options);
179
180 void
182
183 std::vector<std::unique_ptr<ROOT::Internal::RCluster>>
184 LoadClusters(std::span<ROOT::Internal::RCluster::RKey> clusterKeys) final;
185
186 /// Return the object class used for user data OIDs in this ntuple.
187 std::string GetObjectClass() const;
188}; // class RPageSourceDaos
189
190} // namespace Internal
191
192} // namespace Experimental
193} // namespace ROOT
194
195#endif
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char mode
A RDaosContainer provides read/write access to objects in a given container.
Definition RDaos.hxx:157
A RDaosPool provides access to containers in a specific DAOS pool.
Definition RDaos.hxx:66
Storage provider that writes ntuple pages to into a DAOS container.
std::unique_ptr< RDaosContainer > fDaosContainer
Underlying DAOS container.
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const ROOT::Internal::RPage &page) final
std::uint64_t fNBytesCurrentCluster
Tracks the number of bytes committed to the current cluster.
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask) final
Vector commit of preprocessed pages.
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
std::uint64_t StageClusterImpl() final
Returns the number of bytes written to storage (excluding metadata)
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final
std::atomic< std::uint64_t > fPageId
Page identifier for the next committed page; it is automatically incremented in CommitSealedPageImpl(...
std::atomic< std::uint64_t > fClusterGroupId
Cluster group counter for the next committed cluster pagelist; incremented in CommitClusterGroupImpl(...
RNTupleLocator CommitSealedPageImpl(ROOT::DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
Storage provider that reads ntuple pages from a DAOS container.
std::unique_ptr< ROOT::Internal::RClusterPool > fClusterPool
The cluster pool asynchronously preloads the next few clusters.
std::string fURI
A URI to a DAOS pool of the form 'daos://pool-label/container-label'.
ROOT::Internal::RNTupleDescriptorBuilder fDescriptorBuilder
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:148
A helper class for piece-wise construction of an RNTupleDescriptor.
Uses standard C++ memory allocation for the column data pages.
Base class for a sink with a physical storage backend.
virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length)=0
Reference to a page stored in the page pool.
Abstract interface to read data from an ntuple.
RColumnHandle ColumnHandle_t
The column handle identifies a column with the current open page storage.
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:44
The on-storage metadata of an RNTuple.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Generic information about the physical location of data.
Common user-tunable settings for reading RNTuples.
Common user-tunable settings for storing RNTuples.
static constexpr std::uint16_t kVersionPatch
Definition RNTuple.hxx:81
static constexpr std::uint16_t kVersionMajor
Definition RNTuple.hxx:79
static constexpr std::uint16_t kVersionEpoch
Definition RNTuple.hxx:78
static constexpr std::uint16_t kVersionMinor
Definition RNTuple.hxx:80
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
std::uint64_t fVersionAnchor
Allows for evolving the struct in future versions.
bool operator==(const RDaosNTupleAnchor &other) const
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint16_t fVersionEpoch
Version of the binary format supported by the writer.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Summarizes cluster-level information that are necessary to load a certain page.
A sealed page contains the bytes of a page as written to storage (packed & compressed).