Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorageDaos.cxx
Go to the documentation of this file.
1/// \file RPageStorageDaos.cxx
2/// \ingroup NTuple ROOT7
3/// \author Javier Lopez-Gomez <j.lopez@cern.ch>
4/// \date 2020-11-03
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#include <ROOT/RCluster.hxx>
17#include <ROOT/RClusterPool.hxx>
18#include <ROOT/RLogger.hxx>
20#include <ROOT/RNTupleModel.hxx>
23#include <ROOT/RNTupleUtil.hxx>
24#include <ROOT/RNTupleZip.hxx>
25#include <ROOT/RPage.hxx>
27#include <ROOT/RPagePool.hxx>
28#include <ROOT/RDaos.hxx>
30
31#include <RVersion.h>
32#include <TError.h>
33
34#include <algorithm>
35#include <cstdio>
36#include <cstdlib>
37#include <limits>
38#include <utility>
39#include <regex>
40#include <cassert>
41
42namespace {
46
47/// \brief RNTuple page-DAOS mappings
48enum EDaosMapping { kOidPerCluster, kOidPerPage };
49
50struct RDaosKey {
51 daos_obj_id_t fOid;
52 DistributionKey_t fDkey;
53 AttributeKey_t fAkey;
54};
55
56/// \brief Pre-defined keys for object store. `kDistributionKeyDefault` is the distribution key for metadata and
57/// pagelist values; optionally it can be used for ntuple pages (if under the `kOidPerPage` mapping strategy).
58/// `kAttributeKeyDefault` is the attribute key for ntuple pages under `kOidPerPage`.
59/// `kAttributeKey{Anchor,Header,Footer}` are the respective attribute keys for anchor/header/footer metadata elements.
60static constexpr DistributionKey_t kDistributionKeyDefault = 0x5a3c69f0cafe4a11;
61static constexpr AttributeKey_t kAttributeKeyDefault = 0x4243544b53444229;
62static constexpr AttributeKey_t kAttributeKeyAnchor = 0x4243544b5344422a;
63static constexpr AttributeKey_t kAttributeKeyHeader = 0x4243544b5344422b;
64static constexpr AttributeKey_t kAttributeKeyFooter = 0x4243544b5344422c;
65
66/// \brief Pre-defined 64 LSb of the OIDs for ntuple metadata (holds anchor/header/footer) and clusters' pagelists.
67static constexpr decltype(daos_obj_id_t::lo) kOidLowMetadata = -1;
68static constexpr decltype(daos_obj_id_t::lo) kOidLowPageList = -2;
69
70static constexpr daos_oclass_id_t kCidMetadata = OC_SX;
71
72static constexpr EDaosMapping kDefaultDaosMapping = kOidPerCluster;
73
74template <EDaosMapping mapping>
75RDaosKey GetPageDaosKey(ROOT::Experimental::Internal::ntuple_index_t ntplId, long unsigned clusterId,
76 long unsigned columnId, long unsigned pageCount)
77{
78 if constexpr (mapping == kOidPerCluster) {
79 return RDaosKey{daos_obj_id_t{static_cast<decltype(daos_obj_id_t::lo)>(clusterId),
80 static_cast<decltype(daos_obj_id_t::hi)>(ntplId)},
81 static_cast<DistributionKey_t>(columnId), static_cast<AttributeKey_t>(pageCount)};
82 } else if constexpr (mapping == kOidPerPage) {
83 return RDaosKey{daos_obj_id_t{static_cast<decltype(daos_obj_id_t::lo)>(pageCount),
84 static_cast<decltype(daos_obj_id_t::hi)>(ntplId)},
85 kDistributionKeyDefault, kAttributeKeyDefault};
86 }
87}
88
89struct RDaosURI {
90 /// \brief Label of the DAOS pool
91 std::string fPoolLabel;
92 /// \brief Label of the container for this RNTuple
93 std::string fContainerLabel;
94};
95
96/**
97 \brief Parse a DAOS RNTuple URI of the form 'daos://pool_id/container_id'.
98*/
99RDaosURI ParseDaosURI(std::string_view uri)
100{
101 std::regex re("daos://([^/]+)/(.+)");
102 std::cmatch m;
103 if (!std::regex_match(uri.data(), m, re))
104 throw ROOT::Experimental::RException(R__FAIL("Invalid DAOS pool URI."));
105 return {m[1], m[2]};
106}
107
108/// \brief Unpacks a 64-bit RNTuple page locator address for object stores into a pair of 32-bit values:
109/// the attribute key under which the cage is stored and the offset within that cage to access the page.
110std::pair<uint32_t, uint32_t> DecodeDaosPagePosition(const ROOT::Experimental::RNTupleLocatorObject64 &address)
111{
112 auto position = static_cast<uint32_t>(address.fLocation & 0xFFFFFFFF);
113 auto offset = static_cast<uint32_t>(address.fLocation >> 32);
114 return {position, offset};
115}
116
117/// \brief Packs an attribute key together with an offset within its contents into a single 64-bit address.
118/// The offset is kept in the MSb half and defaults to zero, which is the case when caging is disabled.
119ROOT::Experimental::RNTupleLocatorObject64 EncodeDaosPagePosition(uint64_t position, uint64_t offset = 0)
120{
121 uint64_t address = (position & 0xFFFFFFFF) | (offset << 32);
123}
124
125/// \brief Helper structure concentrating the functionality required to locate an ntuple within a DAOS container.
126/// It includes a hashing function that converts the RNTuple's name into a 32-bit identifier; this value is used to
127/// index the subspace for the ntuple among all objects in the container. A zero-value hash value is reserved for
128/// storing any future metadata related to container-wide management; a zero-index ntuple is thus disallowed and
129/// remapped to "1". Once the index is computed, `InitNTupleDescriptorBuilder()` can be called to return a
130/// partially-filled builder with the ntuple's anchor, header and footer, lacking only pagelists. Upon that call,
131/// a copy of the anchor is stored in `fAnchor`.
132struct RDaosContainerNTupleLocator {
133 std::string fName{};
134 ntuple_index_t fIndex{};
135 std::optional<ROOT::Experimental::Internal::RDaosNTupleAnchor> fAnchor;
136 static const ntuple_index_t kReservedIndex = 0;
137
138 RDaosContainerNTupleLocator() = default;
139 explicit RDaosContainerNTupleLocator(const std::string &ntupleName) : fName(ntupleName), fIndex(Hash(ntupleName)){};
140
141 bool IsValid() { return fAnchor.has_value() && fAnchor->fNBytesHeader; }
142 [[nodiscard]] ntuple_index_t GetIndex() const { return fIndex; };
143 static ntuple_index_t Hash(const std::string &ntupleName)
144 {
145 // Convert string to numeric representation via `std::hash`.
146 uint64_t h = std::hash<std::string>{}(ntupleName);
147 // Fold the hash into 32-bit using `boost::hash_combine()` algorithm and magic number.
148 auto seed = static_cast<uint32_t>(h >> 32);
149 seed ^= static_cast<uint32_t>(h & 0xffffffff) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
150 auto hash = static_cast<ntuple_index_t>(seed);
151 return (hash == kReservedIndex) ? kReservedIndex + 1 : hash;
152 }
153
154 int InitNTupleDescriptorBuilder(ROOT::Experimental::Internal::RDaosContainer &cont,
156 {
157 std::unique_ptr<unsigned char[]> buffer, zipBuffer;
158 auto &anchor = fAnchor.emplace();
159 int err;
160
162 daos_obj_id_t oidMetadata{kOidLowMetadata, static_cast<decltype(daos_obj_id_t::hi)>(this->GetIndex())};
163
164 buffer = std::make_unique<unsigned char[]>(anchorSize);
165 if ((err = cont.ReadSingleAkey(buffer.get(), anchorSize, oidMetadata, kDistributionKeyDefault,
166 kAttributeKeyAnchor, kCidMetadata))) {
167 return err;
168 }
169
170 anchor.Deserialize(buffer.get(), anchorSize).Unwrap();
171 if (anchor.fVersionEpoch != ROOT::Experimental::RNTuple::kVersionEpoch) {
173 R__FAIL("unsupported RNTuple epoch version: " + std::to_string(anchor.fVersionEpoch)));
174 }
175 if (anchor.fVersionEpoch == 0) {
176 static std::once_flag once;
177 std::call_once(once, [&anchor]() {
179 << "Pre-release format version: RC " << anchor.fVersionMajor;
180 });
181 }
182
183 builder.SetOnDiskHeaderSize(anchor.fNBytesHeader);
184 buffer = std::make_unique<unsigned char[]>(anchor.fLenHeader);
185 zipBuffer = std::make_unique<unsigned char[]>(anchor.fNBytesHeader);
186 if ((err = cont.ReadSingleAkey(zipBuffer.get(), anchor.fNBytesHeader, oidMetadata, kDistributionKeyDefault,
187 kAttributeKeyHeader, kCidMetadata)))
188 return err;
189 ROOT::Experimental::Internal::RNTupleDecompressor::Unzip(zipBuffer.get(), anchor.fNBytesHeader, anchor.fLenHeader,
190 buffer.get());
191 ROOT::Experimental::Internal::RNTupleSerializer::DeserializeHeader(buffer.get(), anchor.fLenHeader, builder);
192
193 builder.AddToOnDiskFooterSize(anchor.fNBytesFooter);
194 buffer = std::make_unique<unsigned char[]>(anchor.fLenFooter);
195 zipBuffer = std::make_unique<unsigned char[]>(anchor.fNBytesFooter);
196 if ((err = cont.ReadSingleAkey(zipBuffer.get(), anchor.fNBytesFooter, oidMetadata, kDistributionKeyDefault,
197 kAttributeKeyFooter, kCidMetadata)))
198 return err;
199 ROOT::Experimental::Internal::RNTupleDecompressor::Unzip(zipBuffer.get(), anchor.fNBytesFooter, anchor.fLenFooter,
200 buffer.get());
201 ROOT::Experimental::Internal::RNTupleSerializer::DeserializeFooter(buffer.get(), anchor.fLenFooter, builder);
202
203 return 0;
204 }
205
206 static std::pair<RDaosContainerNTupleLocator, ROOT::Experimental::Internal::RNTupleDescriptorBuilder>
207 LocateNTuple(ROOT::Experimental::Internal::RDaosContainer &cont, const std::string &ntupleName)
208 {
209 auto result = std::make_pair(RDaosContainerNTupleLocator(ntupleName),
211
212 auto &loc = result.first;
213 auto &builder = result.second;
214
215 if (int err = loc.InitNTupleDescriptorBuilder(cont, builder); !err) {
216 if (ntupleName.empty() || ntupleName != builder.GetDescriptor().GetName()) {
217 // Hash already taken by a differently-named ntuple.
219 R__FAIL("LocateNTuple: ntuple name '" + ntupleName + "' unavailable in this container."));
220 }
221 }
222 return result;
223 }
224};
225
226} // anonymous namespace
227
228////////////////////////////////////////////////////////////////////////////////
229
231{
233 if (buffer != nullptr) {
234 auto bytes = reinterpret_cast<unsigned char *>(buffer);
245 }
246 return RNTupleSerializer::SerializeString(fObjClass, nullptr) + 32;
247}
248
250ROOT::Experimental::Internal::RDaosNTupleAnchor::Deserialize(const void *buffer, std::uint32_t bufSize)
251{
252 if (bufSize < 32)
253 return R__FAIL("DAOS anchor too short");
254
256 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
258 if (fVersionAnchor != RDaosNTupleAnchor().fVersionAnchor) {
259 return R__FAIL("unsupported DAOS anchor version: " + std::to_string(fVersionAnchor));
260 }
261
270 auto result = RNTupleSerializer::DeserializeString(bytes, bufSize - 32, fObjClass);
271 if (!result)
272 return R__FORWARD_ERROR(result);
273 return result.Unwrap() + 32;
274}
275
277{
279}
280
281////////////////////////////////////////////////////////////////////////////////
282
283ROOT::Experimental::Internal::RPageSinkDaos::RPageSinkDaos(std::string_view ntupleName, std::string_view uri,
284 const RNTupleWriteOptions &options)
285 : RPagePersistentSink(ntupleName, options), fURI(uri)
286{
287 static std::once_flag once;
288 std::call_once(once, []() {
289 R__LOG_WARNING(NTupleLog()) << "The DAOS backend is experimental and still under development. "
290 << "Do not store real data with this version of RNTuple!";
291 });
292 fCompressor = std::make_unique<RNTupleCompressor>();
293 EnableDefaultMetrics("RPageSinkDaos");
294}
295
297
298void ROOT::Experimental::Internal::RPageSinkDaos::InitImpl(unsigned char *serializedHeader, std::uint32_t length)
299{
300 auto opts = dynamic_cast<RNTupleWriteOptionsDaos *>(fOptions.get());
301 fNTupleAnchor.fObjClass = opts ? opts->GetObjectClass() : RNTupleWriteOptionsDaos().GetObjectClass();
302 auto oclass = RDaosObject::ObjClassId(fNTupleAnchor.fObjClass);
303 if (oclass.IsUnknown())
304 throw ROOT::Experimental::RException(R__FAIL("Unknown object class " + fNTupleAnchor.fObjClass));
305
306 size_t cageSz = opts ? opts->GetMaxCageSize() : RNTupleWriteOptionsDaos().GetMaxCageSize();
307 size_t pageSz = opts ? opts->GetMaxUnzippedPageSize() : RNTupleWriteOptionsDaos().GetMaxUnzippedPageSize();
308 fCageSizeLimit = std::max(cageSz, pageSz);
309
310 auto args = ParseDaosURI(fURI);
311 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
312
313 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel, /*create =*/true);
314 fDaosContainer->SetDefaultObjectClass(oclass);
315
316 auto [locator, _] = RDaosContainerNTupleLocator::LocateNTuple(*fDaosContainer, fNTupleName);
317 fNTupleIndex = locator.GetIndex();
318
319 auto zipBuffer = std::make_unique<unsigned char[]>(length);
320 auto szZipHeader = fCompressor->Zip(serializedHeader, length, GetWriteOptions().GetCompression(),
321 RNTupleCompressor::MakeMemCopyWriter(zipBuffer.get()));
322 WriteNTupleHeader(zipBuffer.get(), szZipHeader, length);
323}
324
327{
328 auto element = columnHandle.fColumn->GetElement();
329 RPageStorage::RSealedPage sealedPage;
330 {
331 Detail::RNTupleAtomicTimer timer(fCounters->fTimeWallZip, fCounters->fTimeCpuZip);
332 sealedPage = SealPage(page, *element);
333 }
334
335 fCounters->fSzZip.Add(page.GetNBytes());
336 return CommitSealedPageImpl(columnHandle.fPhysicalId, sealedPage);
337}
338
341 const RPageStorage::RSealedPage &sealedPage)
342{
343 auto offsetData = fPageId.fetch_add(1);
344 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNActiveClusters();
345
346 {
347 Detail::RNTupleAtomicTimer timer(fCounters->fTimeWallWrite, fCounters->fTimeCpuWrite);
348 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, physicalColumnId, offsetData);
349 fDaosContainer->WriteSingleAkey(sealedPage.GetBuffer(), sealedPage.GetBufferSize(), daosKey.fOid, daosKey.fDkey,
350 daosKey.fAkey);
351 }
352
354 result.fPosition = EncodeDaosPagePosition(offsetData);
355 result.fBytesOnStorage = sealedPage.GetDataSize();
357 fCounters->fNPageCommitted.Inc();
358 fCounters->fSzWritePayload.Add(sealedPage.GetBufferSize());
359 fNBytesCurrentCluster += sealedPage.GetBufferSize();
360 return result;
361}
362
363std::vector<ROOT::Experimental::RNTupleLocator>
364ROOT::Experimental::Internal::RPageSinkDaos::CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges,
365 const std::vector<bool> &mask)
366{
368 std::vector<ROOT::Experimental::RNTupleLocator> locators;
369 auto nPages = mask.size();
370 locators.reserve(nPages);
371
372 const uint32_t maxCageSz = fCageSizeLimit;
373 const bool useCaging = fCageSizeLimit > 0;
374 const std::uint8_t locatorFlags = useCaging ? EDaosLocatorFlags::kCagedPage : 0;
375
376 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNActiveClusters();
377 int64_t payloadSz = 0;
378 std::size_t positionOffset;
379 uint32_t positionIndex;
380
381 /// Aggregate batch of requests by object ID and distribution key, determined by the ntuple-DAOS mapping
382 for (auto &range : ranges) {
383 positionOffset = 0;
384 /// Under caging, the atomic page counter is fetch-incremented for every column range to get the position of its
385 /// first cage and indicate the next one, also ensuring subsequent pages of different columns do not end up caged
386 /// together. This increment is not necessary in the absence of caging, as each page is trivially caged.
387 positionIndex = useCaging ? fPageId.fetch_add(1) : fPageId.load();
388
389 for (auto sealedPageIt = range.fFirst; sealedPageIt != range.fLast; ++sealedPageIt) {
390 const RPageStorage::RSealedPage &s = *sealedPageIt;
391
392 if (positionOffset + s.GetBufferSize() > maxCageSz) {
393 positionOffset = 0;
394 positionIndex = fPageId.fetch_add(1);
395 }
396
397 d_iov_t pageIov;
398 d_iov_set(&pageIov, const_cast<void *>(s.GetBuffer()), s.GetBufferSize());
399
400 RDaosKey daosKey =
401 GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, range.fPhysicalColumnId, positionIndex);
402 auto odPair = RDaosContainer::ROidDkeyPair{daosKey.fOid, daosKey.fDkey};
403 auto [it, ret] = writeRequests.emplace(odPair, RDaosContainer::RWOperation(odPair));
404 it->second.Insert(daosKey.fAkey, pageIov);
405
406 RNTupleLocator locator;
407 locator.fPosition = EncodeDaosPagePosition(positionIndex, positionOffset);
408 locator.fBytesOnStorage = s.GetDataSize();
410 locator.fReserved = locatorFlags;
411 locators.push_back(locator);
412
413 positionOffset += s.GetBufferSize();
414 payloadSz += s.GetBufferSize();
415 }
416 }
417 fNBytesCurrentCluster += payloadSz;
418
419 {
420 Detail::RNTupleAtomicTimer timer(fCounters->fTimeWallWrite, fCounters->fTimeCpuWrite);
421 if (int err = fDaosContainer->WriteV(writeRequests))
422 throw ROOT::Experimental::RException(R__FAIL("WriteV: error" + std::string(d_errstr(err))));
423 }
424
425 fCounters->fNPageCommitted.Add(nPages);
426 fCounters->fSzWritePayload.Add(payloadSz);
427
428 return locators;
429}
430
432{
433 return std::exchange(fNBytesCurrentCluster, 0);
434}
435
438 std::uint32_t length)
439{
440 auto bufPageListZip = std::make_unique<unsigned char[]>(length);
441 auto szPageListZip = fCompressor->Zip(serializedPageList, length, GetWriteOptions().GetCompression(),
442 RNTupleCompressor::MakeMemCopyWriter(bufPageListZip.get()));
443
444 auto offsetData = fClusterGroupId.fetch_add(1);
445 fDaosContainer->WriteSingleAkey(
446 bufPageListZip.get(), szPageListZip,
447 daos_obj_id_t{kOidLowPageList, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)}, kDistributionKeyDefault,
448 offsetData, kCidMetadata);
450 result.fPosition = RNTupleLocatorObject64{offsetData};
451 result.fBytesOnStorage = szPageListZip;
453 fCounters->fSzWritePayload.Add(static_cast<int64_t>(szPageListZip));
454 return result;
455}
456
458 std::uint32_t length)
459{
460 auto bufFooterZip = std::make_unique<unsigned char[]>(length);
461 auto szFooterZip = fCompressor->Zip(serializedFooter, length, GetWriteOptions().GetCompression(),
462 RNTupleCompressor::MakeMemCopyWriter(bufFooterZip.get()));
463 WriteNTupleFooter(bufFooterZip.get(), szFooterZip, length);
464 WriteNTupleAnchor();
465}
466
467void ROOT::Experimental::Internal::RPageSinkDaos::WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
468{
469 fDaosContainer->WriteSingleAkey(
470 data, nbytes, daos_obj_id_t{kOidLowMetadata, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)},
471 kDistributionKeyDefault, kAttributeKeyHeader, kCidMetadata);
472 fNTupleAnchor.fLenHeader = lenHeader;
473 fNTupleAnchor.fNBytesHeader = nbytes;
474}
475
476void ROOT::Experimental::Internal::RPageSinkDaos::WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
477{
478 fDaosContainer->WriteSingleAkey(
479 data, nbytes, daos_obj_id_t{kOidLowMetadata, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)},
480 kDistributionKeyDefault, kAttributeKeyFooter, kCidMetadata);
481 fNTupleAnchor.fLenFooter = lenFooter;
482 fNTupleAnchor.fNBytesFooter = nbytes;
483}
484
486{
487 const auto ntplSize = RDaosNTupleAnchor::GetSize();
488 auto buffer = std::make_unique<unsigned char[]>(ntplSize);
489 fNTupleAnchor.Serialize(buffer.get());
490 fDaosContainer->WriteSingleAkey(
491 buffer.get(), ntplSize, daos_obj_id_t{kOidLowMetadata, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)},
492 kDistributionKeyDefault, kAttributeKeyAnchor, kCidMetadata);
493}
494
495////////////////////////////////////////////////////////////////////////////////
496
497ROOT::Experimental::Internal::RPageSourceDaos::RPageSourceDaos(std::string_view ntupleName, std::string_view uri,
498 const RNTupleReadOptions &options)
499 : RPageSource(ntupleName, options),
500 fURI(uri),
501 fClusterPool(std::make_unique<RClusterPool>(*this, options.GetClusterBunchSize()))
502{
503 EnableDefaultMetrics("RPageSourceDaos");
504
505 auto args = ParseDaosURI(uri);
506 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
507 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel);
508}
509
511
513{
515 std::unique_ptr<unsigned char[]> buffer, zipBuffer;
516
517 auto [locator, descBuilder] = RDaosContainerNTupleLocator::LocateNTuple(*fDaosContainer, fNTupleName);
518 if (!locator.IsValid())
520 R__FAIL("Attach: requested ntuple '" + fNTupleName + "' is not present in DAOS container."));
521
522 auto oclass = RDaosObject::ObjClassId(locator.fAnchor->fObjClass);
523 if (oclass.IsUnknown())
524 throw ROOT::Experimental::RException(R__FAIL("Attach: unknown object class " + locator.fAnchor->fObjClass));
525
526 fDaosContainer->SetDefaultObjectClass(oclass);
527 fNTupleIndex = locator.GetIndex();
528 daos_obj_id_t oidPageList{kOidLowPageList, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)};
529
530 auto desc = descBuilder.MoveDescriptor();
531
532 for (const auto &cgDesc : desc.GetClusterGroupIterable()) {
533 buffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLength());
534 zipBuffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLocator().fBytesOnStorage);
535 fDaosContainer->ReadSingleAkey(
536 zipBuffer.get(), cgDesc.GetPageListLocator().fBytesOnStorage, oidPageList, kDistributionKeyDefault,
537 cgDesc.GetPageListLocator().GetPosition<RNTupleLocatorObject64>().fLocation, kCidMetadata);
538 RNTupleDecompressor::Unzip(zipBuffer.get(), cgDesc.GetPageListLocator().fBytesOnStorage,
539 cgDesc.GetPageListLength(), buffer.get());
540
541 RNTupleSerializer::DeserializePageList(buffer.get(), cgDesc.GetPageListLength(), cgDesc.GetId(), desc);
542 }
543
544 return desc;
545}
546
548{
549 return fDaosContainer->GetDefaultObjectClass().ToString();
550}
551
553 RClusterIndex clusterIndex, RSealedPage &sealedPage)
554{
555 const auto clusterId = clusterIndex.GetClusterId();
556
558 {
559 auto descriptorGuard = GetSharedDescriptorGuard();
560 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterId);
561 pageInfo = clusterDescriptor.GetPageRange(physicalColumnId).Find(clusterIndex.GetIndex());
562 }
563
564 sealedPage.SetBufferSize(pageInfo.fLocator.fBytesOnStorage + pageInfo.fHasChecksum * kNBytesPageChecksum);
565 sealedPage.SetNElements(pageInfo.fNElements);
566 sealedPage.SetHasChecksum(pageInfo.fHasChecksum);
567 if (!sealedPage.GetBuffer())
568 return;
569
571 assert(!pageInfo.fHasChecksum);
572 memcpy(const_cast<void *>(sealedPage.GetBuffer()), RPage::GetPageZeroBuffer(), sealedPage.GetBufferSize());
573 return;
574 }
575
577 // Suboptimal but hard to do differently: we load the full cage up to and including the requested page.
578 // In practice, individual LoadSealedPage calls are rare and usually full clusters are buffered.
579 // The support for extracting individual pages from a cage makes testing easier, however.
580 const auto [position, offset] = DecodeDaosPagePosition(pageInfo.fLocator.GetPosition<RNTupleLocatorObject64>());
581 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, physicalColumnId, position);
582 const auto bufSize = offset + sealedPage.GetBufferSize();
583 auto cageHeadBuffer = std::make_unique<unsigned char[]>(bufSize);
584 fDaosContainer->ReadSingleAkey(cageHeadBuffer.get(), bufSize, daosKey.fOid, daosKey.fDkey, daosKey.fAkey);
585 memcpy(const_cast<void *>(sealedPage.GetBuffer()), cageHeadBuffer.get() + offset, sealedPage.GetBufferSize());
586 } else {
587 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(
588 fNTupleIndex, clusterId, physicalColumnId, pageInfo.fLocator.GetPosition<RNTupleLocatorObject64>().fLocation);
589 fDaosContainer->ReadSingleAkey(const_cast<void *>(sealedPage.GetBuffer()), sealedPage.GetBufferSize(),
590 daosKey.fOid, daosKey.fDkey, daosKey.fAkey);
591 }
592
594}
595
598 const RClusterInfo &clusterInfo,
599 ClusterSize_t::ValueType idxInCluster)
600{
601 const auto columnId = columnHandle.fPhysicalId;
602 const auto clusterId = clusterInfo.fClusterId;
603 const auto &pageInfo = clusterInfo.fPageInfo;
604
605 const auto element = columnHandle.fColumn->GetElement();
606 const auto elementSize = element->GetSize();
607
608 if (pageInfo.fLocator.fType == RNTupleLocator::kTypePageZero) {
609 auto pageZero = RPage::MakePageZero(columnId, elementSize);
610 pageZero.GrowUnchecked(pageInfo.fNElements);
611 pageZero.SetWindow(clusterInfo.fColumnOffset + pageInfo.fFirstInPage,
612 RPage::RClusterInfo(clusterId, clusterInfo.fColumnOffset));
613 return fPagePool.RegisterPage(std::move(pageZero));
614 }
615
616 RSealedPage sealedPage;
617 sealedPage.SetNElements(pageInfo.fNElements);
618 sealedPage.SetHasChecksum(pageInfo.fHasChecksum);
619 sealedPage.SetBufferSize(pageInfo.fLocator.fBytesOnStorage + pageInfo.fHasChecksum * kNBytesPageChecksum);
620 std::unique_ptr<unsigned char[]> directReadBuffer; // only used if cluster pool is turned off
621
622 if (fOptions.GetClusterCache() == RNTupleReadOptions::EClusterCache::kOff) {
623 if (pageInfo.fLocator.fReserved & EDaosLocatorFlags::kCagedPage) {
625 R__FAIL("accessing caged pages is only supported in conjunction with cluster cache"));
626 }
627
628 directReadBuffer = std::unique_ptr<unsigned char[]>(new unsigned char[sealedPage.GetBufferSize()]);
629 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(
630 fNTupleIndex, clusterId, columnId, pageInfo.fLocator.GetPosition<RNTupleLocatorObject64>().fLocation);
631 fDaosContainer->ReadSingleAkey(directReadBuffer.get(), sealedPage.GetBufferSize(), daosKey.fOid, daosKey.fDkey,
632 daosKey.fAkey);
633 fCounters->fNPageRead.Inc();
634 fCounters->fNRead.Inc();
635 fCounters->fSzReadPayload.Add(sealedPage.GetBufferSize());
636 sealedPage.SetBuffer(directReadBuffer.get());
637 } else {
638 if (!fCurrentCluster || (fCurrentCluster->GetId() != clusterId) || !fCurrentCluster->ContainsColumn(columnId))
639 fCurrentCluster = fClusterPool->GetCluster(clusterId, fActivePhysicalColumns.ToColumnSet());
640 R__ASSERT(fCurrentCluster->ContainsColumn(columnId));
641
642 auto cachedPageRef = fPagePool.GetPage(columnId, RClusterIndex(clusterId, idxInCluster));
643 if (!cachedPageRef.Get().IsNull())
644 return cachedPageRef;
645
646 ROnDiskPage::Key key(columnId, pageInfo.fPageNo);
647 auto onDiskPage = fCurrentCluster->GetOnDiskPage(key);
648 R__ASSERT(onDiskPage && (sealedPage.GetBufferSize() == onDiskPage->GetSize()));
649 sealedPage.SetBuffer(onDiskPage->GetAddress());
650 }
651
652 RPage newPage;
653 {
654 Detail::RNTupleAtomicTimer timer(fCounters->fTimeWallUnzip, fCounters->fTimeCpuUnzip);
655 newPage = UnsealPage(sealedPage, *element, columnId).Unwrap();
656 fCounters->fSzUnzip.Add(elementSize * pageInfo.fNElements);
657 }
658
659 newPage.SetWindow(clusterInfo.fColumnOffset + pageInfo.fFirstInPage,
660 RPage::RClusterInfo(clusterId, clusterInfo.fColumnOffset));
661 fCounters->fNPageUnsealed.Inc();
662 return fPagePool.RegisterPage(std::move(newPage));
663}
664
665std::unique_ptr<ROOT::Experimental::Internal::RPageSource>
667{
668 auto clone = new RPageSourceDaos(fNTupleName, fURI, fOptions);
669 return std::unique_ptr<RPageSourceDaos>(clone);
670}
671
672std::vector<std::unique_ptr<ROOT::Experimental::Internal::RCluster>>
674{
675 struct RDaosSealedPageLocator {
676 DescriptorId_t fClusterId = 0;
677 DescriptorId_t fColumnId = 0;
678 NTupleSize_t fPageNo = 0;
679 std::uint64_t fPosition = 0;
680 std::uint64_t fCageOffset = 0;
681 std::uint64_t fDataSize = 0; // page payload
682 std::uint64_t fBufferSize = 0; // page payload + checksum (if available)
683 };
684
685 // Prepares read requests for a single cluster; `readRequests` is modified by this function. Requests are coalesced
686 // by OID and distribution key.
687 // TODO(jalopezg): this may be a private member function; that, however, requires additional changes given that
688 // `RDaosContainer::MultiObjectRWOperation_t` cannot be forward-declared
689 auto fnPrepareSingleCluster = [&](const RCluster::RKey &clusterKey,
691 auto clusterId = clusterKey.fClusterId;
692 // Group page locators by their position in the object store; with caging enabled, this facilitates the
693 // processing of cages' requests together into a single IOV to be loaded.
694 std::unordered_map<std::uint32_t, std::vector<RDaosSealedPageLocator>> onDiskPages;
695
696 unsigned clusterBufSz = 0, nPages = 0;
697 auto pageZeroMap = std::make_unique<ROnDiskPageMap>();
698 PrepareLoadCluster(
699 clusterKey, *pageZeroMap,
700 [&](DescriptorId_t physicalColumnId, NTupleSize_t pageNo,
702 const auto &pageLocator = pageInfo.fLocator;
703 uint32_t position, offset;
704 std::tie(position, offset) = DecodeDaosPagePosition(pageLocator.GetPosition<RNTupleLocatorObject64>());
705 auto [itLoc, _] = onDiskPages.emplace(position, std::vector<RDaosSealedPageLocator>());
706 auto pageBufferSize = pageLocator.fBytesOnStorage + pageInfo.fHasChecksum * kNBytesPageChecksum;
707
708 itLoc->second.push_back(
709 {clusterId, physicalColumnId, pageNo, position, offset, pageLocator.fBytesOnStorage, pageBufferSize});
710 ++nPages;
711 clusterBufSz += pageBufferSize;
712 });
713
714 auto clusterBuffer = new unsigned char[clusterBufSz];
715 auto pageMap = std::make_unique<ROnDiskPageMapHeap>(std::unique_ptr<unsigned char[]>(clusterBuffer));
716
717 auto cageBuffer = clusterBuffer;
718 // Fill the cluster page map and the read requests for the RDaosContainer::ReadV() call
719 for (auto &[cageIndex, pageVec] : onDiskPages) {
720 auto columnId = pageVec[0].fColumnId; // All pages in a cage belong to the same column
721 std::size_t cageSz = 0;
722
723 for (auto &s : pageVec) {
724 assert(columnId == s.fColumnId);
725 assert(cageIndex == s.fPosition);
726 // Register the on disk pages in a page map
727 ROnDiskPage::Key key(s.fColumnId, s.fPageNo);
728 pageMap->Register(key, ROnDiskPage(cageBuffer + s.fCageOffset, s.fBufferSize));
729 cageSz += s.fBufferSize;
730 }
731
732 // Prepare new read request batched up by object ID and distribution key
733 d_iov_t iov;
734 d_iov_set(&iov, cageBuffer, cageSz);
735
736 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, columnId, cageIndex);
737 auto odPair = RDaosContainer::ROidDkeyPair{daosKey.fOid, daosKey.fDkey};
738 auto [itReq, ret] = readRequests.emplace(odPair, RDaosContainer::RWOperation(odPair));
739 itReq->second.Insert(daosKey.fAkey, iov);
740
741 cageBuffer += cageSz;
742 }
743 fCounters->fNPageRead.Add(nPages);
744 fCounters->fSzReadPayload.Add(clusterBufSz);
745
746 auto cluster = std::make_unique<RCluster>(clusterId);
747 cluster->Adopt(std::move(pageMap));
748 cluster->Adopt(std::move(pageZeroMap));
749 for (auto colId : clusterKey.fPhysicalColumnSet)
750 cluster->SetColumnAvailable(colId);
751 return cluster;
752 };
753
754 fCounters->fNClusterLoaded.Add(clusterKeys.size());
755
756 std::vector<std::unique_ptr<ROOT::Experimental::Internal::RCluster>> clusters;
758 for (auto key : clusterKeys) {
759 clusters.emplace_back(fnPrepareSingleCluster(key, readRequests));
760 }
761
762 {
763 Detail::RNTupleAtomicTimer timer(fCounters->fTimeWallRead, fCounters->fTimeCpuRead);
764 if (int err = fDaosContainer->ReadV(readRequests))
765 throw ROOT::Experimental::RException(R__FAIL("ReadV: error" + std::string(d_errstr(err))));
766 }
767 fCounters->fNReadV.Inc();
768 fCounters->fNRead.Add(readRequests.size());
769
770 return clusters;
771}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:294
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:290
#define R__LOG_WARNING(...)
Definition RLogger.hxx:363
#define h(i)
Definition RSha256.hxx:106
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
UInt_t Hash(const TString &s)
Definition TString.h:494
#define _(A, B)
Definition cfortran.h:108
Record wall time and CPU time between construction and destruction.
Managed a set of clusters containing compressed and packed pages.
RColumnElementBase * GetElement() const
Definition RColumn.hxx:329
A RDaosContainer provides read/write access to objects in a given container.
Definition RDaos.hxx:157
RDaosObject::DistributionKey_t DistributionKey_t
Definition RDaos.hxx:160
std::unordered_map< ROidDkeyPair, RWOperation, ROidDkeyPair::Hash > MultiObjectRWOperation_t
Definition RDaos.hxx:231
int ReadSingleAkey(void *buffer, std::size_t length, daos_obj_id_t oid, DistributionKey_t dkey, AttributeKey_t akey, ObjClassId_t cid)
Read data from a single object attribute key to the given buffer.
Definition RDaos.cxx:211
RDaosObject::AttributeKey_t AttributeKey_t
Definition RDaos.hxx:161
static Writer_t MakeMemCopyWriter(unsigned char *dest)
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
A helper class for piece-wise construction of an RNTupleDescriptor.
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > DeserializeHeader(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< void > DeserializeFooter(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static RResult< void > DeserializePageList(const void *buffer, std::uint64_t bufSize, DescriptorId_t clusterGroupId, RNTupleDescriptor &desc)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
A page as being stored on disk, that is packed and compressed.
Definition RCluster.hxx:42
Base class for a sink with a physical storage backend.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
Reference to a page stored in the page pool.
Definition RPagePool.hxx:85
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options)
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
std::uint64_t StageClusterImpl() final
Returns the number of bytes written to storage (excluding metadata)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
RNTupleLocator CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask) final
Vector commit of preprocessed pages.
std::unique_ptr< RNTupleCompressor > fCompressor
Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
Storage provider that reads ntuple pages from a DAOS container.
std::string GetObjectClass() const
Return the object class used for user data OIDs in this ntuple.
RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster) final
std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys) final
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options)
void LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage) final
Read the packed and compressed bytes of a page into the memory buffer provided by sealedPage.
RNTupleDescriptor AttachImpl() final
LoadStructureImpl() has been called before AttachImpl() is called
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
std::unique_ptr< RPageSource > CloneImpl() const final
The cloned page source creates a new connection to the pool/container.
Abstract interface to read data from an ntuple.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSource.
Stores information about the cluster in which this page resides.
Definition RPage.hxx:55
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:46
std::size_t GetNBytes() const
The space taken by column elements in the buffer.
Definition RPage.hxx:122
static RPage MakePageZero(ColumnId_t columnId, ClusterSize_t::ValueType elementSize)
Make a 'zero' page for column columnId (that is comprised of 0x00 bytes only).
Definition RPage.hxx:174
void SetWindow(const NTupleSize_t rangeFirst, const RClusterInfo &clusterInfo)
Seek the page to a certain position of the column.
Definition RPage.hxx:164
static const void * GetPageZeroBuffer()
Return a pointer to the page zero buffer used if there is no on-disk data for a particular deferred c...
Definition RPage.cxx:25
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
DescriptorId_t GetClusterId() const
ClusterSize_t::ValueType GetIndex() const
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
The on-storage meta-data of an ntuple.
Common user-tunable settings for reading ntuples.
DAOS-specific user-tunable settings for storing ntuples.
Common user-tunable settings for storing ntuples.
static constexpr std::uint16_t kVersionEpoch
Definition RNTuple.hxx:67
void ThrowOnError()
Short-hand method to throw an exception in the case of errors.
Definition RError.hxx:281
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
const char * d_errstr(int rc)
static void d_iov_set(d_iov_t *iov, void *buf, size_t size)
Definition daos.h:50
uint16_t daos_oclass_id_t
Definition daos.h:135
@ OC_SX
Definition daos.h:129
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
The identifiers that specifies the content of a (partial) cluster.
Definition RCluster.hxx:156
A pair of <object ID, distribution key> that can be used to issue a fetch/update request for multiple...
Definition RDaos.hxx:166
Describes a read/write operation on multiple attribute keys under the same object ID and distribution...
Definition RDaos.hxx:190
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint64_t fVersionAnchor
Allows for evolving the struct in future versions.
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint16_t fVersionEpoch
Version of the binary format supported by the writer.
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
static constexpr std::size_t kOCNameMaxLength
This limit is currently not defined in any header and any call to daos_oclass_id2name() within DAOS u...
Definition RDaos.hxx:108
On-disk pages within a page source are identified by the column and page number.
Definition RCluster.hxx:52
Summarizes cluster-level information that are necessary to load a certain page.
std::uint64_t fColumnOffset
The first element number of the page's column in the given cluster.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
We do not need to store the element size / uncompressed page size because we know to which column the...
std::uint32_t fNElements
The sum of the elements of all the pages must match the corresponding fNElements field in fColumnRang...
bool fHasChecksum
If true, the 8 bytes following the serialized page are an xxhash of the on-disk page data.
RNTupleLocator fLocator
The meaning of fLocator depends on the storage backend.
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
std::uint8_t fReserved
Reserved for use by concrete storage backends.
ELocatorType fType
For non-disk locators, the value for the Type field.
std::variant< std::uint64_t, RNTupleLocatorObject64 > fPosition
Simple on-disk locators consisting of a 64-bit offset use variant type uint64_t; extended locators ha...
iovec for memory buffer
Definition daos.h:37
uint64_t hi
Definition daos.h:147
uint64_t lo
Definition daos.h:146
TMarker m
Definition textangle.C:8