48enum EDaosMapping { kOidPerCluster, kOidPerPage };
52 DistributionKey_t fDkey;
60static constexpr DistributionKey_t kDistributionKeyDefault = 0x5a3c69f0cafe4a11;
61static constexpr AttributeKey_t kAttributeKeyDefault = 0x4243544b53444229;
62static constexpr AttributeKey_t kAttributeKeyAnchor = 0x4243544b5344422a;
63static constexpr AttributeKey_t kAttributeKeyHeader = 0x4243544b5344422b;
64static constexpr AttributeKey_t kAttributeKeyFooter = 0x4243544b5344422c;
72static constexpr EDaosMapping kDefaultDaosMapping = kOidPerCluster;
74template <EDaosMapping mapping>
76 long unsigned columnId,
long unsigned pageCount)
78 if constexpr (mapping == kOidPerCluster) {
81 static_cast<DistributionKey_t
>(columnId),
static_cast<AttributeKey_t
>(pageCount)};
82 }
else if constexpr (mapping == kOidPerPage) {
85 kDistributionKeyDefault, kAttributeKeyDefault};
91 std::string fPoolLabel;
93 std::string fContainerLabel;
99RDaosURI ParseDaosURI(std::string_view uri)
101 std::regex re(
"daos://([^/]+)/(.+)");
103 if (!std::regex_match(uri.data(),
m, re))
112 auto position =
static_cast<uint32_t
>(address.
fLocation & 0xFFFFFFFF);
114 return {position,
offset};
121 uint64_t address = (position & 0xFFFFFFFF) | (
offset << 32);
132struct RDaosContainerNTupleLocator {
135 std::optional<ROOT::Experimental::Internal::RDaosNTupleAnchor> fAnchor;
138 RDaosContainerNTupleLocator() =
default;
139 explicit RDaosContainerNTupleLocator(
const std::string &ntupleName) : fName(ntupleName), fIndex(
Hash(ntupleName)){};
141 bool IsValid() {
return fAnchor.has_value() && fAnchor->fNBytesHeader; }
146 uint64_t
h = std::hash<std::string>{}(ntupleName);
148 auto seed =
static_cast<uint32_t
>(
h >> 32);
149 seed ^=
static_cast<uint32_t
>(
h & 0xffffffff) + 0x9e3779b9 + (seed << 6) + (seed >> 2);
151 return (hash == kReservedIndex) ? kReservedIndex + 1 : hash;
157 std::unique_ptr<unsigned char[]> buffer, zipBuffer;
158 auto &anchor = fAnchor.emplace();
164 buffer = std::make_unique<unsigned char[]>(anchorSize);
165 if ((err = cont.
ReadSingleAkey(buffer.get(), anchorSize, oidMetadata, kDistributionKeyDefault,
166 kAttributeKeyAnchor, kCidMetadata))) {
170 anchor.Deserialize(buffer.get(), anchorSize).Unwrap();
173 R__FAIL(
"unsupported RNTuple epoch version: " + std::to_string(anchor.fVersionEpoch)));
175 if (anchor.fVersionEpoch == 0) {
176 static std::once_flag once;
177 std::call_once(once, [&anchor]() {
179 <<
"Pre-release format version: RC " << anchor.fVersionMajor;
184 buffer = std::make_unique<unsigned char[]>(anchor.fLenHeader);
185 zipBuffer = std::make_unique<unsigned char[]>(anchor.fNBytesHeader);
186 if ((err = cont.
ReadSingleAkey(zipBuffer.get(), anchor.fNBytesHeader, oidMetadata, kDistributionKeyDefault,
187 kAttributeKeyHeader, kCidMetadata)))
194 buffer = std::make_unique<unsigned char[]>(anchor.fLenFooter);
195 zipBuffer = std::make_unique<unsigned char[]>(anchor.fNBytesFooter);
196 if ((err = cont.
ReadSingleAkey(zipBuffer.get(), anchor.fNBytesFooter, oidMetadata, kDistributionKeyDefault,
197 kAttributeKeyFooter, kCidMetadata)))
206 static std::pair<RDaosContainerNTupleLocator, ROOT::Experimental::Internal::RNTupleDescriptorBuilder>
209 auto result = std::make_pair(RDaosContainerNTupleLocator(ntupleName),
213 auto &builder =
result.second;
215 if (
int err = loc.InitNTupleDescriptorBuilder(cont, builder); !err) {
219 R__FAIL(
"LocateNTuple: ntuple name '" + ntupleName +
"' unavailable in this container."));
233 if (buffer !=
nullptr) {
234 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
253 return R__FAIL(
"DAOS anchor too short");
256 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
259 return R__FAIL(
"unsupported DAOS anchor version: " + std::to_string(fVersionAnchor));
273 return result.Unwrap() + 32;
287 static std::once_flag once;
288 std::call_once(once, []() {
290 <<
"Do not store real data with this version of RNTuple!";
292 fCompressor = std::make_unique<RNTupleCompressor>();
303 if (oclass.IsUnknown())
308 fCageSizeLimit = std::max(cageSz, pageSz);
310 auto args = ParseDaosURI(fURI);
311 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
313 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel,
true);
314 fDaosContainer->SetDefaultObjectClass(oclass);
316 auto [locator,
_] = RDaosContainerNTupleLocator::LocateNTuple(*fDaosContainer, fNTupleName);
317 fNTupleIndex = locator.GetIndex();
319 auto zipBuffer = std::make_unique<unsigned char[]>(
length);
320 auto szZipHeader = fCompressor->Zip(serializedHeader,
length, GetWriteOptions().GetCompression(),
322 WriteNTupleHeader(zipBuffer.get(), szZipHeader,
length);
332 sealedPage = SealPage(page, *element);
336 return CommitSealedPageImpl(columnHandle.
fPhysicalId, sealedPage);
343 auto offsetData = fPageId.fetch_add(1);
344 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNActiveClusters();
348 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, physicalColumnId, offsetData);
349 fDaosContainer->WriteSingleAkey(sealedPage.
GetBuffer(), sealedPage.
GetBufferSize(), daosKey.fOid, daosKey.fDkey,
354 result.fPosition = EncodeDaosPagePosition(offsetData);
357 fCounters->fNPageCommitted.Inc();
363std::vector<ROOT::Experimental::RNTupleLocator>
365 const std::vector<bool> &
mask)
368 std::vector<ROOT::Experimental::RNTupleLocator> locators;
369 auto nPages =
mask.size();
370 locators.reserve(nPages);
372 const uint32_t maxCageSz = fCageSizeLimit;
373 const bool useCaging = fCageSizeLimit > 0;
376 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNActiveClusters();
377 int64_t payloadSz = 0;
378 std::size_t positionOffset;
379 uint32_t positionIndex;
382 for (
auto &range : ranges) {
387 positionIndex = useCaging ? fPageId.fetch_add(1) : fPageId.load();
389 for (
auto sealedPageIt = range.fFirst; sealedPageIt != range.fLast; ++sealedPageIt) {
394 positionIndex = fPageId.fetch_add(1);
401 GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, range.fPhysicalColumnId, positionIndex);
404 it->second.Insert(daosKey.fAkey, pageIov);
407 locator.
fPosition = EncodeDaosPagePosition(positionIndex, positionOffset);
411 locators.push_back(locator);
417 fNBytesCurrentCluster += payloadSz;
421 if (
int err = fDaosContainer->WriteV(writeRequests))
425 fCounters->fNPageCommitted.Add(nPages);
426 fCounters->fSzWritePayload.Add(payloadSz);
433 return std::exchange(fNBytesCurrentCluster, 0);
440 auto bufPageListZip = std::make_unique<unsigned char[]>(
length);
441 auto szPageListZip = fCompressor->Zip(serializedPageList,
length, GetWriteOptions().GetCompression(),
444 auto offsetData = fClusterGroupId.fetch_add(1);
445 fDaosContainer->WriteSingleAkey(
446 bufPageListZip.get(), szPageListZip,
447 daos_obj_id_t{kOidLowPageList, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)}, kDistributionKeyDefault,
448 offsetData, kCidMetadata);
451 result.fBytesOnStorage = szPageListZip;
453 fCounters->fSzWritePayload.Add(
static_cast<int64_t
>(szPageListZip));
460 auto bufFooterZip = std::make_unique<unsigned char[]>(
length);
461 auto szFooterZip = fCompressor->Zip(serializedFooter,
length, GetWriteOptions().GetCompression(),
463 WriteNTupleFooter(bufFooterZip.get(), szFooterZip,
length);
469 fDaosContainer->WriteSingleAkey(
471 kDistributionKeyDefault, kAttributeKeyHeader, kCidMetadata);
472 fNTupleAnchor.fLenHeader = lenHeader;
473 fNTupleAnchor.fNBytesHeader = nbytes;
478 fDaosContainer->WriteSingleAkey(
480 kDistributionKeyDefault, kAttributeKeyFooter, kCidMetadata);
481 fNTupleAnchor.fLenFooter = lenFooter;
482 fNTupleAnchor.fNBytesFooter = nbytes;
488 auto buffer = std::make_unique<unsigned char[]>(ntplSize);
489 fNTupleAnchor.Serialize(buffer.get());
490 fDaosContainer->WriteSingleAkey(
491 buffer.get(), ntplSize,
daos_obj_id_t{kOidLowMetadata, static_cast<decltype(daos_obj_id_t::hi)>(fNTupleIndex)},
492 kDistributionKeyDefault, kAttributeKeyAnchor, kCidMetadata);
501 fClusterPool(std::make_unique<
RClusterPool>(*this, options.GetClusterBunchSize()))
505 auto args = ParseDaosURI(uri);
506 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
507 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel);
515 std::unique_ptr<unsigned char[]> buffer, zipBuffer;
517 auto [locator, descBuilder] = RDaosContainerNTupleLocator::LocateNTuple(*fDaosContainer, fNTupleName);
518 if (!locator.IsValid())
520 R__FAIL(
"Attach: requested ntuple '" + fNTupleName +
"' is not present in DAOS container."));
523 if (oclass.IsUnknown())
526 fDaosContainer->SetDefaultObjectClass(oclass);
527 fNTupleIndex = locator.GetIndex();
530 auto desc = descBuilder.MoveDescriptor();
532 for (
const auto &cgDesc : desc.GetClusterGroupIterable()) {
533 buffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLength());
534 zipBuffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLocator().fBytesOnStorage);
535 fDaosContainer->ReadSingleAkey(
536 zipBuffer.get(), cgDesc.GetPageListLocator().fBytesOnStorage, oidPageList, kDistributionKeyDefault,
539 cgDesc.GetPageListLength(), buffer.get());
549 return fDaosContainer->GetDefaultObjectClass().ToString();
559 auto descriptorGuard = GetSharedDescriptorGuard();
560 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterId);
561 pageInfo = clusterDescriptor.GetPageRange(physicalColumnId).Find(clusterIndex.
GetIndex());
581 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, physicalColumnId, position);
583 auto cageHeadBuffer = std::make_unique<unsigned char[]>(bufSize);
584 fDaosContainer->ReadSingleAkey(cageHeadBuffer.get(), bufSize, daosKey.fOid, daosKey.fDkey, daosKey.fAkey);
587 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(
590 daosKey.fOid, daosKey.fDkey, daosKey.fAkey);
602 const auto clusterId = clusterInfo.
fClusterId;
603 const auto &pageInfo = clusterInfo.
fPageInfo;
606 const auto elementSize = element->
GetSize();
610 pageZero.GrowUnchecked(pageInfo.fNElements);
611 pageZero.SetWindow(clusterInfo.
fColumnOffset + pageInfo.fFirstInPage,
613 return fPagePool.RegisterPage(std::move(pageZero));
619 sealedPage.
SetBufferSize(pageInfo.fLocator.fBytesOnStorage + pageInfo.fHasChecksum * kNBytesPageChecksum);
620 std::unique_ptr<unsigned char[]> directReadBuffer;
625 R__FAIL(
"accessing caged pages is only supported in conjunction with cluster cache"));
628 directReadBuffer = std::unique_ptr<unsigned char[]>(
new unsigned char[sealedPage.
GetBufferSize()]);
629 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(
631 fDaosContainer->ReadSingleAkey(directReadBuffer.get(), sealedPage.
GetBufferSize(), daosKey.fOid, daosKey.fDkey,
633 fCounters->fNPageRead.Inc();
634 fCounters->fNRead.Inc();
636 sealedPage.
SetBuffer(directReadBuffer.get());
638 if (!fCurrentCluster || (fCurrentCluster->GetId() != clusterId) || !fCurrentCluster->ContainsColumn(columnId))
639 fCurrentCluster = fClusterPool->GetCluster(clusterId, fActivePhysicalColumns.ToColumnSet());
640 R__ASSERT(fCurrentCluster->ContainsColumn(columnId));
642 auto cachedPageRef = fPagePool.GetPage(columnId,
RClusterIndex(clusterId, idxInCluster));
643 if (!cachedPageRef.Get().IsNull())
644 return cachedPageRef;
647 auto onDiskPage = fCurrentCluster->GetOnDiskPage(key);
649 sealedPage.
SetBuffer(onDiskPage->GetAddress());
655 newPage = UnsealPage(sealedPage, *element, columnId).Unwrap();
656 fCounters->fSzUnzip.Add(elementSize * pageInfo.fNElements);
661 fCounters->fNPageUnsealed.Inc();
662 return fPagePool.RegisterPage(std::move(newPage));
665std::unique_ptr<ROOT::Experimental::Internal::RPageSource>
669 return std::unique_ptr<RPageSourceDaos>(clone);
672std::vector<std::unique_ptr<ROOT::Experimental::Internal::RCluster>>
675 struct RDaosSealedPageLocator {
679 std::uint64_t fPosition = 0;
680 std::uint64_t fCageOffset = 0;
681 std::uint64_t fDataSize = 0;
682 std::uint64_t fBufferSize = 0;
689 auto fnPrepareSingleCluster = [&](
const RCluster::RKey &clusterKey,
694 std::unordered_map<std::uint32_t, std::vector<RDaosSealedPageLocator>> onDiskPages;
696 unsigned clusterBufSz = 0, nPages = 0;
697 auto pageZeroMap = std::make_unique<ROnDiskPageMap>();
699 clusterKey, *pageZeroMap,
702 const auto &pageLocator = pageInfo.
fLocator;
703 uint32_t position,
offset;
705 auto [itLoc,
_] = onDiskPages.emplace(position, std::vector<RDaosSealedPageLocator>());
706 auto pageBufferSize = pageLocator.fBytesOnStorage + pageInfo.
fHasChecksum * kNBytesPageChecksum;
708 itLoc->second.push_back(
709 {clusterId, physicalColumnId, pageNo, position,
offset, pageLocator.fBytesOnStorage, pageBufferSize});
711 clusterBufSz += pageBufferSize;
714 auto clusterBuffer =
new unsigned char[clusterBufSz];
715 auto pageMap = std::make_unique<ROnDiskPageMapHeap>(std::unique_ptr<
unsigned char[]>(clusterBuffer));
717 auto cageBuffer = clusterBuffer;
719 for (
auto &[cageIndex, pageVec] : onDiskPages) {
720 auto columnId = pageVec[0].fColumnId;
721 std::size_t cageSz = 0;
723 for (
auto &s : pageVec) {
724 assert(columnId == s.fColumnId);
725 assert(cageIndex == s.fPosition);
728 pageMap->Register(key,
ROnDiskPage(cageBuffer + s.fCageOffset, s.fBufferSize));
729 cageSz += s.fBufferSize;
736 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(fNTupleIndex, clusterId, columnId, cageIndex);
739 itReq->second.Insert(daosKey.fAkey, iov);
741 cageBuffer += cageSz;
743 fCounters->fNPageRead.Add(nPages);
744 fCounters->fSzReadPayload.Add(clusterBufSz);
746 auto cluster = std::make_unique<RCluster>(clusterId);
747 cluster->Adopt(std::move(pageMap));
748 cluster->Adopt(std::move(pageZeroMap));
750 cluster->SetColumnAvailable(colId);
754 fCounters->fNClusterLoaded.Add(clusterKeys.size());
756 std::vector<std::unique_ptr<ROOT::Experimental::Internal::RCluster>> clusters;
758 for (
auto key : clusterKeys) {
759 clusters.emplace_back(fnPrepareSingleCluster(key, readRequests));
764 if (
int err = fDaosContainer->ReadV(readRequests))
767 fCounters->fNReadV.Inc();
768 fCounters->fNRead.Add(readRequests.size());
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
#define R__LOG_WARNING(...)
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
UInt_t Hash(const TString &s)
Record wall time and CPU time between construction and destruction.
Managed a set of clusters containing compressed and packed pages.
std::size_t GetSize() const
RColumnElementBase * GetElement() const
A RDaosContainer provides read/write access to objects in a given container.
RDaosObject::DistributionKey_t DistributionKey_t
std::unordered_map< ROidDkeyPair, RWOperation, ROidDkeyPair::Hash > MultiObjectRWOperation_t
int ReadSingleAkey(void *buffer, std::size_t length, daos_obj_id_t oid, DistributionKey_t dkey, AttributeKey_t akey, ObjClassId_t cid)
Read data from a single object attribute key to the given buffer.
RDaosObject::AttributeKey_t AttributeKey_t
static Writer_t MakeMemCopyWriter(unsigned char *dest)
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
A helper class for piece-wise construction of an RNTupleDescriptor.
void SetOnDiskHeaderSize(std::uint64_t size)
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
const RNTupleDescriptor & GetDescriptor() const
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > DeserializeHeader(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< void > DeserializeFooter(const void *buffer, std::uint64_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static RResult< void > DeserializePageList(const void *buffer, std::uint64_t bufSize, DescriptorId_t clusterGroupId, RNTupleDescriptor &desc)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
A page as being stored on disk, that is packed and compressed.
Base class for a sink with a physical storage backend.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options)
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
std::uint64_t StageClusterImpl() final
Returns the number of bytes written to storage (excluding metadata)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final
~RPageSinkDaos() override
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
RNTupleLocator CommitSealedPageImpl(DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask) final
Vector commit of preprocessed pages.
void CommitDatasetImpl() final
std::unique_ptr< RNTupleCompressor > fCompressor
Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
Storage provider that reads ntuple pages from a DAOS container.
std::string GetObjectClass() const
Return the object class used for user data OIDs in this ntuple.
RPageRef LoadPageImpl(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster) final
std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys) final
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options)
~RPageSourceDaos() override
void LoadSealedPage(DescriptorId_t physicalColumnId, RClusterIndex clusterIndex, RSealedPage &sealedPage) final
Read the packed and compressed bytes of a page into the memory buffer provided by sealedPage.
RNTupleDescriptor AttachImpl() final
LoadStructureImpl() has been called before AttachImpl() is called
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
std::unique_ptr< RPageSource > CloneImpl() const final
The cloned page source creates a new connection to the pool/container.
Abstract interface to read data from an ntuple.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSource.
Stores information about the cluster in which this page resides.
A page is a slice of a column that is mapped into memory.
std::size_t GetNBytes() const
The space taken by column elements in the buffer.
static RPage MakePageZero(ColumnId_t columnId, ClusterSize_t::ValueType elementSize)
Make a 'zero' page for column columnId (that is comprised of 0x00 bytes only).
void SetWindow(const NTupleSize_t rangeFirst, const RClusterInfo &clusterInfo)
Seek the page to a certain position of the column.
static const void * GetPageZeroBuffer()
Return a pointer to the page zero buffer used if there is no on-disk data for a particular deferred c...
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
DescriptorId_t GetClusterId() const
ClusterSize_t::ValueType GetIndex() const
Base class for all ROOT issued exceptions.
The on-storage meta-data of an ntuple.
const std::string & GetName() const
Common user-tunable settings for reading ntuples.
DAOS-specific user-tunable settings for storing ntuples.
uint32_t GetMaxCageSize() const
const std::string & GetObjectClass() const
Common user-tunable settings for storing ntuples.
std::size_t GetMaxUnzippedPageSize() const
static constexpr std::uint16_t kVersionEpoch
void ThrowOnError()
Short-hand method to throw an exception in the case of errors.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
const char * d_errstr(int rc)
static void d_iov_set(d_iov_t *iov, void *buf, size_t size)
uint16_t daos_oclass_id_t
std::uint32_t ntuple_index_t
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
The identifiers that specifies the content of a (partial) cluster.
DescriptorId_t fClusterId
ColumnSet_t fPhysicalColumnSet
A pair of <object ID, distribution key> that can be used to issue a fetch/update request for multiple...
Describes a read/write operation on multiple attribute keys under the same object ID and distribution...
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint64_t fVersionAnchor
Allows for evolving the struct in future versions.
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint16_t fVersionPatch
std::uint32_t Serialize(void *buffer) const
std::uint16_t fVersionEpoch
Version of the binary format supported by the writer.
std::uint16_t fVersionMinor
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
static std::uint32_t GetSize()
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
std::uint16_t fVersionMajor
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Wrap around a daos_oclass_id_t.
static constexpr std::size_t kOCNameMaxLength
This limit is currently not defined in any header and any call to daos_oclass_id2name() within DAOS u...
On-disk pages within a page source are identified by the column and page number.
Summarizes cluster-level information that are necessary to load a certain page.
DescriptorId_t fClusterId
std::uint64_t fColumnOffset
The first element number of the page's column in the given cluster.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
DescriptorId_t fPhysicalId
A sealed page contains the bytes of a page as written to storage (packed & compressed).
const void * GetBuffer() const
void SetHasChecksum(bool hasChecksum)
void SetNElements(std::uint32_t nElements)
void SetBuffer(const void *buffer)
void SetBufferSize(std::size_t bufferSize)
RResult< void > VerifyChecksumIfEnabled() const
std::size_t GetBufferSize() const
std::size_t GetDataSize() const
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
std::uint64_t fBytesOnStorage
std::uint8_t fReserved
Reserved for use by concrete storage backends.
ELocatorType fType
For non-disk locators, the value for the Type field.
std::variant< std::uint64_t, RNTupleLocatorObject64 > fPosition
Simple on-disk locators consisting of a 64-bit offset use variant type uint64_t; extended locators ha...
const T & GetPosition() const