47enum EDaosMapping { kOidPerCluster, kOidPerPage };
51 DistributionKey_t fDkey;
59static constexpr DistributionKey_t kDistributionKeyDefault = 0x5a3c69f0cafe4a11;
60static constexpr AttributeKey_t kAttributeKeyDefault = 0x4243544b53444229;
61static constexpr AttributeKey_t kAttributeKeyAnchor = 0x4243544b5344422a;
62static constexpr AttributeKey_t kAttributeKeyHeader = 0x4243544b5344422b;
63static constexpr AttributeKey_t kAttributeKeyFooter = 0x4243544b5344422c;
66static constexpr daos_obj_id_t kOidMetadata{std::uint64_t(-1), 0};
67static constexpr daos_obj_id_t kOidPageList{std::uint64_t(-2), 0};
71static constexpr EDaosMapping kDefaultDaosMapping = kOidPerCluster;
73template <EDaosMapping mapping>
74RDaosKey GetPageDaosKey(
long unsigned clusterId,
long unsigned columnId,
long unsigned pageCount)
76 if constexpr (mapping == kOidPerCluster) {
78 static_cast<DistributionKey_t
>(columnId),
static_cast<AttributeKey_t
>(pageCount)};
79 }
else if constexpr (mapping == kOidPerPage) {
81 kAttributeKeyDefault};
87 std::string fPoolLabel;
89 std::string fContainerLabel;
97 std::regex re(
"daos://([^/]+)/(.+)");
99 if (!std::regex_match(uri.data(),
m, re))
112 if (buffer !=
nullptr) {
113 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
121 return RNTupleSerializer::SerializeString(
fObjClass,
nullptr) + 20;
128 return R__FAIL(
"DAOS anchor too short");
131 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
132 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fVersion);
133 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fNBytesHeader);
134 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fLenHeader);
135 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fNBytesFooter);
136 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fLenFooter);
137 auto result = RNTupleSerializer::DeserializeString(
bytes, bufSize - 20, fObjClass);
140 return result.Unwrap() + 20;
161 "Do not store real data with this version of RNTuple!";
162 fCompressor = std::make_unique<RNTupleCompressor>();
170 unsigned char *serializedHeader, std::uint32_t
length)
175 if (oclass.IsUnknown())
178 auto args = ParseDaosURI(fURI);
179 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
180 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel,
true);
181 fDaosContainer->SetDefaultObjectClass(oclass);
183 auto zipBuffer = std::make_unique<unsigned char[]>(
length);
184 auto szZipHeader = fCompressor->Zip(serializedHeader,
length, GetWriteOptions().GetCompression(),
186 WriteNTupleHeader(zipBuffer.get(), szZipHeader,
length);
197 sealedPage = SealPage(page, *element, GetWriteOptions().GetCompression());
201 return CommitSealedPageImpl(columnHandle.
fId, sealedPage);
208 auto offsetData = fPageId.fetch_add(1);
209 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNClusters();
213 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(clusterId, columnId, offsetData);
214 fDaosContainer->WriteSingleAkey(sealedPage.
fBuffer, sealedPage.
fSize, daosKey.fOid, daosKey.fDkey, daosKey.fAkey);
218 result.fPosition = offsetData;
220 fCounters->fNPageCommitted.Inc();
221 fCounters->fSzWritePayload.Add(sealedPage.
fSize);
222 fNBytesCurrentCluster += sealedPage.
fSize;
226std::vector<ROOT::Experimental::RNTupleLocator>
230 std::vector<ROOT::Experimental::RNTupleLocator> locators;
233 return c + std::distance(r.fFirst, r.fLast);
235 locators.reserve(nPages);
237 DescriptorId_t clusterId = fDescriptorBuilder.GetDescriptor().GetNClusters();
238 std::size_t szPayload = 0;
241 for (
auto &range : ranges) {
242 for (
auto sealedPageIt = range.fFirst; sealedPageIt != range.fLast; ++sealedPageIt) {
245 d_iov_set(&pageIov,
const_cast<void *
>(
s.fBuffer),
s.fSize);
246 auto offsetData = fPageId.fetch_add(1);
248 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(clusterId, range.fColumnId, offsetData);
251 it->second.insert(daosKey.fAkey, pageIov);
256 locators.push_back(locator);
258 szPayload +=
s.fSize;
261 fNBytesCurrentCluster += szPayload;
265 if (
int err = fDaosContainer->WriteV(writeRequests))
269 fCounters->fNPageCommitted.Add(nPages);
270 fCounters->fSzWritePayload.Add(szPayload);
278 return std::exchange(fNBytesCurrentCluster, 0);
285 auto bufPageListZip = std::make_unique<unsigned char[]>(
length);
286 auto szPageListZip = fCompressor->Zip(serializedPageList,
length, GetWriteOptions().GetCompression(),
289 auto offsetData = fClusterGroupId.fetch_add(1);
290 fDaosContainer->WriteSingleAkey(bufPageListZip.get(), szPageListZip, kOidPageList, kDistributionKeyDefault,
291 offsetData, kCidMetadata);
293 result.fPosition = offsetData;
294 result.fBytesOnStorage = szPageListZip;
295 fCounters->fSzWritePayload.Add(szPageListZip);
301 auto bufFooterZip = std::make_unique<unsigned char[]>(
length);
302 auto szFooterZip = fCompressor->Zip(serializedFooter,
length, GetWriteOptions().GetCompression(),
304 WriteNTupleFooter(bufFooterZip.get(), szFooterZip,
length);
309 const void *
data,
size_t nbytes,
size_t lenHeader)
311 fDaosContainer->WriteSingleAkey(
data, nbytes, kOidMetadata, kDistributionKeyDefault, kAttributeKeyHeader,
313 fNTupleAnchor.fLenHeader = lenHeader;
314 fNTupleAnchor.fNBytesHeader = nbytes;
318 const void *
data,
size_t nbytes,
size_t lenFooter)
320 fDaosContainer->WriteSingleAkey(
data, nbytes, kOidMetadata, kDistributionKeyDefault, kAttributeKeyFooter,
322 fNTupleAnchor.fLenFooter = lenFooter;
323 fNTupleAnchor.fNBytesFooter = nbytes;
328 auto buffer = std::make_unique<unsigned char[]>(ntplSize);
329 fNTupleAnchor.Serialize(buffer.get());
330 fDaosContainer->WriteSingleAkey(buffer.get(), ntplSize, kOidMetadata, kDistributionKeyDefault, kAttributeKeyAnchor,
340 return fPageAllocator->NewPage(columnHandle.
fId, elementSize, nElements);
345 fPageAllocator->DeletePage(page);
353 ColumnId_t columnId,
void *mem, std::size_t elementSize, std::size_t nElements)
355 RPage newPage(columnId, mem, elementSize, nElements);
364 delete[]
reinterpret_cast<unsigned char *
>(page.
GetBuffer());
373 fPagePool(std::make_shared<
RPagePool>()), fURI(uri),
374 fClusterPool(std::make_unique<
RClusterPool>(*this, options.GetClusterBunchSize()))
379 auto args = ParseDaosURI(uri);
380 auto pool = std::make_shared<RDaosPool>(args.fPoolLabel);
381 fDaosContainer = std::make_unique<RDaosContainer>(pool, args.fContainerLabel);
393 auto buffer = std::make_unique<unsigned char[]>(ntplSize);
394 fDaosContainer->ReadSingleAkey(buffer.get(), ntplSize, kOidMetadata, kDistributionKeyDefault, kAttributeKeyAnchor,
399 if (oclass.IsUnknown())
401 fDaosContainer->SetDefaultObjectClass(oclass);
404 buffer = std::make_unique<unsigned char[]>(ntpl.
fLenHeader);
405 auto zipBuffer = std::make_unique<unsigned char[]>(ntpl.
fNBytesHeader);
406 fDaosContainer->ReadSingleAkey(zipBuffer.get(), ntpl.
fNBytesHeader, kOidMetadata, kDistributionKeyDefault,
407 kAttributeKeyHeader, kCidMetadata);
412 buffer = std::make_unique<unsigned char[]>(ntpl.
fLenFooter);
413 zipBuffer = std::make_unique<unsigned char[]>(ntpl.
fNBytesFooter);
414 fDaosContainer->ReadSingleAkey(zipBuffer.get(), ntpl.
fNBytesFooter, kOidMetadata, kDistributionKeyDefault,
415 kAttributeKeyFooter, kCidMetadata);
421 for (
const auto &cgDesc : ntplDesc.GetClusterGroupIterable()) {
422 buffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLength());
423 zipBuffer = std::make_unique<unsigned char[]>(cgDesc.GetPageListLocator().fBytesOnStorage);
424 fDaosContainer->ReadSingleAkey(zipBuffer.get(), cgDesc.GetPageListLocator().fBytesOnStorage, kOidPageList,
425 kDistributionKeyDefault, cgDesc.GetPageListLocator().fPosition, kCidMetadata);
426 fDecompressor->Unzip(zipBuffer.get(), cgDesc.GetPageListLocator().fBytesOnStorage, cgDesc.GetPageListLength(),
431 for (std::size_t i = 0; i < clusters.size(); ++i) {
432 ntplDesc.AddClusterDetails(clusters[i].MoveDescriptor().Unwrap());
442 return fDaosContainer->GetDefaultObjectClass().ToString();
453 auto descriptorGuard = GetSharedDescriptorGuard();
454 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterId);
455 pageInfo = clusterDescriptor.GetPageRange(columnId).Find(clusterIndex.
GetIndex());
459 sealedPage.
fSize = bytesOnStorage;
462 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(clusterId, columnId, pageInfo.
fLocator.
fPosition);
463 fDaosContainer->ReadSingleAkey(
const_cast<void *
>(sealedPage.
fBuffer), bytesOnStorage, daosKey.fOid,
464 daosKey.fDkey, daosKey.fAkey);
473 const auto columnId = columnHandle.
fId;
474 const auto clusterId = clusterInfo.
fClusterId;
475 const auto &pageInfo = clusterInfo.
fPageInfo;
478 const auto elementSize = element->
GetSize();
479 const auto bytesOnStorage = pageInfo.fLocator.fBytesOnStorage;
481 const void *sealedPageBuffer =
nullptr;
482 std::unique_ptr<unsigned char []> directReadBuffer;
484 if (fOptions.GetClusterCache() == RNTupleReadOptions::EClusterCache::kOff) {
485 directReadBuffer = std::make_unique<unsigned char[]>(bytesOnStorage);
486 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(clusterId, columnId, pageInfo.fLocator.fPosition);
487 fDaosContainer->ReadSingleAkey(directReadBuffer.get(), bytesOnStorage, daosKey.fOid, daosKey.fDkey,
489 fCounters->fNPageLoaded.Inc();
490 fCounters->fNRead.Inc();
491 fCounters->fSzReadPayload.Add(bytesOnStorage);
492 sealedPageBuffer = directReadBuffer.get();
494 if (!fCurrentCluster || (fCurrentCluster->GetId() != clusterId) || !fCurrentCluster->ContainsColumn(columnId))
495 fCurrentCluster = fClusterPool->GetCluster(clusterId, fActiveColumns);
496 R__ASSERT(fCurrentCluster->ContainsColumn(columnId));
498 auto cachedPage = fPagePool->GetPage(columnId,
RClusterIndex(clusterId, idxInCluster));
499 if (!cachedPage.IsNull())
503 auto onDiskPage = fCurrentCluster->GetOnDiskPage(key);
504 R__ASSERT(onDiskPage && (bytesOnStorage == onDiskPage->GetSize()));
505 sealedPageBuffer = onDiskPage->GetAddress();
508 std::unique_ptr<unsigned char []> pageBuffer;
511 pageBuffer = UnsealPage({sealedPageBuffer, bytesOnStorage, pageInfo.fNElements}, *element);
512 fCounters->fSzUnzip.Add(elementSize * pageInfo.fNElements);
515 auto newPage = fPageAllocator->NewPage(columnId, pageBuffer.release(), elementSize, pageInfo.fNElements);
516 newPage.SetWindow(clusterInfo.
fColumnOffset + pageInfo.fFirstInPage,
518 fPagePool->RegisterPage(newPage,
523 fCounters->fNPagePopulated.Inc();
531 const auto columnId = columnHandle.
fId;
532 auto cachedPage = fPagePool->GetPage(columnId, globalIndex);
533 if (!cachedPage.IsNull())
536 std::uint64_t idxInCluster;
539 auto descriptorGuard = GetSharedDescriptorGuard();
540 clusterInfo.
fClusterId = descriptorGuard->FindClusterId(columnId, globalIndex);
543 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterInfo.
fClusterId);
544 clusterInfo.
fColumnOffset = clusterDescriptor.GetColumnRange(columnId).fFirstElementIndex;
547 clusterInfo.
fPageInfo = clusterDescriptor.GetPageRange(columnId).Find(idxInCluster);
549 return PopulatePageFromCluster(columnHandle, clusterInfo, idxInCluster);
557 const auto idxInCluster = clusterIndex.
GetIndex();
558 const auto columnId = columnHandle.
fId;
559 auto cachedPage = fPagePool->GetPage(columnId, clusterIndex);
560 if (!cachedPage.IsNull())
566 auto descriptorGuard = GetSharedDescriptorGuard();
567 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterId);
569 clusterInfo.
fColumnOffset = clusterDescriptor.GetColumnRange(columnId).fFirstElementIndex;
570 clusterInfo.
fPageInfo = clusterDescriptor.GetPageRange(columnId).Find(idxInCluster);
573 return PopulatePageFromCluster(columnHandle, clusterInfo, idxInCluster);
578 fPagePool->ReturnPage(page);
584 return std::unique_ptr<RPageSourceDaos>(clone);
587std::vector<std::unique_ptr<ROOT::Experimental::Detail::RCluster>>
590 std::vector<std::unique_ptr<ROOT::Experimental::Detail::RCluster>>
result;
592 struct RDaosSealedPageLocator {
593 RDaosSealedPageLocator() =
default;
596 : fClusterId(cl), fColumnId(co), fPageNo(
p), fObjectId(o),
fSize(
s), fBufPos(
b)
602 std::uint64_t fObjectId = 0;
603 std::uint64_t
fSize = 0;
604 std::size_t fBufPos = 0;
607 std::vector<unsigned char *> clusterBuffers(clusterKeys.size());
608 std::vector<std::unique_ptr<ROnDiskPageMapHeap>> pageMaps(clusterKeys.size());
611 std::size_t szPayload = 0;
614 for (
unsigned i = 0; i < clusterKeys.size(); ++i) {
615 const auto &clusterKey = clusterKeys[i];
616 auto clusterId = clusterKey.fClusterId;
617 std::vector<RDaosSealedPageLocator> onDiskClusterPages;
619 unsigned clusterBufSz = 0;
620 fCounters->fNClusterLoaded.Inc();
622 auto descriptorGuard = GetSharedDescriptorGuard();
623 const auto &clusterDesc = descriptorGuard->GetClusterDescriptor(clusterId);
626 for (
auto columnId : clusterKey.fColumnSet) {
627 const auto &pageRange = clusterDesc.GetPageRange(columnId);
629 for (
const auto &pageInfo : pageRange.fPageInfos) {
630 const auto &pageLocator = pageInfo.fLocator;
631 onDiskClusterPages.push_back(RDaosSealedPageLocator(clusterId, columnId, columnPageCount,
632 pageLocator.fPosition, pageLocator.fBytesOnStorage,
635 clusterBufSz += pageLocator.fBytesOnStorage;
637 nPages += columnPageCount;
640 szPayload += clusterBufSz;
642 clusterBuffers[i] =
new unsigned char[clusterBufSz];
643 pageMaps[i] = std::make_unique<ROnDiskPageMapHeap>(std::unique_ptr<
unsigned char[]>(clusterBuffers[i]));
646 for (
const auto &
s : onDiskClusterPages) {
649 pageMaps[i]->Register(key,
ROnDiskPage(clusterBuffers[i] +
s.fBufPos,
s.fSize));
653 d_iov_set(&iov, clusterBuffers[i] +
s.fBufPos,
s.fSize);
655 RDaosKey daosKey = GetPageDaosKey<kDefaultDaosMapping>(
s.fClusterId,
s.fColumnId,
s.fObjectId);
658 it->second.insert(daosKey.fAkey, iov);
661 fCounters->fNPageLoaded.Add(nPages);
662 fCounters->fSzReadPayload.Add(szPayload);
666 if (
int err = fDaosContainer->ReadV(readRequests))
669 fCounters->fNReadV.Inc();
670 fCounters->fNRead.Add(nPages);
673 for (
unsigned i = 0; i < clusterKeys.size(); ++i) {
674 auto cluster = std::make_unique<RCluster>(clusterKeys[i].fClusterId);
675 cluster->Adopt(std::move(pageMaps[i]));
676 for (
auto colId : clusterKeys[i].fColumnSet)
677 cluster->SetColumnAvailable(colId);
679 result.emplace_back(std::move(cluster));
687 fTaskScheduler->Reset();
689 const auto clusterId = cluster->
GetId();
690 auto descriptorGuard = GetSharedDescriptorGuard();
691 const auto &clusterDescriptor = descriptorGuard->GetClusterDescriptor(clusterId);
693 std::vector<std::unique_ptr<RColumnElementBase>> allElements;
696 for (
const auto columnId : columnsInCluster) {
697 const auto &columnDesc = descriptorGuard->GetColumnDescriptor(columnId);
701 const auto &pageRange = clusterDescriptor.GetPageRange(columnId);
702 std::uint64_t pageNo = 0;
703 std::uint64_t firstInPage = 0;
704 for (
const auto &
pi : pageRange.fPageInfos) {
707 R__ASSERT(onDiskPage && (onDiskPage->GetSize() ==
pi.fLocator.fBytesOnStorage));
710 [
this, columnId, clusterId, firstInPage, onDiskPage,
711 element = allElements.back().get(),
712 nElements =
pi.fNElements,
713 indexOffset = clusterDescriptor.GetColumnRange(columnId).fFirstElementIndex
715 auto pageBuffer = UnsealPage({onDiskPage->GetAddress(), onDiskPage->GetSize(), nElements}, *element);
716 fCounters->fSzUnzip.Add(element->GetSize() * nElements);
718 auto newPage = fPageAllocator->NewPage(columnId, pageBuffer.release(), element->GetSize(), nElements);
720 fPagePool->PreloadPage(newPage,
727 fTaskScheduler->AddTask(taskFunc);
729 firstInPage +=
pi.fNElements;
736 fTaskScheduler->Wait();
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
#define R__LOG_WARNING(...)
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t b
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t r
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Managed a set of clusters containing compressed and packed pages.
An in-memory subset of the packed and compressed pages of a cluster.
const ColumnSet_t & GetAvailColumns() const
size_t GetNOnDiskPages() const
DescriptorId_t GetId() const
const ROnDiskPage * GetOnDiskPage(const ROnDiskPage::Key &key) const
static std::unique_ptr< RColumnElementBase > Generate(EColumnType type)
std::size_t GetSize() const
RColumnElementBase * GetElement() const
std::unordered_map< ROidDkeyPair, RWOperation, ROidDkeyPair::Hash > MultiObjectRWOperation_t
RDaosObject::DistributionKey_t DistributionKey_t
RDaosObject::AttributeKey_t AttributeKey_t
static Writer_t MakeMemCopyWriter(unsigned char *dest)
Record wall time and CPU time between construction and destruction.
A page as being stored on disk, that is packed and compressed.
Manages pages read from a DAOS container.
static RPage NewPage(ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements)
static void DeletePage(const RPage &page)
Uses standard C++ memory allocation for the column data pages.
A closure that can free the memory associated with a mapped page.
A thread-safe cache of column pages.
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
RPageSinkDaos(std::string_view ntupleName, std::string_view uri, const RNTupleWriteOptions &options)
RNTupleLocator CommitSealedPageImpl(DescriptorId_t columnId, const RPageStorage::RSealedPage &sealedPage) final
void WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
void WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
std::uint64_t CommitClusterImpl(NTupleSize_t nEntries) final
Returns the number of bytes written to storage (excluding metadata)
void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
void CreateImpl(const RNTupleModel &model, unsigned char *serializedHeader, std::uint32_t length) final
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges) final
Vector commit of preprocessed pages.
RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements) final
Get a new, empty page for the given column that can be filled with up to nElements.
Abstract interface to write data into an ntuple.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSink.
std::unique_ptr< RNTupleCompressor > fCompressor
Helper to zip pages and header/footer; includes a 16MB (kMAXZIPBUF) zip buffer.
Storage provider that reads ntuple pages from a DAOS container.
std::unique_ptr< RDaosContainer > fDaosContainer
A container that stores object data (header/footer, pages, etc.)
void LoadSealedPage(DescriptorId_t columnId, const RClusterIndex &clusterIndex, RSealedPage &sealedPage) final
Read the packed and compressed bytes of a page into the memory buffer provided by selaedPage.
RPageSourceDaos(std::string_view ntupleName, std::string_view uri, const RNTupleReadOptions &options)
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
void UnzipClusterImpl(RCluster *cluster) final
std::vector< std::unique_ptr< RCluster > > LoadClusters(std::span< RCluster::RKey > clusterKeys) final
Populates all the pages of the given cluster ids and columns; it is possible that some columns do not...
RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final
Allocates and fills a page that contains the index-th element.
RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterInfo &clusterInfo, ClusterSize_t::ValueType idxInCluster)
std::string GetObjectClass() const
Return the object class used for user data OIDs in this ntuple.
RNTupleDescriptor AttachImpl() final
std::unique_ptr< RPageSource > Clone() const final
The cloned page source creates a new connection to the pool/container.
virtual ~RPageSourceDaos()
Abstract interface to read data from an ntuple.
void EnableDefaultMetrics(const std::string &prefix)
Enables the default set of metrics provided by RPageSource.
std::unique_ptr< RNTupleDecompressor > fDecompressor
Helper to unzip pages and header/footer; comprises a 16MB (kMAXZIPBUF) unzip buffer.
Stores information about the cluster in which this page resides.
A page is a slice of a column that is mapped into memory.
ClusterSize_t::ValueType GetNBytes() const
The space taken by column elements in the buffer.
void * GrowUnchecked(ClusterSize_t::ValueType nElements)
Called during writing: returns a pointer after the last element and increases the element counter in ...
A helper class for serializing and deserialization of the RNTuple binary format.
static RResult< void > DeserializePageListV1(const void *buffer, std::uint32_t bufSize, std::vector< RClusterDescriptorBuilder > &clusters)
static RResult< void > DeserializeFooterV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< void > DeserializeHeaderV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::vector< RClusterDescriptorBuilder > GetClusterSummaries(const RNTupleDescriptor &ntplDesc, DescriptorId_t clusterGroupId)
Used to prepare the cluster descriptor builders when loading the page locations for a certain cluster...
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
DescriptorId_t GetClusterId() const
ClusterSize_t::ValueType GetIndex() const
Base class for all ROOT issued exceptions.
A helper class for piece-wise construction of an RNTupleDescriptor.
RNTupleDescriptor MoveDescriptor()
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
void SetOnDiskHeaderSize(std::uint64_t size)
The on-storage meta-data of an ntuple.
The RNTupleModel encapulates the schema of an ntuple.
Common user-tunable settings for reading ntuples.
DAOS-specific user-tunable settings for storing ntuples.
const std::string & GetObjectClass() const
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
const char * d_errstr(int rc)
static void d_iov_set(d_iov_t *iov, void *buf, size_t size)
uint16_t daos_oclass_id_t
basic_string_view< char > string_view
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
constexpr DescriptorId_t kInvalidDescriptorId
static constexpr double s
static constexpr double pi
A pair of <object ID, distribution key> that can be used to issue a fetch/update request for multiple...
Describes a read/write operation on multiple objects; see the ReadV/WriteV functions.
Entry point for an RNTuple in a DAOS container.
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
std::string fObjClass
The object class for user data OIDs, e.g. SX
std::uint32_t fVersion
Allows for evolving the struct in future versions.
RResult< std::uint32_t > Deserialize(const void *buffer, std::uint32_t bufSize)
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
static std::uint32_t GetSize()
std::uint32_t Serialize(void *buffer) const
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
Wrap around a daos_oclass_id_t.
static constexpr std::size_t kOCNameMaxLength
This limit is currently not defined in any header and any call to daos_oclass_id2name() within DAOS u...
On-disk pages within a page source are identified by the column and page number.
Summarizes cluster-level information that are necessary to populate a certain page.
RClusterDescriptor::RPageRange::RPageInfoExtended fPageInfo
Location of the page on disk.
std::uint64_t fColumnOffset
The first element number of the page's column in the given cluster.
DescriptorId_t fClusterId
A range of sealed pages referring to the same column that can be used for vector commit.
A sealed page contains the bytes of a page as written to storage (packed & compressed).
Generic information about the physical location of data.
std::uint32_t fBytesOnStorage