Logo ROOT  
Reference Guide
RPageStorageFile.cxx
Go to the documentation of this file.
1 /// \file RPageStorageFile.cxx
2 /// \ingroup NTuple ROOT7
3 /// \author Jakob Blomer <jblomer@cern.ch>
4 /// \date 2019-11-25
5 /// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6 /// is welcome!
7 
8 /*************************************************************************
9  * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10  * All rights reserved. *
11  * *
12  * For the licensing terms see $ROOTSYS/LICENSE. *
13  * For the list of contributors see $ROOTSYS/README/CREDITS. *
14  *************************************************************************/
15 
16 #include <ROOT/RCluster.hxx>
17 #include <ROOT/RClusterPool.hxx>
18 #include <ROOT/RField.hxx>
19 #include <ROOT/RLogger.hxx>
21 #include <ROOT/RNTupleModel.hxx>
22 #include <ROOT/RNTupleZip.hxx>
23 #include <ROOT/RPage.hxx>
24 #include <ROOT/RPageAllocator.hxx>
25 #include <ROOT/RPagePool.hxx>
27 #include <ROOT/RRawFile.hxx>
28 
29 #include <RVersion.h>
30 #include <TError.h>
31 
32 #include <algorithm>
33 #include <cstdio>
34 #include <cstdlib>
35 #include <iostream>
36 #include <utility>
37 
38 
40  const RNTupleWriteOptions &options)
41  : RPageSink(ntupleName, options)
42  , fMetrics("RPageSinkRoot")
43  , fPageAllocator(std::make_unique<RPageAllocatorHeap>())
44 {
45  R__WARNING_HERE("NTuple") << "The RNTuple file format will change. " <<
46  "Do not store real data with this version of RNTuple!";
47 
48  fWriter = std::unique_ptr<Internal::RNTupleFileWriter>(Internal::RNTupleFileWriter::Recreate(
49  ntupleName, path, options.GetCompression(), options.GetContainerFormat()));
50 }
51 
52 
54  const RNTupleWriteOptions &options)
55  : RPageSink(ntupleName, options)
56  , fMetrics("RPageSinkRoot")
57  , fPageAllocator(std::make_unique<RPageAllocatorHeap>())
58 {
59  R__WARNING_HERE("NTuple") << "The RNTuple file format will change. " <<
60  "Do not store real data with this version of RNTuple!";
61 
62  fWriter = std::unique_ptr<Internal::RNTupleFileWriter>(Internal::RNTupleFileWriter::Append(ntupleName, file));
63 }
64 
65 
67  const RNTupleWriteOptions &options, std::unique_ptr<TFile> &file)
68  : RPageSink(ntupleName, options)
69  , fMetrics("RPageSinkRoot")
70  , fPageAllocator(std::make_unique<RPageAllocatorHeap>())
71 {
72  R__WARNING_HERE("NTuple") << "The RNTuple file format will change. " <<
73  "Do not store real data with this version of RNTuple!";
74  fWriter = std::unique_ptr<Internal::RNTupleFileWriter>(
75  Internal::RNTupleFileWriter::Recreate(ntupleName, path, file));
76 }
77 
78 
80 {
81 }
82 
83 
85 {
86  const auto &descriptor = fDescriptorBuilder.GetDescriptor();
87  auto szHeader = descriptor.SerializeHeader(nullptr);
88  auto buffer = std::unique_ptr<unsigned char[]>(new unsigned char[szHeader]);
89  descriptor.SerializeHeader(buffer.get());
90 
91  auto zipBuffer = std::unique_ptr<unsigned char[]>(new unsigned char[szHeader]);
92  auto szZipHeader = fCompressor(buffer.get(), szHeader, fOptions.GetCompression(),
93  [&zipBuffer](const void *b, size_t n, size_t o){ memcpy(zipBuffer.get() + o, b, n); } );
94  fWriter->WriteNTupleHeader(zipBuffer.get(), szZipHeader, szHeader);
95 }
96 
97 
100 {
101  unsigned char *buffer = reinterpret_cast<unsigned char *>(page.GetBuffer());
102  bool isAdoptedBuffer = true;
103  auto packedBytes = page.GetSize();
104  auto element = columnHandle.fColumn->GetElement();
105  const auto isMappable = element->IsMappable();
106 
107  if (!isMappable) {
108  packedBytes = (page.GetNElements() * element->GetBitsOnStorage() + 7) / 8;
109  buffer = new unsigned char[packedBytes];
110  isAdoptedBuffer = false;
111  element->Pack(buffer, page.GetBuffer(), page.GetNElements());
112  }
113  auto zippedBytes = packedBytes;
114 
115  if (fOptions.GetCompression() != 0) {
116  zippedBytes = fCompressor(buffer, packedBytes, fOptions.GetCompression());
117  if (!isAdoptedBuffer)
118  delete[] buffer;
119  buffer = const_cast<unsigned char *>(reinterpret_cast<const unsigned char *>(fCompressor.GetZipBuffer()));
120  isAdoptedBuffer = true;
121  }
122 
123  auto offsetData = fWriter->WriteBlob(buffer, zippedBytes, packedBytes);
124  fClusterMinOffset = std::min(offsetData, fClusterMinOffset);
125  fClusterMaxOffset = std::max(offsetData + zippedBytes, fClusterMaxOffset);
126 
127  if (!isAdoptedBuffer)
128  delete[] buffer;
129 
131  result.fPosition = offsetData;
132  result.fBytesOnStorage = zippedBytes;
133  return result;
134 }
135 
136 
139 {
141  result.fPosition = fClusterMinOffset;
142  result.fBytesOnStorage = fClusterMaxOffset - fClusterMinOffset;
143  fClusterMinOffset = std::uint64_t(-1);
144  fClusterMaxOffset = 0;
145  return result;
146 }
147 
148 
150 {
151  const auto &descriptor = fDescriptorBuilder.GetDescriptor();
152  auto szFooter = descriptor.SerializeFooter(nullptr);
153  auto buffer = std::unique_ptr<unsigned char []>(new unsigned char[szFooter]);
154  descriptor.SerializeFooter(buffer.get());
155 
156  auto zipBuffer = std::unique_ptr<unsigned char[]>(new unsigned char[szFooter]);
157  auto szZipFooter = fCompressor(buffer.get(), szFooter, fOptions.GetCompression(),
158  [&zipBuffer](const void *b, size_t n, size_t o){ memcpy(zipBuffer.get() + o, b, n); } );
159  fWriter->WriteNTupleFooter(zipBuffer.get(), szZipFooter, szFooter);
160  fWriter->Commit();
161 }
162 
163 
166 {
167  if (nElements == 0)
168  nElements = kDefaultElementsPerPage;
169  auto elementSize = columnHandle.fColumn->GetElement()->GetSize();
170  return fPageAllocator->NewPage(columnHandle.fId, elementSize, nElements);
171 }
172 
174 {
175  fPageAllocator->DeletePage(page);
176 }
177 
178 
179 ////////////////////////////////////////////////////////////////////////////////
180 
181 
183  ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements)
184 {
185  RPage newPage(columnId, mem, elementSize * nElements, elementSize);
186  newPage.TryGrow(nElements);
187  return newPage;
188 }
189 
191 {
192  if (page.IsNull())
193  return;
194  delete[] reinterpret_cast<unsigned char *>(page.GetBuffer());
195 }
196 
197 
198 ////////////////////////////////////////////////////////////////////////////////
199 
200 
202  const RNTupleReadOptions &options)
203  : RPageSource(ntupleName, options)
204  , fMetrics("RPageSourceFile")
205  , fPageAllocator(std::make_unique<RPageAllocatorFile>())
206  , fPagePool(std::make_shared<RPagePool>())
207  , fClusterPool(std::make_unique<RClusterPool>(*this))
208 {
209  fCounters = std::unique_ptr<RCounters>(new RCounters{
210  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nReadV", "", "number of vector read requests"),
211  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nRead", "", "number of byte ranges read"),
212  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szReadPayload", "B", "volume read from file (required)"),
213  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("szReadOverhead", "B", "volume read from file (overhead)"),
214  *fMetrics.MakeCounter<RNTuplePlainCounter*> ("szUnzip", "B", "volume after unzipping"),
215  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("nClusterLoaded", "",
216  "number of partial clusters preloaded from storage"),
217  *fMetrics.MakeCounter<RNTuplePlainCounter*> ("nPageLoaded", "", "number of pages loaded from storage"),
218  *fMetrics.MakeCounter<RNTuplePlainCounter*> ("nPagePopulated", "", "number of populated pages"),
219  *fMetrics.MakeCounter<RNTupleAtomicCounter*>("timeWallRead", "ns", "wall clock time spent reading"),
220  *fMetrics.MakeCounter<RNTuplePlainCounter*> ("timeWallUnzip", "ns", "wall clock time spent decompressing"),
221  *fMetrics.MakeCounter<RNTupleTickCounter<RNTupleAtomicCounter>*>("timeCpuRead", "ns", "CPU time spent reading"),
223  "CPU time spent decompressing")
224  });
225 }
226 
227 
229  const RNTupleReadOptions &options)
230  : RPageSourceFile(ntupleName, options)
231 {
233  R__ASSERT(fFile);
235 }
236 
237 
239 {
240 }
241 
242 
244 {
245  RNTupleDescriptorBuilder descBuilder;
246  auto ntpl = fReader.GetNTuple(fNTupleName).Unwrap();
247 
248  auto buffer = std::unique_ptr<unsigned char[]>(new unsigned char[ntpl.fLenHeader]);
249  auto zipBuffer = std::unique_ptr<unsigned char[]>(new unsigned char[ntpl.fNBytesHeader]);
250  fReader.ReadBuffer(zipBuffer.get(), ntpl.fNBytesHeader, ntpl.fSeekHeader);
251  fDecompressor(zipBuffer.get(), ntpl.fNBytesHeader, ntpl.fLenHeader, buffer.get());
252  descBuilder.SetFromHeader(buffer.get());
253 
254  buffer = std::unique_ptr<unsigned char[]>(new unsigned char[ntpl.fLenFooter]);
255  zipBuffer = std::unique_ptr<unsigned char[]>(new unsigned char[ntpl.fNBytesFooter]);
256  fReader.ReadBuffer(zipBuffer.get(), ntpl.fNBytesFooter, ntpl.fSeekFooter);
257  fDecompressor(zipBuffer.get(), ntpl.fNBytesFooter, ntpl.fLenFooter, buffer.get());
258  descBuilder.AddClustersFromFooter(buffer.get());
259 
260  return descBuilder.MoveDescriptor();
261 }
262 
263 
265  ColumnHandle_t columnHandle, const RClusterDescriptor &clusterDescriptor, ClusterSize_t::ValueType clusterIndex)
266 {
267  const auto columnId = columnHandle.fId;
268  const auto clusterId = clusterDescriptor.GetId();
269  const auto &pageRange = clusterDescriptor.GetPageRange(columnId);
270 
271  fCounters->fNPagePopulated.Inc();
272 
273  // TODO(jblomer): binary search
275  decltype(clusterIndex) firstInPage = 0;
276  NTupleSize_t pageNo = 0;
277  for (const auto &pi : pageRange.fPageInfos) {
278  if (firstInPage + pi.fNElements > clusterIndex) {
279  pageInfo = pi;
280  break;
281  }
282  firstInPage += pi.fNElements;
283  ++pageNo;
284  }
285  R__ASSERT(firstInPage <= clusterIndex);
286  R__ASSERT((firstInPage + pageInfo.fNElements) > clusterIndex);
287 
288  const auto element = columnHandle.fColumn->GetElement();
289  const auto elementSize = element->GetSize();
290 
291  const auto bytesOnStorage = pageInfo.fLocator.fBytesOnStorage;
292  const auto bytesPacked = (element->GetBitsOnStorage() * pageInfo.fNElements + 7) / 8;
293  const auto pageSize = elementSize * pageInfo.fNElements;
294 
295  auto pageBuffer = new unsigned char[bytesPacked];
296  if (fOptions.GetClusterCache() == RNTupleReadOptions::EClusterCache::kOff) {
297  fReader.ReadBuffer(pageBuffer, bytesOnStorage, pageInfo.fLocator.fPosition);
298  fCounters->fNPageLoaded.Inc();
299  } else {
300  if (!fCurrentCluster || (fCurrentCluster->GetId() != clusterId) || !fCurrentCluster->ContainsColumn(columnId))
301  fCurrentCluster = fClusterPool->GetCluster(clusterId, fActiveColumns);
302  R__ASSERT(fCurrentCluster->ContainsColumn(columnId));
303  ROnDiskPage::Key key(columnId, pageNo);
304  auto onDiskPage = fCurrentCluster->GetOnDiskPage(key);
305  R__ASSERT(onDiskPage);
306  R__ASSERT(bytesOnStorage == onDiskPage->GetSize());
307  memcpy(pageBuffer, onDiskPage->GetAddress(), onDiskPage->GetSize());
308  }
309 
310  if (bytesOnStorage != bytesPacked) {
311  RNTuplePlainTimer timer(fCounters->fTimeWallUnzip, fCounters->fTimeCpuUnzip);
312  fDecompressor(pageBuffer, bytesOnStorage, bytesPacked);
313  fCounters->fSzUnzip.Add(bytesPacked);
314  }
315 
316  if (!element->IsMappable()) {
317  auto unpackedBuffer = new unsigned char[pageSize];
318  element->Unpack(unpackedBuffer, pageBuffer, pageInfo.fNElements);
319  delete[] pageBuffer;
320  pageBuffer = unpackedBuffer;
321  }
322 
323  const auto indexOffset = clusterDescriptor.GetColumnRange(columnId).fFirstElementIndex;
324  auto newPage = fPageAllocator->NewPage(columnId, pageBuffer, elementSize, pageInfo.fNElements);
325  newPage.SetWindow(indexOffset + firstInPage, RPage::RClusterInfo(clusterId, indexOffset));
326  fPagePool->RegisterPage(newPage,
327  RPageDeleter([](const RPage &page, void * /*userData*/)
328  {
330  }, nullptr));
331  return newPage;
332 }
333 
334 
336  ColumnHandle_t columnHandle, NTupleSize_t globalIndex)
337 {
338  const auto columnId = columnHandle.fId;
339  auto cachedPage = fPagePool->GetPage(columnId, globalIndex);
340  if (!cachedPage.IsNull())
341  return cachedPage;
342 
343  const auto clusterId = fDescriptor.FindClusterId(columnId, globalIndex);
344  R__ASSERT(clusterId != kInvalidDescriptorId);
345  const auto &clusterDescriptor = fDescriptor.GetClusterDescriptor(clusterId);
346  const auto selfOffset = clusterDescriptor.GetColumnRange(columnId).fFirstElementIndex;
347  R__ASSERT(selfOffset <= globalIndex);
348  return PopulatePageFromCluster(columnHandle, clusterDescriptor, globalIndex - selfOffset);
349 }
350 
351 
353  ColumnHandle_t columnHandle, const RClusterIndex &clusterIndex)
354 {
355  const auto clusterId = clusterIndex.GetClusterId();
356  const auto index = clusterIndex.GetIndex();
357  const auto columnId = columnHandle.fId;
358  auto cachedPage = fPagePool->GetPage(columnId, clusterIndex);
359  if (!cachedPage.IsNull())
360  return cachedPage;
361 
362  R__ASSERT(clusterId != kInvalidDescriptorId);
363  const auto &clusterDescriptor = fDescriptor.GetClusterDescriptor(clusterId);
364  return PopulatePageFromCluster(columnHandle, clusterDescriptor, index);
365 }
366 
368 {
369  fPagePool->ReturnPage(page);
370 }
371 
372 std::unique_ptr<ROOT::Experimental::Detail::RPageSource> ROOT::Experimental::Detail::RPageSourceFile::Clone() const
373 {
374  auto clone = new RPageSourceFile(fNTupleName, fOptions);
375  clone->fFile = fFile->Clone();
376  clone->fReader = Internal::RMiniFileReader(clone->fFile.get());
377  return std::unique_ptr<RPageSourceFile>(clone);
378 }
379 
380 std::unique_ptr<ROOT::Experimental::Detail::RCluster>
382 {
383  fCounters->fNClusterLoaded.Inc();
384 
385  const auto &clusterDesc = GetDescriptor().GetClusterDescriptor(clusterId);
386  auto clusterLocator = clusterDesc.GetLocator();
387  auto clusterSize = clusterLocator.fBytesOnStorage;
388  R__ASSERT(clusterSize > 0);
389 
390  struct ROnDiskPageLocator {
391  ROnDiskPageLocator() = default;
392  ROnDiskPageLocator(DescriptorId_t c, NTupleSize_t p, std::uint64_t o, std::uint64_t s)
393  : fColumnId(c), fPageNo(p), fOffset(o), fSize(s) {}
394  DescriptorId_t fColumnId = 0;
395  NTupleSize_t fPageNo = 0;
396  std::uint64_t fOffset = 0;
397  std::uint64_t fSize = 0;
398  std::size_t fBufPos = 0;
399  };
400 
401  // Collect the page necessary page meta-data and sum up the total size of the compressed and packed pages
402  std::vector<ROnDiskPageLocator> onDiskPages;
403  auto activeSize = 0;
404  for (auto columnId : columns) {
405  const auto &pageRange = clusterDesc.GetPageRange(columnId);
406  NTupleSize_t pageNo = 0;
407  for (const auto &pageInfo : pageRange.fPageInfos) {
408  const auto &pageLocator = pageInfo.fLocator;
409  activeSize += pageLocator.fBytesOnStorage;
410  onDiskPages.emplace_back(ROnDiskPageLocator(
411  columnId, pageNo, pageLocator.fPosition, pageLocator.fBytesOnStorage));
412  ++pageNo;
413  }
414  }
415 
416  // Linearize the page requests by file offset
417  std::sort(onDiskPages.begin(), onDiskPages.end(),
418  [](const ROnDiskPageLocator &a, const ROnDiskPageLocator &b) {return a.fOffset < b.fOffset;});
419 
420  // In order to coalesce close-by pages, we collect the sizes of the gaps between pages on disk. We then order
421  // the gaps by size, sum them up and find a cutoff for the largest gap that we tolerate when coalescing pages.
422  // The size of the cutoff is given by the fraction of extra bytes we are willing to read in order to reduce
423  // the number of read requests. We thus schedule the lowest number of requests given a tolerable fraction
424  // of extra bytes.
425  // TODO(jblomer): Eventually we may want to select the parameter at runtime according to link latency and speed,
426  // memory consumption, device block size.
427  float maxOverhead = 0.25 * float(activeSize);
428  std::vector<std::size_t> gaps;
429  for (unsigned i = 1; i < onDiskPages.size(); ++i) {
430  gaps.emplace_back(onDiskPages[i].fOffset - (onDiskPages[i-1].fSize + onDiskPages[i-1].fOffset));
431  }
432  std::sort(gaps.begin(), gaps.end());
433  std::size_t gapCut = 0;
434  float szExtra = 0.0;
435  for (auto g : gaps) {
436  szExtra += g;
437  if (szExtra > maxOverhead)
438  break;
439  gapCut = g;
440  }
441 
442  // Prepare the input vector for the RRawFile::ReadV() call
443  struct RReadRequest {
444  RReadRequest() = default;
445  RReadRequest(std::size_t b, std::uint64_t o, std::uint64_t s) : fBufPos(b), fOffset(o), fSize(s) {}
446  std::size_t fBufPos = 0;
447  std::uint64_t fOffset = 0;
448  std::uint64_t fSize = 0;
449  };
450  std::vector<ROOT::Internal::RRawFile::RIOVec> readRequests;
451 
453  std::size_t szPayload = 0;
454  std::size_t szOverhead = 0;
455  for (auto &s : onDiskPages) {
456  R__ASSERT(s.fSize > 0);
457  auto readUpTo = req.fOffset + req.fSize;
458  R__ASSERT(s.fOffset >= readUpTo);
459  auto overhead = s.fOffset - readUpTo;
460  szPayload += s.fSize;
461  if (overhead <= gapCut) {
462  szOverhead += overhead;
463  s.fBufPos = reinterpret_cast<intptr_t>(req.fBuffer) + req.fSize + overhead;
464  req.fSize += overhead + s.fSize;
465  continue;
466  }
467 
468  // close the current request and open new one
469  if (req.fSize > 0)
470  readRequests.emplace_back(req);
471 
472  req.fBuffer = reinterpret_cast<unsigned char *>(req.fBuffer) + req.fSize;
473  s.fBufPos = reinterpret_cast<intptr_t>(req.fBuffer);
474 
475  req.fOffset = s.fOffset;
476  req.fSize = s.fSize;
477  }
478  readRequests.emplace_back(req);
479  fCounters->fSzReadPayload.Add(szPayload);
480  fCounters->fSzReadOverhead.Add(szOverhead);
481 
482  // Register the on disk pages in a page map
483  auto buffer = new unsigned char[reinterpret_cast<intptr_t>(req.fBuffer) + req.fSize];
484  auto pageMap = std::make_unique<ROnDiskPageMapHeap>(std::unique_ptr<unsigned char []>(buffer));
485  for (const auto &s : onDiskPages) {
486  ROnDiskPage::Key key(s.fColumnId, s.fPageNo);
487  pageMap->Register(key, ROnDiskPage(buffer + s.fBufPos, s.fSize));
488  }
489  fCounters->fNPageLoaded.Add(onDiskPages.size());
490  for (auto &r : readRequests) {
491  r.fBuffer = buffer + reinterpret_cast<intptr_t>(r.fBuffer);
492  }
493 
494  auto nReqs = readRequests.size();
495  {
496  RNTupleAtomicTimer timer(fCounters->fTimeWallRead, fCounters->fTimeCpuRead);
497  fFile->ReadV(&readRequests[0], nReqs);
498  }
499  fCounters->fNReadV.Inc();
500  fCounters->fNRead.Add(nReqs);
501 
502  auto cluster = std::make_unique<RCluster>(clusterId);
503  cluster->Adopt(std::move(pageMap));
504  for (auto colId : columns)
505  cluster->SetColumnAvailable(colId);
506  return cluster;
507 }
ROOT::Experimental::RNTupleWriteOptions
Common user-tunable settings for storing ntuples.
Definition: RNTupleOptions.hxx:58
c
#define c(i)
Definition: RSha256.hxx:119
ROOT::Experimental::Detail::RPageSinkFile::RPageSinkFile
RPageSinkFile(std::string_view ntupleName, std::string_view path, const RNTupleWriteOptions &options)
Definition: RPageStorageFile.cxx:39
ROOT::Experimental::Detail::RPageSinkFile::ReleasePage
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
Definition: RPageStorageFile.cxx:173
n
const Int_t n
Definition: legend1.C:16
ROOT::Experimental::RClusterDescriptor::GetColumnRange
const RColumnRange & GetColumnRange(DescriptorId_t columnId) const
Definition: RNTupleDescriptor.hxx:261
ROOT::Experimental::Detail::RNTupleAtomicCounter
A thread-safe integral performance counter.
Definition: RNTupleMetrics.hxx:125
RNTupleModel.hxx
ROOT::Experimental::Detail::RPage
A page is a slice of a column that is mapped into memory.
Definition: RPage.hxx:59
ROOT::Experimental::Detail::RNTupleTickCounter
An either thread-safe or non thread safe counter for CPU ticks.
Definition: RNTupleMetrics.hxx:186
ROOT::Internal::RRawFile::Create
static std::unique_ptr< RRawFile > Create(std::string_view url, ROptions options=ROptions())
Factory method that returns a suitable concrete implementation according to the transport in the url.
Definition: RRawFile.cxx:73
ROOT::Experimental::Detail::RPageAllocatorFile
Manages pages read from a the file.
Definition: RPageStorageFile.hxx:99
ROOT::Experimental::Internal::RNTupleFileWriter::Append
static RNTupleFileWriter * Append(std::string_view ntupleName, TFile &file)
Add a new RNTuple identified by ntupleName to the existing TFile.
Definition: RMiniFile.cxx:1148
ROOT::Experimental::Detail::RPageSourceFile::Clone
std::unique_ptr< RPageSource > Clone() const final
The cloned page source creates a new raw file and reader and opens its own file descriptor to the dat...
Definition: RPageStorageFile.cxx:372
ROOT::Experimental::Detail::RPage::TryGrow
void * TryGrow(ClusterSize_t::ValueType nElements)
Return a pointer after the last element that has space for nElements new elements.
Definition: RPage.hxx:125
ROOT::Experimental::Detail::RPageSourceFile::fMetrics
RNTupleMetrics fMetrics
Wraps the I/O counters and is observed by the RNTupleReader metrics.
Definition: RPageStorageFile.hxx:136
ROOT::Experimental::Detail::RPageAllocatorFile::NewPage
static RPage NewPage(ColumnId_t columnId, void *mem, std::size_t elementSize, std::size_t nElements)
Definition: RPageStorageFile.cxx:182
ROOT::Experimental::Detail::RNTupleMetrics::MakeCounter
CounterPtrT MakeCounter(const std::string &name, const std::string &unit, const std::string &desc)
Definition: RNTupleMetrics.hxx:281
ROOT::Experimental::Detail::RPageSinkFile::CommitClusterImpl
RClusterDescriptor::RLocator CommitClusterImpl(NTupleSize_t nEntries) final
Definition: RPageStorageFile.cxx:138
ROOT::Experimental::Detail::RPageSourceFile::fFile
std::unique_ptr< ROOT::Internal::RRawFile > fFile
An RRawFile is used to request the necessary byte ranges from a local or a remote file.
Definition: RPageStorageFile.hxx:147
ROOT::Experimental::DescriptorId_t
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
Definition: RNTupleUtil.hxx:90
r
ROOT::R::TRInterface & r
Definition: Object.C:4
RPage.hxx
ROOT::Experimental::Detail::RPageSourceFile::ReleasePage
void ReleasePage(RPage &page) final
Every page store needs to be able to free pages it handed out.
Definition: RPageStorageFile.cxx:367
RField.hxx
ROOT::Experimental::Detail::RColumnElementBase::GetSize
std::size_t GetSize() const
Definition: RColumnElement.hxx:118
RPageStorageFile.hxx
RClusterPool.hxx
string_view
basic_string_view< char > string_view
Definition: libcpp_string_view.h:785
ROOT::Experimental::Detail::RPageSourceFile::fReader
Internal::RMiniFileReader fReader
Takes the fFile to read ntuple blobs from it.
Definition: RPageStorageFile.hxx:149
ROOT::Experimental::Detail::RPageSource::ColumnSet_t
std::unordered_set< DescriptorId_t > ColumnSet_t
Derived from the model (fields) that are actually being requested at a given point in time.
Definition: RPageStorage.hxx:183
ROOT::Experimental::RNTupleDescriptorBuilder
A helper class for piece-wise construction of an RNTupleDescriptor.
Definition: RNTupleDescriptor.hxx:480
TGeant4Unit::s
static constexpr double s
Definition: TGeant4SystemOfUnits.h:168
RVersion.h
ROOT::Experimental::RClusterDescriptor::RLocator::fPosition
std::int64_t fPosition
Definition: RNTupleDescriptor.hxx:168
ROOT::Experimental::RClusterDescriptor::RPageRange::RPageInfo
We do not need to store the element size / uncompressed page size because we know to which column the...
Definition: RNTupleDescriptor.hxx:205
ROOT::Experimental::NTupleSize_t
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
Definition: RNTupleUtil.hxx:54
ROOT::Experimental::RNTupleReadOptions
Common user-tunable settings for reading ntuples.
Definition: RNTupleOptions.hxx:83
ROOT::Experimental::Detail::RPageSource
Abstract interface to read data from an ntuple.
Definition: RPageStorage.hxx:180
ROOT::Experimental::RClusterDescriptor::RColumnRange::fFirstElementIndex
NTupleSize_t fFirstElementIndex
A 64bit element index.
Definition: RNTupleDescriptor.hxx:181
ROOT::Experimental::Detail::RPageSourceFile::PopulatePage
RPage PopulatePage(ColumnHandle_t columnHandle, NTupleSize_t globalIndex) final
Allocates and fills a page that contains the index-th element.
Definition: RPageStorageFile.cxx:335
ROOT::Experimental::Detail::ROnDiskPage::Key
On-disk pages within a page source are identified by the column and page number.
Definition: RCluster.hxx:71
ROOT::Experimental::RClusterDescriptor
Meta-data for a set of ntuple clusters.
Definition: RNTupleDescriptor.hxx:160
ROOT::Experimental::RClusterDescriptor::RLocator::fBytesOnStorage
std::uint32_t fBytesOnStorage
Definition: RNTupleDescriptor.hxx:169
ROOT::Internal::RRawFile::RIOVec::fOffset
std::uint64_t fOffset
The file offset.
Definition: RRawFile.hxx:84
b
#define b(i)
Definition: RSha256.hxx:118
ROOT::Experimental::Detail::RPageStorage::RColumnHandle::fColumn
const RColumn * fColumn
Definition: RPageStorage.hxx:87
ROOT::Experimental::Detail::RPageSourceFile::AttachImpl
RNTupleDescriptor AttachImpl() final
Definition: RPageStorageFile.cxx:243
ROOT::Experimental::RClusterIndex::GetIndex
ClusterSize_t::ValueType GetIndex() const
Definition: RNTupleUtil.hxx:115
ROOT::Experimental::RClusterDescriptor::RPageRange::RPageInfo::fLocator
RLocator fLocator
The meaning of fLocator depends on the storage backend.
Definition: RNTupleDescriptor.hxx:209
ROOT::Internal::RRawFile::RIOVec
Used for vector reads from multiple offsets into multiple buffers.
Definition: RRawFile.hxx:80
ROOT::Experimental::RNTupleDescriptor
The on-storage meta-data of an ntuple.
Definition: RNTupleDescriptor.hxx:286
ROOT::Experimental::Detail::RPageSourceFile::RCounters
I/O performance counters that get registered in fMetrics.
Definition: RPageStorageFile.hxx:120
RLogger.hxx
ROOT::Experimental::RClusterDescriptor::GetPageRange
const RPageRange & GetPageRange(DescriptorId_t columnId) const
Definition: RNTupleDescriptor.hxx:262
ROOT::Experimental::RClusterSize::ValueType
std::uint32_t ValueType
Definition: RNTupleUtil.hxx:58
ROOT::Experimental::Detail::ROnDiskPage
A page as being stored on disk, that is packed and compressed.
Definition: RCluster.hxx:61
ROOT::Experimental::Detail::RClusterPool
Managed a set of clusters containing compressed and packed pages.
Definition: RClusterPool.hxx:65
ROOT::Experimental::Detail::RPageSourceFile::LoadCluster
std::unique_ptr< RCluster > LoadCluster(DescriptorId_t clusterId, const ColumnSet_t &columns) final
Populates all the pages of the given cluster id and columns; it is possible that some columns do not ...
Definition: RPageStorageFile.cxx:381
ROOT::Experimental::Detail::RPage::GetBuffer
void * GetBuffer() const
Definition: RPage.hxx:122
ROOT::Experimental::RNTupleDescriptorBuilder::AddClustersFromFooter
void AddClustersFromFooter(void *footerBuffer)
Definition: RNTupleDescriptor.cxx:822
a
auto * a
Definition: textangle.C:12
ROOT::Experimental::RClusterIndex::GetClusterId
DescriptorId_t GetClusterId() const
Definition: RNTupleUtil.hxx:114
ROOT::Experimental::Detail::RPageDeleter
A closure that can free the memory associated with a mapped page.
Definition: RPageAllocator.hxx:57
RRawFile.hxx
ROOT::Experimental::Detail::RPageStorage::RColumnHandle
Definition: RPageStorage.hxx:85
ROOT::Experimental::RNTupleModel
The RNTupleModel encapulates the schema of an ntuple.
Definition: RNTupleModel.hxx:58
ROOT::Experimental::Detail::RPagePool
A thread-safe cache of column pages.
Definition: RPagePool.hxx:64
ROOT::Experimental::RNTupleDescriptorBuilder::MoveDescriptor
RNTupleDescriptor MoveDescriptor()
Definition: RNTupleDescriptor.cxx:746
ROOT::Experimental::Detail::RPageSink
Abstract interface to write data into an ntuple.
Definition: RPageStorage.hxx:122
TFile
Definition: TFile.h:54
ROOT::Experimental::Detail::RPageAllocatorHeap
Uses standard C++ memory allocation for the column data pages.
Definition: RPageAllocator.hxx:89
ROOT::Experimental::Detail::RPageSinkFile::fWriter
std::unique_ptr< Internal::RNTupleFileWriter > fWriter
Definition: RPageStorageFile.hxx:64
ROOT::Experimental::RClusterDescriptor::RPageRange::RPageInfo::fNElements
ClusterSize_t fNElements
The sum of the elements of all the pages must match the corresponding fNElements field in fColumnRang...
Definition: RNTupleDescriptor.hxx:207
ROOT::Experimental::RClusterIndex
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Definition: RNTupleUtil.hxx:94
RNTupleZip.hxx
ROOT::Experimental::Detail::RPageSinkFile::~RPageSinkFile
virtual ~RPageSinkFile()
Definition: RPageStorageFile.cxx:79
ROOT::Experimental::Detail::RPageSinkFile::CommitDatasetImpl
void CommitDatasetImpl() final
Definition: RPageStorageFile.cxx:149
R__ASSERT
#define R__ASSERT(e)
Definition: TError.h:120
RPageAllocator.hxx
TGeant4Unit::pi
static constexpr double pi
Definition: TGeant4SystemOfUnits.h:73
ROOT::Experimental::Detail::RPageSourceFile
Storage provider that reads ntuple pages from a file.
Definition: RPageStorageFile.hxx:113
RPagePool.hxx
file
Definition: file.py:1
ROOT::Experimental::Detail::RPageSinkFile::CreateImpl
void CreateImpl(const RNTupleModel &model) final
Definition: RPageStorageFile.cxx:84
ROOT::Experimental::Detail::RPageSourceFile::fCounters
std::unique_ptr< RCounters > fCounters
Definition: RPageStorageFile.hxx:134
ROOT::Experimental::RNTupleWriteOptions::GetContainerFormat
ENTupleContainerFormat GetContainerFormat() const
Definition: RNTupleOptions.hxx:69
ROOT::Experimental::RClusterDescriptor::RLocator
Generic information about the physical location of data.
Definition: RNTupleDescriptor.hxx:167
ROOT::Experimental::Detail::RPage::RClusterInfo
Stores information about the cluster in which this page resides.
Definition: RPage.hxx:64
ROOT::Experimental::Detail::RPageSourceFile::RPageSourceFile
RPageSourceFile(std::string_view ntupleName, const RNTupleReadOptions &options)
Definition: RPageStorageFile.cxx:201
fSize
size_t fSize
Definition: DeclareConverters.h:342
ROOT::Experimental::Detail::RPageStorage::RColumnHandle::fId
DescriptorId_t fId
Definition: RPageStorage.hxx:86
ROOT::Experimental::kInvalidDescriptorId
constexpr DescriptorId_t kInvalidDescriptorId
Definition: RNTupleUtil.hxx:91
ROOT::Experimental::Internal::RNTupleFileWriter::Recreate
static RNTupleFileWriter * Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression, ENTupleContainerFormat containerFormat)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:1105
ROOT::Internal::RRawFile::RIOVec::fSize
std::size_t fSize
The number of desired bytes.
Definition: RRawFile.hxx:86
R__WARNING_HERE
#define R__WARNING_HERE(GROUP)
Definition: RLogger.hxx:184
ROOT::Experimental::Detail::RColumn::GetElement
RColumnElementBase * GetElement() const
Definition: RColumn.hxx:248
ROOT::Experimental::Detail::RPage::GetNElements
ClusterSize_t::ValueType GetNElements() const
Definition: RPage.hxx:101
ROOT::Experimental::RNTupleWriteOptions::GetCompression
int GetCompression() const
Definition: RNTupleOptions.hxx:63
ROOT::Experimental::Internal::RMiniFileReader
Read RNTuple data blocks from a TFile container, provided by a RRawFile.
Definition: RMiniFile.hxx:102
ROOT::Experimental::Detail::RNTupleTimer
Record wall time and CPU time between construction and destruction.
Definition: RNTupleMetrics.hxx:215
ROOT::Experimental::Detail::RPageAllocatorFile::DeletePage
static void DeletePage(const RPage &page)
Definition: RPageStorageFile.cxx:190
ROOT::Experimental::RNTupleDescriptorBuilder::SetFromHeader
void SetFromHeader(void *headerBuffer)
Definition: RNTupleDescriptor.cxx:753
RCluster.hxx
ROOT::Experimental::Detail::RPage::IsNull
bool IsNull() const
Definition: RPage.hxx:143
ROOT::Experimental::Detail::RPageSinkFile::ReservePage
RPage ReservePage(ColumnHandle_t columnHandle, std::size_t nElements=0) final
Get a new, empty page for the given column that can be filled with up to nElements.
Definition: RPageStorageFile.cxx:165
ROOT::Experimental::Detail::RColumnElementBase::IsMappable
virtual bool IsMappable() const
Derived, typed classes tell whether the on-storage layout is bitwise identical to the memory layout.
Definition: RColumnElement.hxx:102
ROOT::Experimental::Detail::RPage::GetSize
ClusterSize_t::ValueType GetSize() const
The space taken by column elements in the buffer.
Definition: RPage.hxx:99
ROOT::Experimental::ColumnId_t
std::int64_t ColumnId_t
Uniquely identifies a physical column within the scope of the current process, used to tag pages.
Definition: RNTupleUtil.hxx:86
ROOT::Experimental::Detail::RPageSinkFile::CommitPageImpl
RClusterDescriptor::RLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) final
Definition: RPageStorageFile.cxx:99
ROOT::Experimental::Detail::RPageSourceFile::~RPageSourceFile
virtual ~RPageSourceFile()
Definition: RPageStorageFile.cxx:238
ROOT::Experimental::Detail::RPageSourceFile::PopulatePageFromCluster
RPage PopulatePageFromCluster(ColumnHandle_t columnHandle, const RClusterDescriptor &clusterDescriptor, ClusterSize_t::ValueType clusterIndex)
Definition: RPageStorageFile.cxx:264
ROOT::Experimental::RClusterDescriptor::GetId
DescriptorId_t GetId() const
Definition: RNTupleDescriptor.hxx:256
ROOT::Experimental::Detail::RNTuplePlainCounter
A non thread-safe integral performance counter.
Definition: RNTupleMetrics.hxx:98
RNTupleDescriptor.hxx
TError.h
ROOT::Internal::RRawFile::RIOVec::fBuffer
void * fBuffer
The destination for reading.
Definition: RRawFile.hxx:82
g
#define g(i)
Definition: RSha256.hxx:123