Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RPageStorageFile.hxx
Go to the documentation of this file.
1/// \file ROOT/RPageStorageFile.hxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-11-21
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#ifndef ROOT_RPageStorageFile
15#define ROOT_RPageStorageFile
16
17#include <ROOT/RMiniFile.hxx>
18#include <ROOT/RNTuple.hxx>
20#include <ROOT/RNTupleZip.hxx>
21#include <ROOT/RPageStorage.hxx>
22#include <ROOT/RRawFile.hxx>
23#include <string_view>
24
25#include <array>
26#include <cstdio>
27#include <memory>
28#include <optional>
29#include <string>
30#include <utility>
31
32class TDirectory;
33
34namespace ROOT {
35class RNTuple; // for making RPageSourceFile a friend of RNTuple
36class RNTupleLocator;
37
38namespace Internal {
39class RClusterPool;
40class RRawFile;
41class RPageAllocatorHeap;
42
43// clang-format off
44/**
45\class ROOT::Internal::RPageSinkFile
46\ingroup NTuple
47\brief Storage provider that write ntuple pages into a file
48
49The written file can be either in ROOT format or in RNTuple bare format.
50*/
51// clang-format on
53private:
54 // A set of pages to be committed together in a vector write.
55 // Currently we assume they're all sequential (although they may span multiple ranges).
56 struct CommitBatch {
57 /// The list of pages to commit
58 std::vector<const RSealedPage *> fSealedPages;
59 /// Total size in bytes of the batch
60 size_t fSize;
61 /// Total uncompressed size of the elements in the page batch
63 };
64
65 std::unique_ptr<ROOT::Internal::RNTupleFileWriter> fWriter;
66 /// Number of bytes committed to storage in the current cluster
67 std::uint64_t fNBytesCurrentCluster = 0;
68 RPageSinkFile(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options);
69
70 /// We pass bytesPacked so that TFile::ls() reports a reasonable value for the compression ratio of the corresponding
71 /// key. It is not strictly necessary to write and read the sealed page.
73
74 /// Subroutine of CommitSealedPageVImpl, used to perform a vector write of the (multi-)range of pages
75 /// contained in `batch`. The locators for the written pages are appended to `locators`.
76 /// This procedure also updates some internal metrics of the page sink, hence it's not const.
77 /// `batch` gets reset to size 0 after the writing is done (but its begin and end are not updated).
78 void CommitBatchOfPages(CommitBatch &batch, std::vector<RNTupleLocator> &locators);
79
80protected:
82 void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final;
86 std::vector<RNTupleLocator>
87 CommitSealedPageVImpl(std::span<RPageStorage::RSealedPageGroup> ranges, const std::vector<bool> &mask) final;
88 std::uint64_t StageClusterImpl() final;
91 void CommitDatasetImpl(unsigned char *serializedFooter, std::uint32_t length) final;
92
93public:
94 RPageSinkFile(std::string_view ntupleName, std::string_view path, const ROOT::RNTupleWriteOptions &options);
101}; // class RPageSinkFile
102
103// clang-format off
104/**
105\class ROOT::Internal::RPageSourceFile
106\ingroup NTuple
107\brief Storage provider that reads ntuple pages from a file
108*/
109// clang-format on
111 friend class ROOT::RNTuple;
112
113private:
114 /// Holds the uncompressed header and footer
116 std::unique_ptr<unsigned char[]> fBuffer; ///< single buffer for both header and footer
117 void *fPtrHeader = nullptr; ///< either nullptr or points into fBuffer
118 void *fPtrFooter = nullptr; ///< either nullptr or points into fBuffer
119
120 /// Called at the end of Attach(), i.e. when the header and footer are processed
121 void Reset()
122 {
123 RStructureBuffer empty;
124 std::swap(empty, *this);
125 }
126 };
127
128 /// Either provided by CreateFromAnchor, or read from the ROOT file given the ntuple name
129 std::optional<RNTuple> fAnchor;
130 /// The last cluster from which a page got loaded. Points into fClusterPool->fPool
131 ROOT::Internal::RCluster *fCurrentCluster = nullptr;
132 /// An RRawFile is used to request the necessary byte ranges from a local or a remote file
133 std::unique_ptr<RRawFile> fFile;
134 /// Takes the fFile to read ntuple blobs from it
136 /// The descriptor is created from the header and footer either in AttachImpl or in CreateFromAnchor
138 /// The cluster pool asynchronously preloads the next few clusters
139 std::unique_ptr<ROOT::Internal::RClusterPool> fClusterPool;
140 /// Populated by LoadStructureImpl(), reset at the end of Attach()
142
143 RPageSourceFile(std::string_view ntupleName, const ROOT::RNTupleReadOptions &options);
144
145 /// Helper function for LoadClusters: it prepares the memory buffer (page map) and the
146 /// read requests for a given cluster and columns. The reead requests are appended to
147 /// the provided vector. This way, requests can be collected for multiple clusters before
148 /// sending them to RRawFile::ReadV().
149 std::unique_ptr<ROOT::Internal::RCluster>
150 PrepareSingleCluster(const ROOT::Internal::RCluster::RKey &clusterKey, std::vector<RRawFile::RIOVec> &readRequests);
151
152protected:
153 void LoadStructureImpl() final;
154 ROOT::RNTupleDescriptor AttachImpl(RNTupleSerializer::EDescriptorDeserializeMode mode) final;
155 /// The cloned page source creates a new raw file and reader and opens its own file descriptor to the data.
156 std::unique_ptr<RPageSource> CloneImpl() const final;
157
160
161public:
162 RPageSourceFile(std::string_view ntupleName, std::string_view path, const ROOT::RNTupleReadOptions &options);
163 RPageSourceFile(std::string_view ntupleName, std::unique_ptr<RRawFile> file,
164 const ROOT::RNTupleReadOptions &options);
165 /// Used from the RNTuple class to build a datasource if the anchor is already available.
166 /// Requires the RNTuple object to be streamed from a file.
167 static std::unique_ptr<RPageSourceFile>
168 CreateFromAnchor(const RNTuple &anchor, const ROOT::RNTupleReadOptions &options = ROOT::RNTupleReadOptions());
169
175
176 void
178
179 std::vector<std::unique_ptr<ROOT::Internal::RCluster>>
180 LoadClusters(std::span<ROOT::Internal::RCluster::RKey> clusterKeys) final;
181}; // class RPageSourceFile
182
183} // namespace Internal
184} // namespace ROOT
185
186#endif
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t mask
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char mode
An in-memory subset of the packed and compressed pages of a cluster.
Definition RCluster.hxx:148
Read RNTuple data blocks from a TFile container, provided by a RRawFile.
Definition RMiniFile.hxx:55
A helper class for piece-wise construction of an RNTupleDescriptor.
A helper class for serializing and deserialization of the RNTuple binary format.
Base class for a sink with a physical storage backend.
virtual void InitImpl(unsigned char *serializedHeader, std::uint32_t length)=0
Reference to a page stored in the page pool.
Storage provider that write ntuple pages into a file.
std::uint64_t fNBytesCurrentCluster
Number of bytes committed to storage in the current cluster.
void CommitBatchOfPages(CommitBatch &batch, std::vector< RNTupleLocator > &locators)
Subroutine of CommitSealedPageVImpl, used to perform a vector write of the (multi-)range of pages con...
RPageSinkFile(std::string_view ntupleName, const ROOT::RNTupleWriteOptions &options)
std::uint64_t StageClusterImpl() final
Returns the number of bytes written to storage (excluding metadata)
void InitImpl(unsigned char *serializedHeader, std::uint32_t length) final
RNTupleLocator CommitPageImpl(ColumnHandle_t columnHandle, const RPage &page) override
RNTupleLocator WriteSealedPage(const RPageStorage::RSealedPage &sealedPage, std::size_t bytesPacked)
We pass bytesPacked so that TFile::ls() reports a reasonable value for the compression ratio of the c...
RNTupleLocator CommitClusterGroupImpl(unsigned char *serializedPageList, std::uint32_t length) final
Returns the locator of the page list envelope of the given buffer that contains the serialized page l...
RNTupleLocator CommitSealedPageImpl(ROOT::DescriptorId_t physicalColumnId, const RPageStorage::RSealedPage &sealedPage) final
std::unique_ptr< ROOT::Internal::RNTupleFileWriter > fWriter
std::vector< RNTupleLocator > CommitSealedPageVImpl(std::span< RPageStorage::RSealedPageGroup > ranges, const std::vector< bool > &mask) final
Vector commit of preprocessed pages.
Storage provider that reads ntuple pages from a file.
RNTupleDescriptorBuilder fDescriptorBuilder
The descriptor is created from the header and footer either in AttachImpl or in CreateFromAnchor.
RStructureBuffer fStructureBuffer
Populated by LoadStructureImpl(), reset at the end of Attach()
std::unique_ptr< RRawFile > fFile
An RRawFile is used to request the necessary byte ranges from a local or a remote file.
std::optional< RNTuple > fAnchor
Either provided by CreateFromAnchor, or read from the ROOT file given the ntuple name.
ROOT::Internal::RMiniFileReader fReader
Takes the fFile to read ntuple blobs from it.
std::unique_ptr< ROOT::Internal::RClusterPool > fClusterPool
The cluster pool asynchronously preloads the next few clusters.
Abstract interface to read data from an ntuple.
A page is a slice of a column that is mapped into memory.
Definition RPage.hxx:44
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
The on-storage metadata of an RNTuple.
Addresses a column element or field item relative to a particular cluster, instead of a global NTuple...
Generic information about the physical location of data.
Common user-tunable settings for reading RNTuples.
Common user-tunable settings for storing RNTuples.
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:65
Describe directory structure in memory.
Definition TDirectory.h:45
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
std::uint64_t NTupleSize_t
Integer type long enough to hold the maximum number of entries in a column.
The identifiers that specifies the content of a (partial) cluster.
Definition RCluster.hxx:152
size_t fSize
Total size in bytes of the batch.
std::vector< const RSealedPage * > fSealedPages
The list of pages to commit.
size_t fBytesPacked
Total uncompressed size of the elements in the page batch.
Holds the uncompressed header and footer.
void Reset()
Called at the end of Attach(), i.e. when the header and footer are processed.
std::unique_ptr< unsigned char[]> fBuffer
single buffer for both header and footer
Summarizes cluster-level information that are necessary to load a certain page.
A sealed page contains the bytes of a page as written to storage (packed & compressed).