Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RMiniFile.hxx
Go to the documentation of this file.
1/// \file ROOT/RMiniFile.hxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-12-22
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#ifndef ROOT_RMiniFile
15#define ROOT_RMiniFile
16
17#include <ROOT/RError.hxx>
18#include <ROOT/RNTuple.hxx>
20#include <ROOT/RSpan.hxx>
21#include <Compression.h>
22#include <string_view>
23
24#include <cstdint>
25#include <cstdio>
26#include <memory>
27#include <string>
28
29class TDirectory;
30class TFileMergeInfo;
32
33namespace ROOT {
34
35namespace Internal {
36class RRawFile;
37}
38
39class RNTupleWriteOptions;
40
41namespace Internal {
42/// Holds status information of an open ROOT file during writing
43struct RTFileControlBlock;
44
45// clang-format off
46/**
47\class ROOT::Internal::RMiniFileReader
48\ingroup NTuple
49\brief Read RNTuple data blocks from a TFile container, provided by a RRawFile
50
51A RRawFile is used for the byte access. The class implements a minimal subset of TFile, enough to extract
52RNTuple data keys.
53*/
54// clang-format on
56private:
57 /// The raw file used to read byte ranges
59 /// Indicates whether the file is a TFile container or an RNTuple bare file
60 bool fIsBare = false;
61 /// If `fMaxKeySize > 0` and ReadBuffer attempts to read `nbytes > maxKeySize`, it will assume the
62 /// blob being read is chunked and read all the chunks into the buffer. This is symmetrical to
63 /// what happens in `RNTupleFileWriter::WriteBlob()`.
64 std::uint64_t fMaxKeySize = 0;
65
66 /// Used when the file container turns out to be a bare file
68 /// Used when the file turns out to be a TFile container. The ntuplePath variable is either the ntuple name
69 /// or an ntuple name preceded by a directory (`myNtuple` or `foo/bar/myNtuple` or `/foo/bar/myNtuple`)
71
72 /// Searches for a key with the given name and type in the key index of the directory starting at offsetDir.
73 /// The offset points to the start of the TDirectory DATA section, without the key and without the name and title
74 /// of the TFile record (the root directory).
75 /// Return 0 if the key was not found. Otherwise returns the offset of found key.
76 std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName);
77
78public:
79 RMiniFileReader() = default;
80 /// Uses the given raw file to read byte ranges
82 /// Extracts header and footer location for the RNTuple identified by ntupleName
83 RResult<RNTuple> GetNTuple(std::string_view ntupleName);
84 /// Reads a given byte range from the file into the provided memory buffer.
85 /// If `nbytes > fMaxKeySize` it will perform chunked read from multiple blobs,
86 /// whose addresses are listed at the end of the first chunk.
87 void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset);
88
89 std::uint64_t GetMaxKeySize() const { return fMaxKeySize; }
90 /// If the reader is not used to retrieve the anchor, we need to set the max key size manually
92};
93
94// clang-format off
95/**
96\class ROOT::Internal::RNTupleFileWriter
97\ingroup NTuple
98\brief Write RNTuple data blocks in a TFile or a bare file container
99
100The writer can create a new TFile container for an RNTuple or add an RNTuple to an existing TFile.
101Creating a single RNTuple in a new TFile container can be done with a C file stream without a TFile class.
102Updating an existing TFile requires a proper TFile object. Also, writing a remote file requires a proper TFile object.
103A stand-alone version of RNTuple can remove the TFile based writer.
104*/
105// clang-format on
107public:
108 /// The key length of a blob. It is always a big key (version > 1000) with class name RBlob.
109 static constexpr std::size_t kBlobKeyLen = 42;
110
111private:
112 struct RFileProper {
113 /// A sub directory in fFile or nullptr if the data is stored in the root directory of the file
115 /// Low-level writing using a TFile
116 void Write(const void *buffer, size_t nbytes, std::int64_t offset);
117 /// Reserves an RBlob opaque key as data record and returns the offset of the record. If keyBuffer is specified,
118 /// it must be written *before* the returned offset. (Note that the array type is purely documentation, the
119 /// argument is actually just a pointer.)
120 std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen] = nullptr);
121 operator bool() const { return fDirectory; }
122 };
123
124 struct RFileSimple {
125 /// Direct I/O requires that all buffers and write lengths are aligned. It seems 512 byte alignment is the minimum
126 /// for Direct I/O to work, but further testing showed that it results in worse performance than 4kB.
127 static constexpr int kBlockAlign = 4096;
128 /// During commit, WriteTFileKeysList() updates fNBytesKeys and fSeekKeys of the RTFFile located at
129 /// fSeekFileRecord. Given that the TFile key starts at offset 100 and the file name, which is written twice,
130 /// is shorter than 255 characters, we should need at most ~600 bytes. However, the header also needs to be
131 /// aligned to kBlockAlign...
132 static constexpr std::size_t kHeaderBlockSize = 4096;
133
134 // fHeaderBlock and fBlock are raw pointers because we have to manually call operator new and delete.
135 unsigned char *fHeaderBlock = nullptr;
136 std::size_t fBlockSize = 0;
137 std::uint64_t fBlockOffset = 0;
138 unsigned char *fBlock = nullptr;
139
140 /// For the simplest cases, a C file stream can be used for writing
141 FILE *fFile = nullptr;
142 /// Whether the C file stream has been opened with Direct I/O, introducing alignment requirements.
143 bool fDirectIO = false;
144 /// Keeps track of the seek offset
145 std::uint64_t fFilePos = 0;
146 /// Keeps track of the next key offset
147 std::uint64_t fKeyOffset = 0;
148 /// Keeps track of TFile control structures, which need to be updated on committing the data set
149 std::unique_ptr<ROOT::Internal::RTFileControlBlock> fControlBlock;
150
152 RFileSimple(const RFileSimple &other) = delete;
156 ~RFileSimple();
157
158 void AllocateBuffers(std::size_t bufferSize);
159 void Flush();
160
161 /// Writes bytes in the open stream, either at fFilePos or at the given offset
162 void Write(const void *buffer, size_t nbytes, std::int64_t offset = -1);
163 /// Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the payload.
164 /// The payload is already compressed
165 std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset = -1,
166 std::uint64_t directoryOffset = 100, const std::string &className = "",
167 const std::string &objectName = "", const std::string &title = "");
168 /// Reserves an RBlob opaque key as data record and returns the offset of the record. If keyBuffer is specified,
169 /// it must be written *before* the returned offset. (Note that the array type is purely documentation, the
170 /// argument is actually just a pointer.)
171 std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen] = nullptr);
172 operator bool() const { return fFile; }
173 };
174
175 /// RFileSimple: for simple use cases, survives without libRIO dependency
176 /// RFileProper: for updating existing files and for storing more than just an RNTuple in the file
177 std::variant<RFileSimple, RFileProper> fFile;
178 /// A simple file can either be written as TFile container or as NTuple bare file
179 bool fIsBare = false;
180 /// The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple
181 /// writers can operate on the same file if (and only if) they use a proper TFile object for writing.
182 std::string fNTupleName;
183 /// The file name without parent directory; only required when writing with a C file stream
184 std::string fFileName;
185 /// Header and footer location of the ntuple, written on Commit()
187 /// Set of streamer info records that should be written to the file.
188 /// The RNTuple class description is always present.
190
191 explicit RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize);
192
193 /// For a TFile container written by a C file stream, write the header and TFile object
195 /// The only key that will be visible in file->ls()
196 /// Returns the size on disk of the anchor object
197 std::uint64_t WriteTFileNTupleKey(int compression);
198 /// Write the TList with the RNTuple key
199 void WriteTFileKeysList(std::uint64_t anchorSize);
200 /// Write the compressed streamer info record with the description of the RNTuple class
202 /// Last record in the file
203 void WriteTFileFreeList();
204 /// For a bare file, which is necessarily written by a C file stream, write file header
206
207public:
208 /// For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file
209 enum class EContainerFormat {
210 kTFile, // ROOT TFile
211 kBare, // A thin envelope supporting a single RNTuple only
212 };
213
214 /// Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
215 /// Uses a C stream for writing
216 static std::unique_ptr<RNTupleFileWriter> Recreate(std::string_view ntupleName, std::string_view path,
218 const ROOT::RNTupleWriteOptions &options);
219 /// The directory parameter can also be a TFile object (TFile inherits from TDirectory).
220 static std::unique_ptr<RNTupleFileWriter>
221 Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize);
222
228
229 /// Seek a simple writer to offset. Note that previous data is not flushed immediately, but only by the next write
230 /// (if necessary).
231 void Seek(std::uint64_t offset);
232
233 /// Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed header.
234 std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
235 /// Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed footer.
236 std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
237 /// Writes a new record as an RBlob key into the file
238 std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len);
239
240 /// Prepares buffer for a new record as an RBlob key at offset.
241 /// (Note that the array type is purely documentation, the argument is actually just a pointer.)
242 static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen]);
243
244 /// Reserves a new record as an RBlob key in the file. If keyBuffer is specified, it must be written *before* the
245 /// returned offset. (Note that the array type is purely documentation, the argument is actually just a pointer.)
246 std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen] = nullptr);
247 /// Write into a reserved record; the caller is responsible for making sure that the written byte range is in the
248 /// previously reserved key.
249 void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset);
250 /// Ensures that the streamer info records passed as argument are written to the file
252 /// Writes the RNTuple key to the file so that the header and footer keys can be found
254};
255
256} // namespace Internal
257} // namespace ROOT
258
259#endif
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:110
Read RNTuple data blocks from a TFile container, provided by a RRawFile.
Definition RMiniFile.hxx:55
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
std::uint64_t GetMaxKeySize() const
Definition RMiniFile.hxx:89
void SetMaxKeySize(std::uint64_t maxKeySize)
If the reader is not used to retrieve the anchor, we need to set the max key size manually.
Definition RMiniFile.hxx:91
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName)
Searches for a key with the given name and type in the key index of the directory starting at offsetD...
ROOT::Internal::RRawFile * fRawFile
The raw file used to read byte ranges.
Definition RMiniFile.hxx:58
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
RResult< RNTuple > GetNTupleProper(std::string_view ntuplePath)
Used when the file turns out to be a TFile container.
bool fIsBare
Indicates whether the file is a TFile container or an RNTuple bare file.
Definition RMiniFile.hxx:60
std::uint64_t fMaxKeySize
If fMaxKeySize > 0 and ReadBuffer attempts to read nbytes > maxKeySize, it will assume the blob being...
Definition RMiniFile.hxx:64
Write RNTuple data blocks in a TFile or a bare file container.
std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves a new record as an RBlob key in the file.
void WriteTFileStreamerInfo(int compression)
Write the compressed streamer info record with the description of the RNTuple class.
std::string fNTupleName
The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple wr...
RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize)
void WriteTFileKeysList(std::uint64_t anchorSize)
Write the TList with the RNTuple key.
std::uint64_t WriteTFileNTupleKey(int compression)
The only key that will be visible in file->ls() Returns the size on disk of the anchor object.
RNTupleFileWriter(const RNTupleFileWriter &other)=delete
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
RNTupleFileWriter & operator=(const RNTupleFileWriter &other)=delete
void Commit(int compression=RCompressionSetting::EDefaults::kUseGeneralPurpose)
Writes the RNTuple key to the file so that the header and footer keys can be found.
RNTupleFileWriter(RNTupleFileWriter &&other)=delete
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
std::string fFileName
The file name without parent directory; only required when writing with a C file stream.
void WriteTFileFreeList()
Last record in the file.
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the header and TFile object.
void Seek(std::uint64_t offset)
Seek a simple writer to offset.
std::variant< RFileSimple, RFileProper > fFile
RFileSimple: for simple use cases, survives without libRIO dependency RFileProper: for updating exist...
bool fIsBare
A simple file can either be written as TFile container or as NTuple bare file.
ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fStreamerInfoMap
Set of streamer info records that should be written to the file.
RNTupleFileWriter & operator=(RNTupleFileWriter &&other)=delete
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
static std::unique_ptr< RNTupleFileWriter > Recreate(std::string_view ntupleName, std::string_view path, EContainerFormat containerFormat, const ROOT::RNTupleWriteOptions &options)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
Write into a reserved record; the caller is responsible for making sure that the written byte range i...
static constexpr std::size_t kBlobKeyLen
The key length of a blob. It is always a big key (version > 1000) with class name RBlob.
static std::unique_ptr< RNTupleFileWriter > Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize)
The directory parameter can also be a TFile object (TFile inherits from TDirectory).
static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
Prepares buffer for a new record as an RBlob key at offset.
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
void UpdateStreamerInfos(const ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
Ensures that the streamer info records passed as argument are written to the file.
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
EContainerFormat
For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
Common user-tunable settings for storing RNTuples.
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:65
Describe directory structure in memory.
Definition TDirectory.h:45
Abstract Interface class describing Streamer information for one class.
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
TDirectory * fDirectory
A sub directory in fFile or nullptr if the data is stored in the root directory of the file.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
std::uint64_t fFilePos
Keeps track of the seek offset.
static constexpr int kBlockAlign
Direct I/O requires that all buffers and write lengths are aligned.
RFileSimple & operator=(RFileSimple &&other)=delete
std::uint64_t fKeyOffset
Keeps track of the next key offset.
bool fDirectIO
Whether the C file stream has been opened with Direct I/O, introducing alignment requirements.
RFileSimple & operator=(const RFileSimple &other)=delete
void AllocateBuffers(std::size_t bufferSize)
FILE * fFile
For the simplest cases, a C file stream can be used for writing.
std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
static constexpr std::size_t kHeaderBlockSize
During commit, WriteTFileKeysList() updates fNBytesKeys and fSeekKeys of the RTFFile located at fSeek...
std::unique_ptr< ROOT::Internal::RTFileControlBlock > fControlBlock
Keeps track of TFile control structures, which need to be updated on committing the data set.
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
RFileSimple(const RFileSimple &other)=delete
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
@ kUseGeneralPurpose
Use the new recommended general-purpose setting; it is a best trade-off between compression ratio/dec...
Definition Compression.h:58