Logo ROOT   master
Reference Guide
RMiniFile.hxx
Go to the documentation of this file.
1 /// \file ROOT/RMiniFile.hxx
2 /// \ingroup NTuple ROOT7
3 /// \author Jakob Blomer <jblomer@cern.ch>
4 /// \date 2019-12-22
5 /// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6 /// is welcome!
7 
8 /*************************************************************************
9  * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10  * All rights reserved. *
11  * *
12  * For the licensing terms see $ROOTSYS/LICENSE. *
13  * For the list of contributors see $ROOTSYS/README/CREDITS. *
14  *************************************************************************/
15 
16 #ifndef ROOT7_RMiniFile
17 #define ROOT7_RMiniFile
18 
19 #include <ROOT/RNTupleOptions.hxx>
20 #include <ROOT/RStringView.hxx>
21 
22 #include <cstdint>
23 #include <cstdio>
24 #include <memory>
25 #include <string>
26 
27 class TFile;
28 
29 namespace ROOT {
30 
31 namespace Internal {
32 class RRawFile;
33 }
34 
35 namespace Experimental {
36 
37 // clang-format off
38 /**
39 \class ROOT::Experimental::RNTuple
40 \ingroup NTuple
41 \brief Entry point for an RNTuple in a ROOT file
42 
43 The class points to the header and footer keys, which in turn have the references to the pages.
44 Only the RNTuple key will be listed in the list of keys. Like TBaskets, the pages are "invisible" keys.
45 Byte offset references in the RNTuple header and footer reference directly the data part of page records,
46 skipping the TFile key part.
47 
48 While the class is central to anchoring an RNTuple in a TFile, it is an internal detail not exposed to users.
49 Note that there is no user-facing RNTuple class but RNTupleReader and RNTupleWriter.
50 */
51 // clang-format on
52 struct RNTuple {
53  /// Allows for evolving the struct in future versions
54  std::uint32_t fVersion = 0;
55  /// Allows for skipping the struct
56  std::uint32_t fSize = sizeof(RNTuple);
57  /// The file offset of the header excluding the TKey part
58  std::uint64_t fSeekHeader = 0;
59  /// The size of the compressed ntuple header
60  std::uint32_t fNBytesHeader = 0;
61  /// The size of the uncompressed ntuple header
62  std::uint32_t fLenHeader = 0;
63  /// The file offset of the footer excluding the TKey part
64  std::uint64_t fSeekFooter = 0;
65  /// The size of the compressed ntuple footer
66  std::uint32_t fNBytesFooter = 0;
67  /// The size of the uncompressed ntuple footer
68  std::uint32_t fLenFooter = 0;
69  /// Currently unused, reserved for later use
70  std::uint64_t fReserved = 0;
71 
72  /// The canonical, member-wise equality test
73  bool operator ==(const RNTuple &other) const {
74  return fVersion == other.fVersion &&
75  fSize == other.fSize &&
76  fSeekHeader == other.fSeekHeader &&
77  fNBytesHeader == other.fNBytesHeader &&
78  fLenHeader == other.fLenHeader &&
79  fSeekFooter == other.fSeekFooter &&
80  fNBytesFooter == other.fNBytesFooter &&
81  fLenFooter == other.fLenFooter &&
82  fReserved == other.fReserved;
83  }
84 };
85 
86 namespace Internal {
87 
88 /// Holds status information of an open ROOT file during writing
89 struct RTFileControlBlock;
90 
91 // clang-format off
92 /**
93 \class ROOT::Experimental::Internal::RMiniFileReader
94 \ingroup NTuple
95 \brief Read RNTuple data blocks from a TFile container, provided by a RRawFile
96 
97 A RRawFile is used for the byte access. The class implements a minimal subset of TFile, enough to extract
98 RNTuple data keys.
99 */
100 // clang-format on
102 private:
103  /// The raw file used to read byte ranges
105  /// Indicates whether the file is a TFile container or an RNTuple bare file
106  bool fIsBare = false;
107  /// Used when the file container turns out to be a bare file
108  RNTuple GetNTupleBare(std::string_view ntupleName);
109  /// Used when the file turns out to be a TFile container
110  RNTuple GetNTupleProper(std::string_view ntupleName);
111 
112 public:
113  RMiniFileReader() = default;
114  /// Uses the given raw file to read byte ranges
115  explicit RMiniFileReader(ROOT::Internal::RRawFile *rawFile);
116  /// Extracts header and footer location for the RNTuple identified by ntupleName
117  RNTuple GetNTuple(std::string_view ntupleName);
118  /// Reads a given byte range from the file into the provided memory buffer
119  void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset);
120 };
121 
122 
123 // clang-format off
124 /**
125 \class ROOT::Experimental::Internal::RNTupleFileWriter
126 \ingroup NTuple
127 \brief Write RNTuple data blocks in a TFile or a bare file container
128 
129 The writer can create a new TFile container for an RNTuple or add an RNTuple to an existing TFile.
130 Creating a single RNTuple in a new TFile container can be done with a C file stream without a TFile class.
131 Updating an existing TFile requires a proper TFile object. Also, writing a remote file requires a proper TFile object.
132 A stand-alone version of RNTuple can remove the TFile based writer.
133 */
134 // clang-format on
136 private:
137  struct RFileProper {
138  TFile *fFile = nullptr;
139  /// Low-level writing using a TFile
140  void Write(const void *buffer, size_t nbytes, std::int64_t offset);
141  /// Writes an RBlob opaque key with the provided buffer as data record and returns the offset of the record
142  std::uint64_t WriteKey(const void *buffer, size_t nbytes, size_t len);
143  operator bool() const { return fFile; }
144  };
145 
146  struct RFileSimple {
147  /// For the simplest cases, a C file stream can be used for writing
148  FILE *fFile = nullptr;
149  /// Keeps track of the seek offset
150  std::uint64_t fFilePos = 0;
151  /// Keeps track of TFile control structures, which need to be updated on committing the data set
152  std::unique_ptr<ROOT::Experimental::Internal::RTFileControlBlock> fControlBlock;
153 
154  RFileSimple() = default;
155  RFileSimple(const RFileSimple &other) = delete;
156  RFileSimple(RFileSimple &&other) = delete;
157  RFileSimple &operator =(const RFileSimple &other) = delete;
158  RFileSimple &operator =(RFileSimple &&other) = delete;
159  ~RFileSimple();
160 
161  /// Writes bytes in the open stream, either at fFilePos or at the given offset
162  void Write(const void *buffer, size_t nbytes, std::int64_t offset = -1);
163  /// Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the payload.
164  /// The payload is already compressed
165  std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset = -1,
166  std::uint64_t directoryOffset = 100,
167  const std::string &className = "",
168  const std::string &objectName = "",
169  const std::string &title = "");
170  operator bool() const { return fFile; }
171  };
172 
173  // TODO(jblomer): wrap in an std::variant with C++17
174  /// For updating existing files and for storing more than just an RNTuple in the file
176  /// For simple use cases, survives without libRIO dependency
178  /// A simple file can either be written as TFile container or as NTuple bare file
179  bool fIsBare = false;
180  /// The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple
181  /// writers can operate on the same file if (and only if) they use a proper TFile object for writing.
182  std::string fNTupleName;
183  /// The file name without parent directory; only required when writing with a C file stream
184  std::string fFileName;
185  /// Header and footer location of the ntuple, written on Commit()
187 
188  explicit RNTupleFileWriter(std::string_view name);
189 
190  /// For a TFile container written by a C file stream, write the records that constitute an empty file
191  void WriteTFileSkeleton(int defaultCompression);
192  /// For a bare file, which is necessarily written by a C file stream, write file header
193  void WriteBareFileSkeleton(int defaultCompression);
194 
195 public:
196  /// Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
197  /// Uses a C stream for writing
198  static RNTupleFileWriter *Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression,
199  ENTupleContainerFormat containerFormat);
200  /// Create or truncate the local or remote file given by path with the new empty RNTuple identified by ntupleName.
201  /// Creates a new TFile object for writing and hands over ownership of the object to the user.
202  static RNTupleFileWriter *Recreate(std::string_view ntupleName, std::string_view path,
203  std::unique_ptr<TFile> &file);
204  /// Add a new RNTuple identified by ntupleName to the existing TFile.
205  static RNTupleFileWriter *Append(std::string_view ntupleName, TFile &file);
206 
207  RNTupleFileWriter(const RNTupleFileWriter &other) = delete;
208  RNTupleFileWriter(RNTupleFileWriter &&other) = delete;
209  RNTupleFileWriter &operator =(const RNTupleFileWriter &other) = delete;
212 
213  /// Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed header.
214  std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
215  /// Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed footer.
216  std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
217  /// Writes a new record as an RBlob key into the file
218  std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len);
219  /// Writes the RNTuple key to the file so that the header and footer keys can be found
220  void Commit();
221 };
222 
223 } // namespace Internal
224 } // namespace Experimental
225 } // namespace ROOT
226 
227 #endif
RNTuple GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:904
Returns the available number of logical cores.
Definition: StringConv.hxx:21
Describes the options for wrapping RNTuple data in files
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
Definition: RMiniFile.cxx:979
std::string fFileName
The file name without parent directory; only required when writing with a C file stream.
Definition: RMiniFile.hxx:184
std::uint64_t WriteKey(const void *buffer, size_t nbytes, size_t len)
Writes an RBlob opaque key with the provided buffer as data record and returns the offset of the reco...
Definition: RMiniFile.cxx:1051
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:53
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
Definition: RMiniFile.hxx:66
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
Definition: RMiniFile.hxx:186
static RNTupleFileWriter * Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression, ENTupleContainerFormat containerFormat)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName...
Definition: RMiniFile.cxx:1094
bool fIsBare
A simple file can either be written as TFile container or as NTuple bare file.
Definition: RMiniFile.hxx:179
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
Definition: RMiniFile.cxx:1041
std::uint64_t fFilePos
Keeps track of the seek offset.
Definition: RMiniFile.hxx:150
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
Definition: RMiniFile.cxx:1205
std::uint64_t fSeekFooter
The file offset of the footer excluding the TKey part.
Definition: RMiniFile.hxx:64
Write RNTuple data blocks in a TFile or a bare file container
Definition: RMiniFile.hxx:135
void Commit()
Writes the RNTuple key to the file so that the header and footer keys can be found.
Definition: RMiniFile.cxx:1146
std::uint32_t fVersion
Allows for evolving the struct in future versions.
Definition: RMiniFile.hxx:54
Entry point for an RNTuple in a ROOT file
Definition: RMiniFile.hxx:52
RFileSimple fFileSimple
For simple use cases, survives without libRIO dependency.
Definition: RMiniFile.hxx:177
RNTupleFileWriter & operator=(const RNTupleFileWriter &other)=delete
std::uint32_t fSize
Allows for skipping the struct.
Definition: RMiniFile.hxx:56
std::uint64_t fReserved
Currently unused, reserved for later use.
Definition: RMiniFile.hxx:70
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
Definition: RMiniFile.hxx:68
bool fIsBare
Indicates whether the file is a TFile container or an RNTuple bare file.
Definition: RMiniFile.hxx:106
FILE * fFile
For the simplest cases, a C file stream can be used for writing.
Definition: RMiniFile.hxx:148
std::string fNTupleName
The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple wr...
Definition: RMiniFile.hxx:182
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
Definition: RMiniFile.hxx:62
std::unique_ptr< ROOT::Experimental::Internal::RTFileControlBlock > fControlBlock
Keeps track of TFile control structures, which need to be updated on committing the data set...
Definition: RMiniFile.hxx:152
bool operator==(const RNTuple &other) const
The canonical, member-wise equality test.
Definition: RMiniFile.hxx:73
RNTuple GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
Definition: RMiniFile.cxx:962
RNTuple GetNTupleProper(std::string_view ntupleName)
Used when the file turns out to be a TFile container.
Definition: RMiniFile.cxx:915
static RNTupleFileWriter * Append(std::string_view ntupleName, TFile &file)
Add a new RNTuple identified by ntupleName to the existing TFile.
Definition: RMiniFile.cxx:1137
RFileProper fFileProper
For updating existing files and for storing more than just an RNTuple in the file.
Definition: RMiniFile.hxx:175
RFileSimple & operator=(const RFileSimple &other)=delete
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the records that constitute an empty file...
Definition: RMiniFile.cxx:1242
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
Definition: RMiniFile.cxx:1216
std::uint64_t fSeekHeader
The file offset of the header excluding the TKey part.
Definition: RMiniFile.hxx:58
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
Definition: RMiniFile.cxx:1227
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Definition: RMiniFile.hxx:60
Definition: file.py:1
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
Definition: RMiniFile.cxx:1188
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
Definition: RMiniFile.cxx:1012
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
Definition: RMiniFile.cxx:996
Read RNTuple data blocks from a TFile container, provided by a RRawFile
Definition: RMiniFile.hxx:101
The RRawFile provides read-only access to local and remote files.
Definition: RRawFile.hxx:40
ROOT::Internal::RRawFile * fRawFile
The raw file used to read byte ranges.
Definition: RMiniFile.hxx:104
char name[80]
Definition: TGX11.cxx:109