Logo ROOT  
Reference Guide
RMiniFile.hxx
Go to the documentation of this file.
1 /// \file ROOT/RMiniFile.hxx
2 /// \ingroup NTuple ROOT7
3 /// \author Jakob Blomer <jblomer@cern.ch>
4 /// \date 2019-12-22
5 /// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6 /// is welcome!
7 
8 /*************************************************************************
9  * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10  * All rights reserved. *
11  * *
12  * For the licensing terms see $ROOTSYS/LICENSE. *
13  * For the list of contributors see $ROOTSYS/README/CREDITS. *
14  *************************************************************************/
15 
16 #ifndef ROOT7_RMiniFile
17 #define ROOT7_RMiniFile
18 
19 #include <ROOT/RError.hxx>
20 #include <ROOT/RNTupleOptions.hxx>
21 #include <ROOT/RStringView.hxx>
22 
23 #include <cstdint>
24 #include <cstdio>
25 #include <memory>
26 #include <string>
27 
28 class TCollection;
29 class TFile;
30 class TFileMergeInfo;
31 
32 namespace ROOT {
33 
34 namespace Internal {
35 class RRawFile;
36 }
37 
38 namespace Experimental {
39 
40 // clang-format off
41 /**
42 \class ROOT::Experimental::RNTuple
43 \ingroup NTuple
44 \brief Entry point for an RNTuple in a ROOT file
45 
46 The class points to the header and footer keys, which in turn have the references to the pages.
47 Only the RNTuple key will be listed in the list of keys. Like TBaskets, the pages are "invisible" keys.
48 Byte offset references in the RNTuple header and footer reference directly the data part of page records,
49 skipping the TFile key part.
50 
51 While the class is central to anchoring an RNTuple in a TFile, it is an internal detail not exposed to users.
52 Note that there is no user-facing RNTuple class but RNTupleReader and RNTupleWriter.
53 */
54 // clang-format on
55 struct RNTuple {
56  /// Allows for evolving the struct in future versions
57  std::uint32_t fVersion = 0;
58  /// Allows for skipping the struct
59  std::uint32_t fSize = sizeof(RNTuple);
60  /// The file offset of the header excluding the TKey part
61  std::uint64_t fSeekHeader = 0;
62  /// The size of the compressed ntuple header
63  std::uint32_t fNBytesHeader = 0;
64  /// The size of the uncompressed ntuple header
65  std::uint32_t fLenHeader = 0;
66  /// The file offset of the footer excluding the TKey part
67  std::uint64_t fSeekFooter = 0;
68  /// The size of the compressed ntuple footer
69  std::uint32_t fNBytesFooter = 0;
70  /// The size of the uncompressed ntuple footer
71  std::uint32_t fLenFooter = 0;
72  /// Currently unused, reserved for later use
73  std::uint64_t fReserved = 0;
74 
75  /// The canonical, member-wise equality test
76  bool operator ==(const RNTuple &other) const {
77  return fVersion == other.fVersion &&
78  fSize == other.fSize &&
79  fSeekHeader == other.fSeekHeader &&
80  fNBytesHeader == other.fNBytesHeader &&
81  fLenHeader == other.fLenHeader &&
82  fSeekFooter == other.fSeekFooter &&
83  fNBytesFooter == other.fNBytesFooter &&
84  fLenFooter == other.fLenFooter &&
85  fReserved == other.fReserved;
86  }
87 
88  // RNTuple implements the hadd MergeFile interface
89  /// Merge this NTuple with the input list entries
90  Long64_t Merge(TCollection *input, TFileMergeInfo *mergeInfo);
91 };
92 
93 namespace Internal {
94 
95 /// Holds status information of an open ROOT file during writing
96 struct RTFileControlBlock;
97 
98 // clang-format off
99 /**
100 \class ROOT::Experimental::Internal::RMiniFileReader
101 \ingroup NTuple
102 \brief Read RNTuple data blocks from a TFile container, provided by a RRawFile
103 
104 A RRawFile is used for the byte access. The class implements a minimal subset of TFile, enough to extract
105 RNTuple data keys.
106 */
107 // clang-format on
109 private:
110  /// The raw file used to read byte ranges
112  /// Indicates whether the file is a TFile container or an RNTuple bare file
113  bool fIsBare = false;
114  /// Used when the file container turns out to be a bare file
116  /// Used when the file turns out to be a TFile container
118 
119 public:
120  RMiniFileReader() = default;
121  /// Uses the given raw file to read byte ranges
122  explicit RMiniFileReader(ROOT::Internal::RRawFile *rawFile);
123  /// Extracts header and footer location for the RNTuple identified by ntupleName
125  /// Reads a given byte range from the file into the provided memory buffer
126  void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset);
127 };
128 
129 
130 // clang-format off
131 /**
132 \class ROOT::Experimental::Internal::RNTupleFileWriter
133 \ingroup NTuple
134 \brief Write RNTuple data blocks in a TFile or a bare file container
135 
136 The writer can create a new TFile container for an RNTuple or add an RNTuple to an existing TFile.
137 Creating a single RNTuple in a new TFile container can be done with a C file stream without a TFile class.
138 Updating an existing TFile requires a proper TFile object. Also, writing a remote file requires a proper TFile object.
139 A stand-alone version of RNTuple can remove the TFile based writer.
140 */
141 // clang-format on
143 private:
144  struct RFileProper {
145  TFile *fFile = nullptr;
146  /// Low-level writing using a TFile
147  void Write(const void *buffer, size_t nbytes, std::int64_t offset);
148  /// Writes an RBlob opaque key with the provided buffer as data record and returns the offset of the record
149  std::uint64_t WriteKey(const void *buffer, size_t nbytes, size_t len);
150  operator bool() const { return fFile; }
151  };
152 
153  struct RFileSimple {
154  /// For the simplest cases, a C file stream can be used for writing
155  FILE *fFile = nullptr;
156  /// Keeps track of the seek offset
157  std::uint64_t fFilePos = 0;
158  /// Keeps track of TFile control structures, which need to be updated on committing the data set
159  std::unique_ptr<ROOT::Experimental::Internal::RTFileControlBlock> fControlBlock;
160 
161  RFileSimple() = default;
162  RFileSimple(const RFileSimple &other) = delete;
163  RFileSimple(RFileSimple &&other) = delete;
164  RFileSimple &operator =(const RFileSimple &other) = delete;
165  RFileSimple &operator =(RFileSimple &&other) = delete;
166  ~RFileSimple();
167 
168  /// Writes bytes in the open stream, either at fFilePos or at the given offset
169  void Write(const void *buffer, size_t nbytes, std::int64_t offset = -1);
170  /// Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the payload.
171  /// The payload is already compressed
172  std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset = -1,
173  std::uint64_t directoryOffset = 100,
174  const std::string &className = "",
175  const std::string &objectName = "",
176  const std::string &title = "");
177  operator bool() const { return fFile; }
178  };
179 
180  // TODO(jblomer): wrap in an std::variant with C++17
181  /// For updating existing files and for storing more than just an RNTuple in the file
183  /// For simple use cases, survives without libRIO dependency
185  /// A simple file can either be written as TFile container or as NTuple bare file
186  bool fIsBare = false;
187  /// The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple
188  /// writers can operate on the same file if (and only if) they use a proper TFile object for writing.
189  std::string fNTupleName;
190  /// The file name without parent directory; only required when writing with a C file stream
191  std::string fFileName;
192  /// Header and footer location of the ntuple, written on Commit()
194 
196 
197  /// For a TFile container written by a C file stream, write the records that constitute an empty file
198  void WriteTFileSkeleton(int defaultCompression);
199  /// For a bare file, which is necessarily written by a C file stream, write file header
200  void WriteBareFileSkeleton(int defaultCompression);
201 
202 public:
203  /// Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
204  /// Uses a C stream for writing
205  static RNTupleFileWriter *Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression,
206  ENTupleContainerFormat containerFormat);
207  /// Create or truncate the local or remote file given by path with the new empty RNTuple identified by ntupleName.
208  /// Creates a new TFile object for writing and hands over ownership of the object to the user.
210  std::unique_ptr<TFile> &file);
211  /// Add a new RNTuple identified by ntupleName to the existing TFile.
212  static RNTupleFileWriter *Append(std::string_view ntupleName, TFile &file);
213 
214  RNTupleFileWriter(const RNTupleFileWriter &other) = delete;
219 
220  /// Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed header.
221  std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader);
222  /// Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed footer.
223  std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter);
224  /// Writes a new record as an RBlob key into the file
225  std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len);
226  /// Writes the RNTuple key to the file so that the header and footer keys can be found
227  void Commit();
228 };
229 
230 } // namespace Internal
231 } // namespace Experimental
232 } // namespace ROOT
233 
234 #endif
ROOT::Experimental::Internal::RNTupleFileWriter::fFileName
std::string fFileName
The file name without parent directory; only required when writing with a C file stream.
Definition: RMiniFile.hxx:191
TFileMergeInfo
Definition: TFileMergeInfo.h:42
ROOT::Experimental::Internal::RMiniFileReader::fRawFile
ROOT::Internal::RRawFile * fRawFile
The raw file used to read byte ranges.
Definition: RMiniFile.hxx:111
ROOT::Experimental::Internal::RNTupleFileWriter::Append
static RNTupleFileWriter * Append(std::string_view ntupleName, TFile &file)
Add a new RNTuple identified by ntupleName to the existing TFile.
Definition: RMiniFile.cxx:1174
ROOT::Experimental::Internal::RNTupleFileWriter
Write RNTuple data blocks in a TFile or a bare file container.
Definition: RMiniFile.hxx:142
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::RFileSimple
RFileSimple(const RFileSimple &other)=delete
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple
Definition: RMiniFile.hxx:153
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
ROOT::Experimental::Internal::RNTupleFileWriter::RNTupleFileWriter
RNTupleFileWriter(RNTupleFileWriter &&other)=delete
string_view
basic_string_view< char > string_view
Definition: libcpp_string_view.h:785
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::~RFileSimple
~RFileSimple()
Definition: RMiniFile.cxx:1018
ROOT::Experimental::Internal::RNTupleFileWriter::RNTupleFileWriter
RNTupleFileWriter(const RNTupleFileWriter &other)=delete
ROOT::Internal::RRawFile
The RRawFile provides read-only access to local and remote files.
Definition: RRawFile.hxx:43
ROOT::Experimental::RNTuple::fSeekHeader
std::uint64_t fSeekHeader
The file offset of the header excluding the TKey part.
Definition: RMiniFile.hxx:61
ROOT::Experimental::Internal::RNTupleFileWriter::WriteTFileSkeleton
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the records that constitute an empty file.
Definition: RMiniFile.cxx:1279
ROOT::Experimental::Internal::RNTupleFileWriter::fFileSimple
RFileSimple fFileSimple
For simple use cases, survives without libRIO dependency.
Definition: RMiniFile.hxx:184
ROOT::Experimental::Internal::RNTupleFileWriter::~RNTupleFileWriter
~RNTupleFileWriter()
Definition: RMiniFile.cxx:1122
ROOT::Experimental::Internal::RNTupleFileWriter::RFileProper::fFile
TFile * fFile
Definition: RMiniFile.hxx:145
ROOT::Experimental::RNTuple::fSeekFooter
std::uint64_t fSeekFooter
The file offset of the footer excluding the TKey part.
Definition: RMiniFile.hxx:67
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::WriteKey
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
Definition: RMiniFile.cxx:1045
ROOT::Experimental::RNTuple::fReserved
std::uint64_t fReserved
Currently unused, reserved for later use.
Definition: RMiniFile.hxx:73
ROOT::Experimental::Internal::RMiniFileReader::ReadBuffer
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
Definition: RMiniFile.cxx:1008
ROOT::Experimental::Internal::RNTupleFileWriter::WriteNTupleFooter
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
Definition: RMiniFile.cxx:1253
ROOT::Experimental::Internal::RMiniFileReader::GetNTupleBare
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
Definition: RMiniFile.cxx:986
bool
ROOT::Experimental::RNTuple::fNBytesHeader
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Definition: RMiniFile.hxx:63
ROOT::Experimental::Internal::RNTupleFileWriter::WriteBlob
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
Definition: RMiniFile.cxx:1225
ROOT::Experimental::RResult
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition: RError.hxx:196
ROOT::Experimental::RNTuple::fNBytesFooter
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
Definition: RMiniFile.hxx:69
ROOT::Experimental::Internal::RNTupleFileWriter::fFileProper
RFileProper fFileProper
For updating existing files and for storing more than just an RNTuple in the file.
Definition: RMiniFile.hxx:182
ROOT::Experimental::ENTupleContainerFormat
ENTupleContainerFormat
Definition: RNTupleOptions.hxx:31
ROOT::Experimental::Internal::RNTupleFileWriter::RFileProper
Definition: RMiniFile.hxx:144
ROOT::Experimental::Internal::RNTupleFileWriter::fIsBare
bool fIsBare
A simple file can either be written as TFile container or as NTuple bare file.
Definition: RMiniFile.hxx:186
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::Write
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
Definition: RMiniFile.cxx:1025
RStringView.hxx
ROOT::Experimental::Internal::RMiniFileReader::GetNTuple
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:923
ROOT::Experimental::Internal::RNTupleFileWriter::WriteBareFileSkeleton
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
Definition: RMiniFile.cxx:1264
ROOT::Experimental::RNTuple::fVersion
std::uint32_t fVersion
Allows for evolving the struct in future versions.
Definition: RMiniFile.hxx:57
ROOT::Experimental::RNTuple::fSize
std::uint32_t fSize
Allows for skipping the struct.
Definition: RMiniFile.hxx:59
ROOT::Experimental::Internal::RMiniFileReader::fIsBare
bool fIsBare
Indicates whether the file is a TFile container or an RNTuple bare file.
Definition: RMiniFile.hxx:113
ROOT::Experimental::Internal::RNTupleFileWriter::WriteNTupleHeader
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
Definition: RMiniFile.cxx:1242
ROOT::Experimental::RNTuple::fLenFooter
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
Definition: RMiniFile.hxx:71
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::fFile
FILE * fFile
For the simplest cases, a C file stream can be used for writing.
Definition: RMiniFile.hxx:155
ROOT::Experimental::Internal::RMiniFileReader::GetNTupleProper
RResult< RNTuple > GetNTupleProper(std::string_view ntupleName)
Used when the file turns out to be a TFile container.
Definition: RMiniFile.cxx:935
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::fControlBlock
std::unique_ptr< ROOT::Experimental::Internal::RTFileControlBlock > fControlBlock
Keeps track of TFile control structures, which need to be updated on committing the data set.
Definition: RMiniFile.hxx:159
TFile
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format.
Definition: TFile.h:54
ROOT::Experimental::RNTuple::fLenHeader
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
Definition: RMiniFile.hxx:65
ROOT::Experimental::Internal::RMiniFileReader::RMiniFileReader
RMiniFileReader()=default
ROOT::Experimental::Internal::RNTupleFileWriter::RFileProper::Write
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
Definition: RMiniFile.cxx:1074
ROOT::Experimental::Internal::RNTupleFileWriter::fNTupleName
std::string fNTupleName
The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple wr...
Definition: RMiniFile.hxx:189
ROOT::Experimental::RNTuple
Entry point for an RNTuple in a ROOT file.
Definition: RMiniFile.hxx:55
ROOT::Experimental::Internal::RNTupleFileWriter::RNTupleFileWriter
RNTupleFileWriter(std::string_view name)
Definition: RMiniFile.cxx:1115
ROOT::Experimental::Internal::RNTupleFileWriter::operator=
RNTupleFileWriter & operator=(const RNTupleFileWriter &other)=delete
file
Definition: file.py:1
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::fFilePos
std::uint64_t fFilePos
Keeps track of the seek offset.
Definition: RMiniFile.hxx:157
ROOT::Experimental::Internal::RNTupleFileWriter::Commit
void Commit()
Writes the RNTuple key to the file so that the header and footer keys can be found.
Definition: RMiniFile.cxx:1183
name
char name[80]
Definition: TGX11.cxx:110
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::RFileSimple
RFileSimple(RFileSimple &&other)=delete
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::operator=
RFileSimple & operator=(const RFileSimple &other)=delete
RError.hxx
ROOT::Experimental::Internal::RNTupleFileWriter::Recreate
static RNTupleFileWriter * Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression, ENTupleContainerFormat containerFormat)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:1127
TCollection
Collection abstract base class.
Definition: TCollection.h:63
ROOT::Experimental::Internal::RMiniFileReader
Read RNTuple data blocks from a TFile container, provided by a RRawFile.
Definition: RMiniFile.hxx:108
RNTupleOptions.hxx
ROOT::Experimental::Internal::RNTupleFileWriter::fNTupleAnchor
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
Definition: RMiniFile.hxx:193
ROOT
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Definition: EExecutionPolicy.hxx:4
ROOT::Experimental::Internal::RNTupleFileWriter::RFileProper::WriteKey
std::uint64_t WriteKey(const void *buffer, size_t nbytes, size_t len)
Writes an RBlob opaque key with the provided buffer as data record and returns the offset of the reco...
Definition: RMiniFile.cxx:1084
ROOT::Experimental::RNTuple::Merge
Long64_t Merge(TCollection *input, TFileMergeInfo *mergeInfo)
Merge this NTuple with the input list entries.
Definition: RNTupleMerger.cxx:22
ROOT::Experimental::RNTuple::operator==
bool operator==(const RNTuple &other) const
The canonical, member-wise equality test.
Definition: RMiniFile.hxx:76
ROOT::Experimental::Internal::RNTupleFileWriter::RFileSimple::RFileSimple
RFileSimple()=default