Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleImporter.hxx
Go to the documentation of this file.
1/// \file ROOT/RNTuplerImporter.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2022-11-22
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2022, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RNTuplerImporter
17#define ROOT7_RNTuplerImporter
18
19#include <ROOT/REntry.hxx>
20#include <ROOT/RError.hxx>
21#include <ROOT/RField.hxx>
22#include <ROOT/RNTuple.hxx>
23#include <ROOT/RNTupleModel.hxx>
25#include <string_view>
26
27#include <TFile.h>
28#include <TTree.h>
29
30#include <cstdlib>
31#include <map>
32#include <memory>
33#include <vector>
34
35class TLeaf;
36
37namespace ROOT {
38namespace Experimental {
39
40// clang-format off
41/**
42\class ROOT::Experimental::RNTupleImporter
43\ingroup NTuple
44\brief Converts a TTree into an RNTuple
45
46Example usage (see the ntpl008_import.C tutorial for a full example):
47
48~~~ {.cpp}
49#include <ROOT/RNTupleImporter.hxx>
50using ROOT::Experimental::RNTupleImporter;
51
52auto importer = RNTupleImporter::Create("data.root", "TreeName", "output.root");
53// As required: importer->SetNTupleName(), importer->SetWriteOptions(), ...
54importer->Import();
55~~~
56
57The output file is created if it does not exist, otherwise the ntuple is added to the existing file.
58Note that input file and output file can be identical if the ntuple is stored under a different name than the tree
59(use `SetNTupleName()`).
60
61By default, the RNTuple is compressed with zstd, independent of the input compression. The compression settings
62(and other output parameters) can be changed by `SetWriteOptions()`. For example, to compress the imported RNTuple
63using lz4 (with compression level 4) instead:
64
65~~~ {.cpp}
66auto writeOptions = importer->GetWriteOptions();
67writeOptions.SetCompression(404);
68importer->SetWriteOptions(writeOptions);
69~~~
70
71Most RNTuple fields have a type identical to the corresponding TTree input branch. Exceptions are
72 - C string branches are translated to `std::string` fields
73 - C style arrays are translated to `std::array<...>` fields
74 - Leaf lists are translated to untyped records
75 - Leaf count arrays are translated to anonymous collections with generic names (`_collection0`, `_collection1`, etc.).
76 In order to keep field names and branch names aligned, RNTuple projects the members of these collections and
77 its collection counter to the input branch names. For instance, the following input leafs:
78~~~
79Int_t njets
80float jet_pt[njets]
81float jet_eta[njets]
82~~~
83 will be converted to the following RNTuple schema:
84~~~
85 _collection0 (untyped collection)
86 |- float jet_pt
87 |- float jet_eta
88 std::size_t (RNTupleCardinality) njets (projected from _collection0 without subfields)
89 ROOT::RVec<float> jet_pt (projected from _collection0.jet_pt)
90 ROOT::RVec<float> jet_eta (projected from _collection0.jet_eta)
91~~~
92 These projections are meta-data only operations and don't involve duplicating the data.
93
94Current limitations of the importer:
95 - No support for trees containing TClonesArray collections
96 - Due to RNTuple currently storing data fully split, "don't split" markers are ignored
97 - Some types are not available in RNTuple. Please refer to the
98 [RNTuple specification](https://github.com/root-project/root/blob/master/tree/ntuple/v7/doc/specifications.md) for
99 an overview of all types currently supported.
100*/
101// clang-format on
103public:
104 /// Used to report every ~50MB (compressed), and at the end about the status of the import.
106 public:
107 virtual ~RProgressCallback() = default;
108 void operator()(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
109 {
110 Call(nbytesWritten, neventsWritten);
111 }
112 virtual void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten) = 0;
113 virtual void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten) = 0;
114 };
115
116private:
117 /// By default, compress RNTuple with zstd, level 5
118 static constexpr int kDefaultCompressionSettings = 505;
119
121 RImportBranch() = default;
122 RImportBranch(const RImportBranch &other) = delete;
123 RImportBranch(RImportBranch &&other) = default;
124 RImportBranch &operator=(const RImportBranch &other) = delete;
126 std::string fBranchName; ///< Top-level branch name from the input TTree
127 std::unique_ptr<unsigned char[]> fBranchBuffer; ///< The destination of SetBranchAddress() for `fBranchName`
128 };
129
131 RImportField() = default;
132 ~RImportField() = default;
133 RImportField(const RImportField &other) = delete;
134 RImportField(RImportField &&other) = default;
135 RImportField &operator=(const RImportField &other) = delete;
137
138 /// The field is kept during schema preparation and transferred to the fModel before the writing starts
140 std::unique_ptr<Detail::RFieldBase::RValue> fValue; ///< Set if a value is generated, only for transformed fields
141 void *fFieldBuffer = nullptr; ///< Usually points to the corresponding RImportBranch::fBranchBuffer but not always
142 bool fIsInUntypedCollection = false; ///< Sub-fields of untyped collections (leaf count arrays in the input)
143 bool fIsClass = false; ///< Field imported from a branch with stramer info (e.g., STL, user-defined class)
144 };
145
146 /// Base class to perform data transformations from TTree branches to RNTuple fields if necessary
148 std::size_t fImportBranchIdx = 0;
149 std::size_t fImportFieldIdx = 0;
150
151 RImportTransformation(std::size_t branchIdx, std::size_t fieldIdx)
152 : fImportBranchIdx(branchIdx), fImportFieldIdx(fieldIdx)
153 {
154 }
155 virtual ~RImportTransformation() = default;
156 virtual RResult<void> Transform(const RImportBranch &branch, RImportField &field) = 0;
157 virtual void ResetEntry() = 0; // called at the end of an entry
158 };
159
160 /// When the schema is set up and the import started, it needs to be reset before the next Import() call
161 /// can start. This RAII guard ensures that ResetSchema is called.
164
165 explicit RImportGuard(RNTupleImporter &importer) : fImporter(importer) {}
166 RImportGuard(const RImportGuard &) = delete;
171 };
172
173 /// Leaf count arrays require special treatment. They are translated into RNTuple untyped collections.
174 /// This class does the bookkeeping of the sub-schema for these collections.
181 std::unique_ptr<RNTupleModel> fCollectionModel; ///< The model for the collection itself
182 std::shared_ptr<RCollectionNTupleWriter> fCollectionWriter; ///< Used to fill the collection elements per event
183 std::unique_ptr<REntry> fCollectionEntry; ///< Keeps the memory location of the collection members
184 /// The number of elements for the collection for a particular event. Used as a destination for SetBranchAddress()
185 /// of the count leaf
186 std::unique_ptr<Int_t> fCountVal;
187 std::vector<size_t> fImportFieldIndexes; ///< Points to the correspondings fields in fImportFields
188 /// One transformation for every field, to copy the content of the array one by one
189 std::vector<std::unique_ptr<RImportTransformation>> fTransformations;
190 Int_t fMaxLength = 0; ///< Stores count leaf GetMaximum() to create large enough buffers for the array leafs
191 std::string fFieldName; ///< name of the untyped collection, e.g. `_collection0`, `_collection1`, etc.
192 };
193
194 /// Transform a NULL terminated C string branch into an `std::string` field
196 RCStringTransformation(std::size_t b, std::size_t f) : RImportTransformation(b, f) {}
197 ~RCStringTransformation() override = default;
198 RResult<void> Transform(const RImportBranch &branch, RImportField &field) final;
199 void ResetEntry() final {}
200 };
201
202 /// When writing the elements of a leaf count array, moves the data from the input array one-by-one
203 /// to the memory locations of the fields of the corresponding untyped collection.
204 /// TODO(jblomer): write arrays as a whole to RNTuple
206 std::int64_t fNum = 0;
207 RLeafArrayTransformation(std::size_t b, std::size_t f) : RImportTransformation(b, f) {}
208 ~RLeafArrayTransformation() override = default;
209 RResult<void> Transform(const RImportBranch &branch, RImportField &field) final;
210 void ResetEntry() final { fNum = 0; }
211 };
212
213 RNTupleImporter() = default;
214
215 std::unique_ptr<TFile> fSourceFile;
217
218 std::string fDestFileName;
219 std::string fNTupleName;
220 std::unique_ptr<TFile> fDestFile;
222
223 /// Whether or not dot characters in branch names should be converted to underscores. If this option is not set and a
224 /// branch with a '.' is encountered, the importer will throw an exception.
226
227 /// The maximum number of entries to import. When this value is -1 (default), import all entries.
228 std::int64_t fMaxEntries = -1;
229
230 /// No standard output, conversely if set to false, schema information and progress is printed.
231 bool fIsQuiet = false;
232 std::unique_ptr<RProgressCallback> fProgressCallback;
233
234 std::unique_ptr<RNTupleModel> fModel;
235 std::unique_ptr<REntry> fEntry;
236 std::vector<RImportBranch> fImportBranches;
237 std::vector<RImportField> fImportFields;
238 /// Maps the count leaf to the information about the corresponding untyped collection
239 std::map<std::string, RImportLeafCountCollection> fLeafCountCollections;
240 /// The list of transformations to be performed for every entry
241 std::vector<std::unique_ptr<RImportTransformation>> fImportTransformations;
242
243 ROOT::Experimental::RResult<void> InitDestination(std::string_view destFileName);
244
245 void ResetSchema();
246 /// Sets up the connection from TTree branches to RNTuple fields, including initialization of the memory
247 /// buffers used for reading and writing.
249 void ReportSchema();
250
251public:
252 RNTupleImporter(const RNTupleImporter &other) = delete;
256 ~RNTupleImporter() = default;
257
258 /// Opens the input file for reading and the output file for writing (update).
259 static std::unique_ptr<RNTupleImporter>
260 Create(std::string_view sourceFileName, std::string_view treeName, std::string_view destFileName);
261
262 /// Directly uses the provided tree and opens the output file for writing (update).
263 static std::unique_ptr<RNTupleImporter> Create(TTree *sourceTree, std::string_view destFileName);
264
267 void SetNTupleName(const std::string &name) { fNTupleName = name; }
268 void SetMaxEntries(std::uint64_t maxEntries) { fMaxEntries = maxEntries; };
269
270 /// Whereas branch names may contain dots, RNTuple field names may not. By setting this option, dot characters are
271 /// automatically converted into underscores to prevent the importer from throwing an exception.
273
274 /// Whether or not information and progress is printed to stdout.
275 void SetIsQuiet(bool value) { fIsQuiet = value; }
276
277 /// Import works in two steps:
278 /// 1. PrepareSchema() calls SetBranchAddress() on all the TTree branches and creates the corresponding RNTuple
279 /// fields and the model
280 /// 2. An event loop reads every entry from the TTree, applies transformations where necessary, and writes the
281 /// output entry to the RNTuple.
282 void Import();
283}; // class RNTupleImporter
284
285} // namespace Experimental
286} // namespace ROOT
287
288#endif
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
char name[80]
Definition TGX11.cxx:110
A field translates read and write calls from/to underlying columns to/from tree values.
Definition RField.hxx:85
Used to report every ~50MB (compressed), and at the end about the status of the import.
virtual void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
void operator()(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
virtual void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
Converts a TTree into an RNTuple.
void SetWriteOptions(RNTupleWriteOptions options)
bool fConvertDotsInBranchNames
Whether or not dot characters in branch names should be converted to underscores.
std::int64_t fMaxEntries
The maximum number of entries to import. When this value is -1 (default), import all entries.
std::map< std::string, RImportLeafCountCollection > fLeafCountCollections
Maps the count leaf to the information about the corresponding untyped collection.
RNTupleImporter & operator=(const RNTupleImporter &other)=delete
std::vector< RImportBranch > fImportBranches
void SetNTupleName(const std::string &name)
static constexpr int kDefaultCompressionSettings
By default, compress RNTuple with zstd, level 5.
RNTupleImporter(const RNTupleImporter &other)=delete
void SetConvertDotsInBranchNames(bool value)
Whereas branch names may contain dots, RNTuple field names may not.
RNTupleImporter & operator=(RNTupleImporter &&other)=delete
static std::unique_ptr< RNTupleImporter > Create(std::string_view sourceFileName, std::string_view treeName, std::string_view destFileName)
Opens the input file for reading and the output file for writing (update).
std::unique_ptr< RProgressCallback > fProgressCallback
RNTupleImporter(RNTupleImporter &&other)=delete
RResult< void > PrepareSchema()
Sets up the connection from TTree branches to RNTuple fields, including initialization of the memory ...
ROOT::Experimental::RResult< void > InitDestination(std::string_view destFileName)
void Import()
Import works in two steps:
RNTupleWriteOptions GetWriteOptions() const
bool fIsQuiet
No standard output, conversely if set to false, schema information and progress is printed.
std::vector< RImportField > fImportFields
void SetIsQuiet(bool value)
Whether or not information and progress is printed to stdout.
void SetMaxEntries(std::uint64_t maxEntries)
std::unique_ptr< RNTupleModel > fModel
std::vector< std::unique_ptr< RImportTransformation > > fImportTransformations
The list of transformations to be performed for every entry.
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:207
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition TLeaf.h:57
A TTree represents a columnar dataset.
Definition TTree.h:79
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
Transform a NULL terminated C string branch into an std::string field.
RResult< void > Transform(const RImportBranch &branch, RImportField &field) final
std::string fBranchName
Top-level branch name from the input TTree.
RImportBranch(const RImportBranch &other)=delete
RImportBranch & operator=(RImportBranch &&other)=default
RImportBranch & operator=(const RImportBranch &other)=delete
std::unique_ptr< unsigned char[]> fBranchBuffer
The destination of SetBranchAddress() for fBranchName
RImportBranch(RImportBranch &&other)=default
void * fFieldBuffer
Usually points to the corresponding RImportBranch::fBranchBuffer but not always.
bool fIsClass
Field imported from a branch with stramer info (e.g., STL, user-defined class)
std::unique_ptr< Detail::RFieldBase::RValue > fValue
Set if a value is generated, only for transformed fields.
RImportField(RImportField &&other)=default
bool fIsInUntypedCollection
Sub-fields of untyped collections (leaf count arrays in the input)
RImportField & operator=(const RImportField &other)=delete
RImportField & operator=(RImportField &&other)=default
Detail::RFieldBase * fField
The field is kept during schema preparation and transferred to the fModel before the writing starts.
RImportField(const RImportField &other)=delete
When the schema is set up and the import started, it needs to be reset before the next Import() call ...
RImportGuard & operator=(const RImportGuard &)=delete
RImportGuard & operator=(RImportGuard &&)=delete
std::string fFieldName
name of the untyped collection, e.g. _collection0, _collection1, etc.
Int_t fMaxLength
Stores count leaf GetMaximum() to create large enough buffers for the array leafs.
std::vector< size_t > fImportFieldIndexes
Points to the correspondings fields in fImportFields.
std::unique_ptr< RNTupleModel > fCollectionModel
The model for the collection itself.
RImportLeafCountCollection & operator=(const RImportLeafCountCollection &other)=delete
RImportLeafCountCollection(RImportLeafCountCollection &&other)=default
std::shared_ptr< RCollectionNTupleWriter > fCollectionWriter
Used to fill the collection elements per event.
std::vector< std::unique_ptr< RImportTransformation > > fTransformations
One transformation for every field, to copy the content of the array one by one.
RImportLeafCountCollection(const RImportLeafCountCollection &other)=delete
RImportLeafCountCollection & operator=(RImportLeafCountCollection &&other)=default
std::unique_ptr< Int_t > fCountVal
The number of elements for the collection for a particular event.
std::unique_ptr< REntry > fCollectionEntry
Keeps the memory location of the collection members.
Base class to perform data transformations from TTree branches to RNTuple fields if necessary.
virtual RResult< void > Transform(const RImportBranch &branch, RImportField &field)=0
RImportTransformation(std::size_t branchIdx, std::size_t fieldIdx)
When writing the elements of a leaf count array, moves the data from the input array one-by-one to th...
RResult< void > Transform(const RImportBranch &branch, RImportField &field) final