Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleImporter.hxx
Go to the documentation of this file.
1/// \file ROOT/RNTuplerImporter.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2022-11-22
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2022, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RNTuplerImporter
17#define ROOT7_RNTuplerImporter
18
19#include <ROOT/REntry.hxx>
20#include <ROOT/RError.hxx>
21#include <ROOT/RField.hxx>
23#include <ROOT/RNTupleModel.hxx>
26#include <string_view>
27
28#include <TFile.h>
29#include <TTree.h>
30
31#include <cstdlib>
32#include <map>
33#include <memory>
34#include <vector>
35
36class TLeaf;
37
38namespace ROOT {
39namespace Experimental {
40
41// clang-format off
42/**
43\class ROOT::Experimental::RNTupleImporter
44\ingroup NTuple
45\brief Converts a TTree into an RNTuple
46
47Example usage (see the ntpl008_import.C tutorial for a full example):
48
49~~~ {.cpp}
50#include <ROOT/RNTupleImporter.hxx>
51using ROOT::Experimental::RNTupleImporter;
52
53auto importer = RNTupleImporter::Create("data.root", "TreeName", "output.root");
54// As required: importer->SetNTupleName(), importer->SetWriteOptions(), ...
55importer->Import();
56~~~
57
58The output file is created if it does not exist, otherwise the ntuple is added to the existing file.
59Note that input file and output file can be identical if the ntuple is stored under a different name than the tree
60(use `SetNTupleName()`).
61
62By default, the RNTuple is compressed with zstd, independent of the input compression. The compression settings
63(and other output parameters) can be changed by `SetWriteOptions()`. For example, to compress the imported RNTuple
64using lz4 (with compression level 4) instead:
65
66~~~ {.cpp}
67auto writeOptions = importer->GetWriteOptions();
68writeOptions.SetCompression(404);
69importer->SetWriteOptions(writeOptions);
70~~~
71
72Most RNTuple fields have a type identical to the corresponding TTree input branch. Exceptions are
73 - C string branches are translated to `std::string` fields
74 - C style arrays are translated to `std::array<...>` fields
75 - Leaf lists are translated to untyped records
76 - Leaf count arrays are translated to anonymous collections with generic names (`_collection0`, `_collection1`, etc.).
77 In order to keep field names and branch names aligned, RNTuple projects the members of these collections and
78 its collection counter to the input branch names. For instance, the following input leafs:
79~~~
80Int_t njets
81float jet_pt[njets]
82float jet_eta[njets]
83~~~
84 will be converted to the following RNTuple schema:
85~~~
86 _collection0 (untyped collection)
87 |- float jet_pt
88 |- float jet_eta
89 std::size_t (RNTupleCardinality) njets (projected from _collection0 without subfields)
90 ROOT::RVec<float> jet_pt (projected from _collection0.jet_pt)
91 ROOT::RVec<float> jet_eta (projected from _collection0.jet_eta)
92~~~
93 These projections are meta-data only operations and don't involve duplicating the data.
94
95Current limitations of the importer:
96 - No support for trees containing TClonesArray collections
97 - Due to RNTuple currently storing data fully split, "don't split" markers are ignored
98 - Some types are not available in RNTuple. Please refer to the
99 [RNTuple specification](https://github.com/root-project/root/blob/master/tree/ntuple/v7/doc/specifications.md) for
100 an overview of all types currently supported.
101*/
102// clang-format on
104public:
105 /// Used to report every ~50MB (compressed), and at the end about the status of the import.
107 public:
108 virtual ~RProgressCallback() = default;
109 void operator()(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
110 {
111 Call(nbytesWritten, neventsWritten);
112 }
113 virtual void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten) = 0;
114 virtual void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten) = 0;
115 };
116
117private:
119 RImportBranch() = default;
120 RImportBranch(const RImportBranch &other) = delete;
121 RImportBranch(RImportBranch &&other) = default;
122 RImportBranch &operator=(const RImportBranch &other) = delete;
124 std::string fBranchName; ///< Top-level branch name from the input TTree
125 std::unique_ptr<unsigned char[]> fBranchBuffer; ///< The destination of SetBranchAddress() for `fBranchName`
126 };
127
129 RImportField() = default;
130 ~RImportField() = default;
131 RImportField(const RImportField &other) = delete;
132 RImportField(RImportField &&other) = default;
133 RImportField &operator=(const RImportField &other) = delete;
135
136 /// The field is kept during schema preparation and transferred to the fModel before the writing starts
137 RFieldBase *fField = nullptr;
138 std::unique_ptr<RFieldBase::RValue> fValue; ///< Set if a value is generated, only for transformed fields
139 void *fFieldBuffer = nullptr; ///< Usually points to the corresponding RImportBranch::fBranchBuffer but not always
140 bool fIsInUntypedCollection = false; ///< Sub-fields of untyped collections (leaf count arrays in the input)
141 bool fIsClass = false; ///< Field imported from a branch with stramer info (e.g., STL, user-defined class)
142 };
143
144 /// Base class to perform data transformations from TTree branches to RNTuple fields if necessary
146 std::size_t fImportBranchIdx = 0;
147 std::size_t fImportFieldIdx = 0;
148
149 RImportTransformation(std::size_t branchIdx, std::size_t fieldIdx)
150 : fImportBranchIdx(branchIdx), fImportFieldIdx(fieldIdx)
151 {
152 }
153 virtual ~RImportTransformation() = default;
154 virtual RResult<void> Transform(const RImportBranch &branch, RImportField &field) = 0;
155 virtual void ResetEntry() = 0; // called at the end of an entry
156 };
157
158 /// When the schema is set up and the import started, it needs to be reset before the next Import() call
159 /// can start. This RAII guard ensures that ResetSchema is called.
162
163 explicit RImportGuard(RNTupleImporter &importer) : fImporter(importer) {}
164 RImportGuard(const RImportGuard &) = delete;
169 };
170
171 /// Leaf count arrays require special treatment. They are translated into RNTuple untyped collections.
172 /// This class does the bookkeeping of the sub-schema for these collections.
179 std::unique_ptr<RNTupleModel> fCollectionModel; ///< The model for the collection itself
180 std::shared_ptr<RNTupleCollectionWriter> fCollectionWriter; ///< Used to fill the collection elements per event
181 std::unique_ptr<REntry> fCollectionEntry; ///< Keeps the memory location of the collection members
182 /// The number of elements for the collection for a particular event. Used as a destination for SetBranchAddress()
183 /// of the count leaf
184 std::unique_ptr<Int_t> fCountVal;
185 std::vector<size_t> fImportFieldIndexes; ///< Points to the correspondings fields in fImportFields
186 /// One transformation for every field, to copy the content of the array one by one
187 std::vector<std::unique_ptr<RImportTransformation>> fTransformations;
188 Int_t fMaxLength = 0; ///< Stores count leaf GetMaximum() to create large enough buffers for the array leafs
189 std::string fFieldName; ///< name of the untyped collection, e.g. `_collection0`, `_collection1`, etc.
190 };
191
192 /// Transform a NULL terminated C string branch into an `std::string` field
194 RCStringTransformation(std::size_t b, std::size_t f) : RImportTransformation(b, f) {}
195 ~RCStringTransformation() override = default;
196 RResult<void> Transform(const RImportBranch &branch, RImportField &field) final;
197 void ResetEntry() final {}
198 };
199
200 /// When writing the elements of a leaf count array, moves the data from the input array one-by-one
201 /// to the memory locations of the fields of the corresponding untyped collection.
202 /// TODO(jblomer): write arrays as a whole to RNTuple
204 std::int64_t fNum = 0;
205 RLeafArrayTransformation(std::size_t b, std::size_t f) : RImportTransformation(b, f) {}
206 ~RLeafArrayTransformation() override = default;
207 RResult<void> Transform(const RImportBranch &branch, RImportField &field) final;
208 void ResetEntry() final { fNum = 0; }
209 };
210
211 RNTupleImporter() = default;
212
213 std::unique_ptr<TFile> fSourceFile;
215
216 std::string fDestFileName;
217 std::string fNTupleName;
218 std::unique_ptr<TFile> fDestFile;
220
221 /// Whether or not dot characters in branch names should be converted to underscores. If this option is not set and a
222 /// branch with a '.' is encountered, the importer will throw an exception.
224
225 /// The maximum number of entries to import. When this value is -1 (default), import all entries.
226 std::int64_t fMaxEntries = -1;
227
228 /// No standard output, conversely if set to false, schema information and progress is printed.
229 bool fIsQuiet = false;
230 std::unique_ptr<RProgressCallback> fProgressCallback;
231
232 std::unique_ptr<RNTupleModel> fModel;
233 std::unique_ptr<REntry> fEntry;
234 std::vector<RImportBranch> fImportBranches;
235 std::vector<RImportField> fImportFields;
236 /// Maps the count leaf to the information about the corresponding untyped collection
237 std::map<std::string, RImportLeafCountCollection> fLeafCountCollections;
238 /// The list of transformations to be performed for every entry
239 std::vector<std::unique_ptr<RImportTransformation>> fImportTransformations;
240
241 ROOT::Experimental::RResult<void> InitDestination(std::string_view destFileName);
242
243 void ResetSchema();
244 /// Sets up the connection from TTree branches to RNTuple fields, including initialization of the memory
245 /// buffers used for reading and writing.
247 void ReportSchema();
248
249public:
250 RNTupleImporter(const RNTupleImporter &other) = delete;
254 ~RNTupleImporter() = default;
255
256 /// Opens the input file for reading and the output file for writing (update).
257 static std::unique_ptr<RNTupleImporter>
258 Create(std::string_view sourceFileName, std::string_view treeName, std::string_view destFileName);
259
260 /// Directly uses the provided tree and opens the output file for writing (update).
261 static std::unique_ptr<RNTupleImporter> Create(TTree *sourceTree, std::string_view destFileName);
262
265 void SetNTupleName(const std::string &name) { fNTupleName = name; }
266 void SetMaxEntries(std::uint64_t maxEntries) { fMaxEntries = maxEntries; };
267
268 /// Whereas branch names may contain dots, RNTuple field names may not. By setting this option, dot characters are
269 /// automatically converted into underscores to prevent the importer from throwing an exception.
271
272 /// Whether or not information and progress is printed to stdout.
273 void SetIsQuiet(bool value) { fIsQuiet = value; }
274
275 /// Import works in two steps:
276 /// 1. PrepareSchema() calls SetBranchAddress() on all the TTree branches and creates the corresponding RNTuple
277 /// fields and the model
278 /// 2. An event loop reads every entry from the TTree, applies transformations where necessary, and writes the
279 /// output entry to the RNTuple.
280 void Import();
281}; // class RNTupleImporter
282
283} // namespace Experimental
284} // namespace ROOT
285
286#endif
#define b(i)
Definition RSha256.hxx:100
#define f(i)
Definition RSha256.hxx:104
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
char name[80]
Definition TGX11.cxx:110
A field translates read and write calls from/to underlying columns to/from tree values.
Definition RField.hxx:96
Used to report every ~50MB (compressed), and at the end about the status of the import.
virtual void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
void operator()(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
virtual void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
Converts a TTree into an RNTuple.
void SetWriteOptions(RNTupleWriteOptions options)
bool fConvertDotsInBranchNames
Whether or not dot characters in branch names should be converted to underscores.
std::int64_t fMaxEntries
The maximum number of entries to import. When this value is -1 (default), import all entries.
std::map< std::string, RImportLeafCountCollection > fLeafCountCollections
Maps the count leaf to the information about the corresponding untyped collection.
RNTupleImporter & operator=(const RNTupleImporter &other)=delete
std::vector< RImportBranch > fImportBranches
void SetNTupleName(const std::string &name)
RNTupleImporter(const RNTupleImporter &other)=delete
void SetConvertDotsInBranchNames(bool value)
Whereas branch names may contain dots, RNTuple field names may not.
RNTupleImporter & operator=(RNTupleImporter &&other)=delete
static std::unique_ptr< RNTupleImporter > Create(std::string_view sourceFileName, std::string_view treeName, std::string_view destFileName)
Opens the input file for reading and the output file for writing (update).
std::unique_ptr< RProgressCallback > fProgressCallback
RNTupleImporter(RNTupleImporter &&other)=delete
RResult< void > PrepareSchema()
Sets up the connection from TTree branches to RNTuple fields, including initialization of the memory ...
ROOT::Experimental::RResult< void > InitDestination(std::string_view destFileName)
void Import()
Import works in two steps:
RNTupleWriteOptions GetWriteOptions() const
bool fIsQuiet
No standard output, conversely if set to false, schema information and progress is printed.
std::vector< RImportField > fImportFields
void SetIsQuiet(bool value)
Whether or not information and progress is printed to stdout.
void SetMaxEntries(std::uint64_t maxEntries)
std::unique_ptr< RNTupleModel > fModel
std::vector< std::unique_ptr< RImportTransformation > > fImportTransformations
The list of transformations to be performed for every entry.
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition TLeaf.h:57
A TTree represents a columnar dataset.
Definition TTree.h:79
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Transform a NULL terminated C string branch into an std::string field.
RResult< void > Transform(const RImportBranch &branch, RImportField &field) final
std::string fBranchName
Top-level branch name from the input TTree.
RImportBranch(const RImportBranch &other)=delete
RImportBranch & operator=(RImportBranch &&other)=default
RImportBranch & operator=(const RImportBranch &other)=delete
std::unique_ptr< unsigned char[]> fBranchBuffer
The destination of SetBranchAddress() for fBranchName
RImportBranch(RImportBranch &&other)=default
void * fFieldBuffer
Usually points to the corresponding RImportBranch::fBranchBuffer but not always.
RFieldBase * fField
The field is kept during schema preparation and transferred to the fModel before the writing starts.
bool fIsClass
Field imported from a branch with stramer info (e.g., STL, user-defined class)
std::unique_ptr< RFieldBase::RValue > fValue
Set if a value is generated, only for transformed fields.
RImportField(RImportField &&other)=default
bool fIsInUntypedCollection
Sub-fields of untyped collections (leaf count arrays in the input)
RImportField & operator=(const RImportField &other)=delete
RImportField & operator=(RImportField &&other)=default
RImportField(const RImportField &other)=delete
When the schema is set up and the import started, it needs to be reset before the next Import() call ...
RImportGuard & operator=(const RImportGuard &)=delete
RImportGuard & operator=(RImportGuard &&)=delete
std::string fFieldName
name of the untyped collection, e.g. _collection0, _collection1, etc.
Int_t fMaxLength
Stores count leaf GetMaximum() to create large enough buffers for the array leafs.
std::vector< size_t > fImportFieldIndexes
Points to the correspondings fields in fImportFields.
std::unique_ptr< RNTupleModel > fCollectionModel
The model for the collection itself.
RImportLeafCountCollection & operator=(const RImportLeafCountCollection &other)=delete
RImportLeafCountCollection(RImportLeafCountCollection &&other)=default
std::vector< std::unique_ptr< RImportTransformation > > fTransformations
One transformation for every field, to copy the content of the array one by one.
RImportLeafCountCollection(const RImportLeafCountCollection &other)=delete
std::shared_ptr< RNTupleCollectionWriter > fCollectionWriter
Used to fill the collection elements per event.
RImportLeafCountCollection & operator=(RImportLeafCountCollection &&other)=default
std::unique_ptr< Int_t > fCountVal
The number of elements for the collection for a particular event.
std::unique_ptr< REntry > fCollectionEntry
Keeps the memory location of the collection members.
Base class to perform data transformations from TTree branches to RNTuple fields if necessary.
virtual RResult< void > Transform(const RImportBranch &branch, RImportField &field)=0
RImportTransformation(std::size_t branchIdx, std::size_t fieldIdx)
When writing the elements of a leaf count array, moves the data from the input array one-by-one to th...
RResult< void > Transform(const RImportBranch &branch, RImportField &field) final