45 static constexpr std::uint64_t gUpdateFrequencyBytes = 50 * 1000 * 1000;
46 std::uint64_t fNbytesNext = gUpdateFrequencyBytes;
49 ~RDefaultProgressCallback()
override {}
50 void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
final
53 if (nbytesWritten < fNbytesNext)
55 std::cout <<
"Wrote " << nbytesWritten / 1000 / 1000 <<
"MB, " << neventsWritten <<
" entries" << std::endl;
56 fNbytesNext += gUpdateFrequencyBytes;
59 void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)
final
61 std::cout <<
"Done, wrote " << nbytesWritten / 1000 / 1000 <<
"MB, " << neventsWritten <<
" entries" << std::endl;
84std::unique_ptr<ROOT::Experimental::RNTupleImporter>
86 std::string_view destFileName)
88 auto importer = std::unique_ptr<RNTupleImporter>(
new RNTupleImporter());
89 importer->fNTupleName = treeName;
90 importer->fSourceFile = std::unique_ptr<TFile>(
TFile::Open(std::string(sourceFileName).c_str()));
91 if (!importer->fSourceFile || importer->fSourceFile->IsZombie()) {
92 throw RException(
R__FAIL(
"cannot open source file " + std::string(sourceFileName)));
95 importer->fSourceTree = importer->fSourceFile->Get<
TTree>(std::string(treeName).c_str());
96 if (!importer->fSourceTree) {
97 throw RException(
R__FAIL(
"cannot read TTree " + std::string(treeName) +
" from " + std::string(sourceFileName)));
101 importer->fSourceTree->SetImplicitMT(
false);
102 auto result = importer->InitDestination(destFileName);
110std::unique_ptr<ROOT::Experimental::RNTupleImporter>
113 auto importer = std::unique_ptr<RNTupleImporter>(
new RNTupleImporter());
120 importer->fNTupleName = sourceTree->
GetName();
123 importer->fSourceTree = sourceTree;
127 auto result = importer->InitDestination(destFileName);
150 std::cout <<
"Importing '" <<
f.fField->GetName() <<
"' [" <<
f.fField->GetType() <<
']' << std::endl;
173 const auto firstLeaf =
static_cast<TLeaf *
>(
b->GetListOfLeaves()->First());
176 const bool isLeafList =
b->GetNleaves() > 1;
177 const bool isCountLeaf = firstLeaf->IsRange();
179 if (isLeafList && isClass)
180 return R__FAIL(
"unsupported: classes in leaf list, branch " + std::string(
b->GetName()));
181 if (isLeafList && isCountLeaf)
182 return R__FAIL(
"unsupported: count leaf arrays in leaf list, branch " + std::string(
b->GetName()));
187 Int_t firstLeafCountval;
188 const bool isCString = !isLeafList && (firstLeaf->IsA() ==
TLeafC::Class()) &&
189 (!firstLeaf->GetLeafCounter(firstLeafCountval)) && (firstLeafCountval == 1);
198 c.fMaxLength = firstLeaf->GetMaximum();
199 c.fCountVal = std::make_unique<Int_t>();
206 std::size_t branchBufferSize = 0;
208 std::vector<std::unique_ptr<Detail::RFieldBase>> recordItems;
211 return R__FAIL(
"unsupported: TObject branches, branch: " + std::string(
b->GetName()));
216 Int_t countval =
l->GetLenStatic();
217 auto *countleaf =
l->GetLeafCount();
218 const bool isLeafCountArray = (countleaf !=
nullptr);
219 const bool isFixedSizeArray = !isCString && (countleaf ==
nullptr) && (countval > 1);
224 std::string fieldName =
b->GetName();
225 std::string fieldType =
l->GetTypeName();
231 fieldType =
"std::string";
234 fieldType =
b->GetClassName();
236 if (isFixedSizeArray)
237 fieldType =
"std::array<" + fieldType +
"," + std::to_string(countval) +
">";
241 std::replace(fieldName.begin(), fieldName.end(),
'.',
'_');
249 auto field = fieldOrError.Unwrap();
251 branchBufferSize =
l->GetMaximum();
252 f.fValue = std::make_unique<Detail::RFieldBase::RValue>(field->GenerateValue());
253 f.fFieldBuffer =
f.fValue->GetRawPtr();
260 branchBufferSize =
sizeof(
void *) * countval;
261 }
else if (isLeafCountArray) {
264 branchBufferSize =
l->GetOffset() + field->GetValueSize();
267 f.fField = field.get();
270 recordItems.emplace_back(std::move(field));
271 }
else if (isLeafCountArray) {
272 f.fValue = std::make_unique<Detail::RFieldBase::RValue>(field->GenerateValue());
273 f.fFieldBuffer =
f.fValue->GetRawPtr();
274 f.fIsInUntypedCollection =
true;
275 const std::string countleafName = countleaf->GetName();
282 fModel->AddField(std::move(field));
286 if (!recordItems.empty()) {
287 auto recordField = std::make_unique<RRecordField>(
b->GetName(), std::move(recordItems));
291 fModel->AddField(std::move(recordField));
296 ib.
fBranchBuffer = std::make_unique<unsigned char[]>(branchBufferSize);
300 return R__FAIL(
"unable to load class " + std::string(
b->GetClassName()) +
" for branch " +
301 std::string(
b->GetName()));
303 auto ptrBuf =
reinterpret_cast<void **
>(ib.
fBranchBuffer.get());
318 int iLeafCountCollection = 0;
322 auto &countLeafName =
p.first;
324 c.fCollectionModel->Freeze();
325 c.fCollectionEntry =
c.fCollectionModel->CreateBareEntry();
326 for (
auto idx :
c.fImportFieldIndexes) {
329 c.fCollectionEntry->CaptureValueUnsafe(
name, buffer);
331 c.fFieldName =
"_collection" + std::to_string(iLeafCountCollection);
332 c.fCollectionWriter =
fModel->MakeCollection(
c.fFieldName, std::move(
c.fCollectionModel));
334 for (
auto idx :
c.fImportFieldIndexes) {
336 auto projectedField =
338 fModel->AddProjectedField(std::move(projectedField), [&
name, &
c](
const std::string &fieldName) {
339 if (fieldName ==
name)
342 return c.fFieldName +
"." +
name;
346 auto projectedField =
348 fModel->AddProjectedField(std::move(projectedField), [&
c](
const std::string &) {
return c.fFieldName; });
349 iLeafCountCollection++;
355 if (
f.fIsInUntypedCollection)
357 fEntry->CaptureValueUnsafe(
f.fField->GetName(),
f.fFieldBuffer);
360 fEntry->CaptureValueUnsafe(
c.fFieldName,
c.fCollectionWriter->GetOffsetPtr());
377 sink->GetMetrics().Enable();
378 auto ctrZippedBytes = sink->GetMetrics().GetCounter(
"RPageSinkFile.szWritePayload");
380 auto ntplWriter = std::make_unique<RNTupleWriter>(std::move(
fModel), std::move(sink));
392 for (
decltype(nEntries) i = 0; i < nEntries; ++i) {
396 for (
Int_t l = 0;
l < *
c.fCountVal; ++
l) {
397 for (
auto &t :
c.fTransformations) {
402 c.fCollectionWriter->Fill(
c.fCollectionEntry.get());
404 for (
auto &t :
c.fTransformations)
415 ntplWriter->Fill(*
fEntry);
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
winID h TVirtualViewer3D TVirtualGLPainter p
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
static RResult< std::unique_ptr< RFieldBase > > Create(const std::string &fieldName, const std::string &canonicalType, const std::string &typeAlias)
Factory method to resurrect a field from the stored on-disk type information.
virtual size_t GetValueSize() const =0
The number of bytes taken by a value of the appropriate type.
Base class for all ROOT issued exceptions.
Used to report every ~50MB (compressed), and at the end about the status of the import.
virtual void Finish(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
virtual void Call(std::uint64_t nbytesWritten, std::uint64_t neventsWritten)=0
bool fConvertDotsInBranchNames
Whether or not dot characters in branch names should be converted to underscores.
std::unique_ptr< TFile > fDestFile
RNTupleWriteOptions fWriteOptions
std::int64_t fMaxEntries
The maximum number of entries to import. When this value is -1 (default), import all entries.
std::map< std::string, RImportLeafCountCollection > fLeafCountCollections
Maps the count leaf to the information about the corresponding untyped collection.
RNTupleImporter()=default
std::vector< RImportBranch > fImportBranches
static constexpr int kDefaultCompressionSettings
By default, compress RNTuple with zstd, level 5.
static std::unique_ptr< RNTupleImporter > Create(std::string_view sourceFileName, std::string_view treeName, std::string_view destFileName)
Opens the input file for reading and the output file for writing (update).
std::unique_ptr< RProgressCallback > fProgressCallback
RResult< void > PrepareSchema()
Sets up the connection from TTree branches to RNTuple fields, including initialization of the memory ...
ROOT::Experimental::RResult< void > InitDestination(std::string_view destFileName)
void Import()
Import works in two steps:
std::unique_ptr< REntry > fEntry
std::string fDestFileName
bool fIsQuiet
No standard output, conversely if set to false, schema information and progress is printed.
std::vector< RImportField > fImportFields
std::unique_ptr< RNTupleModel > fModel
std::vector< std::unique_ptr< RImportTransformation > > fImportTransformations
The list of transformations to be performed for every entry.
static std::unique_ptr< RNTupleModel > CreateBare()
A bare model has no default entry.
void SetCompression(int val)
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
TClass * IsA() const override
const char * GetName() const override
Returns name of object.
virtual const char * GetName() const
Returns name of object.
A TTree represents a columnar dataset.
virtual Int_t GetEntry(Long64_t entry, Int_t getall=0)
Read all branches of entry and return total number of bytes read.
virtual Int_t SetBranchAddress(const char *bname, void *add, TBranch **ptr=nullptr)
Change branch address, dealing with clone trees properly.
virtual void SetImplicitMT(Bool_t enabled)
virtual Long64_t GetEntries() const
virtual TObjArray * GetListOfBranches()
virtual TTree * GetTree() const
virtual Long64_t LoadTree(Long64_t entry)
Set current entry.
TClass * IsA() const override
std::string fBranchName
Top-level branch name from the input TTree.
std::unique_ptr< unsigned char[]> fBranchBuffer
The destination of SetBranchAddress() for fBranchName
void * fFieldBuffer
Usually points to the corresponding RImportBranch::fBranchBuffer but not always.
bool fIsClass
Field imported from a branch with stramer info (e.g., STL, user-defined class)
Detail::RFieldBase * fField
The field is kept during schema preparation and transferred to the fModel before the writing starts.
When the schema is set up and the import started, it needs to be reset before the next Import() call ...
Leaf count arrays require special treatment.
std::unique_ptr< RNTupleModel > fCollectionModel
The model for the collection itself.