Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RMiniFile.cxx
Go to the documentation of this file.
1/// \file RMiniFile.cxx
2/// \author Jakob Blomer <jblomer@cern.ch>
3/// \date 2019-12-22
4
5/*************************************************************************
6 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
7 * All rights reserved. *
8 * *
9 * For the licensing terms see $ROOTSYS/LICENSE. *
10 * For the list of contributors see $ROOTSYS/README/CREDITS. *
11 *************************************************************************/
12
13#include "Rtypes.h"
14#include <ROOT/RConfig.hxx>
15#include <ROOT/RError.hxx>
16#include <ROOT/RMiniFile.hxx>
17#include <ROOT/RRawFile.hxx>
18#include <ROOT/RNTupleUtils.hxx>
19#include <ROOT/RNTupleZip.hxx>
22#include <ROOT/RFile.hxx>
23
24#include <Byteswap.h>
25#include <TBufferFile.h>
26#include <TDirectory.h>
27#include <TError.h>
28#include <TFile.h>
29#include <TKey.h>
30#include <TObjString.h>
31#include <TUUID.h>
32#include <TStreamerInfo.h>
33
34#include <xxhash.h>
35
36#include <algorithm>
37#include <cassert>
38#include <cerrno>
39#include <cstdio>
40#include <cstring>
41#include <memory>
42#include <string>
43#include <chrono>
44
45#ifdef R__LINUX
46#include <fcntl.h>
47#endif
48
49#ifndef R__LITTLE_ENDIAN
50#ifdef R__BYTESWAP
51// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise
52#define R__LITTLE_ENDIAN 1
53#else
54#define R__LITTLE_ENDIAN 0
55#endif
56#endif /* R__LITTLE_ENDIAN */
57
58namespace {
59
60// The following types are used to read and write the TFile binary format
61
62/// Big-endian 16-bit unsigned integer
63class RUInt16BE {
64private:
65 std::uint16_t fValBE = 0;
66 static std::uint16_t Swap(std::uint16_t val)
67 {
68#if R__LITTLE_ENDIAN == 1
69 return RByteSwap<sizeof(val)>::bswap(val);
70#else
71 return val;
72#endif
73 }
74
75public:
76 RUInt16BE() = default;
77 explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {}
78 operator std::uint16_t() const { return Swap(fValBE); }
79 RUInt16BE &operator=(const std::uint16_t val)
80 {
81 fValBE = Swap(val);
82 return *this;
83 }
84};
85
86/// Big-endian 32-bit unsigned integer
87class RUInt32BE {
88private:
89 std::uint32_t fValBE = 0;
90 static std::uint32_t Swap(std::uint32_t val)
91 {
92#if R__LITTLE_ENDIAN == 1
93 return RByteSwap<sizeof(val)>::bswap(val);
94#else
95 return val;
96#endif
97 }
98
99public:
100 RUInt32BE() = default;
101 explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {}
102 operator std::uint32_t() const { return Swap(fValBE); }
103 RUInt32BE &operator=(const std::uint32_t val)
104 {
105 fValBE = Swap(val);
106 return *this;
107 }
108};
109
110/// Big-endian 32-bit signed integer
111class RInt32BE {
112private:
113 std::int32_t fValBE = 0;
114 static std::int32_t Swap(std::int32_t val)
115 {
116#if R__LITTLE_ENDIAN == 1
117 return RByteSwap<sizeof(val)>::bswap(val);
118#else
119 return val;
120#endif
121 }
122
123public:
124 RInt32BE() = default;
125 explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {}
126 operator std::int32_t() const { return Swap(fValBE); }
127 RInt32BE &operator=(const std::int32_t val)
128 {
129 fValBE = Swap(val);
130 return *this;
131 }
132};
133
134/// Big-endian 64-bit unsigned integer
135class RUInt64BE {
136private:
137 std::uint64_t fValBE = 0;
138 static std::uint64_t Swap(std::uint64_t val)
139 {
140#if R__LITTLE_ENDIAN == 1
141 return RByteSwap<sizeof(val)>::bswap(val);
142#else
143 return val;
144#endif
145 }
146
147public:
148 RUInt64BE() = default;
149 explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {}
150 operator std::uint64_t() const { return Swap(fValBE); }
151 RUInt64BE &operator=(const std::uint64_t val)
152 {
153 fValBE = Swap(val);
154 return *this;
155 }
156};
157
158#pragma pack(push, 1)
159/// A name (type, identifier, ...) in the TFile binary format
160struct RTFString {
161 unsigned char fLName{0};
162 char fData[255];
163 RTFString() = default;
164 RTFString(const std::string &str)
165 {
166 // The length of strings with 255 characters and longer are encoded with a 32-bit integer following the first
167 // byte. This is currently not handled.
168 R__ASSERT(str.length() < 255);
169 fLName = static_cast<unsigned char>(str.length());
170 memcpy(fData, str.data(), fLName);
171 }
172 std::size_t GetSize() const
173 {
174 // A length of 255 is special and means that the first byte is followed by a 32-bit integer with the actual
175 // length.
176 R__ASSERT(fLName != 255);
177 return 1 + fLName;
178 }
179};
180
181/// The timestamp format used in TFile; the default constructor initializes with the current time
182struct RTFDatetime {
183 RUInt32BE fDatetime;
184 RTFDatetime()
185 {
186 auto now = std::chrono::system_clock::now();
187 auto tt = std::chrono::system_clock::to_time_t(now);
188 auto tm = *localtime(&tt);
189 fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 | tm.tm_hour << 12 |
190 tm.tm_min << 6 | tm.tm_sec;
191 }
192 explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {}
193};
194
195/// The key part of a TFile record excluding the class, object, and title names
196struct RTFKey {
197 static constexpr unsigned kBigKeyVersion = 1000;
198
199 RInt32BE fNbytes{0};
200 RUInt16BE fVersion{4};
201 RUInt32BE fObjLen{0};
202 RTFDatetime fDatetime;
203 RUInt16BE fKeyLen{0};
204 RUInt16BE fCycle{1};
205 union {
206 struct {
207 RUInt32BE fSeekKey{0};
208 RUInt32BE fSeekPdir{0};
209 } fInfoShort;
210 struct {
211 RUInt64BE fSeekKey{0};
212 RUInt64BE fSeekPdir{0};
213 } fInfoLong;
214 };
215
216 RTFKey() : fInfoLong() {}
217 RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName,
218 const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0)
219 {
220 R__ASSERT(szObjInMem <= std::numeric_limits<std::uint32_t>::max());
221 R__ASSERT(szObjOnDisk <= std::numeric_limits<std::uint32_t>::max());
222 // For writing, we always produce "big" keys with 64-bit SeekKey and SeekPdir.
223 fVersion = fVersion + kBigKeyVersion;
224 fObjLen = szObjInMem;
225 fKeyLen = static_cast<RUInt16BE>(GetHeaderSize() + clName.GetSize() + objName.GetSize() + titleName.GetSize());
226 fInfoLong.fSeekKey = seekKey;
227 fInfoLong.fSeekPdir = seekPdir;
228 // Depends on fKeyLen being set
229 fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk);
230 }
231
232 std::uint32_t GetSize() const
233 {
234 // Negative size indicates a gap in the file
235 if (fNbytes < 0)
236 return -fNbytes;
237 return fNbytes;
238 }
239
240 std::uint32_t GetHeaderSize() const
241 {
242 if (fVersion >= kBigKeyVersion)
243 return 18 + sizeof(fInfoLong);
244 return 18 + sizeof(fInfoShort);
245 }
246
247 std::uint64_t GetSeekKey() const
248 {
249 if (fVersion >= kBigKeyVersion)
250 return fInfoLong.fSeekKey;
251 return fInfoShort.fSeekKey;
252 }
253};
254
255/// The TFile global header
256struct RTFHeader {
257 static constexpr unsigned kBEGIN = 100;
258 static constexpr unsigned kBigHeaderVersion = 1000000;
259
260 char fMagic[4]{'r', 'o', 'o', 't'};
261 RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
262 (ROOT_VERSION_CODE & 0xFF)};
263 RUInt32BE fBEGIN{kBEGIN};
264 union {
265 struct {
266 RUInt32BE fEND{0};
267 RUInt32BE fSeekFree{0};
268 RUInt32BE fNbytesFree{0};
269 RUInt32BE fNfree{1};
270 RUInt32BE fNbytesName{0};
271 unsigned char fUnits{4};
272 RUInt32BE fCompress{0};
273 RUInt32BE fSeekInfo{0};
274 RUInt32BE fNbytesInfo{0};
275 } fInfoShort;
276 struct {
277 RUInt64BE fEND{0};
278 RUInt64BE fSeekFree{0};
279 RUInt32BE fNbytesFree{0};
280 RUInt32BE fNfree{1};
281 RUInt32BE fNbytesName{0};
282 unsigned char fUnits{8};
283 RUInt32BE fCompress{0};
284 RUInt64BE fSeekInfo{0};
285 RUInt32BE fNbytesInfo{0};
286 } fInfoLong;
287 };
288
289 RTFHeader() : fInfoShort() {}
290 RTFHeader(int compression) : fInfoShort() { fInfoShort.fCompress = compression; }
291
292 void SetBigFile()
293 {
294 if (fVersion >= kBigHeaderVersion)
295 return;
296
297 // clang-format off
298 std::uint32_t end = fInfoShort.fEND;
299 std::uint32_t seekFree = fInfoShort.fSeekFree;
300 std::uint32_t nbytesFree = fInfoShort.fNbytesFree;
301 std::uint32_t nFree = fInfoShort.fNfree;
302 std::uint32_t nbytesName = fInfoShort.fNbytesName;
303 std::uint32_t compress = fInfoShort.fCompress;
304 std::uint32_t seekInfo = fInfoShort.fSeekInfo;
305 std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo;
306 fInfoLong.fEND = end;
307 fInfoLong.fSeekFree = seekFree;
308 fInfoLong.fNbytesFree = nbytesFree;
309 fInfoLong.fNfree = nFree;
310 fInfoLong.fNbytesName = nbytesName;
311 fInfoLong.fUnits = 8;
312 fInfoLong.fCompress = compress;
313 fInfoLong.fSeekInfo = seekInfo;
314 fInfoLong.fNbytesInfo = nbytesInfo;
315 fVersion = fVersion + kBigHeaderVersion;
316 // clang-format on
317 }
318
319 bool IsBigFile(std::uint64_t offset = 0) const
320 {
321 return (fVersion >= kBigHeaderVersion) ||
322 (offset > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()));
323 }
324
325 std::uint32_t GetSize() const
326 {
327 std::uint32_t sizeHead = sizeof(fMagic) + sizeof(fVersion) + sizeof(fBEGIN);
328 if (IsBigFile())
329 return sizeHead + sizeof(fInfoLong);
330 return sizeHead + sizeof(fInfoShort);
331 }
332
333 std::uint64_t GetEnd() const
334 {
335 if (IsBigFile())
336 return fInfoLong.fEND;
337 return fInfoShort.fEND;
338 }
339
340 void SetEnd(std::uint64_t value)
341 {
342 if (IsBigFile(value)) {
343 SetBigFile();
344 fInfoLong.fEND = value;
345 } else {
346 fInfoShort.fEND = value;
347 }
348 }
349
350 std::uint64_t GetSeekFree() const
351 {
352 if (IsBigFile())
353 return fInfoLong.fSeekFree;
354 return fInfoShort.fSeekFree;
355 }
356
357 void SetSeekFree(std::uint64_t value)
358 {
359 if (IsBigFile(value)) {
360 SetBigFile();
361 fInfoLong.fSeekFree = value;
362 } else {
363 fInfoShort.fSeekFree = value;
364 }
365 }
366
367 void SetNbytesFree(std::uint32_t value)
368 {
369 if (IsBigFile()) {
370 fInfoLong.fNbytesFree = value;
371 } else {
372 fInfoShort.fNbytesFree = value;
373 }
374 }
375
376 void SetNbytesName(std::uint32_t value)
377 {
378 if (IsBigFile()) {
379 fInfoLong.fNbytesName = value;
380 } else {
381 fInfoShort.fNbytesName = value;
382 }
383 }
384
385 std::uint64_t GetSeekInfo() const
386 {
387 if (IsBigFile())
388 return fInfoLong.fSeekInfo;
389 return fInfoShort.fSeekInfo;
390 }
391
392 void SetSeekInfo(std::uint64_t value)
393 {
394 if (IsBigFile(value)) {
395 SetBigFile();
396 fInfoLong.fSeekInfo = value;
397 } else {
398 fInfoShort.fSeekInfo = value;
399 }
400 }
401
402 std::uint64_t GetNbytesInfo() const
403 {
404 if (IsBigFile())
405 return fInfoLong.fNbytesInfo;
406 return fInfoShort.fNbytesInfo;
407 }
408
409 void SetNbytesInfo(std::uint32_t value)
410 {
411 if (IsBigFile()) {
412 fInfoLong.fNbytesInfo = value;
413 } else {
414 fInfoShort.fNbytesInfo = value;
415 }
416 }
417
418 void SetCompression(std::uint32_t value)
419 {
420 if (IsBigFile()) {
421 fInfoLong.fCompress = value;
422 } else {
423 fInfoShort.fCompress = value;
424 }
425 }
426};
427
428/// A reference to an unused byte-range in a TFile
429struct RTFFreeEntry {
430 static constexpr unsigned kBigFreeEntryVersion = 1000;
431
432 RUInt16BE fVersion{1};
433 union {
434 struct {
435 RUInt32BE fFirst{0};
436 RUInt32BE fLast{0};
437 } fInfoShort;
438 struct {
439 RUInt64BE fFirst{0};
440 RUInt64BE fLast{0};
441 } fInfoLong;
442 };
443
444 RTFFreeEntry() : fInfoShort() {}
445 void Set(std::uint64_t first, std::uint64_t last)
446 {
447 if (last > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
448 fVersion = fVersion + kBigFreeEntryVersion;
449 fInfoLong.fFirst = first;
450 fInfoLong.fLast = last;
451 } else {
452 fInfoShort.fFirst = first;
453 fInfoShort.fLast = last;
454 }
455 }
456 std::uint32_t GetSize() { return (fVersion >= kBigFreeEntryVersion) ? 18 : 10; }
457};
458
459/// The header of the directory key index
460struct RTFKeyList {
461 RUInt32BE fNKeys;
462 std::uint32_t GetSize() const { return sizeof(RTFKeyList); }
463 explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {}
464};
465
466/// A streamed TDirectory (TFile) object
467struct RTFDirectory {
468 static constexpr unsigned kBigFileVersion = 1000;
469
470 RUInt16BE fClassVersion{5};
471 RTFDatetime fDateC;
472 RTFDatetime fDateM;
473 RUInt32BE fNBytesKeys{0};
474 RUInt32BE fNBytesName{0};
475 // The version of the key has to tell whether offsets are 32bit or 64bit long
476 union {
477 struct {
478 RUInt32BE fSeekDir{RTFHeader::kBEGIN};
479 RUInt32BE fSeekParent{0};
480 RUInt32BE fSeekKeys{0};
481 } fInfoShort;
482 struct {
483 RUInt64BE fSeekDir{RTFHeader::kBEGIN};
484 RUInt64BE fSeekParent{0};
485 RUInt64BE fSeekKeys{0};
486 } fInfoLong;
487 };
488
489 RTFDirectory() : fInfoShort() {}
490
491 // In case of a short TFile record (<2G), 3 padding ints are written after the UUID
492 std::uint32_t GetSize() const
493 {
494 if (fClassVersion >= kBigFileVersion)
495 return sizeof(RTFDirectory);
496 return 18 + sizeof(fInfoShort);
497 }
498
499 std::uint64_t GetSeekKeys() const
500 {
501 if (fClassVersion >= kBigFileVersion)
502 return fInfoLong.fSeekKeys;
503 return fInfoShort.fSeekKeys;
504 }
505
506 void SetSeekKeys(std::uint64_t seekKeys)
507 {
508 if (seekKeys > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
509 std::uint32_t seekDir = fInfoShort.fSeekDir;
510 std::uint32_t seekParent = fInfoShort.fSeekParent;
511 fInfoLong.fSeekDir = seekDir;
512 fInfoLong.fSeekParent = seekParent;
513 fInfoLong.fSeekKeys = seekKeys;
514 fClassVersion = fClassVersion + kBigFileVersion;
515 } else {
516 fInfoShort.fSeekKeys = seekKeys;
517 }
518 }
519};
520
521/// A zero UUID stored at the end of the TFile record
522struct RTFUUID {
523 RUInt16BE fVersionClass{1};
524 unsigned char fUUID[16];
525
526 RTFUUID()
527 {
528 TUUID uuid{TUUID::UUIDv4()};
529 char *buffer = reinterpret_cast<char *>(this);
530 uuid.FillBuffer(buffer);
531 assert(reinterpret_cast<RTFUUID *>(buffer) <= (this + 1));
532 }
533 std::uint32_t GetSize() const { return sizeof(RTFUUID); }
534};
535
536/// A streamed RNTuple class
537///
538/// NOTE: this must be kept in sync with RNTuple.hxx.
539/// Aside ensuring consistency between the two classes' members, you need to make sure
540/// that fVersionClass matches the class version of RNTuple.
541struct RTFNTuple {
542 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))};
543 RUInt16BE fVersionClass{2};
544 RUInt16BE fVersionEpoch{0};
545 RUInt16BE fVersionMajor{0};
546 RUInt16BE fVersionMinor{0};
547 RUInt16BE fVersionPatch{0};
548 RUInt64BE fSeekHeader{0};
549 RUInt64BE fNBytesHeader{0};
550 RUInt64BE fLenHeader{0};
551 RUInt64BE fSeekFooter{0};
552 RUInt64BE fNBytesFooter{0};
553 RUInt64BE fLenFooter{0};
554 RUInt64BE fMaxKeySize{0};
555
556 static constexpr std::uint32_t GetSizePlusChecksum() { return sizeof(RTFNTuple) + sizeof(std::uint64_t); }
557
558 RTFNTuple() = default;
559 explicit RTFNTuple(const ROOT::RNTuple &inMemoryAnchor)
560 {
561 fVersionEpoch = inMemoryAnchor.GetVersionEpoch();
562 fVersionMajor = inMemoryAnchor.GetVersionMajor();
563 fVersionMinor = inMemoryAnchor.GetVersionMinor();
564 fVersionPatch = inMemoryAnchor.GetVersionPatch();
565 fSeekHeader = inMemoryAnchor.GetSeekHeader();
566 fNBytesHeader = inMemoryAnchor.GetNBytesHeader();
567 fLenHeader = inMemoryAnchor.GetLenHeader();
568 fSeekFooter = inMemoryAnchor.GetSeekFooter();
569 fNBytesFooter = inMemoryAnchor.GetNBytesFooter();
570 fLenFooter = inMemoryAnchor.GetLenFooter();
571 fMaxKeySize = inMemoryAnchor.GetMaxKeySize();
572 }
573 std::uint32_t GetSize() const { return sizeof(RTFNTuple); }
574 // The byte count and class version members are not checksummed
575 std::uint32_t GetOffsetCkData() { return sizeof(fByteCount) + sizeof(fVersionClass); }
576 std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); }
577 unsigned char *GetPtrCkData() { return reinterpret_cast<unsigned char *>(this) + GetOffsetCkData(); }
578};
579
580/// The bare file global header
581struct RBareFileHeader {
582 char fMagic[7]{'r', 'n', 't', 'u', 'p', 'l', 'e'};
583 RUInt32BE fRootVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
584 (ROOT_VERSION_CODE & 0xFF)};
585 RUInt32BE fFormatVersion{1};
586 RUInt32BE fCompress{0};
587 RTFNTuple fNTuple;
588 // followed by the ntuple name
589};
590#pragma pack(pop)
591
592/// The artificial class name shown for opaque RNTuple keys (see TBasket)
593constexpr char const *kBlobClassName = "RBlob";
594/// The class name of the RNTuple anchor
595constexpr char const *kNTupleClassName = "ROOT::RNTuple";
596
597} // anonymous namespace
598
599namespace ROOT {
600namespace Internal {
601/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header
602/// and the TFile record need to be updated
604 RTFHeader fHeader;
605 RTFDirectory fFileRecord;
606 std::uint64_t fSeekNTuple{0}; // Remember the offset for the keys list
607 std::uint64_t fSeekFileRecord{0};
608};
609
610/// The RKeyBlob writes an invisible key into a TFile. That is, a key that is not indexed in the list of keys,
611/// like a TBasket.
612/// NOTE: out of anonymous namespace because otherwise ClassDefInline fails to compile
613/// on some platforms.
614class RKeyBlob : public TKey { // NOLINT(misc-use-internal-linkage)
615public:
616 RKeyBlob() = default;
617
618 explicit RKeyBlob(TFile *file) : TKey(file)
619 {
620 fClassName = kBlobClassName;
621 fVersion += RTFKey::kBigKeyVersion;
622 fKeylen = Sizeof();
623 }
624
625 /// Register a new key for a data record of size nbytes
626 void Reserve(size_t nbytes, std::uint64_t *seekKey)
627 {
628 Create(nbytes);
629 *seekKey = fSeekKey;
630 }
631
632 bool WasAllocatedInAFreeSlot() const { return fLeft > 0; }
633
635};
636
637} // namespace Internal
638} // namespace ROOT
639
640// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
641// first chunk also needs to house the offsets of the other chunks and no chunk can
642// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
643static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
644{
645 constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
646
647 assert(nbytes > maxChunkSize);
648 size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
649 assert(nChunks > 1);
650 size_t nbytesTail = nbytes % maxChunkSize;
651 size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
652 size_t nbytesChunkOffsets = (nChunks - 1) * kChunkOffsetSize;
653 if (nbytesChunkOffsets > nbytesExtra) {
654 ++nChunks;
655 nbytesChunkOffsets += kChunkOffsetSize;
656 }
657
658 // We don't support having more chunkOffsets than what fits in one chunk.
659 // For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
660 // than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
661 // 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
662 R__ASSERT(nbytesChunkOffsets <= maxChunkSize);
663
664 return nChunks;
665}
666
668
670{
671 char ident[4];
672 ReadBuffer(ident, 4, 0);
673 if (std::string(ident, 4) == "root")
674 return GetNTupleProper(ntupleName);
675 fIsBare = true;
676 return GetNTupleBare(ntupleName);
677}
678
679/// Searches for a key with the given name and type in the key index of the given directory.
680/// Return 0 if the key was not found.
681std::uint64_t ROOT::Internal::RMiniFileReader::SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName,
682 std::string_view typeName)
683{
684 RTFDirectory directory;
685 ReadBuffer(&directory, sizeof(directory), offsetDir);
686
687 RTFKey key;
688 RUInt32BE nKeys;
689 std::uint64_t offset = directory.GetSeekKeys();
690 ReadBuffer(&key, sizeof(key), offset);
691 offset += key.fKeyLen;
692 ReadBuffer(&nKeys, sizeof(nKeys), offset);
693 offset += sizeof(nKeys);
694
695 for (unsigned int i = 0; i < nKeys; ++i) {
696 ReadBuffer(&key, sizeof(key), offset);
697 auto offsetNextKey = offset + key.fKeyLen;
698
699 offset += key.GetHeaderSize();
700 RTFString name;
701 ReadBuffer(&name, 1, offset);
702 ReadBuffer(&name, name.GetSize(), offset);
703 if (std::string_view(name.fData, name.fLName) != typeName) {
704 offset = offsetNextKey;
705 continue;
706 }
707 offset += name.GetSize();
708 ReadBuffer(&name, 1, offset);
709 ReadBuffer(&name, name.GetSize(), offset);
710 if (std::string_view(name.fData, name.fLName) == keyName) {
711 return key.GetSeekKey();
712 }
713 offset = offsetNextKey;
714 }
715
716 // Not found
717 return 0;
718}
719
721{
722 if (fIsBare)
723 return;
724
725 RTFKey key;
726 ReadBuffer(&key, sizeof(key), fSeekKeyInfo);
727
728 R__ASSERT(fNbytesKeyAndInfo >= key.fKeyLen);
729 const std::uint64_t nbytesInfo = fNbytesKeyAndInfo - key.fKeyLen;
730 const std::uint64_t seekInfo = fSeekKeyInfo + key.fKeyLen;
731 const std::uint32_t uncompLenInfo = key.fObjLen;
732 auto streamerInfo = MakeUninitArray<char>(uncompLenInfo);
733 if (nbytesInfo == uncompLenInfo) {
734 // Uncompressed
735 ReadBuffer(streamerInfo.get(), nbytesInfo, seekInfo);
736 } else {
737 auto buffer = MakeUninitArray<std::byte>(nbytesInfo);
738 ReadBuffer(buffer.get(), nbytesInfo, seekInfo);
739 RNTupleDecompressor::Unzip(buffer.get(), nbytesInfo, uncompLenInfo, streamerInfo.get());
740 }
741
742 TBufferFile buffer(TBuffer::kRead, uncompLenInfo, streamerInfo.release());
743 // This is necessary to allow the "class tags" inside the StreamerInfo list to refer to the proper offset into
744 // the buffer. Normally TFile loads the StreamerInfo via TKey::ReadObjWithBuffer, whose buffer also includes the
745 // key itself. Since we dealt with the key above already, we are only passing the payload to TBufferFile so offsets
746 // need to be patched up.
747 buffer.SetBufferDisplacement(key.fKeyLen);
748 TList streamerInfoList;
749 streamerInfoList.Streamer(buffer);
750 TObjLink *lnk = streamerInfoList.FirstLink();
751 while (lnk) {
752 auto obj = lnk->GetObject();
753 // NOTE: the last element of the streamer info list may be a TList with the IO customization rules, so we need
754 // to check before static casting.
755 if (obj->IsA() == TStreamerInfo::Class()) {
756 auto info = static_cast<TStreamerInfo *>(obj);
757 info->BuildCheck();
758 }
759 lnk = lnk->Next();
760 }
761}
762
764{
765 RTFHeader fileHeader;
766 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
767
768 fSeekKeyInfo = fileHeader.GetSeekInfo();
769 fNbytesKeyAndInfo = fileHeader.GetNbytesInfo();
770
771 RTFKey key;
772 RTFString name;
773 ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN);
774 // Skip over the entire key length, including the class name, object name, and title stored in it.
775 std::uint64_t offset = fileHeader.fBEGIN + key.fKeyLen;
776 // Skip over the name and title of the TNamed preceding the TFile (root TDirectory) entry.
777 ReadBuffer(&name, 1, offset);
778 offset += name.GetSize();
779 ReadBuffer(&name, 1, offset);
780 offset += name.GetSize();
781
782 // split ntupleName by '/' character to open datasets in subdirectories.
783 std::string ntuplePathTail(ntuplePath);
784 if (!ntuplePathTail.empty() && ntuplePathTail[0] == '/')
785 ntuplePathTail = ntuplePathTail.substr(1);
786 auto pos = std::string::npos;
787 while ((pos = ntuplePathTail.find('/')) != std::string::npos) {
788 auto directoryName = ntuplePathTail.substr(0, pos);
789 ntuplePathTail.erase(0, pos + 1);
790
791 offset = SearchInDirectory(offset, directoryName, "TDirectory");
792 if (offset == 0) {
793 return R__FAIL("no directory named '" + std::string(directoryName) + "' in file '" + fRawFile->GetUrl() + "'");
794 }
795 ReadBuffer(&key, sizeof(key), offset);
796 offset = key.GetSeekKey() + key.fKeyLen;
797 }
798 // no more '/' delimiter in ntuplePath
799 auto ntupleName = ntuplePathTail;
800
801 offset = SearchInDirectory(offset, ntupleName, kNTupleClassName);
802 if (offset == 0) {
803 return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'");
804 }
805
806 ReadBuffer(&key, sizeof(key), offset);
807 offset = key.GetSeekKey() + key.fKeyLen;
808
809 // size of a RTFNTuple version 2 (min supported version); future anchor versions can grow.
810 constexpr size_t kMinNTupleSize = 78;
811 static_assert(kMinNTupleSize == RTFNTuple::GetSizePlusChecksum());
812 if (key.fObjLen < kMinNTupleSize) {
813 return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
814 }
815
816 const auto objNbytes = key.GetSize() - key.fKeyLen;
817 auto res = GetNTupleProperAtOffset(offset, objNbytes, key.fObjLen);
818
819 return res;
820}
821
823 std::uint64_t compSize,
824 std::uint64_t uncompLen)
825{
826 if (compSize > uncompLen) {
827 return R__FAIL("Invalid size and length combination of anchor at offset " + std::to_string(payloadOffset) +
828 " (length: " + std::to_string(uncompLen) + ", bytes on disk: " + std::to_string(compSize) + ")");
829 }
830
831 // The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version,
832 // or larger than it if it comes from a future RNTuple class version.
833 auto bufAnchor = MakeUninitArray<unsigned char>(std::max<size_t>(uncompLen, sizeof(RTFNTuple)));
834 RTFNTuple *ntuple = new (bufAnchor.get()) RTFNTuple;
835
836 if (compSize < uncompLen) {
837 // Read into a temporary buffer
838 auto zipBuf = MakeUninitArray<unsigned char>(compSize);
839 ReadBuffer(zipBuf.get(), compSize, payloadOffset);
840 // Unzip into the final buffer
841 RNTupleDecompressor::Unzip(zipBuf.get(), compSize, uncompLen, ntuple);
842 } else {
843 // We can directly decompress into ntuple because this RTFNTuple object has been constructed in a memory area
844 // of (max(uncompLen, sizeof(RTFNTuple))) bytes.
845 ReadBuffer(ntuple, uncompLen, payloadOffset);
846 }
847
848 // We require that future class versions only append members and store the checksum in the last 8 bytes
849 // Checksum calculation: strip byte count, class version, fChecksum member
850 const auto lenCkData = uncompLen - ntuple->GetOffsetCkData() - sizeof(uint64_t);
851 const auto ckCalc = XXH3_64bits(ntuple->GetPtrCkData(), lenCkData);
852 uint64_t ckOnDisk;
853
854 RUInt64BE *ckOnDiskPtr = reinterpret_cast<RUInt64BE *>(bufAnchor.get() + uncompLen - sizeof(uint64_t));
855 ckOnDisk = static_cast<uint64_t>(*ckOnDiskPtr);
856 if (ckCalc != ckOnDisk) {
857 return R__FAIL("RNTuple anchor checksum mismatch");
858 }
859
860 return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
861 ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
862 ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize);
863}
864
866{
867 RBareFileHeader fileHeader;
868 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
869 RTFString name;
870 auto offset = sizeof(fileHeader);
871 ReadBuffer(&name, 1, offset);
872 ReadBuffer(&name, name.GetSize(), offset);
873 std::string_view foundName(name.fData, name.fLName);
874 if (foundName != ntupleName) {
875 return R__FAIL("expected RNTuple named '" + std::string(ntupleName) + "' but instead found '" +
876 std::string(foundName) + "' in file '" + fRawFile->GetUrl() + "'");
877 }
878 offset += name.GetSize();
879
880 RTFNTuple ntuple;
881 ReadBuffer(&ntuple, sizeof(ntuple), offset);
882 std::uint64_t onDiskChecksum;
883 ReadBuffer(&onDiskChecksum, sizeof(onDiskChecksum), offset + sizeof(ntuple));
884 auto checksum = XXH3_64bits(ntuple.GetPtrCkData(), ntuple.GetSizeCkData());
885 if (checksum != static_cast<uint64_t>(onDiskChecksum))
886 return R__FAIL("RNTuple bare file: anchor checksum mismatch");
887
888 return CreateAnchor(ntuple.fVersionEpoch, ntuple.fVersionMajor, ntuple.fVersionMinor, ntuple.fVersionPatch,
889 ntuple.fSeekHeader, ntuple.fNBytesHeader, ntuple.fLenHeader, ntuple.fSeekFooter,
890 ntuple.fNBytesFooter, ntuple.fLenFooter, ntuple.fMaxKeySize);
891}
892
893void ROOT::Internal::RMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
894{
895 TryReadBuffer(buffer, nbytes, offset).ThrowOnError();
896}
897
899{
900 const auto ByteReadErr = [](std::size_t expected, std::size_t nread) {
901 return R__FAIL("invalid read (expected bytes: " + std::to_string(expected) + ", read: " + std::to_string(nread) +
902 ")");
903 };
904
905 size_t nread;
906 if (fMaxKeySize == 0 || nbytes <= fMaxKeySize) {
907 // Fast path: read single blob
908 nread = fRawFile->ReadAt(buffer, nbytes, offset);
909 } else {
910 // Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
911 const size_t nChunks = ComputeNumChunks(nbytes, fMaxKeySize);
912 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
913 const size_t nbytesFirstChunk = fMaxKeySize - nbytesChunkOffsets;
914 uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
915
916 // Read first chunk
917 nread = fRawFile->ReadAt(bufCur, fMaxKeySize, offset);
918 if (nread != fMaxKeySize)
919 return ByteReadErr(fMaxKeySize, nread);
920
921 // NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
922 // since the last part of `bufCur` will later be overwritten by the next chunk's payload.
923 // We do this to avoid a second ReadAt to read in the chunk offsets.
924 bufCur += nbytesFirstChunk;
925 nread -= nbytesChunkOffsets;
926
927 const auto chunkOffsets = MakeUninitArray<std::uint64_t>(nChunks - 1);
928 memcpy(chunkOffsets.get(), bufCur, nbytesChunkOffsets);
929
930 size_t remainingBytes = nbytes - nbytesFirstChunk;
931 std::uint64_t *curChunkOffset = &chunkOffsets[0];
932
933 do {
934 std::uint64_t chunkOffset;
935 RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset);
936 ++curChunkOffset;
937
938 const size_t bytesToRead = std::min<size_t>(fMaxKeySize, remainingBytes);
939 // Ensure we don't read outside of the buffer
940 R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
941
942 auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
943 if (nbytesRead != bytesToRead)
944 return ByteReadErr(bytesToRead, nbytesRead);
945
946 nread += bytesToRead;
947 bufCur += bytesToRead;
948 remainingBytes -= bytesToRead;
949 } while (remainingBytes > 0);
950 }
951
952 if (nread != nbytes)
953 return ByteReadErr(nbytes, nread);
954
955 return RResult<void>::Success();
956}
957
958////////////////////////////////////////////////////////////////////////////////
959
960/// Prepare a blob key in the provided buffer, which must provide space for kBlobKeyLen bytes. Note that the array type
961/// is purely documentation, the argument is actually just a pointer.
962void ROOT::Internal::RNTupleFileWriter::PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len,
963 unsigned char buffer[kBlobKeyLen])
964{
965 RTFString strClass{kBlobClassName};
966 RTFString strObject;
967 RTFString strTitle;
968 RTFKey keyHeader(offset, RTFHeader::kBEGIN, strClass, strObject, strTitle, len, nbytes);
969 R__ASSERT(keyHeader.fKeyLen == kBlobKeyLen);
970
971 // Copy structures into the buffer.
972 unsigned char *writeBuffer = buffer;
973 memcpy(writeBuffer, &keyHeader, keyHeader.GetHeaderSize());
974 writeBuffer += keyHeader.GetHeaderSize();
975 memcpy(writeBuffer, &strClass, strClass.GetSize());
976 writeBuffer += strClass.GetSize();
977 memcpy(writeBuffer, &strObject, strObject.GetSize());
978 writeBuffer += strObject.GetSize();
979 memcpy(writeBuffer, &strTitle, strTitle.GetSize());
980 writeBuffer += strTitle.GetSize();
981 R__ASSERT(writeBuffer == buffer + kBlobKeyLen);
982}
983
984////////////////////////////////////////////////////////////////////////////////
985
987
989{
990 static_assert(kHeaderBlockSize % kBlockAlign == 0, "invalid header block size");
991 if (bufferSize % kBlockAlign != 0)
992 throw RException(R__FAIL("Buffer size not a multiple of alignment: " + std::to_string(bufferSize)));
993
994 assert(!fShared);
995
996 fShared = std::make_shared<RSharedData>(nullptr);
997
998 fShared->fBlockSize = bufferSize;
999 fShared->fControlBlock = std::make_unique<ROOT::Internal::RTFileControlBlock>();
1000
1001 std::align_val_t blockAlign{kBlockAlign};
1002 fShared->fHeaderBlock = static_cast<unsigned char *>(::operator new[](kHeaderBlockSize, blockAlign));
1003 memset(fShared->fHeaderBlock, 0, kHeaderBlockSize);
1004 fShared->fBlock = static_cast<unsigned char *>(::operator new[](fShared->fBlockSize, blockAlign));
1005 memset(fShared->fBlock, 0, fShared->fBlockSize);
1006}
1007
1009
1011{
1012 if (fFile)
1013 fclose(fFile);
1014
1015 std::align_val_t blockAlign{kBlockAlign};
1016 if (fHeaderBlock)
1017 ::operator delete[](fHeaderBlock, blockAlign);
1018 if (fBlock)
1019 ::operator delete[](fBlock, blockAlign);
1020}
1021
1022namespace {
1023int FSeek64(FILE *stream, std::int64_t offset, int origin)
1024{
1025#ifdef R__SEEK64
1026 return fseeko64(stream, offset, origin);
1027#else
1028 return fseek(stream, offset, origin);
1029#endif
1030}
1031} // namespace
1032
1034{
1035 auto &shared = *fShared;
1036
1037 // Write the last partially filled block, which may still need appropriate alignment for Direct I/O.
1038 // If it is the first block, get the updated header block.
1039 if (shared.fBlockOffset == 0) {
1040 std::size_t headerBlockSize = kHeaderBlockSize;
1041 if (headerBlockSize > shared.fFilePos) {
1042 headerBlockSize = shared.fFilePos;
1043 }
1044 memcpy(shared.fBlock, shared.fHeaderBlock, headerBlockSize);
1045 }
1046
1047 std::size_t retval = FSeek64(shared.fFile, shared.fBlockOffset, SEEK_SET);
1048 if (retval)
1049 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1050
1051 std::size_t lastBlockSize = shared.fFilePos - shared.fBlockOffset;
1052 R__ASSERT(lastBlockSize <= shared.fBlockSize);
1053 if (shared.fDirectIO) {
1054 // Round up to a multiple of kBlockAlign.
1055 lastBlockSize += kBlockAlign - 1;
1056 lastBlockSize = (lastBlockSize / kBlockAlign) * kBlockAlign;
1057 R__ASSERT(lastBlockSize <= shared.fBlockSize);
1058 }
1059 retval = fwrite(shared.fBlock, 1, lastBlockSize, shared.fFile);
1060 if (retval != lastBlockSize)
1061 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1062
1063 // Write the (updated) header block, unless it was part of the write above.
1064 if (shared.fBlockOffset > 0) {
1065 retval = FSeek64(shared.fFile, 0, SEEK_SET);
1066 if (retval)
1067 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1068
1069 retval = fwrite(shared.fHeaderBlock, 1, kHeaderBlockSize, shared.fFile);
1070 if (retval != RImplSimple::kHeaderBlockSize)
1071 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1072 }
1073
1074 retval = fflush(shared.fFile);
1075 if (retval)
1076 throw RException(R__FAIL(std::string("Flush failed: ") + strerror(errno)));
1077}
1078
1079void ROOT::Internal::RNTupleFileWriter::RImplSimple::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1080{
1081 auto &shared = *fShared;
1082
1083 R__ASSERT(shared.fFile);
1084 size_t retval;
1085 if ((offset >= 0) && (static_cast<std::uint64_t>(offset) != shared.fFilePos)) {
1086 shared.fFilePos = offset;
1087 }
1088
1089 // Keep header block to overwrite on commit.
1090 if (shared.fFilePos < kHeaderBlockSize) {
1091 std::size_t headerBytes = nbytes;
1092 if (shared.fFilePos + headerBytes > kHeaderBlockSize) {
1093 headerBytes = kHeaderBlockSize - shared.fFilePos;
1094 }
1095 memcpy(shared.fHeaderBlock + shared.fFilePos, buffer, headerBytes);
1096 }
1097
1098 R__ASSERT(shared.fFilePos >= shared.fBlockOffset);
1099
1100 while (nbytes > 0) {
1101 std::uint64_t posInBlock = shared.fFilePos % shared.fBlockSize;
1102 std::uint64_t blockOffset = shared.fFilePos - posInBlock;
1103 if (blockOffset != shared.fBlockOffset) {
1104 // Write the block.
1105 retval = FSeek64(shared.fFile, shared.fBlockOffset, SEEK_SET);
1106 if (retval)
1107 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1108
1109 retval = fwrite(shared.fBlock, 1, shared.fBlockSize, shared.fFile);
1110 if (retval != shared.fBlockSize)
1111 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1112
1113 // Null the buffer contents for good measure.
1114 memset(shared.fBlock, 0, shared.fBlockSize);
1115 }
1116
1117 shared.fBlockOffset = blockOffset;
1118 std::size_t blockSize = nbytes;
1119 if (blockSize > shared.fBlockSize - posInBlock) {
1120 blockSize = shared.fBlockSize - posInBlock;
1121 }
1122 memcpy(shared.fBlock + posInBlock, buffer, blockSize);
1123 buffer = static_cast<const unsigned char *>(buffer) + blockSize;
1124 nbytes -= blockSize;
1125 shared.fFilePos += blockSize;
1126 }
1127}
1128
1129std::uint64_t
1130ROOT::Internal::RNTupleFileWriter::RImplSimple::WriteKey(const void *buffer, std::size_t nbytes, std::size_t len,
1131 std::int64_t offset, std::uint64_t directoryOffset,
1132 const std::string &className, const std::string &objectName,
1133 const std::string &title)
1134{
1135 auto &shared = *fShared;
1136
1137 if (offset > 0)
1138 shared.fKeyOffset = offset;
1139 RTFString strClass{className};
1140 RTFString strObject{objectName};
1141 RTFString strTitle{title};
1142
1143 RTFKey key(shared.fKeyOffset, directoryOffset, strClass, strObject, strTitle, len, nbytes);
1144 Write(&key, key.GetHeaderSize(), shared.fKeyOffset);
1145 Write(&strClass, strClass.GetSize());
1146 Write(&strObject, strObject.GetSize());
1147 Write(&strTitle, strTitle.GetSize());
1148 auto offsetData = shared.fFilePos;
1149 // The next key starts after the data.
1150 shared.fKeyOffset = offsetData + nbytes;
1151 if (buffer)
1152 Write(buffer, nbytes);
1153
1154 return offsetData;
1155}
1156
1157std::uint64_t ROOT::Internal::RNTupleFileWriter::RImplSimple::ReserveBlobKey(std::size_t nbytes, std::size_t len,
1158 unsigned char keyBuffer[kBlobKeyLen])
1159{
1160 auto &shared = *fShared;
1161
1162 if (keyBuffer) {
1163 PrepareBlobKey(shared.fKeyOffset, nbytes, len, keyBuffer);
1164 } else {
1165 unsigned char localKeyBuffer[kBlobKeyLen];
1166 PrepareBlobKey(shared.fKeyOffset, nbytes, len, localKeyBuffer);
1167 Write(localKeyBuffer, kBlobKeyLen, shared.fKeyOffset);
1168 }
1169
1170 auto offsetData = shared.fKeyOffset + kBlobKeyLen;
1171 // The next key starts after the data.
1172 shared.fKeyOffset = offsetData + nbytes;
1173
1174 return offsetData;
1175}
1176
1177////////////////////////////////////////////////////////////////////////////////
1178
1179template <typename T>
1180std::uint64_t ROOT::Internal::RNTupleFileWriter::ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes,
1181 std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
1182{
1183 std::uint64_t offsetKey;
1184 ROOT::Internal::RKeyBlob keyBlob(&file);
1185 // Since it is unknown beforehand if offsetKey is beyond the 2GB limit or not,
1186 // RKeyBlob will always reserve space for a big key (version >= 1000)
1187 keyBlob.Reserve(nbytes, &offsetKey);
1188
1189 if (keyBuffer) {
1190 PrepareBlobKey(offsetKey, nbytes, len, keyBuffer);
1191 } else {
1192 unsigned char localKeyBuffer[kBlobKeyLen];
1193 PrepareBlobKey(offsetKey, nbytes, len, localKeyBuffer);
1194 caller.Write(localKeyBuffer, kBlobKeyLen, offsetKey);
1195 }
1196
1197 if (keyBlob.WasAllocatedInAFreeSlot()) {
1198 // If the key was allocated in a free slot, the last 4 bytes of its buffer contain the new size
1199 // of the remaining free slot and we need to write it to disk before the key gets destroyed at the end of the
1200 // function.
1201 caller.Write(keyBlob.GetBuffer() + nbytes, sizeof(Int_t), offsetKey + kBlobKeyLen + nbytes);
1202 }
1203
1204 auto offsetData = offsetKey + kBlobKeyLen;
1205
1206 return offsetData;
1207}
1208
1209void ROOT::Internal::RNTupleFileWriter::RImplTFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1210{
1211 fDirectory->GetFile()->Seek(offset);
1212 bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
1213 if (rv)
1214 throw RException(R__FAIL("WriteBuffer failed."));
1215}
1216
1218 unsigned char keyBuffer[kBlobKeyLen])
1219{
1220 auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *fDirectory->GetFile(), nbytes, len, keyBuffer);
1221 return offsetData;
1222}
1223
1224////////////////////////////////////////////////////////////////////////////////
1225
1226void ROOT::Internal::RNTupleFileWriter::RImplRFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1227{
1228 auto *file = ROOT::Internal::GetRFileTFile(*fFile);
1229 file->Seek(offset);
1230 bool rv = file->WriteBuffer((char *)(buffer), nbytes);
1231 if (rv)
1232 throw RException(R__FAIL("WriteBuffer failed."));
1233}
1234
1236 unsigned char keyBuffer[kBlobKeyLen])
1237{
1238 auto *file = ROOT::Internal::GetRFileTFile(*fFile);
1239 auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *file, nbytes, len, keyBuffer);
1240 return offsetData;
1241}
1242
1243////////////////////////////////////////////////////////////////////////////////
1244
1245ROOT::Internal::RNTupleFileWriter::RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool isHidden)
1246 : fIsHidden(isHidden), fNTupleName(name)
1247{
1248 fFile.emplace<RImplSimple>();
1249 fNTupleAnchor.fMaxKeySize = maxKeySize;
1250 auto infoRNTuple = RNTuple::Class()->GetStreamerInfo();
1251 fStreamerInfoMap[infoRNTuple->GetNumber()] = infoRNTuple;
1252}
1253
1255
1256std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1257ROOT::Internal::RNTupleFileWriter::Recreate(std::string_view ntupleName, std::string_view path,
1258 EContainerFormat containerFormat, const ROOT::RNTupleWriteOptions &options)
1259{
1260 std::string fileName(path);
1261 size_t idxDirSep = fileName.find_last_of("\\/");
1262 if (idxDirSep != std::string::npos) {
1263 fileName.erase(0, idxDirSep + 1);
1264 }
1265#ifdef R__LINUX
1266 int flags = O_WRONLY | O_CREAT | O_TRUNC;
1267#ifdef O_LARGEFILE
1268 // Add the equivalent flag that is passed by fopen64.
1269 flags |= O_LARGEFILE;
1270#endif
1271 if (options.GetUseDirectIO()) {
1272 flags |= O_DIRECT;
1273 }
1274 int fd = open(std::string(path).c_str(), flags, 0666);
1275 if (fd == -1) {
1276 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1277 }
1278 FILE *fileStream = fdopen(fd, "wb");
1279#else
1280#ifdef R__SEEK64
1281 FILE *fileStream = fopen64(std::string(path.data(), path.size()).c_str(), "wb");
1282#else
1283 FILE *fileStream = fopen(std::string(path.data(), path.size()).c_str(), "wb");
1284#endif
1285#endif
1286 if (!fileStream) {
1287 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1288 }
1289 // RNTupleFileWriter::RImplSimple does its own buffering, turn off additional buffering from C stdio.
1290 std::setvbuf(fileStream, nullptr, _IONBF, 0);
1291
1292 auto writer = std::unique_ptr<RNTupleFileWriter>(
1293 new RNTupleFileWriter(ntupleName, options.GetMaxKeySize(), /*isHidden=*/false));
1294 RImplSimple &fileSimple = std::get<RImplSimple>(writer->fFile);
1295 fileSimple.AllocateBuffers(options.GetWriteBufferSize());
1296 fileSimple.fShared->fFile = fileStream;
1297 fileSimple.fShared->fDirectIO = options.GetUseDirectIO();
1298 writer->fFileName = fileName;
1299
1300 int defaultCompression = options.GetCompression();
1301 switch (containerFormat) {
1302 case EContainerFormat::kTFile: writer->WriteTFileSkeleton(defaultCompression); break;
1303 case EContainerFormat::kBare:
1304 writer->fIsBare = true;
1305 writer->WriteBareFileSkeleton(defaultCompression);
1306 break;
1307 default: R__ASSERT(false && "Internal error: unhandled container format");
1308 }
1309
1310 return writer;
1311}
1312
1313std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1314ROOT::Internal::RNTupleFileWriter::Append(std::string_view ntupleName, TDirectory &fileOrDirectory,
1315 std::uint64_t maxKeySize, bool hidden)
1316{
1317 TFile *file = fileOrDirectory.GetFile();
1318 if (!file)
1319 throw RException(R__FAIL("invalid attempt to add an RNTuple to a directory that is not backed by a file"));
1320 assert(file->IsBinary());
1321
1322 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, hidden));
1323 auto &fileProper = writer->fFile.emplace<RImplTFile>();
1324 fileProper.fDirectory = &fileOrDirectory;
1325 return writer;
1326}
1327
1328std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1329ROOT::Internal::RNTupleFileWriter::Append(std::string_view ntupleName, ROOT::RFile &file, std::string_view ntupleDir,
1330 std::uint64_t maxKeySize)
1331{
1332 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, /*isHidden=*/false));
1333 auto &rfile = writer->fFile.emplace<RImplRFile>();
1334 rfile.fFile = &file;
1335 R__ASSERT(ntupleDir.empty() || ntupleDir[ntupleDir.size() - 1] == '/');
1336 rfile.fDir = ntupleDir;
1337 return writer;
1338}
1339
1340std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1342{
1343 if (auto *tfile = std::get_if<RImplTFile>(&fFile)) {
1344 return Append(ntupleName, *tfile->fDirectory, fNTupleAnchor.fMaxKeySize, /* isHidden= */ true);
1345 } else if (auto *file = std::get_if<RImplSimple>(&fFile)) {
1346 if (fIsBare)
1347 throw ROOT::RException(R__FAIL("cloning a bare file is currently unsupported"));
1348
1349 auto writer = std::unique_ptr<RNTupleFileWriter>(
1350 new RNTupleFileWriter(ntupleName, fNTupleAnchor.GetMaxKeySize(), /* isHidden= */ true));
1351 auto &clonedFile = std::get<RImplSimple>(writer->fFile);
1352 clonedFile.fShared = file->fShared;
1353 return writer;
1354 }
1355 // TODO: support also RFile-based writers
1356 throw ROOT::RException(R__FAIL("cannot clone an RFile-based RNTupleFileWriter."));
1357}
1358
1360{
1361 RImplSimple *fileSimple = std::get_if<RImplSimple>(&fFile);
1362 if (!fileSimple)
1363 throw RException(R__FAIL("invalid attempt to seek non-simple writer"));
1364
1365 fileSimple->fShared->fFilePos = offset;
1366 fileSimple->fShared->fKeyOffset = offset;
1367 // The next Write() will Flush() if necessary.
1368}
1369
1371{
1372 fStreamerInfoMap.insert(streamerInfos.cbegin(), streamerInfos.cend());
1373}
1374
1376{
1377 const auto WriteStreamerInfoToFile = [&](TFile *file) {
1378 // Make sure the streamer info records used in the RNTuple are written to the file
1380 buf.SetParent(file);
1381 for (auto [_, info] : fStreamerInfoMap)
1382 buf.TagStreamerInfo(info);
1383 };
1384
1385 ROOT::Internal::RNTupleLink anchorInfo;
1386 // NOTE: checksum length is included in the uncompressed len
1387 anchorInfo.fLength = RTFNTuple{}.GetSize() + sizeof(std::uint64_t);
1389
1390 if (auto fileProper = std::get_if<RImplTFile>(&fFile)) {
1391 // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
1392 fileProper->fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
1393 WriteStreamerInfoToFile(fileProper->fDirectory->GetFile());
1394 auto key = static_cast<TKey *>(fileProper->fDirectory->GetListOfKeys()->FindObject(fNTupleName.c_str()));
1395 R__ASSERT(key);
1396 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetKeylen());
1397 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNbytes() - key->GetKeylen());
1398 // NOTE: this must happen after FindObject(), otherwise some TFile implementations, such as TBufferMergerFile,
1399 // may reset the keys list upon write.
1400 fileProper->fDirectory->GetFile()->Write();
1401
1402 if (fIsHidden) {
1403 // Remove the anchor's key from the directory's KeysList to disallow retrieving directly the
1404 // attribute RNTuple from the TFile.
1405 fileProper->fDirectory->GetListOfKeys()->Remove(key);
1406 }
1407 } else if (auto fileRFile = std::get_if<RImplRFile>(&fFile)) {
1408 // Same as the case above but handled via RFile
1409 fileRFile->fFile->Put(fileRFile->fDir + fNTupleName, fNTupleAnchor);
1410 WriteStreamerInfoToFile(ROOT::Internal::GetRFileTFile(*fileRFile->fFile));
1411 auto key = fileRFile->fFile->GetKeyInfo(fNTupleName);
1412 R__ASSERT(key);
1413 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetNBytesKey());
1414 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNBytesObj());
1415 fileRFile->fFile->Flush();
1416 } else {
1417 // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
1418 auto &fileSimple = std::get<RImplSimple>(fFile);
1419 auto &shared = *fileSimple.fShared;
1420
1421 if (fIsBare) {
1422 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1423
1424 // Compute the checksum
1425 std::uint64_t checksum = XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData());
1426 memcpy(shared.fHeaderBlock + shared.fControlBlock->fSeekNTuple, &ntupleOnDisk, ntupleOnDisk.GetSize());
1427 memcpy(shared.fHeaderBlock + shared.fControlBlock->fSeekNTuple + ntupleOnDisk.GetSize(), &checksum,
1428 sizeof(checksum));
1429 fileSimple.Flush();
1430
1431 anchorInfo.fLocator.SetPosition(shared.fControlBlock->fSeekNTuple);
1432 anchorInfo.fLocator.SetNBytesOnStorage(ntupleOnDisk.GetSize());
1433 } else {
1434 anchorInfo = WriteTFileNTupleKey(compression);
1435 if (!fIsHidden) {
1436 WriteTFileKeysList(anchorInfo.fLocator.GetNBytesOnStorage()); // NOTE: this is written uncompressed
1437 WriteTFileStreamerInfo(compression);
1438 WriteTFileFreeList(); // NOTE: this is written uncompressed
1439
1440 // Update header and TFile record
1441 memcpy(shared.fHeaderBlock, &shared.fControlBlock->fHeader, shared.fControlBlock->fHeader.GetSize());
1442 R__ASSERT(shared.fControlBlock->fSeekFileRecord + shared.fControlBlock->fFileRecord.GetSize() <
1443 RImplSimple::kHeaderBlockSize);
1444 memcpy(shared.fHeaderBlock + shared.fControlBlock->fSeekFileRecord, &shared.fControlBlock->fFileRecord,
1445 shared.fControlBlock->fFileRecord.GetSize());
1446 }
1447
1448 fileSimple.Flush();
1449 }
1450 }
1451
1452 return anchorInfo;
1453}
1454
1455std::uint64_t ROOT::Internal::RNTupleFileWriter::WriteBlob(const void *data, size_t nbytes, size_t len)
1456{
1457 auto writeKey = [this](const void *payload, size_t nBytes, size_t length) {
1458 std::uint64_t offset = ReserveBlob(nBytes, length);
1459 WriteIntoReservedBlob(payload, nBytes, offset);
1460 return offset;
1461 };
1462
1463 const std::uint64_t maxKeySize = fNTupleAnchor.fMaxKeySize;
1464 R__ASSERT(maxKeySize > 0);
1465 // We don't need the object length except for seeing compression ratios in TFile::Map()
1466 // Make sure that the on-disk object length fits into the TKey header.
1467 if (static_cast<std::uint64_t>(len) > static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()))
1468 len = nbytes;
1469
1470 if (nbytes <= maxKeySize) {
1471 // Fast path: only write 1 key.
1472 return writeKey(data, nbytes, len);
1473 }
1474
1475 /**
1476 * Writing a key bigger than the max allowed size. In this case we split the payload
1477 * into multiple keys, reserving the end of the first key payload for pointers to the
1478 * next ones. E.g. if a key needs to be split into 3 chunks, the first chunk will have
1479 * the format:
1480 * +--------------------+
1481 * | |
1482 * | Data |
1483 * |--------------------|
1484 * | pointer to chunk 2 |
1485 * | pointer to chunk 3 |
1486 * +--------------------+
1487 */
1488 const size_t nChunks = ComputeNumChunks(nbytes, maxKeySize);
1489 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
1490 const size_t nbytesFirstChunk = maxKeySize - nbytesChunkOffsets;
1491 // Skip writing the first chunk, it will be written last (in the file) below.
1492
1493 const uint8_t *chunkData = reinterpret_cast<const uint8_t *>(data) + nbytesFirstChunk;
1494 size_t remainingBytes = nbytes - nbytesFirstChunk;
1495
1496 const auto chunkOffsetsToWrite = MakeUninitArray<std::uint64_t>(nChunks - 1);
1497 std::uint64_t chunkOffsetIdx = 0;
1498
1499 do {
1500 const size_t bytesNextChunk = std::min<size_t>(remainingBytes, maxKeySize);
1501 const std::uint64_t offset = writeKey(chunkData, bytesNextChunk, bytesNextChunk);
1502
1503 RNTupleSerializer::SerializeUInt64(offset, &chunkOffsetsToWrite[chunkOffsetIdx]);
1504 ++chunkOffsetIdx;
1505
1506 remainingBytes -= bytesNextChunk;
1507 chunkData += bytesNextChunk;
1508
1509 } while (remainingBytes > 0);
1510
1511 // Write the first key, with part of the data and the pointers to (logically) following keys appended.
1512 const std::uint64_t firstOffset = ReserveBlob(maxKeySize, maxKeySize);
1513 WriteIntoReservedBlob(data, nbytesFirstChunk, firstOffset);
1514 const std::uint64_t chunkOffsetsOffset = firstOffset + nbytesFirstChunk;
1515 WriteIntoReservedBlob(chunkOffsetsToWrite.get(), nbytesChunkOffsets, chunkOffsetsOffset);
1516
1517 return firstOffset;
1518}
1519
1520std::uint64_t
1521ROOT::Internal::RNTupleFileWriter::ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen])
1522{
1523 // ReserveBlob cannot be used to reserve a multi-key blob
1524 R__ASSERT(nbytes <= fNTupleAnchor.GetMaxKeySize());
1525
1526 std::uint64_t offset;
1527 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1528 if (fIsBare) {
1529 offset = fileSimple->fShared->fKeyOffset;
1530 fileSimple->fShared->fKeyOffset += nbytes;
1531 } else {
1532 offset = fileSimple->ReserveBlobKey(nbytes, len, keyBuffer);
1533 }
1534 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1535 offset = fileProper->ReserveBlobKey(nbytes, len, keyBuffer);
1536 } else {
1537 auto &fileRFile = std::get<RImplRFile>(fFile);
1538 offset = fileRFile.ReserveBlobKey(nbytes, len, keyBuffer);
1539 }
1540 return offset;
1541}
1542
1543void ROOT::Internal::RNTupleFileWriter::WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
1544{
1545 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1546 fileSimple->Write(buffer, nbytes, offset);
1547 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1548 fileProper->Write(buffer, nbytes, offset);
1549 } else {
1550 auto &fileRFile = std::get<RImplRFile>(fFile);
1551 fileRFile.Write(buffer, nbytes, offset);
1552 }
1553}
1554
1555std::uint64_t ROOT::Internal::RNTupleFileWriter::WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
1556{
1557 auto offset = WriteBlob(data, nbytes, lenHeader);
1558 fNTupleAnchor.fLenHeader = lenHeader;
1559 fNTupleAnchor.fNBytesHeader = nbytes;
1560 fNTupleAnchor.fSeekHeader = offset;
1561 return offset;
1562}
1563
1564std::uint64_t ROOT::Internal::RNTupleFileWriter::WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
1565{
1566 auto offset = WriteBlob(data, nbytes, lenFooter);
1567 fNTupleAnchor.fLenFooter = lenFooter;
1568 fNTupleAnchor.fNBytesFooter = nbytes;
1569 fNTupleAnchor.fSeekFooter = offset;
1570 return offset;
1571}
1572
1574{
1575 RBareFileHeader bareHeader;
1576 bareHeader.fCompress = defaultCompression;
1577 auto &fileSimple = std::get<RImplSimple>(fFile);
1578 fileSimple.Write(&bareHeader, sizeof(bareHeader), 0);
1579 RTFString ntupleName{fNTupleName};
1580 fileSimple.Write(&ntupleName, ntupleName.GetSize());
1581
1582 // Write zero-initialized ntuple to reserve the space; will be overwritten on commit
1583 RTFNTuple ntupleOnDisk;
1584 fileSimple.fShared->fControlBlock->fSeekNTuple = fileSimple.fShared->fFilePos;
1585 fileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize());
1586 std::uint64_t checksum = 0;
1587 fileSimple.Write(&checksum, sizeof(checksum));
1588 fileSimple.fShared->fKeyOffset = fileSimple.fShared->fFilePos;
1589}
1590
1592{
1593 // The streamer info record is a TList of TStreamerInfo object. We cannot use
1594 // RNTupleSerializer::SerializeStreamerInfos because that uses TBufferIO::WriteObject.
1595 // This would prepend the streamed TList with self-decription information.
1596 // The streamer info record is just the streamed TList.
1597
1598 TList streamerInfoList;
1599 for (auto [_, info] : fStreamerInfoMap) {
1600 streamerInfoList.Add(info);
1601 }
1602
1603 // We will stream the list with a TBufferFile. When reading the streamer info records back,
1604 // the read buffer includes the key and the streamed list. Therefore, we need to start streaming
1605 // with an offset of the key length. Otherwise, the offset for referencing duplicate objects in the
1606 // buffer will point to the wrong places.
1607
1608 // Figure out key length
1609 RTFString strTList{"TList"};
1610 RTFString strStreamerInfo{"StreamerInfo"};
1611 RTFString strStreamerTitle{"Doubly linked list"};
1612 auto &fileSimple = std::get<RImplSimple>(fFile);
1613 fileSimple.fShared->fControlBlock->fHeader.SetSeekInfo(fileSimple.fShared->fKeyOffset);
1614 auto keyLen = RTFKey(fileSimple.fShared->fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, strTList,
1615 strStreamerInfo, strStreamerTitle, 0)
1616 .fKeyLen;
1617
1618 TBufferFile buffer(TBuffer::kWrite, keyLen + 1);
1619 buffer.SetBufferOffset(keyLen);
1620 streamerInfoList.Streamer(buffer);
1621 assert(buffer.Length() > keyLen);
1622 const auto bufPayload = buffer.Buffer() + keyLen;
1623 const auto lenPayload = buffer.Length() - keyLen;
1624
1625 auto zipStreamerInfos = MakeUninitArray<unsigned char>(lenPayload);
1626 auto szZipStreamerInfos = RNTupleCompressor::Zip(bufPayload, lenPayload, compression, zipStreamerInfos.get());
1627
1628 fileSimple.WriteKey(zipStreamerInfos.get(), szZipStreamerInfos, lenPayload,
1629 fileSimple.fShared->fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, "TList",
1630 "StreamerInfo", "Doubly linked list");
1631 fileSimple.fShared->fControlBlock->fHeader.SetNbytesInfo(fileSimple.fShared->fFilePos -
1632 fileSimple.fShared->fControlBlock->fHeader.GetSeekInfo());
1633}
1634
1636{
1637 RTFString strEmpty;
1638 RTFString strRNTupleClass{"ROOT::RNTuple"};
1639 RTFString strRNTupleName{fNTupleName};
1640 RTFString strFileName{fFileName};
1641
1642 auto &fileSimple = std::get<RImplSimple>(fFile);
1643 RTFKey keyRNTuple(fileSimple.fShared->fControlBlock->fSeekNTuple, RTFHeader::kBEGIN, strRNTupleClass, strRNTupleName,
1644 strEmpty, RTFNTuple::GetSizePlusChecksum(), anchorSize);
1645
1646 auto &fileShared = *fileSimple.fShared;
1647 fileSimple.fShared->fControlBlock->fFileRecord.SetSeekKeys(fileShared.fKeyOffset);
1648 RTFKeyList keyList{1};
1649 RTFKey keyKeyList(fileSimple.fShared->fControlBlock->fFileRecord.GetSeekKeys(), RTFHeader::kBEGIN, strEmpty,
1650 strFileName, strEmpty, keyList.GetSize() + keyRNTuple.fKeyLen);
1651 fileSimple.Write(&keyKeyList, keyKeyList.GetHeaderSize(),
1652 fileSimple.fShared->fControlBlock->fFileRecord.GetSeekKeys());
1653 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1654 fileSimple.Write(&strFileName, strFileName.GetSize());
1655 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1656 fileSimple.Write(&keyList, keyList.GetSize());
1657 fileSimple.Write(&keyRNTuple, keyRNTuple.GetHeaderSize());
1658 // Write class name, object name, and title for this key.
1659 fileSimple.Write(&strRNTupleClass, strRNTupleClass.GetSize());
1660 fileSimple.Write(&strRNTupleName, strRNTupleName.GetSize());
1661 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1662 fileSimple.fShared->fControlBlock->fFileRecord.fNBytesKeys =
1663 fileShared.fFilePos - fileSimple.fShared->fControlBlock->fFileRecord.GetSeekKeys();
1664 fileShared.fKeyOffset = fileShared.fFilePos;
1665}
1666
1668{
1669 auto &fileSimple = std::get<RImplSimple>(fFile);
1670 auto &fileShared = *fileSimple.fShared;
1671 fileSimple.fShared->fControlBlock->fHeader.SetSeekFree(fileShared.fKeyOffset);
1672 RTFString strEmpty;
1673 RTFString strFileName{fFileName};
1674 RTFFreeEntry freeEntry;
1675 RTFKey keyFreeList(fileSimple.fShared->fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, strEmpty,
1676 strFileName, strEmpty, freeEntry.GetSize());
1677 std::uint64_t firstFree = fileSimple.fShared->fControlBlock->fHeader.GetSeekFree() + keyFreeList.GetSize();
1678 freeEntry.Set(firstFree, std::max(2000000000ULL, ((firstFree / 1000000000ULL) + 1) * 1000000000ULL));
1679 fileSimple.WriteKey(&freeEntry, freeEntry.GetSize(), freeEntry.GetSize(),
1680 fileSimple.fShared->fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, "", fFileName, "");
1681 fileSimple.fShared->fControlBlock->fHeader.SetNbytesFree(fileShared.fFilePos -
1682 fileSimple.fShared->fControlBlock->fHeader.GetSeekFree());
1683 fileSimple.fShared->fControlBlock->fHeader.SetEnd(fileShared.fFilePos);
1684}
1685
1687{
1688 RTFString strRNTupleClass{"ROOT::RNTuple"};
1689 RTFString strRNTupleName{fNTupleName};
1690 RTFString strEmpty;
1691
1692 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1693 RUInt64BE checksum{XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData())};
1694 auto &fileSimple = std::get<RImplSimple>(fFile);
1695 fileSimple.fShared->fControlBlock->fSeekNTuple = fileSimple.fShared->fKeyOffset;
1696
1697 char keyBuf[RTFNTuple::GetSizePlusChecksum()];
1698
1699 // concatenate the RNTuple anchor with its checksum
1700 memcpy(keyBuf, &ntupleOnDisk, sizeof(RTFNTuple));
1701 memcpy(keyBuf + sizeof(RTFNTuple), &checksum, sizeof(checksum));
1702
1703 const auto sizeAnchor = sizeof(RTFNTuple) + sizeof(checksum);
1704 char zipAnchor[RTFNTuple::GetSizePlusChecksum()];
1705 auto szZipAnchor = RNTupleCompressor::Zip(keyBuf, sizeAnchor, compression, zipAnchor);
1706
1707 ROOT::Internal::RNTupleLink anchorLink;
1708 auto anchorOffset =
1709 fileSimple.WriteKey(zipAnchor, szZipAnchor, sizeof(keyBuf), fileSimple.fShared->fControlBlock->fSeekNTuple,
1710 RTFHeader::kBEGIN, "ROOT::RNTuple", fNTupleName, "");
1711
1712 assert(szZipAnchor < std::numeric_limits<decltype(anchorLink.fLength)>::max());
1713 anchorLink.fLength = sizeof(keyBuf);
1714 anchorLink.fLocator.SetPosition(anchorOffset);
1715 anchorLink.fLocator.SetNBytesOnStorage(szZipAnchor);
1716 return anchorLink;
1717}
1718
1720{
1721 RTFString strTFile{"TFile"};
1722 RTFString strFileName{fFileName};
1723 RTFString strEmpty;
1724
1725 auto &fileSimple = std::get<RImplSimple>(fFile);
1726 fileSimple.fShared->fControlBlock->fHeader = RTFHeader(defaultCompression);
1727
1728 RTFUUID uuid;
1729
1730 // First record of the file: the TFile object at offset kBEGIN (= 100)
1731 RTFKey keyRoot(RTFHeader::kBEGIN, 0, strTFile, strFileName, strEmpty,
1732 sizeof(RTFDirectory) + strFileName.GetSize() + strEmpty.GetSize() + uuid.GetSize());
1733 std::uint32_t nbytesName = keyRoot.fKeyLen + strFileName.GetSize() + 1;
1734 fileSimple.fShared->fControlBlock->fFileRecord.fNBytesName = nbytesName;
1735 fileSimple.fShared->fControlBlock->fHeader.SetNbytesName(nbytesName);
1736
1737 fileSimple.Write(&keyRoot, keyRoot.GetHeaderSize(), RTFHeader::kBEGIN);
1738 // Write class name, object name, and title for the TFile key.
1739 fileSimple.Write(&strTFile, strTFile.GetSize());
1740 fileSimple.Write(&strFileName, strFileName.GetSize());
1741 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1742 // Write the name and title of the TNamed preceding the TFile entry.
1743 fileSimple.Write(&strFileName, strFileName.GetSize());
1744 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1745 // Will be overwritten on commit
1746 fileSimple.fShared->fControlBlock->fSeekFileRecord = fileSimple.fShared->fFilePos;
1747 fileSimple.Write(&fileSimple.fShared->fControlBlock->fFileRecord,
1748 fileSimple.fShared->fControlBlock->fFileRecord.GetSize());
1749 fileSimple.Write(&uuid, uuid.GetSize());
1750
1751 // Padding bytes to allow the TFile record to grow for a big file
1752 RUInt32BE padding{0};
1753 for (int i = 0; i < 3; ++i)
1754 fileSimple.Write(&padding, sizeof(padding));
1755 fileSimple.fShared->fKeyOffset = fileSimple.fShared->fFilePos;
1756}
T ReadBuffer(TBufferFile *buf)
One of the template functions used to read objects from messages.
Definition MPSendRecv.h:158
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:322
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
#define ROOT_VERSION_CODE
Definition RVersion.hxx:27
#define ClassDefInlineOverride(name, id)
Definition Rtypes.h:359
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:139
const Int_t kBEGIN
Definition TFile.cxx:207
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:142
T1 fFirst
Definition X11Events.mm:86
#define _(A, B)
Definition cfortran.h:112
The RKeyBlob writes an invisible key into a TFile.
bool WasAllocatedInAFreeSlot() const
void Reserve(size_t nbytes, std::uint64_t *seekKey)
Register a new key for a data record of size nbytes.
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
void LoadStreamerInfo()
Load the streamer info from the file into the global list of streamer infos.
RResult< RNTuple > GetNTupleProperAtOffset(std::uint64_t payloadOffset, std::uint64_t compSize, std::uint64_t uncompLen)
Loads an RNTuple anchor from a TFile at the given file offset (unzipping it if necessary).
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
ROOT::RResult< void > TryReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Like ReadBuffer but returns a RResult instead of throwing.
std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName)
Searches for a key with the given name and type in the key index of the directory starting at offsetD...
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
RResult< RNTuple > GetNTupleProper(std::string_view ntuplePath)
Used when the file turns out to be a TFile container.
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Write RNTuple data blocks in a TFile or a bare file container.
std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves a new record as an RBlob key in the file.
ROOT::Internal::RNTupleLink WriteTFileNTupleKey(int compression)
The only key that will be visible in file->ls() Returns the link to the RNTuple anchor.
void WriteTFileStreamerInfo(int compression)
Write the compressed streamer info record with the description of the RNTuple class.
std::string fNTupleName
The identifier of the RNTuple; A single writer object can only write a single RNTuple but multiple wr...
void WriteTFileKeysList(std::uint64_t anchorSize)
Write the TList with the RNTuple key.
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
static std::uint64_t ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
static std::unique_ptr< RNTupleFileWriter > Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize, bool isHidden)
The directory parameter can also be a TFile object (TFile inherits from TDirectory).
void WriteTFileFreeList()
Last record in the file.
std::unique_ptr< RNTupleFileWriter > CloneAsHidden(std::string_view ntupleName) const
Creates a new RNTupleFileWriter with the same underlying TDirectory as this but writing to a differen...
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the header and TFile object.
RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool isHidden)
Private constructor used by all factory methods.
void Seek(std::uint64_t offset)
Seek a simple writer to offset.
ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fStreamerInfoMap
Set of streamer info records that should be written to the file.
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
static std::unique_ptr< RNTupleFileWriter > Recreate(std::string_view ntupleName, std::string_view path, EContainerFormat containerFormat, const ROOT::RNTupleWriteOptions &options)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
Write into a reserved record; the caller is responsible for making sure that the written byte range i...
static constexpr std::size_t kBlobKeyLen
The key length of a blob. It is always a big key (version > 1000) with class name RBlob.
RNTupleLink Commit(int compression=RCompressionSetting::EDefaults::kUseGeneralPurpose)
Writes the RNTuple key to the file so that the header and footer keys can be found.
static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
Prepares buffer for a new record as an RBlob key at offset.
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
bool fIsHidden
True if this RNTuple's anchor must be stored as a hidden key (this is the case e.g....
void UpdateStreamerInfos(const ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
Ensures that the streamer info records passed as argument are written to the file.
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
EContainerFormat
For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
An interface to read from, or write to, a ROOT file, as well as performing other common operations.
Definition RFile.hxx:254
std::uint64_t GetNBytesOnStorage() const
void SetType(ELocatorType type)
void SetPosition(std::uint64_t position)
void SetNBytesOnStorage(std::uint64_t nBytesOnStorage)
Common user-tunable settings for storing RNTuples.
std::size_t GetWriteBufferSize() const
std::uint64_t GetMaxKeySize() const
std::uint32_t GetCompression() const
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:67
std::uint64_t GetLenFooter() const
Definition RNTuple.hxx:140
std::uint64_t GetSeekHeader() const
Definition RNTuple.hxx:134
std::uint16_t GetVersionMajor() const
Definition RNTuple.hxx:130
std::uint64_t GetNBytesFooter() const
Definition RNTuple.hxx:139
std::uint64_t GetNBytesHeader() const
Definition RNTuple.hxx:135
std::uint64_t fMaxKeySize
The maximum size for a TKey payload. Payloads bigger than this size will be written as multiple blobs...
Definition RNTuple.hxx:121
std::uint16_t GetVersionMinor() const
Definition RNTuple.hxx:131
std::uint16_t GetVersionPatch() const
Definition RNTuple.hxx:132
std::uint64_t GetMaxKeySize() const
Definition RNTuple.hxx:141
std::uint64_t GetLenHeader() const
Definition RNTuple.hxx:136
static TClass * Class()
std::uint64_t GetSeekFooter() const
Definition RNTuple.hxx:138
std::uint16_t GetVersionEpoch() const
Definition RNTuple.hxx:129
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:222
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
void TagStreamerInfo(TVirtualStreamerInfo *info) override
Mark the classindex of the current file as using this TStreamerInfo.
void SetBufferDisplacement() override
Definition TBufferIO.h:82
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition TBuffer.cxx:269
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
void SetBufferOffset(Int_t offset=0)
Definition TBuffer.h:93
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
TVirtualStreamerInfo * GetStreamerInfo(Int_t version=0, Bool_t isTransient=kFALSE) const
returns a pointer to the TVirtualStreamerInfo object for version If the object does not exist,...
Definition TClass.cxx:4720
Describe directory structure in memory.
Definition TDirectory.h:45
virtual TFile * GetFile() const
Definition TDirectory.h:221
A file, usually with extension .root, that stores data and code in the form of serialized objects in ...
Definition TFile.h:130
Bool_t IsBinary() const
Definition TFile.h:347
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition TKey.h:28
Int_t Sizeof() const override
Return the size in bytes of the key header structure.
Definition TKey.cxx:1504
Int_t fVersion
Key version identifier.
Definition TKey.h:41
Int_t fLeft
Number of bytes left in current segment.
Definition TKey.h:50
Short_t fKeylen
Number of bytes for the key itself.
Definition TKey.h:45
Long64_t fSeekKey
Location of object on file.
Definition TKey.h:47
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
Definition TKey.cxx:513
virtual char * GetBuffer() const
Definition TKey.h:80
TString fClassName
Object Class name.
Definition TKey.h:49
A doubly linked list.
Definition TList.h:38
void Streamer(TBuffer &) override
Stream all objects in the collection to or from the I/O buffer.
Definition TList.cxx:1323
void Add(TObject *obj) override
Definition TList.h:81
virtual TObjLink * FirstLink() const
Definition TList.h:107
Describes a persistent version of a class.
void BuildCheck(TFile *file=nullptr, Bool_t load=kTRUE) override
Check if built and consistent with the class dictionary.
static TClass * Class()
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition TUUID.h:42
static TUUID UUIDv4()
Create a UUID version 4 (variant 1) UUID according to RFC 4122.
Definition TUUID.cxx:157
void FillBuffer(char *&buffer)
Stream UUID into output buffer.
Definition TUUID.cxx:306
RNTuple CreateAnchor(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
Definition RNTuple.cxx:51
TFile * GetRFileTFile(RFile &rfile)
Definition RFile.cxx:595
Helper templated class for swapping bytes; specializations for N={2,4,8} are provided below.
Definition Byteswap.h:124
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
void AllocateBuffers(std::size_t bufferSize)
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
static constexpr int kBlockAlign
Direct I/O requires that all buffers and write lengths are aligned.
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
static constexpr std::size_t kHeaderBlockSize
During commit, WriteTFileKeysList() updates fNBytesKeys and fSeekKeys of the RTFFile located at fSeek...
std::shared_ptr< RSharedData > fShared
std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
TDirectory * fDirectory
A sub directory in fFile or nullptr if the data is stored in the root directory of the file.
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
If a TFile container is written by a C stream (simple file), on dataset commit, the file header and t...
auto * tt
Definition textangle.C:16