Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RMiniFile.cxx
Go to the documentation of this file.
1/// \file RMiniFile.cxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-12-22
5
6/*************************************************************************
7 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
8 * All rights reserved. *
9 * *
10 * For the licensing terms see $ROOTSYS/LICENSE. *
11 * For the list of contributors see $ROOTSYS/README/CREDITS. *
12 *************************************************************************/
13
14#include "Rtypes.h"
15#include <ROOT/RConfig.hxx>
16#include <ROOT/RError.hxx>
17#include <ROOT/RMiniFile.hxx>
18#include <ROOT/RRawFile.hxx>
19#include <ROOT/RNTupleUtils.hxx>
20#include <ROOT/RNTupleZip.hxx>
23#include <ROOT/RFile.hxx>
24
25#include <Byteswap.h>
26#include <TBufferFile.h>
27#include <TDirectory.h>
28#include <TError.h>
29#include <TFile.h>
30#include <TKey.h>
31#include <TObjString.h>
32#include <TUUID.h>
33#include <TStreamerInfo.h>
34
35#include <xxhash.h>
36
37#include <algorithm>
38#include <cassert>
39#include <cerrno>
40#include <cstdio>
41#include <cstring>
42#include <memory>
43#include <string>
44#include <chrono>
45
46#ifdef R__LINUX
47#include <fcntl.h>
48#endif
49
50#ifndef R__LITTLE_ENDIAN
51#ifdef R__BYTESWAP
52// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise
53#define R__LITTLE_ENDIAN 1
54#else
55#define R__LITTLE_ENDIAN 0
56#endif
57#endif /* R__LITTLE_ENDIAN */
58
63
64namespace {
65
66// The following types are used to read and write the TFile binary format
67
68/// Big-endian 16-bit unsigned integer
69class RUInt16BE {
70private:
71 std::uint16_t fValBE = 0;
72 static std::uint16_t Swap(std::uint16_t val)
73 {
74#if R__LITTLE_ENDIAN == 1
75 return RByteSwap<sizeof(val)>::bswap(val);
76#else
77 return val;
78#endif
79 }
80
81public:
82 RUInt16BE() = default;
83 explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {}
84 operator std::uint16_t() const { return Swap(fValBE); }
85 RUInt16BE &operator=(const std::uint16_t val)
86 {
87 fValBE = Swap(val);
88 return *this;
89 }
90};
91
92/// Big-endian 32-bit unsigned integer
93class RUInt32BE {
94private:
95 std::uint32_t fValBE = 0;
96 static std::uint32_t Swap(std::uint32_t val)
97 {
98#if R__LITTLE_ENDIAN == 1
99 return RByteSwap<sizeof(val)>::bswap(val);
100#else
101 return val;
102#endif
103 }
104
105public:
106 RUInt32BE() = default;
107 explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {}
108 operator std::uint32_t() const { return Swap(fValBE); }
109 RUInt32BE &operator=(const std::uint32_t val)
110 {
111 fValBE = Swap(val);
112 return *this;
113 }
114};
115
116/// Big-endian 32-bit signed integer
117class RInt32BE {
118private:
119 std::int32_t fValBE = 0;
120 static std::int32_t Swap(std::int32_t val)
121 {
122#if R__LITTLE_ENDIAN == 1
123 return RByteSwap<sizeof(val)>::bswap(val);
124#else
125 return val;
126#endif
127 }
128
129public:
130 RInt32BE() = default;
131 explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {}
132 operator std::int32_t() const { return Swap(fValBE); }
133 RInt32BE &operator=(const std::int32_t val)
134 {
135 fValBE = Swap(val);
136 return *this;
137 }
138};
139
140/// Big-endian 64-bit unsigned integer
141class RUInt64BE {
142private:
143 std::uint64_t fValBE = 0;
144 static std::uint64_t Swap(std::uint64_t val)
145 {
146#if R__LITTLE_ENDIAN == 1
147 return RByteSwap<sizeof(val)>::bswap(val);
148#else
149 return val;
150#endif
151 }
152
153public:
154 RUInt64BE() = default;
155 explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {}
156 operator std::uint64_t() const { return Swap(fValBE); }
157 RUInt64BE &operator=(const std::uint64_t val)
158 {
159 fValBE = Swap(val);
160 return *this;
161 }
162};
163
164#pragma pack(push, 1)
165/// A name (type, identifies, ...) in the TFile binary format
166struct RTFString {
167 unsigned char fLName{0};
168 char fData[255];
169 RTFString() = default;
170 RTFString(const std::string &str)
171 {
172 // The length of strings with 255 characters and longer are encoded with a 32-bit integer following the first
173 // byte. This is currently not handled.
174 R__ASSERT(str.length() < 255);
175 fLName = static_cast<unsigned char>(str.length());
176 memcpy(fData, str.data(), fLName);
177 }
178 std::size_t GetSize() const
179 {
180 // A length of 255 is special and means that the first byte is followed by a 32-bit integer with the actual
181 // length.
182 R__ASSERT(fLName != 255);
183 return 1 + fLName;
184 }
185};
186
187/// The timestamp format used in TFile; the default constructor initializes with the current time
188struct RTFDatetime {
189 RUInt32BE fDatetime;
190 RTFDatetime()
191 {
192 auto now = std::chrono::system_clock::now();
193 auto tt = std::chrono::system_clock::to_time_t(now);
194 auto tm = *localtime(&tt);
195 fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 | tm.tm_hour << 12 |
196 tm.tm_min << 6 | tm.tm_sec;
197 }
198 explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {}
199};
200
201/// The key part of a TFile record excluding the class, object, and title names
202struct RTFKey {
203 static constexpr unsigned kBigKeyVersion = 1000;
204
205 RInt32BE fNbytes{0};
206 RUInt16BE fVersion{4};
207 RUInt32BE fObjLen{0};
208 RTFDatetime fDatetime;
209 RUInt16BE fKeyLen{0};
210 RUInt16BE fCycle{1};
211 union {
212 struct {
213 RUInt32BE fSeekKey{0};
214 RUInt32BE fSeekPdir{0};
215 } fInfoShort;
216 struct {
217 RUInt64BE fSeekKey{0};
218 RUInt64BE fSeekPdir{0};
219 } fInfoLong;
220 };
221
222 RTFKey() : fInfoLong() {}
223 RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName,
224 const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0)
225 {
226 R__ASSERT(szObjInMem <= std::numeric_limits<std::uint32_t>::max());
227 R__ASSERT(szObjOnDisk <= std::numeric_limits<std::uint32_t>::max());
228 // For writing, we alywas produce "big" keys with 64-bit SeekKey and SeekPdir.
229 fVersion = fVersion + kBigKeyVersion;
230 fObjLen = szObjInMem;
231 fKeyLen = static_cast<RUInt16BE>(GetHeaderSize() + clName.GetSize() + objName.GetSize() + titleName.GetSize());
232 fInfoLong.fSeekKey = seekKey;
233 fInfoLong.fSeekPdir = seekPdir;
234 // Depends on fKeyLen being set
235 fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk);
236 }
237
238 std::uint32_t GetSize() const
239 {
240 // Negative size indicates a gap in the file
241 if (fNbytes < 0)
242 return -fNbytes;
243 return fNbytes;
244 }
245
246 std::uint32_t GetHeaderSize() const
247 {
248 if (fVersion >= kBigKeyVersion)
249 return 18 + sizeof(fInfoLong);
250 return 18 + sizeof(fInfoShort);
251 }
252
253 std::uint64_t GetSeekKey() const
254 {
255 if (fVersion >= kBigKeyVersion)
256 return fInfoLong.fSeekKey;
257 return fInfoShort.fSeekKey;
258 }
259};
260
261/// The TFile global header
262struct RTFHeader {
263 static constexpr unsigned kBEGIN = 100;
264 static constexpr unsigned kBigHeaderVersion = 1000000;
265
266 char fMagic[4]{'r', 'o', 'o', 't'};
267 RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
268 (ROOT_VERSION_CODE & 0xFF)};
269 RUInt32BE fBEGIN{kBEGIN};
270 union {
271 struct {
272 RUInt32BE fEND{0};
273 RUInt32BE fSeekFree{0};
274 RUInt32BE fNbytesFree{0};
275 RUInt32BE fNfree{1};
276 RUInt32BE fNbytesName{0};
277 unsigned char fUnits{4};
278 RUInt32BE fCompress{0};
279 RUInt32BE fSeekInfo{0};
280 RUInt32BE fNbytesInfo{0};
281 } fInfoShort;
282 struct {
283 RUInt64BE fEND{0};
284 RUInt64BE fSeekFree{0};
285 RUInt32BE fNbytesFree{0};
286 RUInt32BE fNfree{1};
287 RUInt32BE fNbytesName{0};
288 unsigned char fUnits{8};
289 RUInt32BE fCompress{0};
290 RUInt64BE fSeekInfo{0};
291 RUInt32BE fNbytesInfo{0};
292 } fInfoLong;
293 };
294
295 RTFHeader() : fInfoShort() {}
296 RTFHeader(int compression) : fInfoShort() { fInfoShort.fCompress = compression; }
297
298 void SetBigFile()
299 {
300 if (fVersion >= kBigHeaderVersion)
301 return;
302
303 // clang-format off
304 std::uint32_t end = fInfoShort.fEND;
305 std::uint32_t seekFree = fInfoShort.fSeekFree;
306 std::uint32_t nbytesFree = fInfoShort.fNbytesFree;
307 std::uint32_t nFree = fInfoShort.fNfree;
308 std::uint32_t nbytesName = fInfoShort.fNbytesName;
309 std::uint32_t compress = fInfoShort.fCompress;
310 std::uint32_t seekInfo = fInfoShort.fSeekInfo;
311 std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo;
312 fInfoLong.fEND = end;
313 fInfoLong.fSeekFree = seekFree;
314 fInfoLong.fNbytesFree = nbytesFree;
315 fInfoLong.fNfree = nFree;
316 fInfoLong.fNbytesName = nbytesName;
317 fInfoLong.fUnits = 8;
318 fInfoLong.fCompress = compress;
319 fInfoLong.fSeekInfo = seekInfo;
320 fInfoLong.fNbytesInfo = nbytesInfo;
321 fVersion = fVersion + kBigHeaderVersion;
322 // clang-format on
323 }
324
325 bool IsBigFile(std::uint64_t offset = 0) const
326 {
327 return (fVersion >= kBigHeaderVersion) ||
328 (offset > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()));
329 }
330
331 std::uint32_t GetSize() const
332 {
333 std::uint32_t sizeHead = sizeof(fMagic) + sizeof(fVersion) + sizeof(fBEGIN);
334 if (IsBigFile())
335 return sizeHead + sizeof(fInfoLong);
336 return sizeHead + sizeof(fInfoShort);
337 }
338
339 std::uint64_t GetEnd() const
340 {
341 if (IsBigFile())
342 return fInfoLong.fEND;
343 return fInfoShort.fEND;
344 }
345
346 void SetEnd(std::uint64_t value)
347 {
348 if (IsBigFile(value)) {
349 SetBigFile();
350 fInfoLong.fEND = value;
351 } else {
352 fInfoShort.fEND = value;
353 }
354 }
355
356 std::uint64_t GetSeekFree() const
357 {
358 if (IsBigFile())
359 return fInfoLong.fSeekFree;
360 return fInfoShort.fSeekFree;
361 }
362
363 void SetSeekFree(std::uint64_t value)
364 {
365 if (IsBigFile(value)) {
366 SetBigFile();
367 fInfoLong.fSeekFree = value;
368 } else {
369 fInfoShort.fSeekFree = value;
370 }
371 }
372
373 void SetNbytesFree(std::uint32_t value)
374 {
375 if (IsBigFile()) {
376 fInfoLong.fNbytesFree = value;
377 } else {
378 fInfoShort.fNbytesFree = value;
379 }
380 }
381
382 void SetNbytesName(std::uint32_t value)
383 {
384 if (IsBigFile()) {
385 fInfoLong.fNbytesName = value;
386 } else {
387 fInfoShort.fNbytesName = value;
388 }
389 }
390
391 std::uint64_t GetSeekInfo() const
392 {
393 if (IsBigFile())
394 return fInfoLong.fSeekInfo;
395 return fInfoShort.fSeekInfo;
396 }
397
398 void SetSeekInfo(std::uint64_t value)
399 {
400 if (IsBigFile(value)) {
401 SetBigFile();
402 fInfoLong.fSeekInfo = value;
403 } else {
404 fInfoShort.fSeekInfo = value;
405 }
406 }
407
408 std::uint64_t GetNbytesInfo() const
409 {
410 if (IsBigFile())
411 return fInfoLong.fNbytesInfo;
412 return fInfoShort.fNbytesInfo;
413 }
414
415 void SetNbytesInfo(std::uint32_t value)
416 {
417 if (IsBigFile()) {
418 fInfoLong.fNbytesInfo = value;
419 } else {
420 fInfoShort.fNbytesInfo = value;
421 }
422 }
423
424 void SetCompression(std::uint32_t value)
425 {
426 if (IsBigFile()) {
427 fInfoLong.fCompress = value;
428 } else {
429 fInfoShort.fCompress = value;
430 }
431 }
432};
433
434/// A reference to an unused byte-range in a TFile
435struct RTFFreeEntry {
436 static constexpr unsigned kBigFreeEntryVersion = 1000;
437
438 RUInt16BE fVersion{1};
439 union {
440 struct {
441 RUInt32BE fFirst{0};
442 RUInt32BE fLast{0};
443 } fInfoShort;
444 struct {
445 RUInt64BE fFirst{0};
446 RUInt64BE fLast{0};
447 } fInfoLong;
448 };
449
450 RTFFreeEntry() : fInfoShort() {}
451 void Set(std::uint64_t first, std::uint64_t last)
452 {
453 if (last > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
454 fVersion = fVersion + kBigFreeEntryVersion;
455 fInfoLong.fFirst = first;
456 fInfoLong.fLast = last;
457 } else {
458 fInfoShort.fFirst = first;
459 fInfoShort.fLast = last;
460 }
461 }
462 std::uint32_t GetSize() { return (fVersion >= kBigFreeEntryVersion) ? 18 : 10; }
463};
464
465/// The header of the directory key index
466struct RTFKeyList {
467 RUInt32BE fNKeys;
468 std::uint32_t GetSize() const { return sizeof(RTFKeyList); }
469 explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {}
470};
471
472/// A streamed TDirectory (TFile) object
473struct RTFDirectory {
474 static constexpr unsigned kBigFileVersion = 1000;
475
476 RUInt16BE fClassVersion{5};
477 RTFDatetime fDateC;
478 RTFDatetime fDateM;
479 RUInt32BE fNBytesKeys{0};
480 RUInt32BE fNBytesName{0};
481 // The version of the key has to tell whether offsets are 32bit or 64bit long
482 union {
483 struct {
484 RUInt32BE fSeekDir{RTFHeader::kBEGIN};
485 RUInt32BE fSeekParent{0};
486 RUInt32BE fSeekKeys{0};
487 } fInfoShort;
488 struct {
489 RUInt64BE fSeekDir{RTFHeader::kBEGIN};
490 RUInt64BE fSeekParent{0};
491 RUInt64BE fSeekKeys{0};
492 } fInfoLong;
493 };
494
495 RTFDirectory() : fInfoShort() {}
496
497 // In case of a short TFile record (<2G), 3 padding ints are written after the UUID
498 std::uint32_t GetSize() const
499 {
500 if (fClassVersion >= kBigFileVersion)
501 return sizeof(RTFDirectory);
502 return 18 + sizeof(fInfoShort);
503 }
504
505 std::uint64_t GetSeekKeys() const
506 {
507 if (fClassVersion >= kBigFileVersion)
508 return fInfoLong.fSeekKeys;
509 return fInfoShort.fSeekKeys;
510 }
511
512 void SetSeekKeys(std::uint64_t seekKeys)
513 {
514 if (seekKeys > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
515 std::uint32_t seekDir = fInfoShort.fSeekDir;
516 std::uint32_t seekParent = fInfoShort.fSeekParent;
517 fInfoLong.fSeekDir = seekDir;
518 fInfoLong.fSeekParent = seekParent;
519 fInfoLong.fSeekKeys = seekKeys;
520 fClassVersion = fClassVersion + kBigFileVersion;
521 } else {
522 fInfoShort.fSeekKeys = seekKeys;
523 }
524 }
525};
526
527/// A zero UUID stored at the end of the TFile record
528struct RTFUUID {
529 RUInt16BE fVersionClass{1};
530 unsigned char fUUID[16];
531
532 RTFUUID()
533 {
534 TUUID uuid;
535 char *buffer = reinterpret_cast<char *>(this);
536 uuid.FillBuffer(buffer);
537 assert(reinterpret_cast<RTFUUID *>(buffer) <= (this + 1));
538 }
539 std::uint32_t GetSize() const { return sizeof(RTFUUID); }
540};
541
542/// A streamed RNTuple class
543///
544/// NOTE: this must be kept in sync with RNTuple.hxx.
545/// Aside ensuring consistency between the two classes' members, you need to make sure
546/// that fVersionClass matches the class version of RNTuple.
547struct RTFNTuple {
548 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))};
549 RUInt16BE fVersionClass{2};
550 RUInt16BE fVersionEpoch{0};
551 RUInt16BE fVersionMajor{0};
552 RUInt16BE fVersionMinor{0};
553 RUInt16BE fVersionPatch{0};
554 RUInt64BE fSeekHeader{0};
555 RUInt64BE fNBytesHeader{0};
556 RUInt64BE fLenHeader{0};
557 RUInt64BE fSeekFooter{0};
558 RUInt64BE fNBytesFooter{0};
559 RUInt64BE fLenFooter{0};
560 RUInt64BE fMaxKeySize{0};
561
562 static constexpr std::uint32_t GetSizePlusChecksum() { return sizeof(RTFNTuple) + sizeof(std::uint64_t); }
563
564 RTFNTuple() = default;
565 explicit RTFNTuple(const ROOT::RNTuple &inMemoryAnchor)
566 {
567 fVersionEpoch = inMemoryAnchor.GetVersionEpoch();
568 fVersionMajor = inMemoryAnchor.GetVersionMajor();
569 fVersionMinor = inMemoryAnchor.GetVersionMinor();
570 fVersionPatch = inMemoryAnchor.GetVersionPatch();
571 fSeekHeader = inMemoryAnchor.GetSeekHeader();
572 fNBytesHeader = inMemoryAnchor.GetNBytesHeader();
573 fLenHeader = inMemoryAnchor.GetLenHeader();
574 fSeekFooter = inMemoryAnchor.GetSeekFooter();
575 fNBytesFooter = inMemoryAnchor.GetNBytesFooter();
576 fLenFooter = inMemoryAnchor.GetLenFooter();
577 fMaxKeySize = inMemoryAnchor.GetMaxKeySize();
578 }
579 std::uint32_t GetSize() const { return sizeof(RTFNTuple); }
580 // The byte count and class version members are not checksummed
581 std::uint32_t GetOffsetCkData() { return sizeof(fByteCount) + sizeof(fVersionClass); }
582 std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); }
583 unsigned char *GetPtrCkData() { return reinterpret_cast<unsigned char *>(this) + GetOffsetCkData(); }
584};
585
586/// The bare file global header
587struct RBareFileHeader {
588 char fMagic[7]{'r', 'n', 't', 'u', 'p', 'l', 'e'};
589 RUInt32BE fRootVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
590 (ROOT_VERSION_CODE & 0xFF)};
591 RUInt32BE fFormatVersion{1};
592 RUInt32BE fCompress{0};
593 RTFNTuple fNTuple;
594 // followed by the ntuple name
595};
596#pragma pack(pop)
597
598/// The artifical class name shown for opaque RNTuple keys (see TBasket)
599constexpr char const *kBlobClassName = "RBlob";
600/// The class name of the RNTuple anchor
601constexpr char const *kNTupleClassName = "ROOT::RNTuple";
602
603} // anonymous namespace
604
605namespace ROOT {
606namespace Internal {
607/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header
608/// and the TFile record need to be updated
610 RTFHeader fHeader;
611 RTFDirectory fFileRecord;
612 std::uint64_t fSeekNTuple{0}; // Remember the offset for the keys list
613 std::uint64_t fSeekFileRecord{0};
614};
615
616/// The RKeyBlob writes an invisible key into a TFile. That is, a key that is not indexed in the list of keys,
617/// like a TBasket.
618/// NOTE: out of anonymous namespace because otherwise ClassDefInline fails to compile
619/// on some platforms.
620class RKeyBlob : public TKey {
621public:
622 RKeyBlob() = default;
623
624 explicit RKeyBlob(TFile *file) : TKey(file)
625 {
627 fVersion += RTFKey::kBigKeyVersion;
628 fKeylen = Sizeof();
629 }
630
631 /// Register a new key for a data record of size nbytes
632 void Reserve(size_t nbytes, std::uint64_t *seekKey)
633 {
634 Create(nbytes);
635 *seekKey = fSeekKey;
636 }
637
638 bool WasAllocatedInAFreeSlot() const { return fLeft > 0; }
639
641};
642
643} // namespace Internal
644} // namespace ROOT
645
646// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
647// first chunk also needs to house the offsets of the other chunks and no chunk can
648// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
649static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
650{
651 constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
652
654 size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
655 assert(nChunks > 1);
656 size_t nbytesTail = nbytes % maxChunkSize;
657 size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
660 ++nChunks;
662 }
663
664 // We don't support having more chunkOffsets than what fits in one chunk.
665 // For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
666 // than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
667 // 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
669
670 return nChunks;
671}
672
674
676{
677 char ident[4];
678 ReadBuffer(ident, 4, 0);
679 if (std::string(ident, 4) == "root")
680 return GetNTupleProper(ntupleName);
681 fIsBare = true;
682 return GetNTupleBare(ntupleName);
683}
684
685/// Searches for a key with the given name and type in the key index of the given directory.
686/// Return 0 if the key was not found.
687std::uint64_t ROOT::Internal::RMiniFileReader::SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName,
688 std::string_view typeName)
689{
690 RTFDirectory directory;
692
693 RTFKey key;
694 RUInt32BE nKeys;
695 std::uint64_t offset = directory.GetSeekKeys();
696 ReadBuffer(&key, sizeof(key), offset);
697 offset += key.fKeyLen;
698 ReadBuffer(&nKeys, sizeof(nKeys), offset);
699 offset += sizeof(nKeys);
700
701 for (unsigned int i = 0; i < nKeys; ++i) {
702 ReadBuffer(&key, sizeof(key), offset);
703 auto offsetNextKey = offset + key.fKeyLen;
704
705 offset += key.GetHeaderSize();
706 RTFString name;
707 ReadBuffer(&name, 1, offset);
708 ReadBuffer(&name, name.GetSize(), offset);
709 if (std::string_view(name.fData, name.fLName) != typeName) {
711 continue;
712 }
713 offset += name.GetSize();
714 ReadBuffer(&name, 1, offset);
715 ReadBuffer(&name, name.GetSize(), offset);
716 if (std::string_view(name.fData, name.fLName) == keyName) {
717 return key.GetSeekKey();
718 }
720 }
721
722 // Not found
723 return 0;
724}
725
727{
728 RTFHeader fileHeader;
729 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
730
731 const std::uint64_t seekKeyInfo = fileHeader.GetSeekInfo();
732
733 RTFKey key;
734 ReadBuffer(&key, sizeof(key), seekKeyInfo);
735
736 const std::uint64_t nbytesInfo = fileHeader.GetNbytesInfo() - key.fKeyLen;
737 const std::uint64_t seekInfo = seekKeyInfo + key.fKeyLen;
738 const std::uint32_t uncompLenInfo = key.fObjLen;
740 if (nbytesInfo == uncompLenInfo) {
741 // Uncompressed
743 } else {
745 ReadBuffer(buffer.get(), nbytesInfo, seekInfo);
747 }
748
750 // This is necessary to allow the "class tags" inside the StreamerInfo list to refer to the proper offset into
751 // the buffer. Normally TFile loads the StreamerInfo via TKey::ReadObjWithBuffer, whose buffer also includes the
752 // key itself. Since we dealt with the key above already, we are only passing the payload to TBufferFile so offsets
753 // need to be patched up.
754 buffer.SetBufferDisplacement(key.fKeyLen);
756 streamerInfoList.Streamer(buffer);
757 TObjLink *lnk = streamerInfoList.FirstLink();
758 while (lnk) {
759 auto obj = lnk->GetObject();
760 // NOTE: the last element of the streamer info list may be a TList with the IO customization rules, so we need
761 // to check before static casting.
762 if (obj->IsA() == TStreamerInfo::Class()) {
763 auto info = static_cast<TStreamerInfo *>(obj);
764 info->BuildCheck();
765 }
766 lnk = lnk->Next();
767 }
768}
769
771{
772 RTFHeader fileHeader;
773 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
774
775 RTFKey key;
776 RTFString name;
777 ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN);
778 // Skip over the entire key length, including the class name, object name, and title stored in it.
779 std::uint64_t offset = fileHeader.fBEGIN + key.fKeyLen;
780 // Skip over the name and title of the TNamed preceding the TFile (root TDirectory) entry.
781 ReadBuffer(&name, 1, offset);
782 offset += name.GetSize();
783 ReadBuffer(&name, 1, offset);
784 offset += name.GetSize();
785
786 // split ntupleName by '/' character to open datasets in subdirectories.
787 std::string ntuplePathTail(ntuplePath);
788 if (!ntuplePathTail.empty() && ntuplePathTail[0] == '/')
789 ntuplePathTail = ntuplePathTail.substr(1);
790 auto pos = std::string::npos;
791 while ((pos = ntuplePathTail.find('/')) != std::string::npos) {
792 auto directoryName = ntuplePathTail.substr(0, pos);
793 ntuplePathTail.erase(0, pos + 1);
794
795 offset = SearchInDirectory(offset, directoryName, "TDirectory");
796 if (offset == 0) {
797 return R__FAIL("no directory named '" + std::string(directoryName) + "' in file '" + fRawFile->GetUrl() + "'");
798 }
799 ReadBuffer(&key, sizeof(key), offset);
800 offset = key.GetSeekKey() + key.fKeyLen;
801 }
802 // no more '/' delimiter in ntuplePath
804
805 offset = SearchInDirectory(offset, ntupleName, kNTupleClassName);
806 if (offset == 0) {
807 return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'");
808 }
809
810 ReadBuffer(&key, sizeof(key), offset);
811 offset = key.GetSeekKey() + key.fKeyLen;
812
813 // size of a RTFNTuple version 2 (min supported version); future anchor versions can grow.
814 constexpr size_t kMinNTupleSize = 78;
815 static_assert(kMinNTupleSize == RTFNTuple::GetSizePlusChecksum());
816 if (key.fObjLen < kMinNTupleSize) {
817 return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
818 }
819
820 const auto objNbytes = key.GetSize() - key.fKeyLen;
821 auto res = GetNTupleProperAtOffset(offset, objNbytes, key.fObjLen);
822 return res;
823}
824
826 std::uint64_t compSize,
827 std::uint64_t uncompLen)
828{
829 // The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version,
830 // or larger than it if it comes from a future RNTuple class version.
831 auto bufAnchor = MakeUninitArray<unsigned char>(std::max<size_t>(uncompLen, sizeof(RTFNTuple)));
832 RTFNTuple *ntuple = new (bufAnchor.get()) RTFNTuple;
833
834 if (compSize != uncompLen) {
835 // Read into a temporary buffer
836 auto unzipBuf = MakeUninitArray<unsigned char>(std::max<size_t>(uncompLen, sizeof(RTFNTuple)));
838 // Unzip into the final buffer
840 } else {
842 }
843
844 // We require that future class versions only append members and store the checksum in the last 8 bytes
845 // Checksum calculation: strip byte count, class version, fChecksum member
846 const auto lenCkData = uncompLen - ntuple->GetOffsetCkData() - sizeof(uint64_t);
847 const auto ckCalc = XXH3_64bits(ntuple->GetPtrCkData(), lenCkData);
848 uint64_t ckOnDisk;
849
850 RUInt64BE *ckOnDiskPtr = reinterpret_cast<RUInt64BE *>(bufAnchor.get() + uncompLen - sizeof(uint64_t));
851 ckOnDisk = static_cast<uint64_t>(*ckOnDiskPtr);
852 if (ckCalc != ckOnDisk) {
853 return R__FAIL("RNTuple anchor checksum mismatch");
854 }
855
856 return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
857 ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
858 ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize);
859}
860
862{
863 RBareFileHeader fileHeader;
864 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
865 RTFString name;
866 auto offset = sizeof(fileHeader);
867 ReadBuffer(&name, 1, offset);
868 ReadBuffer(&name, name.GetSize(), offset);
869 std::string_view foundName(name.fData, name.fLName);
870 if (foundName != ntupleName) {
871 return R__FAIL("expected RNTuple named '" + std::string(ntupleName) + "' but instead found '" +
872 std::string(foundName) + "' in file '" + fRawFile->GetUrl() + "'");
873 }
874 offset += name.GetSize();
875
876 RTFNTuple ntuple;
877 ReadBuffer(&ntuple, sizeof(ntuple), offset);
878 std::uint64_t onDiskChecksum;
880 auto checksum = XXH3_64bits(ntuple.GetPtrCkData(), ntuple.GetSizeCkData());
881 if (checksum != static_cast<uint64_t>(onDiskChecksum))
882 return R__FAIL("RNTuple bare file: anchor checksum mismatch");
883
884 return CreateAnchor(ntuple.fVersionEpoch, ntuple.fVersionMajor, ntuple.fVersionMinor, ntuple.fVersionPatch,
885 ntuple.fSeekHeader, ntuple.fNBytesHeader, ntuple.fLenHeader, ntuple.fSeekFooter,
886 ntuple.fNBytesFooter, ntuple.fLenFooter, ntuple.fMaxKeySize);
887}
888
889void ROOT::Internal::RMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
890{
891 TryReadBuffer(buffer, nbytes, offset).ThrowOnError();
892}
893
895{
896 const auto ByteReadErr = [](std::size_t expected, std::size_t nread) {
897 return R__FAIL("invalid read (expected bytes: " + std::to_string(expected) + ", read: " + std::to_string(nread) +
898 ")");
899 };
900
901 size_t nread;
902 if (fMaxKeySize == 0 || nbytes <= fMaxKeySize) {
903 // Fast path: read single blob
904 nread = fRawFile->ReadAt(buffer, nbytes, offset);
905 } else {
906 // Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
907 const size_t nChunks = ComputeNumChunks(nbytes, fMaxKeySize);
908 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
909 const size_t nbytesFirstChunk = fMaxKeySize - nbytesChunkOffsets;
910 uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
911
912 // Read first chunk
913 nread = fRawFile->ReadAt(bufCur, fMaxKeySize, offset);
914 if (nread != fMaxKeySize)
915 return ByteReadErr(fMaxKeySize, nread);
916
917 // NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
918 // since the last part of `bufCur` will later be overwritten by the next chunk's payload.
919 // We do this to avoid a second ReadAt to read in the chunk offsets.
922
925
927 std::uint64_t *curChunkOffset = &chunkOffsets[0];
928
929 do {
930 std::uint64_t chunkOffset;
933
934 const size_t bytesToRead = std::min<size_t>(fMaxKeySize, remainingBytes);
935 // Ensure we don't read outside of the buffer
936 R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
937
938 auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
939 if (nbytesRead != bytesToRead)
941
945 } while (remainingBytes > 0);
946 }
947
948 if (nread != nbytes)
949 return ByteReadErr(nbytes, nread);
950
951 return RResult<void>::Success();
952}
953
954////////////////////////////////////////////////////////////////////////////////
955
956/// Prepare a blob key in the provided buffer, which must provide space for kBlobKeyLen bytes. Note that the array type
957/// is purely documentation, the argument is actually just a pointer.
959 unsigned char buffer[kBlobKeyLen])
960{
961 RTFString strClass{kBlobClassName};
962 RTFString strObject;
963 RTFString strTitle;
964 RTFKey keyHeader(offset, RTFHeader::kBEGIN, strClass, strObject, strTitle, len, nbytes);
965 R__ASSERT(keyHeader.fKeyLen == kBlobKeyLen);
966
967 // Copy structures into the buffer.
968 unsigned char *writeBuffer = buffer;
969 memcpy(writeBuffer, &keyHeader, keyHeader.GetHeaderSize());
970 writeBuffer += keyHeader.GetHeaderSize();
971 memcpy(writeBuffer, &strClass, strClass.GetSize());
972 writeBuffer += strClass.GetSize();
974 writeBuffer += strObject.GetSize();
975 memcpy(writeBuffer, &strTitle, strTitle.GetSize());
976 writeBuffer += strTitle.GetSize();
977 R__ASSERT(writeBuffer == buffer + kBlobKeyLen);
978}
979
980////////////////////////////////////////////////////////////////////////////////
981
983
985{
986 static_assert(kHeaderBlockSize % kBlockAlign == 0, "invalid header block size");
987 if (bufferSize % kBlockAlign != 0)
988 throw RException(R__FAIL("Buffer size not a multiple of alignment: " + std::to_string(bufferSize)));
989 fBlockSize = bufferSize;
990
991 std::align_val_t blockAlign{kBlockAlign};
992 fHeaderBlock = static_cast<unsigned char *>(::operator new[](kHeaderBlockSize, blockAlign));
993 memset(fHeaderBlock, 0, kHeaderBlockSize);
994 fBlock = static_cast<unsigned char *>(::operator new[](fBlockSize, blockAlign));
995 memset(fBlock, 0, fBlockSize);
996}
997
999{
1000 if (fFile)
1001 fclose(fFile);
1002
1003 std::align_val_t blockAlign{kBlockAlign};
1004 if (fHeaderBlock)
1005 ::operator delete[](fHeaderBlock, blockAlign);
1006 if (fBlock)
1007 ::operator delete[](fBlock, blockAlign);
1008}
1009
1010namespace {
1011int FSeek64(FILE *stream, std::int64_t offset, int origin)
1012{
1013#ifdef R__SEEK64
1014 return fseeko64(stream, offset, origin);
1015#else
1016 return fseek(stream, offset, origin);
1017#endif
1018}
1019} // namespace
1020
1022{
1023 // Write the last partially filled block, which may still need appropriate alignment for Direct I/O.
1024 // If it is the first block, get the updated header block.
1025 if (fBlockOffset == 0) {
1026 std::size_t headerBlockSize = kHeaderBlockSize;
1027 if (headerBlockSize > fFilePos) {
1028 headerBlockSize = fFilePos;
1029 }
1030 memcpy(fBlock, fHeaderBlock, headerBlockSize);
1031 }
1032
1033 std::size_t retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
1034 if (retval)
1035 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1036
1037 std::size_t lastBlockSize = fFilePos - fBlockOffset;
1038 R__ASSERT(lastBlockSize <= fBlockSize);
1039 if (fDirectIO) {
1040 // Round up to a multiple of kBlockAlign.
1041 lastBlockSize += kBlockAlign - 1;
1042 lastBlockSize = (lastBlockSize / kBlockAlign) * kBlockAlign;
1043 R__ASSERT(lastBlockSize <= fBlockSize);
1044 }
1045 retval = fwrite(fBlock, 1, lastBlockSize, fFile);
1046 if (retval != lastBlockSize)
1047 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1048
1049 // Write the (updated) header block, unless it was part of the write above.
1050 if (fBlockOffset > 0) {
1051 retval = FSeek64(fFile, 0, SEEK_SET);
1052 if (retval)
1053 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1054
1055 retval = fwrite(fHeaderBlock, 1, kHeaderBlockSize, fFile);
1056 if (retval != RImplSimple::kHeaderBlockSize)
1057 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1058 }
1059
1060 retval = fflush(fFile);
1061 if (retval)
1062 throw RException(R__FAIL(std::string("Flush failed: ") + strerror(errno)));
1063}
1064
1065void ROOT::Internal::RNTupleFileWriter::RImplSimple::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1066{
1067 R__ASSERT(fFile);
1068 size_t retval;
1069 if ((offset >= 0) && (static_cast<std::uint64_t>(offset) != fFilePos)) {
1070 fFilePos = offset;
1071 }
1072
1073 // Keep header block to overwrite on commit.
1074 if (fFilePos < kHeaderBlockSize) {
1075 std::size_t headerBytes = nbytes;
1076 if (fFilePos + headerBytes > kHeaderBlockSize) {
1077 headerBytes = kHeaderBlockSize - fFilePos;
1078 }
1079 memcpy(fHeaderBlock + fFilePos, buffer, headerBytes);
1080 }
1081
1082 R__ASSERT(fFilePos >= fBlockOffset);
1083
1084 while (nbytes > 0) {
1085 std::uint64_t posInBlock = fFilePos % fBlockSize;
1086 std::uint64_t blockOffset = fFilePos - posInBlock;
1087 if (blockOffset != fBlockOffset) {
1088 // Write the block.
1089 retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
1090 if (retval)
1091 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1092
1093 retval = fwrite(fBlock, 1, fBlockSize, fFile);
1094 if (retval != fBlockSize)
1095 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1096
1097 // Null the buffer contents for good measure.
1098 memset(fBlock, 0, fBlockSize);
1099 }
1100
1101 fBlockOffset = blockOffset;
1102 std::size_t blockSize = nbytes;
1103 if (blockSize > fBlockSize - posInBlock) {
1104 blockSize = fBlockSize - posInBlock;
1105 }
1106 memcpy(fBlock + posInBlock, buffer, blockSize);
1107 buffer = static_cast<const unsigned char *>(buffer) + blockSize;
1108 nbytes -= blockSize;
1109 fFilePos += blockSize;
1110 }
1111}
1112
1113std::uint64_t
1114ROOT::Internal::RNTupleFileWriter::RImplSimple::WriteKey(const void *buffer, std::size_t nbytes, std::size_t len,
1115 std::int64_t offset, std::uint64_t directoryOffset,
1116 const std::string &className, const std::string &objectName,
1117 const std::string &title)
1118{
1119 if (offset > 0)
1120 fKeyOffset = offset;
1121 RTFString strClass{className};
1122 RTFString strObject{objectName};
1123 RTFString strTitle{title};
1124
1125 RTFKey key(fKeyOffset, directoryOffset, strClass, strObject, strTitle, len, nbytes);
1126 Write(&key, key.GetHeaderSize(), fKeyOffset);
1127 Write(&strClass, strClass.GetSize());
1128 Write(&strObject, strObject.GetSize());
1129 Write(&strTitle, strTitle.GetSize());
1130 auto offsetData = fFilePos;
1131 // The next key starts after the data.
1132 fKeyOffset = offsetData + nbytes;
1133 if (buffer)
1134 Write(buffer, nbytes);
1135
1136 return offsetData;
1137}
1138
1140 unsigned char keyBuffer[kBlobKeyLen])
1141{
1142 if (keyBuffer) {
1143 PrepareBlobKey(fKeyOffset, nbytes, len, keyBuffer);
1144 } else {
1145 unsigned char localKeyBuffer[kBlobKeyLen];
1146 PrepareBlobKey(fKeyOffset, nbytes, len, localKeyBuffer);
1147 Write(localKeyBuffer, kBlobKeyLen, fKeyOffset);
1148 }
1149
1150 auto offsetData = fKeyOffset + kBlobKeyLen;
1151 // The next key starts after the data.
1152 fKeyOffset = offsetData + nbytes;
1153
1154 return offsetData;
1155}
1156
1157////////////////////////////////////////////////////////////////////////////////
1158
1159template <typename T>
1161 std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
1162{
1163 std::uint64_t offsetKey;
1165 // Since it is unknown beforehand if offsetKey is beyond the 2GB limit or not,
1166 // RKeyBlob will always reserve space for a big key (version >= 1000)
1167 keyBlob.Reserve(nbytes, &offsetKey);
1168
1169 if (keyBuffer) {
1170 PrepareBlobKey(offsetKey, nbytes, len, keyBuffer);
1171 } else {
1172 unsigned char localKeyBuffer[kBlobKeyLen];
1173 PrepareBlobKey(offsetKey, nbytes, len, localKeyBuffer);
1174 caller.Write(localKeyBuffer, kBlobKeyLen, offsetKey);
1175 }
1176
1177 if (keyBlob.WasAllocatedInAFreeSlot()) {
1178 // If the key was allocated in a free slot, the last 4 bytes of its buffer contain the new size
1179 // of the remaining free slot and we need to write it to disk before the key gets destroyed at the end of the
1180 // function.
1181 caller.Write(keyBlob.GetBuffer() + nbytes, sizeof(Int_t), offsetKey + kBlobKeyLen + nbytes);
1182 }
1183
1184 auto offsetData = offsetKey + kBlobKeyLen;
1185
1186 return offsetData;
1187}
1188
1189void ROOT::Internal::RNTupleFileWriter::RImplTFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1190{
1191 fDirectory->GetFile()->Seek(offset);
1192 bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
1193 if (rv)
1194 throw RException(R__FAIL("WriteBuffer failed."));
1195}
1196
1198 unsigned char keyBuffer[kBlobKeyLen])
1199{
1200 auto offsetData = RNTupleFileWriter::ReserveBlobKey(*this, *fDirectory->GetFile(), nbytes, len, keyBuffer);
1201 return offsetData;
1202}
1203
1204////////////////////////////////////////////////////////////////////////////////
1205
1206void ROOT::Internal::RNTupleFileWriter::RImplRFile::Write(const void *buffer, size_t nbytes, std::int64_t offset)
1207{
1209 file->Seek(offset);
1210 bool rv = file->WriteBuffer((char *)(buffer), nbytes);
1211 if (rv)
1212 throw RException(R__FAIL("WriteBuffer failed."));
1213}
1214
1216 unsigned char keyBuffer[kBlobKeyLen])
1217{
1220 return offsetData;
1221}
1222
1223////////////////////////////////////////////////////////////////////////////////
1224
1225ROOT::Internal::RNTupleFileWriter::RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool hidden)
1226 : fIsHidden(hidden), fNTupleName(name)
1227{
1228 auto &fileSimple = fFile.emplace<RImplSimple>();
1229 fileSimple.fControlBlock = std::make_unique<ROOT::Internal::RTFileControlBlock>();
1231 auto infoRNTuple = RNTuple::Class()->GetStreamerInfo();
1233}
1234
1236
1237std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1238ROOT::Internal::RNTupleFileWriter::Recreate(std::string_view ntupleName, std::string_view path,
1240{
1241 std::string fileName(path);
1242 size_t idxDirSep = fileName.find_last_of("\\/");
1243 if (idxDirSep != std::string::npos) {
1244 fileName.erase(0, idxDirSep + 1);
1245 }
1246#ifdef R__LINUX
1247 int flags = O_WRONLY | O_CREAT | O_TRUNC;
1248#ifdef O_LARGEFILE
1249 // Add the equivalent flag that is passed by fopen64.
1250 flags |= O_LARGEFILE;
1251#endif
1252 if (options.GetUseDirectIO()) {
1253 flags |= O_DIRECT;
1254 }
1255 int fd = open(std::string(path).c_str(), flags, 0666);
1256 if (fd == -1) {
1257 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1258 }
1259 FILE *fileStream = fdopen(fd, "wb");
1260#else
1261#ifdef R__SEEK64
1262 FILE *fileStream = fopen64(std::string(path.data(), path.size()).c_str(), "wb");
1263#else
1264 FILE *fileStream = fopen(std::string(path.data(), path.size()).c_str(), "wb");
1265#endif
1266#endif
1267 if (!fileStream) {
1268 throw RException(R__FAIL(std::string("open failed for file \"") + std::string(path) + "\": " + strerror(errno)));
1269 }
1270 // RNTupleFileWriter::RImplSimple does its own buffering, turn off additional buffering from C stdio.
1271 std::setvbuf(fileStream, nullptr, _IONBF, 0);
1272
1273 auto writer =
1274 std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, options.GetMaxKeySize(), /*hidden=*/false));
1275 RImplSimple &fileSimple = std::get<RImplSimple>(writer->fFile);
1276 fileSimple.fFile = fileStream;
1277 fileSimple.fDirectIO = options.GetUseDirectIO();
1278 fileSimple.AllocateBuffers(options.GetWriteBufferSize());
1279 writer->fFileName = fileName;
1280
1281 int defaultCompression = options.GetCompression();
1282 switch (containerFormat) {
1283 case EContainerFormat::kTFile: writer->WriteTFileSkeleton(defaultCompression); break;
1284 case EContainerFormat::kBare:
1285 writer->fIsBare = true;
1286 writer->WriteBareFileSkeleton(defaultCompression);
1287 break;
1288 default: R__ASSERT(false && "Internal error: unhandled container format");
1289 }
1290
1291 return writer;
1292}
1293
1294std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1296 std::uint64_t maxKeySize, bool hidden)
1297{
1298 TFile *file = fileOrDirectory.GetFile();
1299 if (!file)
1300 throw RException(R__FAIL("invalid attempt to add an RNTuple to a directory that is not backed by a file"));
1301 assert(file->IsBinary());
1302
1303 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, hidden));
1304 auto &fileProper = writer->fFile.emplace<RImplTFile>();
1305 fileProper.fDirectory = &fileOrDirectory;
1306 return writer;
1307}
1308
1309std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1311 std::string_view ntupleDir, std::uint64_t maxKeySize)
1312{
1313 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize, /*hidden=*/false));
1314 auto &rfile = writer->fFile.emplace<RImplRFile>();
1315 rfile.fFile = &file;
1316 R__ASSERT(ntupleDir.empty() || ntupleDir[ntupleDir.size() - 1] == '/');
1317 rfile.fDir = ntupleDir;
1318 return writer;
1319}
1320
1321std::unique_ptr<ROOT::Internal::RNTupleFileWriter>
1323{
1324 if (auto *file = std::get_if<RImplTFile>(&fFile)) {
1325 return Append(ntupleName, *file->fDirectory, fNTupleAnchor.fMaxKeySize, /* hidden= */ true);
1326 }
1327 // TODO: support also non-TFile-based writers
1328 throw ROOT::RException(R__FAIL("cannot clone a non-TFile-based RNTupleFileWriter."));
1329}
1330
1332{
1333 RImplSimple *fileSimple = std::get_if<RImplSimple>(&fFile);
1334 if (!fileSimple)
1335 throw RException(R__FAIL("invalid attempt to seek non-simple writer"));
1336
1337 fileSimple->fFilePos = offset;
1338 fileSimple->fKeyOffset = offset;
1339 // The next Write() will Flush() if necessary.
1340}
1341
1346
1348{
1349 const auto WriteStreamerInfoToFile = [&](TFile *file) {
1350 // Make sure the streamer info records used in the RNTuple are written to the file
1352 buf.SetParent(file);
1353 for (auto [_, info] : fStreamerInfoMap)
1354 buf.TagStreamerInfo(info);
1355 };
1356
1358 // NOTE: checksum length is included in the uncompressed len
1359 anchorInfo.fLength = RTFNTuple{}.GetSize() + sizeof(std::uint64_t);
1360 anchorInfo.fLocator.SetType(RNTupleLocator::kTypeFile);
1361
1362 if (auto fileProper = std::get_if<RImplTFile>(&fFile)) {
1363 // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
1364 fileProper->fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
1365 WriteStreamerInfoToFile(fileProper->fDirectory->GetFile());
1366 auto key = static_cast<TKey *>(fileProper->fDirectory->GetListOfKeys()->FindObject(fNTupleName.c_str()));
1367 R__ASSERT(key);
1368 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetKeylen());
1369 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNbytes() - key->GetKeylen());
1370 // NOTE: this must happen after FindObject(), otherwise some TFile implementations, such as TBufferMergerFile,
1371 // may reset the keys list upon write.
1372 fileProper->fDirectory->GetFile()->Write();
1373
1374 if (fIsHidden) {
1375 // Remove the anchor's key from the directory's KeysList to disallow retrieving directly the
1376 // attribute RNTuple from the TFile.
1377 fileProper->fDirectory->GetListOfKeys()->Remove(key);
1378 }
1379 } else if (auto fileRFile = std::get_if<RImplRFile>(&fFile)) {
1380 // Same as the case above but handled via RFile
1381 fileRFile->fFile->Put(fileRFile->fDir + fNTupleName, fNTupleAnchor);
1383 auto key = fileRFile->fFile->GetKeyInfo(fNTupleName);
1384 R__ASSERT(key);
1385 anchorInfo.fLocator.SetPosition(key->GetSeekKey() + key->GetNBytesKey());
1386 anchorInfo.fLocator.SetNBytesOnStorage(key->GetNBytesObj());
1387 fileRFile->fFile->Flush();
1388 } else {
1389 // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
1390 auto &fileSimple = std::get<RImplSimple>(fFile);
1391
1392 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1393 anchorInfo.fLocator.SetPosition(fileSimple.fControlBlock->fSeekNTuple);
1394
1395 if (fIsBare) {
1396 // Compute the checksum
1397 std::uint64_t checksum = XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData());
1398 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekNTuple, &ntupleOnDisk, ntupleOnDisk.GetSize());
1399 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekNTuple + ntupleOnDisk.GetSize(), &checksum,
1400 sizeof(checksum));
1401 fileSimple.Flush();
1402
1403 anchorInfo.fLocator.SetNBytesOnStorage(ntupleOnDisk.GetSize());
1404 } else {
1405 auto anchorSize = WriteTFileNTupleKey(compression);
1406 WriteTFileKeysList(anchorSize); // NOTE: this is written uncompressed
1407 WriteTFileStreamerInfo(compression);
1408 WriteTFileFreeList(); // NOTE: this is written uncompressed
1409
1410 // Update header and TFile record
1411 memcpy(fileSimple.fHeaderBlock, &fileSimple.fControlBlock->fHeader,
1412 fileSimple.fControlBlock->fHeader.GetSize());
1413 R__ASSERT(fileSimple.fControlBlock->fSeekFileRecord + fileSimple.fControlBlock->fFileRecord.GetSize() <
1414 RImplSimple::kHeaderBlockSize);
1415 memcpy(fileSimple.fHeaderBlock + fileSimple.fControlBlock->fSeekFileRecord,
1416 &fileSimple.fControlBlock->fFileRecord, fileSimple.fControlBlock->fFileRecord.GetSize());
1417
1418 fileSimple.Flush();
1419
1420 anchorInfo.fLocator.SetNBytesOnStorage(anchorSize);
1421 }
1422 }
1423
1424 return anchorInfo;
1425}
1426
1427std::uint64_t ROOT::Internal::RNTupleFileWriter::WriteBlob(const void *data, size_t nbytes, size_t len)
1428{
1429 auto writeKey = [this](const void *payload, size_t nBytes, size_t length) {
1430 std::uint64_t offset = ReserveBlob(nBytes, length);
1431 WriteIntoReservedBlob(payload, nBytes, offset);
1432 return offset;
1433 };
1434
1435 const std::uint64_t maxKeySize = fNTupleAnchor.fMaxKeySize;
1436 R__ASSERT(maxKeySize > 0);
1437 // We don't need the object length except for seeing compression ratios in TFile::Map()
1438 // Make sure that the on-disk object length fits into the TKey header.
1439 if (static_cast<std::uint64_t>(len) > static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()))
1440 len = nbytes;
1441
1442 if (nbytes <= maxKeySize) {
1443 // Fast path: only write 1 key.
1444 return writeKey(data, nbytes, len);
1445 }
1446
1447 /**
1448 * Writing a key bigger than the max allowed size. In this case we split the payload
1449 * into multiple keys, reserving the end of the first key payload for pointers to the
1450 * next ones. E.g. if a key needs to be split into 3 chunks, the first chunk will have
1451 * the format:
1452 * +--------------------+
1453 * | |
1454 * | Data |
1455 * |--------------------|
1456 * | pointer to chunk 2 |
1457 * | pointer to chunk 3 |
1458 * +--------------------+
1459 */
1460 const size_t nChunks = ComputeNumChunks(nbytes, maxKeySize);
1461 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
1463 // Skip writing the first chunk, it will be written last (in the file) below.
1464
1465 const uint8_t *chunkData = reinterpret_cast<const uint8_t *>(data) + nbytesFirstChunk;
1467
1469 std::uint64_t chunkOffsetIdx = 0;
1470
1471 do {
1472 const size_t bytesNextChunk = std::min<size_t>(remainingBytes, maxKeySize);
1473 const std::uint64_t offset = writeKey(chunkData, bytesNextChunk, bytesNextChunk);
1474
1477
1480
1481 } while (remainingBytes > 0);
1482
1483 // Write the first key, with part of the data and the pointers to (logically) following keys appended.
1484 const std::uint64_t firstOffset = ReserveBlob(maxKeySize, maxKeySize);
1485 WriteIntoReservedBlob(data, nbytesFirstChunk, firstOffset);
1486 const std::uint64_t chunkOffsetsOffset = firstOffset + nbytesFirstChunk;
1487 WriteIntoReservedBlob(chunkOffsetsToWrite.get(), nbytesChunkOffsets, chunkOffsetsOffset);
1488
1489 return firstOffset;
1490}
1491
1492std::uint64_t
1493ROOT::Internal::RNTupleFileWriter::ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen])
1494{
1495 // ReserveBlob cannot be used to reserve a multi-key blob
1496 R__ASSERT(nbytes <= fNTupleAnchor.GetMaxKeySize());
1497
1498 std::uint64_t offset;
1499 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1500 if (fIsBare) {
1501 offset = fileSimple->fKeyOffset;
1502 fileSimple->fKeyOffset += nbytes;
1503 } else {
1504 offset = fileSimple->ReserveBlobKey(nbytes, len, keyBuffer);
1505 }
1506 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1507 offset = fileProper->ReserveBlobKey(nbytes, len, keyBuffer);
1508 } else {
1509 auto &fileRFile = std::get<RImplRFile>(fFile);
1510 offset = fileRFile.ReserveBlobKey(nbytes, len, keyBuffer);
1511 }
1512 return offset;
1513}
1514
1515void ROOT::Internal::RNTupleFileWriter::WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
1516{
1517 if (auto *fileSimple = std::get_if<RImplSimple>(&fFile)) {
1518 fileSimple->Write(buffer, nbytes, offset);
1519 } else if (auto *fileProper = std::get_if<RImplTFile>(&fFile)) {
1520 fileProper->Write(buffer, nbytes, offset);
1521 } else {
1522 auto &fileRFile = std::get<RImplRFile>(fFile);
1523 fileRFile.Write(buffer, nbytes, offset);
1524 }
1525}
1526
1528{
1529 auto offset = WriteBlob(data, nbytes, lenHeader);
1530 fNTupleAnchor.fLenHeader = lenHeader;
1531 fNTupleAnchor.fNBytesHeader = nbytes;
1532 fNTupleAnchor.fSeekHeader = offset;
1533 return offset;
1534}
1535
1537{
1538 auto offset = WriteBlob(data, nbytes, lenFooter);
1539 fNTupleAnchor.fLenFooter = lenFooter;
1540 fNTupleAnchor.fNBytesFooter = nbytes;
1541 fNTupleAnchor.fSeekFooter = offset;
1542 return offset;
1543}
1544
1546{
1547 RBareFileHeader bareHeader;
1548 bareHeader.fCompress = defaultCompression;
1549 auto &fileSimple = std::get<RImplSimple>(fFile);
1550 fileSimple.Write(&bareHeader, sizeof(bareHeader), 0);
1551 RTFString ntupleName{fNTupleName};
1552 fileSimple.Write(&ntupleName, ntupleName.GetSize());
1553
1554 // Write zero-initialized ntuple to reserve the space; will be overwritten on commit
1555 RTFNTuple ntupleOnDisk;
1556 fileSimple.fControlBlock->fSeekNTuple = fileSimple.fFilePos;
1557 fileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize());
1558 std::uint64_t checksum = 0;
1559 fileSimple.Write(&checksum, sizeof(checksum));
1560 fileSimple.fKeyOffset = fileSimple.fFilePos;
1561}
1562
1564{
1565 // The streamer info record is a TList of TStreamerInfo object. We cannot use
1566 // RNTupleSerializer::SerializeStreamerInfos because that uses TBufferIO::WriteObject.
1567 // This would prepend the streamed TList with self-decription information.
1568 // The streamer info record is just the streamed TList.
1569
1571 for (auto [_, info] : fStreamerInfoMap) {
1573 }
1574
1575 // We will stream the list with a TBufferFile. When reading the streamer info records back,
1576 // the read buffer includes the key and the streamed list. Therefore, we need to start streaming
1577 // with an offset of the key length. Otherwise, the offset for referencing duplicate objects in the
1578 // buffer will point to the wrong places.
1579
1580 // Figure out key length
1581 RTFString strTList{"TList"};
1582 RTFString strStreamerInfo{"StreamerInfo"};
1583 RTFString strStreamerTitle{"Doubly linked list"};
1584 auto &fileSimple = std::get<RImplSimple>(fFile);
1585 fileSimple.fControlBlock->fHeader.SetSeekInfo(fileSimple.fKeyOffset);
1586 auto keyLen = RTFKey(fileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, strTList, strStreamerInfo,
1588 .fKeyLen;
1589
1590 TBufferFile buffer(TBuffer::kWrite, keyLen + 1);
1591 buffer.SetBufferOffset(keyLen);
1592 streamerInfoList.Streamer(buffer);
1593 assert(buffer.Length() > keyLen);
1594 const auto bufPayload = buffer.Buffer() + keyLen;
1595 const auto lenPayload = buffer.Length() - keyLen;
1596
1599
1601 fileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, "TList", "StreamerInfo",
1602 "Doubly linked list");
1603 fileSimple.fControlBlock->fHeader.SetNbytesInfo(fileSimple.fFilePos -
1604 fileSimple.fControlBlock->fHeader.GetSeekInfo());
1605}
1606
1608{
1609 RTFString strEmpty;
1610 RTFString strRNTupleClass{"ROOT::RNTuple"};
1611 RTFString strRNTupleName{fNTupleName};
1612 RTFString strFileName{fFileName};
1613
1614 auto &fileSimple = std::get<RImplSimple>(fFile);
1615 RTFKey keyRNTuple(fileSimple.fControlBlock->fSeekNTuple, RTFHeader::kBEGIN, strRNTupleClass, strRNTupleName,
1616 strEmpty, RTFNTuple::GetSizePlusChecksum(), anchorSize);
1617
1618 fileSimple.fControlBlock->fFileRecord.SetSeekKeys(fileSimple.fKeyOffset);
1619 RTFKeyList keyList{1};
1620 RTFKey keyKeyList(fileSimple.fControlBlock->fFileRecord.GetSeekKeys(), RTFHeader::kBEGIN, strEmpty, strFileName,
1621 strEmpty, keyList.GetSize() + keyRNTuple.fKeyLen);
1622 fileSimple.Write(&keyKeyList, keyKeyList.GetHeaderSize(), fileSimple.fControlBlock->fFileRecord.GetSeekKeys());
1623 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1624 fileSimple.Write(&strFileName, strFileName.GetSize());
1625 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1626 fileSimple.Write(&keyList, keyList.GetSize());
1627 fileSimple.Write(&keyRNTuple, keyRNTuple.GetHeaderSize());
1628 // Write class name, object name, and title for this key.
1629 fileSimple.Write(&strRNTupleClass, strRNTupleClass.GetSize());
1630 fileSimple.Write(&strRNTupleName, strRNTupleName.GetSize());
1631 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1632 fileSimple.fControlBlock->fFileRecord.fNBytesKeys =
1633 fileSimple.fFilePos - fileSimple.fControlBlock->fFileRecord.GetSeekKeys();
1634 fileSimple.fKeyOffset = fileSimple.fFilePos;
1635}
1636
1638{
1639 auto &fileSimple = std::get<RImplSimple>(fFile);
1640 fileSimple.fControlBlock->fHeader.SetSeekFree(fileSimple.fKeyOffset);
1641 RTFString strEmpty;
1642 RTFString strFileName{fFileName};
1643 RTFFreeEntry freeEntry;
1644 RTFKey keyFreeList(fileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, strEmpty, strFileName,
1645 strEmpty, freeEntry.GetSize());
1646 std::uint64_t firstFree = fileSimple.fControlBlock->fHeader.GetSeekFree() + keyFreeList.GetSize();
1647 freeEntry.Set(firstFree, std::max(2000000000ULL, ((firstFree / 1000000000ULL) + 1) * 1000000000ULL));
1648 fileSimple.WriteKey(&freeEntry, freeEntry.GetSize(), freeEntry.GetSize(),
1649 fileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, "", fFileName, "");
1650 fileSimple.fControlBlock->fHeader.SetNbytesFree(fileSimple.fFilePos -
1651 fileSimple.fControlBlock->fHeader.GetSeekFree());
1652 fileSimple.fControlBlock->fHeader.SetEnd(fileSimple.fFilePos);
1653}
1654
1656{
1657 RTFString strRNTupleClass{"ROOT::RNTuple"};
1658 RTFString strRNTupleName{fNTupleName};
1659 RTFString strEmpty;
1660
1661 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1662 RUInt64BE checksum{XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData())};
1663 auto &fileSimple = std::get<RImplSimple>(fFile);
1664 fileSimple.fControlBlock->fSeekNTuple = fileSimple.fKeyOffset;
1665
1666 char keyBuf[RTFNTuple::GetSizePlusChecksum()];
1667
1668 // concatenate the RNTuple anchor with its checksum
1669 memcpy(keyBuf, &ntupleOnDisk, sizeof(RTFNTuple));
1670 memcpy(keyBuf + sizeof(RTFNTuple), &checksum, sizeof(checksum));
1671
1672 const auto sizeAnchor = sizeof(RTFNTuple) + sizeof(checksum);
1673 char zipAnchor[RTFNTuple::GetSizePlusChecksum()];
1675
1676 fileSimple.WriteKey(zipAnchor, szZipAnchor, sizeof(keyBuf), fileSimple.fControlBlock->fSeekNTuple, RTFHeader::kBEGIN,
1677 "ROOT::RNTuple", fNTupleName, "");
1678 return szZipAnchor;
1679}
1680
1682{
1683 RTFString strTFile{"TFile"};
1684 RTFString strFileName{fFileName};
1685 RTFString strEmpty;
1686
1687 auto &fileSimple = std::get<RImplSimple>(fFile);
1688 fileSimple.fControlBlock->fHeader = RTFHeader(defaultCompression);
1689
1690 RTFUUID uuid;
1691
1692 // First record of the file: the TFile object at offset kBEGIN (= 100)
1693 RTFKey keyRoot(RTFHeader::kBEGIN, 0, strTFile, strFileName, strEmpty,
1694 sizeof(RTFDirectory) + strFileName.GetSize() + strEmpty.GetSize() + uuid.GetSize());
1695 std::uint32_t nbytesName = keyRoot.fKeyLen + strFileName.GetSize() + 1;
1696 fileSimple.fControlBlock->fFileRecord.fNBytesName = nbytesName;
1697 fileSimple.fControlBlock->fHeader.SetNbytesName(nbytesName);
1698
1699 fileSimple.Write(&keyRoot, keyRoot.GetHeaderSize(), RTFHeader::kBEGIN);
1700 // Write class name, object name, and title for the TFile key.
1701 fileSimple.Write(&strTFile, strTFile.GetSize());
1702 fileSimple.Write(&strFileName, strFileName.GetSize());
1703 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1704 // Write the name and title of the TNamed preceding the TFile entry.
1705 fileSimple.Write(&strFileName, strFileName.GetSize());
1706 fileSimple.Write(&strEmpty, strEmpty.GetSize());
1707 // Will be overwritten on commit
1708 fileSimple.fControlBlock->fSeekFileRecord = fileSimple.fFilePos;
1709 fileSimple.Write(&fileSimple.fControlBlock->fFileRecord, fileSimple.fControlBlock->fFileRecord.GetSize());
1710 fileSimple.Write(&uuid, uuid.GetSize());
1711
1712 // Padding bytes to allow the TFile record to grow for a big file
1713 RUInt32BE padding{0};
1714 for (int i = 0; i < 3; ++i)
1715 fileSimple.Write(&padding, sizeof(padding));
1716 fileSimple.fKeyOffset = fileSimple.fFilePos;
1717}
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:300
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
#define ROOT_VERSION_CODE
Definition RVersion.hxx:24
#define ClassDefInlineOverride(name, id)
Definition Rtypes.h:360
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
const Int_t kBEGIN
Definition TFile.cxx:203
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:148
Binding & operator=(OUT(*fun)(void))
void ReadBuffer(char *&buffer) override
T1 fFirst
Definition X11Events.mm:86
#define _(A, B)
Definition cfortran.h:108
An interface to read from, or write to, a ROOT file, as well as performing other common operations.
Definition RFile.hxx:253
The RKeyBlob writes an invisible key into a TFile.
bool WasAllocatedInAFreeSlot() const
void Reserve(size_t nbytes, std::uint64_t *seekKey)
Register a new key for a data record of size nbytes.
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
void LoadStreamerInfo()
Attempts to load the streamer info from the file.
RResult< RNTuple > GetNTupleProperAtOffset(std::uint64_t payloadOffset, std::uint64_t compSize, std::uint64_t uncompLen)
Loads an RNTuple anchor from a TFile at the given file offset (unzipping it if necessary).
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
ROOT::RResult< void > TryReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Like ReadBuffer but returns a RResult instead of throwing.
std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName)
Searches for a key with the given name and type in the key index of the directory starting at offsetD...
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
RResult< RNTuple > GetNTupleProper(std::string_view ntuplePath)
Used when the file turns out to be a TFile container.
Helper class to compress data blocks in the ROOT compression frame format.
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
Helper class to uncompress data blocks in the ROOT compression frame format.
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Write RNTuple data blocks in a TFile or a bare file container.
std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves a new record as an RBlob key in the file.
void WriteTFileStreamerInfo(int compression)
Write the compressed streamer info record with the description of the RNTuple class.
void WriteTFileKeysList(std::uint64_t anchorSize)
Write the TList with the RNTuple key.
std::uint64_t WriteTFileNTupleKey(int compression)
The only key that will be visible in file->ls() Returns the size on disk of the anchor object.
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
static std::uint64_t ReserveBlobKey(T &caller, TFile &file, std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen])
static std::unique_ptr< RNTupleFileWriter > Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize, bool isHidden)
The directory parameter can also be a TFile object (TFile inherits from TDirectory).
void WriteTFileFreeList()
Last record in the file.
std::unique_ptr< RNTupleFileWriter > CloneAsHidden(std::string_view ntupleName) const
Creates a new RNTupleFileWriter with the same underlying TDirectory as this but writing to a differen...
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the header and TFile object.
RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize, bool isHidden)
void Seek(std::uint64_t offset)
Seek a simple writer to offset.
ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t fStreamerInfoMap
Set of streamer info records that should be written to the file.
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
static std::unique_ptr< RNTupleFileWriter > Recreate(std::string_view ntupleName, std::string_view path, EContainerFormat containerFormat, const ROOT::RNTupleWriteOptions &options)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
Write into a reserved record; the caller is responsible for making sure that the written byte range i...
RNTupleLink Commit(int compression=RCompressionSetting::EDefaults::kUseGeneralPurpose)
Writes the RNTuple key to the file so that the header and footer keys can be found.
static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
Prepares buffer for a new record as an RBlob key at offset.
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
void UpdateStreamerInfos(const ROOT::Internal::RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
Ensures that the streamer info records passed as argument are written to the file.
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
EContainerFormat
For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file.
A helper class for serializing and deserialization of the RNTuple binary format.
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
Base class for all ROOT issued exceptions.
Definition RError.hxx:79
Common user-tunable settings for storing RNTuples.
std::size_t GetWriteBufferSize() const
std::uint64_t GetMaxKeySize() const
std::uint32_t GetCompression() const
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:68
std::uint64_t fMaxKeySize
The maximum size for a TKey payload. Payloads bigger than this size will be written as multiple blobs...
Definition RNTuple.hxx:121
static TClass * Class()
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:198
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
void TagStreamerInfo(TVirtualStreamerInfo *info) override
Mark the classindex of the current file as using this TStreamerInfo.
void SetBufferDisplacement() override
Definition TBufferIO.h:82
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition TBuffer.cxx:269
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
void SetBufferOffset(Int_t offset=0)
Definition TBuffer.h:93
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
Describe directory structure in memory.
Definition TDirectory.h:45
A file, usually with extension .root, that stores data and code in the form of serialized objects in ...
Definition TFile.h:130
Bool_t IsBinary() const
Definition TFile.h:347
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition TKey.h:28
Int_t Sizeof() const override
Return the size in bytes of the key header structure.
Definition TKey.cxx:1358
Int_t fVersion
Key version identifier.
Definition TKey.h:41
Int_t fLeft
Number of bytes left in current segment.
Definition TKey.h:50
Short_t fKeylen
Number of bytes for the key itself.
Definition TKey.h:45
Long64_t fSeekKey
Location of object on file.
Definition TKey.h:47
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
Definition TKey.cxx:493
TString fClassName
Object Class name.
Definition TKey.h:49
A doubly linked list.
Definition TList.h:38
Describes a persistent version of a class.
static TClass * Class()
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition TUUID.h:42
void FillBuffer(char *&buffer)
Stream UUID into output buffer.
Definition TUUID.cxx:274
TFile * GetRFileTFile(RFile &rfile)
Definition RFile.cxx:574
RNTuple CreateAnchor(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
Definition RNTuple.cxx:52
std::unique_ptr< T[]> MakeUninitArray(std::size_t size)
Make an array of default-initialized elements.
Helper templated class for swapping bytes; specializations for N={2,4,8} are provided below.
Definition Byteswap.h:124
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
void AllocateBuffers(std::size_t bufferSize)
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
If a TFile container is written by a C stream (simple file), on dataset commit, the file header and t...
auto * tt
Definition textangle.C:16