Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RMiniFile.cxx
Go to the documentation of this file.
1/// \file RMiniFile.cxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-12-22
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#include "Rtypes.h"
17#include <ROOT/RConfig.hxx>
18#include <ROOT/RError.hxx>
19#include <ROOT/RMiniFile.hxx>
20#include <ROOT/RRawFile.hxx>
21#include <ROOT/RNTupleZip.hxx>
24
25#include <Byteswap.h>
26#include <TBufferFile.h>
27#include <TDirectory.h>
28#include <TError.h>
29#include <TFile.h>
30#include <TKey.h>
31#include <TObjString.h>
32#include <TUUID.h>
34
35#include <xxhash.h>
36
37#include <algorithm>
38#include <cassert>
39#include <cerrno>
40#include <cstdio>
41#include <cstring>
42#include <memory>
43#include <string>
44#include <chrono>
45
46#ifdef R__LINUX
47#include <fcntl.h>
48#endif
49
50#ifndef R__LITTLE_ENDIAN
51#ifdef R__BYTESWAP
52// `R__BYTESWAP` is defined in RConfig.hxx for little-endian architectures; undefined otherwise
53#define R__LITTLE_ENDIAN 1
54#else
55#define R__LITTLE_ENDIAN 0
56#endif
57#endif /* R__LITTLE_ENDIAN */
58
59namespace {
60
61// The following types are used to read and write the TFile binary format
62
63/// Big-endian 16-bit unsigned integer
64class RUInt16BE {
65private:
66 std::uint16_t fValBE = 0;
67 static std::uint16_t Swap(std::uint16_t val)
68 {
69#if R__LITTLE_ENDIAN == 1
70 return RByteSwap<sizeof(val)>::bswap(val);
71#else
72 return val;
73#endif
74 }
75
76public:
77 RUInt16BE() = default;
78 explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {}
79 operator std::uint16_t() const { return Swap(fValBE); }
80 RUInt16BE &operator=(const std::uint16_t val)
81 {
82 fValBE = Swap(val);
83 return *this;
84 }
85};
86
87/// Big-endian 32-bit unsigned integer
88class RUInt32BE {
89private:
90 std::uint32_t fValBE = 0;
91 static std::uint32_t Swap(std::uint32_t val)
92 {
93#if R__LITTLE_ENDIAN == 1
94 return RByteSwap<sizeof(val)>::bswap(val);
95#else
96 return val;
97#endif
98 }
99
100public:
101 RUInt32BE() = default;
102 explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {}
103 operator std::uint32_t() const { return Swap(fValBE); }
104 RUInt32BE &operator=(const std::uint32_t val)
105 {
106 fValBE = Swap(val);
107 return *this;
108 }
109};
110
111/// Big-endian 32-bit signed integer
112class RInt32BE {
113private:
114 std::int32_t fValBE = 0;
115 static std::int32_t Swap(std::int32_t val)
116 {
117#if R__LITTLE_ENDIAN == 1
118 return RByteSwap<sizeof(val)>::bswap(val);
119#else
120 return val;
121#endif
122 }
123
124public:
125 RInt32BE() = default;
126 explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {}
127 operator std::int32_t() const { return Swap(fValBE); }
128 RInt32BE &operator=(const std::int32_t val)
129 {
130 fValBE = Swap(val);
131 return *this;
132 }
133};
134
135/// Big-endian 64-bit unsigned integer
136class RUInt64BE {
137private:
138 std::uint64_t fValBE = 0;
139 static std::uint64_t Swap(std::uint64_t val)
140 {
141#if R__LITTLE_ENDIAN == 1
142 return RByteSwap<sizeof(val)>::bswap(val);
143#else
144 return val;
145#endif
146 }
147
148public:
149 RUInt64BE() = default;
150 explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {}
151 operator std::uint64_t() const { return Swap(fValBE); }
152 RUInt64BE &operator=(const std::uint64_t val)
153 {
154 fValBE = Swap(val);
155 return *this;
156 }
157};
158
159#pragma pack(push, 1)
160/// A name (type, identifies, ...) in the TFile binary format
161struct RTFString {
162 unsigned char fLName{0};
163 char fData[255];
164 RTFString() = default;
165 RTFString(const std::string &str)
166 {
167 // The length of strings with 255 characters and longer are encoded with a 32-bit integer following the first
168 // byte. This is currently not handled.
169 R__ASSERT(str.length() < 255);
170 fLName = str.length();
171 memcpy(fData, str.data(), fLName);
172 }
173 std::size_t GetSize() const
174 {
175 // A length of 255 is special and means that the first byte is followed by a 32-bit integer with the actual
176 // length.
177 R__ASSERT(fLName != 255);
178 return 1 + fLName;
179 }
180};
181
182/// The timestamp format used in TFile; the default constructor initializes with the current time
183struct RTFDatetime {
184 RUInt32BE fDatetime;
185 RTFDatetime()
186 {
187 auto now = std::chrono::system_clock::now();
188 auto tt = std::chrono::system_clock::to_time_t(now);
189 auto tm = *localtime(&tt);
190 fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 | tm.tm_hour << 12 |
191 tm.tm_min << 6 | tm.tm_sec;
192 }
193 explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {}
194};
195
196/// The key part of a TFile record excluding the class, object, and title names
197struct RTFKey {
198 static constexpr unsigned kBigKeyVersion = 1000;
199
200 RInt32BE fNbytes{0};
201 RUInt16BE fVersion{4};
202 RUInt32BE fObjLen{0};
203 RTFDatetime fDatetime;
204 RUInt16BE fKeyLen{0};
205 RUInt16BE fCycle{1};
206 union {
207 struct {
208 RUInt32BE fSeekKey{0};
209 RUInt32BE fSeekPdir{0};
210 } fInfoShort;
211 struct {
212 RUInt64BE fSeekKey{0};
213 RUInt64BE fSeekPdir{0};
214 } fInfoLong;
215 };
216
217 RTFKey() : fInfoLong() {}
218 RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir, const RTFString &clName, const RTFString &objName,
219 const RTFString &titleName, std::size_t szObjInMem, std::size_t szObjOnDisk = 0)
220 {
221 R__ASSERT(szObjInMem <= std::numeric_limits<std::uint32_t>::max());
222 R__ASSERT(szObjOnDisk <= std::numeric_limits<std::uint32_t>::max());
223 // For writing, we alywas produce "big" keys with 64-bit SeekKey and SeekPdir.
224 fVersion = fVersion + kBigKeyVersion;
225 fObjLen = szObjInMem;
226 fKeyLen = GetHeaderSize() + clName.GetSize() + objName.GetSize() + titleName.GetSize();
227 fInfoLong.fSeekKey = seekKey;
228 fInfoLong.fSeekPdir = seekPdir;
229 // Depends on fKeyLen being set
230 fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk);
231 }
232
233 std::uint32_t GetSize() const
234 {
235 // Negative size indicates a gap in the file
236 if (fNbytes < 0)
237 return -fNbytes;
238 return fNbytes;
239 }
240
241 std::uint32_t GetHeaderSize() const
242 {
243 if (fVersion >= kBigKeyVersion)
244 return 18 + sizeof(fInfoLong);
245 return 18 + sizeof(fInfoShort);
246 }
247
248 std::uint64_t GetSeekKey() const
249 {
250 if (fVersion >= kBigKeyVersion)
251 return fInfoLong.fSeekKey;
252 return fInfoShort.fSeekKey;
253 }
254};
255
256/// The TFile global header
257struct RTFHeader {
258 static constexpr unsigned kBEGIN = 100;
259 static constexpr unsigned kBigHeaderVersion = 1000000;
260
261 char fMagic[4]{'r', 'o', 'o', 't'};
262 RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
263 (ROOT_VERSION_CODE & 0xFF)};
264 RUInt32BE fBEGIN{kBEGIN};
265 union {
266 struct {
267 RUInt32BE fEND{0};
268 RUInt32BE fSeekFree{0};
269 RUInt32BE fNbytesFree{0};
270 RUInt32BE fNfree{1};
271 RUInt32BE fNbytesName{0};
272 unsigned char fUnits{4};
273 RUInt32BE fCompress{0};
274 RUInt32BE fSeekInfo{0};
275 RUInt32BE fNbytesInfo{0};
276 } fInfoShort;
277 struct {
278 RUInt64BE fEND{0};
279 RUInt64BE fSeekFree{0};
280 RUInt32BE fNbytesFree{0};
281 RUInt32BE fNfree{1};
282 RUInt32BE fNbytesName{0};
283 unsigned char fUnits{8};
284 RUInt32BE fCompress{0};
285 RUInt64BE fSeekInfo{0};
286 RUInt32BE fNbytesInfo{0};
287 } fInfoLong;
288 };
289
290 RTFHeader() : fInfoShort() {}
291 RTFHeader(int compression) : fInfoShort() { fInfoShort.fCompress = compression; }
292
293 void SetBigFile()
294 {
295 if (fVersion >= kBigHeaderVersion)
296 return;
297
298 // clang-format off
299 std::uint32_t end = fInfoShort.fEND;
300 std::uint32_t seekFree = fInfoShort.fSeekFree;
301 std::uint32_t nbytesFree = fInfoShort.fNbytesFree;
302 std::uint32_t nFree = fInfoShort.fNfree;
303 std::uint32_t nbytesName = fInfoShort.fNbytesName;
304 std::uint32_t compress = fInfoShort.fCompress;
305 std::uint32_t seekInfo = fInfoShort.fSeekInfo;
306 std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo;
307 fInfoLong.fEND = end;
308 fInfoLong.fSeekFree = seekFree;
309 fInfoLong.fNbytesFree = nbytesFree;
310 fInfoLong.fNfree = nFree;
311 fInfoLong.fNbytesName = nbytesName;
312 fInfoLong.fUnits = 8;
313 fInfoLong.fCompress = compress;
314 fInfoLong.fSeekInfo = seekInfo;
315 fInfoLong.fNbytesInfo = nbytesInfo;
316 fVersion = fVersion + kBigHeaderVersion;
317 // clang-format on
318 }
319
320 bool IsBigFile(std::uint64_t offset = 0) const
321 {
322 return (fVersion >= kBigHeaderVersion) ||
323 (offset > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()));
324 }
325
326 std::uint32_t GetSize() const
327 {
328 std::uint32_t sizeHead = sizeof(fMagic) + sizeof(fVersion) + sizeof(fBEGIN);
329 if (IsBigFile())
330 return sizeHead + sizeof(fInfoLong);
331 return sizeHead + sizeof(fInfoShort);
332 }
333
334 std::uint64_t GetEnd() const
335 {
336 if (IsBigFile())
337 return fInfoLong.fEND;
338 return fInfoShort.fEND;
339 }
340
341 void SetEnd(std::uint64_t value)
342 {
343 if (IsBigFile(value)) {
344 SetBigFile();
345 fInfoLong.fEND = value;
346 } else {
347 fInfoShort.fEND = value;
348 }
349 }
350
351 std::uint64_t GetSeekFree() const
352 {
353 if (IsBigFile())
354 return fInfoLong.fSeekFree;
355 return fInfoShort.fSeekFree;
356 }
357
358 void SetSeekFree(std::uint64_t value)
359 {
360 if (IsBigFile(value)) {
361 SetBigFile();
362 fInfoLong.fSeekFree = value;
363 } else {
364 fInfoShort.fSeekFree = value;
365 }
366 }
367
368 void SetNbytesFree(std::uint32_t value)
369 {
370 if (IsBigFile()) {
371 fInfoLong.fNbytesFree = value;
372 } else {
373 fInfoShort.fNbytesFree = value;
374 }
375 }
376
377 void SetNbytesName(std::uint32_t value)
378 {
379 if (IsBigFile()) {
380 fInfoLong.fNbytesName = value;
381 } else {
382 fInfoShort.fNbytesName = value;
383 }
384 }
385
386 std::uint64_t GetSeekInfo() const
387 {
388 if (IsBigFile())
389 return fInfoLong.fSeekInfo;
390 return fInfoShort.fSeekInfo;
391 }
392
393 void SetSeekInfo(std::uint64_t value)
394 {
395 if (IsBigFile(value)) {
396 SetBigFile();
397 fInfoLong.fSeekInfo = value;
398 } else {
399 fInfoShort.fSeekInfo = value;
400 }
401 }
402
403 void SetNbytesInfo(std::uint32_t value)
404 {
405 if (IsBigFile()) {
406 fInfoLong.fNbytesInfo = value;
407 } else {
408 fInfoShort.fNbytesInfo = value;
409 }
410 }
411
412 void SetCompression(std::uint32_t value)
413 {
414 if (IsBigFile()) {
415 fInfoLong.fCompress = value;
416 } else {
417 fInfoShort.fCompress = value;
418 }
419 }
420};
421
422/// A reference to an unused byte-range in a TFile
423struct RTFFreeEntry {
424 static constexpr unsigned kBigFreeEntryVersion = 1000;
425
426 RUInt16BE fVersion{1};
427 union {
428 struct {
429 RUInt32BE fFirst{0};
430 RUInt32BE fLast{0};
431 } fInfoShort;
432 struct {
433 RUInt64BE fFirst{0};
434 RUInt64BE fLast{0};
435 } fInfoLong;
436 };
437
438 RTFFreeEntry() : fInfoShort() {}
439 void Set(std::uint64_t first, std::uint64_t last)
440 {
441 if (last > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
442 fVersion = fVersion + kBigFreeEntryVersion;
443 fInfoLong.fFirst = first;
444 fInfoLong.fLast = last;
445 } else {
446 fInfoShort.fFirst = first;
447 fInfoShort.fLast = last;
448 }
449 }
450 std::uint32_t GetSize() { return (fVersion >= kBigFreeEntryVersion) ? 18 : 10; }
451};
452
453/// The header of the directory key index
454struct RTFKeyList {
455 RUInt32BE fNKeys;
456 std::uint32_t GetSize() const { return sizeof(RTFKeyList); }
457 explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {}
458};
459
460/// A streamed TDirectory (TFile) object
461struct RTFDirectory {
462 static constexpr unsigned kBigFileVersion = 1000;
463
464 RUInt16BE fClassVersion{5};
465 RTFDatetime fDateC;
466 RTFDatetime fDateM;
467 RUInt32BE fNBytesKeys{0};
468 RUInt32BE fNBytesName{0};
469 // The version of the key has to tell whether offsets are 32bit or 64bit long
470 union {
471 struct {
472 RUInt32BE fSeekDir{RTFHeader::kBEGIN};
473 RUInt32BE fSeekParent{0};
474 RUInt32BE fSeekKeys{0};
475 } fInfoShort;
476 struct {
477 RUInt64BE fSeekDir{RTFHeader::kBEGIN};
478 RUInt64BE fSeekParent{0};
479 RUInt64BE fSeekKeys{0};
480 } fInfoLong;
481 };
482
483 RTFDirectory() : fInfoShort() {}
484
485 // In case of a short TFile record (<2G), 3 padding ints are written after the UUID
486 std::uint32_t GetSize() const
487 {
488 if (fClassVersion >= kBigFileVersion)
489 return sizeof(RTFDirectory);
490 return 18 + sizeof(fInfoShort);
491 }
492
493 std::uint64_t GetSeekKeys() const
494 {
495 if (fClassVersion >= kBigFileVersion)
496 return fInfoLong.fSeekKeys;
497 return fInfoShort.fSeekKeys;
498 }
499
500 void SetSeekKeys(std::uint64_t seekKeys)
501 {
502 if (seekKeys > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
503 std::uint32_t seekDir = fInfoShort.fSeekDir;
504 std::uint32_t seekParent = fInfoShort.fSeekParent;
505 fInfoLong.fSeekDir = seekDir;
506 fInfoLong.fSeekParent = seekParent;
507 fInfoLong.fSeekKeys = seekKeys;
508 fClassVersion = fClassVersion + kBigFileVersion;
509 } else {
510 fInfoShort.fSeekKeys = seekKeys;
511 }
512 }
513};
514
515/// A zero UUID stored at the end of the TFile record
516struct RTFUUID {
517 RUInt16BE fVersionClass{1};
518 unsigned char fUUID[16];
519
520 RTFUUID()
521 {
522 TUUID uuid;
523 char *buffer = reinterpret_cast<char *>(this);
524 uuid.FillBuffer(buffer);
525 assert(reinterpret_cast<RTFUUID *>(buffer) <= (this + 1));
526 }
527 std::uint32_t GetSize() const { return sizeof(RTFUUID); }
528};
529
530/// A streamed RNTuple class
531///
532/// NOTE: this must be kept in sync with RNTuple.hxx.
533/// Aside ensuring consistency between the two classes' members, you need to make sure
534/// that fVersionClass matches the class version of RNTuple.
535struct RTFNTuple {
536 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))};
537 RUInt16BE fVersionClass{2};
538 RUInt16BE fVersionEpoch{0};
539 RUInt16BE fVersionMajor{0};
540 RUInt16BE fVersionMinor{0};
541 RUInt16BE fVersionPatch{0};
542 RUInt64BE fSeekHeader{0};
543 RUInt64BE fNBytesHeader{0};
544 RUInt64BE fLenHeader{0};
545 RUInt64BE fSeekFooter{0};
546 RUInt64BE fNBytesFooter{0};
547 RUInt64BE fLenFooter{0};
548 RUInt64BE fMaxKeySize{0};
549
550 static constexpr std::uint32_t GetSizePlusChecksum() { return sizeof(RTFNTuple) + sizeof(std::uint64_t); }
551
552 RTFNTuple() = default;
553 explicit RTFNTuple(const ROOT::RNTuple &inMemoryAnchor)
554 {
555 fVersionEpoch = inMemoryAnchor.GetVersionEpoch();
556 fVersionMajor = inMemoryAnchor.GetVersionMajor();
557 fVersionMinor = inMemoryAnchor.GetVersionMinor();
558 fVersionPatch = inMemoryAnchor.GetVersionPatch();
559 fSeekHeader = inMemoryAnchor.GetSeekHeader();
560 fNBytesHeader = inMemoryAnchor.GetNBytesHeader();
561 fLenHeader = inMemoryAnchor.GetLenHeader();
562 fSeekFooter = inMemoryAnchor.GetSeekFooter();
563 fNBytesFooter = inMemoryAnchor.GetNBytesFooter();
564 fLenFooter = inMemoryAnchor.GetLenFooter();
565 fMaxKeySize = inMemoryAnchor.GetMaxKeySize();
566 }
567 std::uint32_t GetSize() const { return sizeof(RTFNTuple); }
568 // The byte count and class version members are not checksummed
569 std::uint32_t GetOffsetCkData() { return sizeof(fByteCount) + sizeof(fVersionClass); }
570 std::uint32_t GetSizeCkData() { return GetSize() - GetOffsetCkData(); }
571 unsigned char *GetPtrCkData() { return reinterpret_cast<unsigned char *>(this) + GetOffsetCkData(); }
572};
573
574/// The bare file global header
575struct RBareFileHeader {
576 char fMagic[7]{'r', 'n', 't', 'u', 'p', 'l', 'e'};
577 RUInt32BE fRootVersion{(ROOT_VERSION_CODE >> 16) * 10000 + ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
578 (ROOT_VERSION_CODE & 0xFF)};
579 RUInt32BE fFormatVersion{1};
580 RUInt32BE fCompress{0};
581 RTFNTuple fNTuple;
582 // followed by the ntuple name
583};
584#pragma pack(pop)
585
586/// The artifical class name shown for opaque RNTuple keys (see TBasket)
587constexpr char const *kBlobClassName = "RBlob";
588/// The class name of the RNTuple anchor
589constexpr char const *kNTupleClassName = "ROOT::RNTuple";
590
591} // anonymous namespace
592
593namespace ROOT {
594namespace Experimental {
595namespace Internal {
596/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header
597/// and the TFile record need to be updated
599 RTFHeader fHeader;
600 RTFDirectory fFileRecord;
601 std::uint64_t fSeekNTuple{0}; // Remember the offset for the keys list
602 std::uint64_t fSeekFileRecord{0};
603};
604
605/// The RKeyBlob writes an invisible key into a TFile. That is, a key that is not indexed in the list of keys,
606/// like a TBasket.
607/// NOTE: out of anonymous namespace because otherwise ClassDefInline fails to compile
608/// on some platforms.
609class RKeyBlob : public TKey {
610public:
611 RKeyBlob() = default;
612
613 explicit RKeyBlob(TFile *file) : TKey(file)
614 {
615 fClassName = kBlobClassName;
616 fVersion += RTFKey::kBigKeyVersion;
617 fKeylen = Sizeof();
618 }
619
620 /// Register a new key for a data record of size nbytes
621 void Reserve(size_t nbytes, std::uint64_t *seekKey)
622 {
623 Create(nbytes);
624 *seekKey = fSeekKey;
625 }
626
628};
629
630} // namespace Internal
631} // namespace Experimental
632} // namespace ROOT
633
634// Computes how many chunks do we need to fit `nbytes` of payload, considering that the
635// first chunk also needs to house the offsets of the other chunks and no chunk can
636// be bigger than `maxChunkSize`. When saved to a TFile, each chunk is part of a separate TKey.
637static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
638{
639 constexpr size_t kChunkOffsetSize = sizeof(std::uint64_t);
640
641 assert(nbytes > maxChunkSize);
642 size_t nChunks = (nbytes + maxChunkSize - 1) / maxChunkSize;
643 assert(nChunks > 1);
644 size_t nbytesTail = nbytes % maxChunkSize;
645 size_t nbytesExtra = (nbytesTail > 0) * (maxChunkSize - nbytesTail);
646 size_t nbytesChunkOffsets = (nChunks - 1) * kChunkOffsetSize;
647 if (nbytesChunkOffsets > nbytesExtra) {
648 ++nChunks;
649 nbytesChunkOffsets += kChunkOffsetSize;
650 }
651
652 // We don't support having more chunkOffsets than what fits in one chunk.
653 // For a reasonable-sized maxKeySize it looks very unlikely that we can have more chunks
654 // than we can fit in the first `maxKeySize` bytes. E.g. for maxKeySize = 1GiB we can fit
655 // 134217728 chunk offsets, making our multi-key blob's capacity exactly 128 PiB.
656 R__ASSERT(nbytesChunkOffsets <= maxChunkSize);
657
658 return nChunks;
659}
660
662
665{
666 char ident[4];
667 ReadBuffer(ident, 4, 0);
668 if (std::string(ident, 4) == "root")
669 return GetNTupleProper(ntupleName);
670 fIsBare = true;
671 return GetNTupleBare(ntupleName);
672}
673
674/// Searches for a key with the given name and type in the key index of the given directory.
675/// Return 0 if the key was not found.
677 std::string_view keyName,
678 std::string_view typeName)
679{
680 RTFDirectory directory;
681 ReadBuffer(&directory, sizeof(directory), offsetDir);
682
683 RTFKey key;
684 RUInt32BE nKeys;
685 std::uint64_t offset = directory.GetSeekKeys();
686 ReadBuffer(&key, sizeof(key), offset);
687 offset += key.fKeyLen;
688 ReadBuffer(&nKeys, sizeof(nKeys), offset);
689 offset += sizeof(nKeys);
690
691 for (unsigned int i = 0; i < nKeys; ++i) {
692 ReadBuffer(&key, sizeof(key), offset);
693 auto offsetNextKey = offset + key.fKeyLen;
694
695 offset += key.GetHeaderSize();
696 RTFString name;
697 ReadBuffer(&name, 1, offset);
698 ReadBuffer(&name, name.GetSize(), offset);
699 if (std::string_view(name.fData, name.fLName) != typeName) {
700 offset = offsetNextKey;
701 continue;
702 }
703 offset += name.GetSize();
704 ReadBuffer(&name, 1, offset);
705 ReadBuffer(&name, name.GetSize(), offset);
706 if (std::string_view(name.fData, name.fLName) == keyName) {
707 return key.GetSeekKey();
708 }
709 offset = offsetNextKey;
710 }
711
712 // Not found
713 return 0;
714}
715
718{
719 RTFHeader fileHeader;
720 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
721
722 RTFKey key;
723 RTFString name;
724 ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN);
725 // Skip over the entire key length, including the class name, object name, and title stored in it.
726 std::uint64_t offset = fileHeader.fBEGIN + key.fKeyLen;
727 // Skip over the name and title of the TNamed preceding the TFile (root TDirectory) entry.
728 ReadBuffer(&name, 1, offset);
729 offset += name.GetSize();
730 ReadBuffer(&name, 1, offset);
731 offset += name.GetSize();
732
733 // split ntupleName by '/' character to open datasets in subdirectories.
734 std::string ntuplePathTail(ntuplePath);
735 if (!ntuplePathTail.empty() && ntuplePathTail[0] == '/')
736 ntuplePathTail = ntuplePathTail.substr(1);
737 auto pos = std::string::npos;
738 while ((pos = ntuplePathTail.find('/')) != std::string::npos) {
739 auto directoryName = ntuplePathTail.substr(0, pos);
740 ntuplePathTail.erase(0, pos + 1);
741
742 offset = SearchInDirectory(offset, directoryName, "TDirectory");
743 if (offset == 0) {
744 return R__FAIL("no directory named '" + std::string(directoryName) + "' in file '" + fRawFile->GetUrl() + "'");
745 }
746 ReadBuffer(&key, sizeof(key), offset);
747 offset = key.GetSeekKey() + key.fKeyLen;
748 }
749 // no more '/' delimiter in ntuplePath
750 auto ntupleName = ntuplePathTail;
751
752 offset = SearchInDirectory(offset, ntupleName, kNTupleClassName);
753 if (offset == 0) {
754 return R__FAIL("no RNTuple named '" + std::string(ntupleName) + "' in file '" + fRawFile->GetUrl() + "'");
755 }
756
757 ReadBuffer(&key, sizeof(key), offset);
758 offset = key.GetSeekKey() + key.fKeyLen;
759
760 // size of a RTFNTuple version 2 (min supported version); future anchor versions can grow.
761 constexpr size_t kMinNTupleSize = 78;
762 static_assert(kMinNTupleSize == RTFNTuple::GetSizePlusChecksum());
763 if (key.fObjLen < kMinNTupleSize) {
764 return R__FAIL("invalid anchor size: " + std::to_string(key.fObjLen) + " < " + std::to_string(sizeof(RTFNTuple)));
765 }
766 // The object length can be smaller than the size of RTFNTuple if it comes from a past RNTuple class version,
767 // or larger than it if it comes from a future RNTuple class version.
768 auto bufAnchor = MakeUninitArray<unsigned char>(std::max<size_t>(key.fObjLen, sizeof(RTFNTuple)));
769 RTFNTuple *ntuple = new (bufAnchor.get()) RTFNTuple;
770
771 auto objNbytes = key.GetSize() - key.fKeyLen;
772 ReadBuffer(ntuple, objNbytes, offset);
773 if (objNbytes != key.fObjLen) {
774 RNTupleDecompressor decompressor;
775 decompressor.Unzip(bufAnchor.get(), objNbytes, key.fObjLen);
776 }
777
778 // We require that future class versions only append members and store the checksum in the last 8 bytes
779 // Checksum calculation: strip byte count, class version, fChecksum member
780 auto lenCkData = key.fObjLen - ntuple->GetOffsetCkData() - sizeof(uint64_t);
781 auto ckCalc = XXH3_64bits(ntuple->GetPtrCkData(), lenCkData);
782 uint64_t ckOnDisk;
783
784 RUInt64BE *ckOnDiskPtr = reinterpret_cast<RUInt64BE *>(bufAnchor.get() + key.fObjLen - sizeof(uint64_t));
785 ckOnDisk = static_cast<uint64_t>(*ckOnDiskPtr);
786 if (ckCalc != ckOnDisk) {
787 return R__FAIL("RNTuple anchor checksum mismatch");
788 }
789
790 return CreateAnchor(ntuple->fVersionEpoch, ntuple->fVersionMajor, ntuple->fVersionMinor, ntuple->fVersionPatch,
791 ntuple->fSeekHeader, ntuple->fNBytesHeader, ntuple->fLenHeader, ntuple->fSeekFooter,
792 ntuple->fNBytesFooter, ntuple->fLenFooter, ntuple->fMaxKeySize);
793}
794
797{
798 RBareFileHeader fileHeader;
799 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
800 RTFString name;
801 auto offset = sizeof(fileHeader);
802 ReadBuffer(&name, 1, offset);
803 ReadBuffer(&name, name.GetSize(), offset);
804 std::string_view foundName(name.fData, name.fLName);
805 if (foundName != ntupleName) {
806 return R__FAIL("expected RNTuple named '" + std::string(ntupleName) + "' but instead found '" +
807 std::string(foundName) + "' in file '" + fRawFile->GetUrl() + "'");
808 }
809 offset += name.GetSize();
810
811 RTFNTuple ntuple;
812 ReadBuffer(&ntuple, sizeof(ntuple), offset);
813 std::uint64_t onDiskChecksum;
814 ReadBuffer(&onDiskChecksum, sizeof(onDiskChecksum), offset + sizeof(ntuple));
815 auto checksum = XXH3_64bits(ntuple.GetPtrCkData(), ntuple.GetSizeCkData());
816 if (checksum != static_cast<uint64_t>(onDiskChecksum))
817 return R__FAIL("RNTuple bare file: anchor checksum mismatch");
818
819 return CreateAnchor(ntuple.fVersionEpoch, ntuple.fVersionMajor, ntuple.fVersionMinor, ntuple.fVersionPatch,
820 ntuple.fSeekHeader, ntuple.fNBytesHeader, ntuple.fLenHeader, ntuple.fSeekFooter,
821 ntuple.fNBytesFooter, ntuple.fLenFooter, ntuple.fMaxKeySize);
822}
823
824void ROOT::Experimental::Internal::RMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
825{
826 size_t nread;
827 if (fMaxKeySize == 0 || nbytes <= fMaxKeySize) {
828 // Fast path: read single blob
829 nread = fRawFile->ReadAt(buffer, nbytes, offset);
830 } else {
831 // Read chunked blob. See RNTupleFileWriter::WriteBlob() for details.
832 const size_t nChunks = ComputeNumChunks(nbytes, fMaxKeySize);
833 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
834 const size_t nbytesFirstChunk = fMaxKeySize - nbytesChunkOffsets;
835 uint8_t *bufCur = reinterpret_cast<uint8_t *>(buffer);
836
837 // Read first chunk
838 nread = fRawFile->ReadAt(bufCur, fMaxKeySize, offset);
839 R__ASSERT(nread == fMaxKeySize);
840 // NOTE: we read the entire chunk in `bufCur`, but we only advance the pointer by `nbytesFirstChunk`,
841 // since the last part of `bufCur` will later be overwritten by the next chunk's payload.
842 // We do this to avoid a second ReadAt to read in the chunk offsets.
843 bufCur += nbytesFirstChunk;
844 nread -= nbytesChunkOffsets;
845
846 const auto chunkOffsets = MakeUninitArray<std::uint64_t>(nChunks - 1);
847 memcpy(chunkOffsets.get(), bufCur, nbytesChunkOffsets);
848
849 size_t remainingBytes = nbytes - nbytesFirstChunk;
850 std::uint64_t *curChunkOffset = &chunkOffsets[0];
851
852 do {
853 std::uint64_t chunkOffset;
854 RNTupleSerializer::DeserializeUInt64(curChunkOffset, chunkOffset);
855 ++curChunkOffset;
856
857 const size_t bytesToRead = std::min<size_t>(fMaxKeySize, remainingBytes);
858 // Ensure we don't read outside of the buffer
859 R__ASSERT(static_cast<size_t>(bufCur - reinterpret_cast<uint8_t *>(buffer)) <= nbytes - bytesToRead);
860
861 auto nbytesRead = fRawFile->ReadAt(bufCur, bytesToRead, chunkOffset);
862 R__ASSERT(nbytesRead == bytesToRead);
863
864 nread += bytesToRead;
865 bufCur += bytesToRead;
866 remainingBytes -= bytesToRead;
867 } while (remainingBytes > 0);
868 }
869 R__ASSERT(nread == nbytes);
870}
871
872////////////////////////////////////////////////////////////////////////////////
873
875
876/// Prepare a blob key in the provided buffer, which must provide space for kBlobKeyLen bytes. Note that the array type
877/// is purely documentation, the argument is actually just a pointer.
878static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
879{
880 RTFString strClass{kBlobClassName};
881 RTFString strObject;
882 RTFString strTitle;
883 RTFKey keyHeader(offset, RTFHeader::kBEGIN, strClass, strObject, strTitle, len, nbytes);
884 R__ASSERT(keyHeader.fKeyLen == kBlobKeyLen);
885
886 // Copy structures into the buffer.
887 unsigned char *writeBuffer = buffer;
888 memcpy(writeBuffer, &keyHeader, keyHeader.GetHeaderSize());
889 writeBuffer += keyHeader.GetHeaderSize();
890 memcpy(writeBuffer, &strClass, strClass.GetSize());
891 writeBuffer += strClass.GetSize();
892 memcpy(writeBuffer, &strObject, strObject.GetSize());
893 writeBuffer += strObject.GetSize();
894 memcpy(writeBuffer, &strTitle, strTitle.GetSize());
895 writeBuffer += strTitle.GetSize();
896 R__ASSERT(writeBuffer == buffer + kBlobKeyLen);
897}
898
899////////////////////////////////////////////////////////////////////////////////
900
902
904{
905 static_assert(kHeaderBlockSize % kBlockAlign == 0, "invalid header block size");
906 if (bufferSize % kBlockAlign != 0)
907 throw RException(R__FAIL("Buffer size not a multiple of alignment: " + std::to_string(bufferSize)));
908 fBlockSize = bufferSize;
909
910 std::align_val_t blockAlign{kBlockAlign};
911 fHeaderBlock = static_cast<unsigned char *>(::operator new[](kHeaderBlockSize, blockAlign));
912 memset(fHeaderBlock, 0, kHeaderBlockSize);
913 fBlock = static_cast<unsigned char *>(::operator new[](fBlockSize, blockAlign));
914 memset(fBlock, 0, fBlockSize);
915}
916
918{
919 if (fFile)
920 fclose(fFile);
921
922 std::align_val_t blockAlign{kBlockAlign};
923 if (fHeaderBlock)
924 ::operator delete[](fHeaderBlock, blockAlign);
925 if (fBlock)
926 ::operator delete[](fBlock, blockAlign);
927}
928
929namespace {
930int FSeek64(FILE *stream, std::int64_t offset, int origin)
931{
932#ifdef R__SEEK64
933 return fseeko64(stream, offset, origin);
934#else
935 return fseek(stream, offset, origin);
936#endif
937}
938} // namespace
939
941{
942 // Write the last partially filled block, which may still need appropriate alignment for Direct I/O.
943 // If it is the first block, get the updated header block.
944 if (fBlockOffset == 0) {
945 std::size_t headerBlockSize = kHeaderBlockSize;
946 if (headerBlockSize > fFilePos) {
947 headerBlockSize = fFilePos;
948 }
949 memcpy(fBlock, fHeaderBlock, headerBlockSize);
950 }
951
952 std::size_t retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
953 if (retval)
954 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
955
956 std::size_t lastBlockSize = fFilePos - fBlockOffset;
957 R__ASSERT(lastBlockSize <= fBlockSize);
958 if (fDirectIO) {
959 // Round up to a multiple of kBlockAlign.
960 lastBlockSize += kBlockAlign - 1;
961 lastBlockSize = (lastBlockSize / kBlockAlign) * kBlockAlign;
962 R__ASSERT(lastBlockSize <= fBlockSize);
963 }
964 retval = fwrite(fBlock, 1, lastBlockSize, fFile);
965 if (retval != lastBlockSize)
966 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
967
968 // Write the (updated) header block, unless it was part of the write above.
969 if (fBlockOffset > 0) {
970 retval = FSeek64(fFile, 0, SEEK_SET);
971 if (retval)
972 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
973
974 retval = fwrite(fHeaderBlock, 1, kHeaderBlockSize, fFile);
975 if (retval != RFileSimple::kHeaderBlockSize)
976 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
977 }
978
979 retval = fflush(fFile);
980 if (retval)
981 throw RException(R__FAIL(std::string("Flush failed: ") + strerror(errno)));
982}
983
985 std::int64_t offset)
986{
987 R__ASSERT(fFile);
988 size_t retval;
989 if ((offset >= 0) && (static_cast<std::uint64_t>(offset) != fFilePos)) {
990 fFilePos = offset;
991 }
992
993 // Keep header block to overwrite on commit.
994 if (fFilePos < kHeaderBlockSize) {
995 std::size_t headerBytes = nbytes;
996 if (fFilePos + headerBytes > kHeaderBlockSize) {
997 headerBytes = kHeaderBlockSize - fFilePos;
998 }
999 memcpy(fHeaderBlock + fFilePos, buffer, headerBytes);
1000 }
1001
1002 R__ASSERT(fFilePos >= fBlockOffset);
1003
1004 while (nbytes > 0) {
1005 std::uint64_t posInBlock = fFilePos % fBlockSize;
1006 std::uint64_t blockOffset = fFilePos - posInBlock;
1007 if (blockOffset != fBlockOffset) {
1008 // Write the block.
1009 retval = FSeek64(fFile, fBlockOffset, SEEK_SET);
1010 if (retval)
1011 throw RException(R__FAIL(std::string("Seek failed: ") + strerror(errno)));
1012
1013 retval = fwrite(fBlock, 1, fBlockSize, fFile);
1014 if (retval != fBlockSize)
1015 throw RException(R__FAIL(std::string("write failed: ") + strerror(errno)));
1016
1017 // Null the buffer contents for good measure.
1018 memset(fBlock, 0, fBlockSize);
1019 }
1020
1021 fBlockOffset = blockOffset;
1022 std::size_t blockSize = nbytes;
1023 if (blockSize > fBlockSize - posInBlock) {
1024 blockSize = fBlockSize - posInBlock;
1025 }
1026 memcpy(fBlock + posInBlock, buffer, blockSize);
1027 buffer = static_cast<const unsigned char *>(buffer) + blockSize;
1028 nbytes -= blockSize;
1029 fFilePos += blockSize;
1030 }
1031}
1032
1034 const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset, std::uint64_t directoryOffset,
1035 const std::string &className, const std::string &objectName, const std::string &title)
1036{
1037 if (offset > 0)
1038 fKeyOffset = offset;
1039 RTFString strClass{className};
1040 RTFString strObject{objectName};
1041 RTFString strTitle{title};
1042
1043 RTFKey key(fKeyOffset, directoryOffset, strClass, strObject, strTitle, len, nbytes);
1044 Write(&key, key.GetHeaderSize(), fKeyOffset);
1045 Write(&strClass, strClass.GetSize());
1046 Write(&strObject, strObject.GetSize());
1047 Write(&strTitle, strTitle.GetSize());
1048 auto offsetData = fFilePos;
1049 // The next key starts after the data.
1050 fKeyOffset = offsetData + nbytes;
1051 if (buffer)
1052 Write(buffer, nbytes);
1053
1054 return offsetData;
1055}
1056
1057std::uint64_t
1059 unsigned char keyBuffer[kBlobKeyLen])
1060{
1061 if (keyBuffer) {
1062 PrepareBlobKey(fKeyOffset, nbytes, len, keyBuffer);
1063 } else {
1064 unsigned char localKeyBuffer[kBlobKeyLen];
1065 PrepareBlobKey(fKeyOffset, nbytes, len, localKeyBuffer);
1066 Write(localKeyBuffer, kBlobKeyLen, fKeyOffset);
1067 }
1068
1069 auto offsetData = fKeyOffset + kBlobKeyLen;
1070 // The next key starts after the data.
1071 fKeyOffset = offsetData + nbytes;
1072
1073 return offsetData;
1074}
1075
1076////////////////////////////////////////////////////////////////////////////////
1077
1079 std::int64_t offset)
1080{
1081 fDirectory->GetFile()->Seek(offset);
1082 bool rv = fDirectory->GetFile()->WriteBuffer((char *)(buffer), nbytes);
1083 if (rv)
1084 throw RException(R__FAIL("WriteBuffer failed."));
1085}
1086
1087std::uint64_t
1089 unsigned char keyBuffer[kBlobKeyLen])
1090{
1091 std::uint64_t offsetKey;
1092 RKeyBlob keyBlob(fDirectory->GetFile());
1093 // Since it is unknown beforehand if offsetKey is beyond the 2GB limit or not,
1094 // RKeyBlob will always reserve space for a big key (version >= 1000)
1095 keyBlob.Reserve(nbytes, &offsetKey);
1096
1097 if (keyBuffer) {
1098 PrepareBlobKey(offsetKey, nbytes, len, keyBuffer);
1099 } else {
1100 unsigned char localKeyBuffer[kBlobKeyLen];
1101 PrepareBlobKey(offsetKey, nbytes, len, localKeyBuffer);
1102 Write(localKeyBuffer, kBlobKeyLen, offsetKey);
1103 }
1104
1105 auto offsetData = offsetKey + kBlobKeyLen;
1106
1107 return offsetData;
1108}
1109
1110////////////////////////////////////////////////////////////////////////////////
1111
1113 : fNTupleName(name)
1114{
1115 fFileSimple.fControlBlock = std::make_unique<ROOT::Experimental::Internal::RTFileControlBlock>();
1116 fNTupleAnchor.fMaxKeySize = maxKeySize;
1117 auto infoRNTuple = RNTuple::Class()->GetStreamerInfo();
1118 fStreamerInfoMap[infoRNTuple->GetNumber()] = infoRNTuple;
1119}
1120
1122
1123std::unique_ptr<ROOT::Experimental::Internal::RNTupleFileWriter>
1124ROOT::Experimental::Internal::RNTupleFileWriter::Recreate(std::string_view ntupleName, std::string_view path,
1125 EContainerFormat containerFormat,
1126 const RNTupleWriteOptions &options)
1127{
1128 std::string fileName(path);
1129 size_t idxDirSep = fileName.find_last_of("\\/");
1130 if (idxDirSep != std::string::npos) {
1131 fileName.erase(0, idxDirSep + 1);
1132 }
1133#ifdef R__LINUX
1134 int flags = O_WRONLY | O_CREAT | O_TRUNC;
1135#ifdef O_LARGEFILE
1136 // Add the equivalent flag that is passed by fopen64.
1137 flags |= O_LARGEFILE;
1138#endif
1139 if (options.GetUseDirectIO()) {
1140 flags |= O_DIRECT;
1141 }
1142 int fd = open(std::string(path).c_str(), flags, 0666);
1143 FILE *fileStream = fdopen(fd, "wb");
1144#else
1145#ifdef R__SEEK64
1146 FILE *fileStream = fopen64(std::string(path.data(), path.size()).c_str(), "wb");
1147#else
1148 FILE *fileStream = fopen(std::string(path.data(), path.size()).c_str(), "wb");
1149#endif
1150#endif
1151 R__ASSERT(fileStream);
1152 // RNTupleFileWriter::RFileSimple does its own buffering, turn off additional buffering from C stdio.
1153 std::setvbuf(fileStream, nullptr, _IONBF, 0);
1154
1155 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, options.GetMaxKeySize()));
1156 writer->fFileSimple.fFile = fileStream;
1157 writer->fFileSimple.fDirectIO = options.GetUseDirectIO();
1158 writer->fFileSimple.AllocateBuffers(options.GetWriteBufferSize());
1159 writer->fFileName = fileName;
1160
1161 int defaultCompression = options.GetCompression();
1162 switch (containerFormat) {
1163 case EContainerFormat::kTFile: writer->WriteTFileSkeleton(defaultCompression); break;
1164 case EContainerFormat::kBare:
1165 writer->fIsBare = true;
1166 writer->WriteBareFileSkeleton(defaultCompression);
1167 break;
1168 default: R__ASSERT(false && "Internal error: unhandled container format");
1169 }
1170
1171 return writer;
1172}
1173
1174std::unique_ptr<ROOT::Experimental::Internal::RNTupleFileWriter>
1175ROOT::Experimental::Internal::RNTupleFileWriter::Append(std::string_view ntupleName, TDirectory &fileOrDirectory,
1176 std::uint64_t maxKeySize)
1177{
1178 TFile *file = fileOrDirectory.GetFile();
1179 if (!file)
1180 throw RException(R__FAIL("invalid attempt to add an RNTuple to a directory that is not backed by a file"));
1181 assert(file->IsBinary());
1182
1183 auto writer = std::unique_ptr<RNTupleFileWriter>(new RNTupleFileWriter(ntupleName, maxKeySize));
1184 writer->fFileProper.fDirectory = &fileOrDirectory;
1185 return writer;
1186}
1187
1189 const RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
1190{
1191 fStreamerInfoMap.insert(streamerInfos.cbegin(), streamerInfos.cend());
1192}
1193
1195{
1196 if (fFileProper) {
1197 // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
1198 fFileProper.fDirectory->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
1199
1200 // Make sure the streamer info records used in the RNTuple are written to the file
1202 buf.SetParent(fFileProper.fDirectory->GetFile());
1203 for (auto [_, info] : fStreamerInfoMap)
1204 buf.TagStreamerInfo(info);
1205
1206 fFileProper.fDirectory->GetFile()->Write();
1207 return;
1208 }
1209
1210 // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
1211 R__ASSERT(fFileSimple);
1212
1213 if (fIsBare) {
1214 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1215 // Compute the checksum
1216 std::uint64_t checksum = XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData());
1217 memcpy(fFileSimple.fHeaderBlock + fFileSimple.fControlBlock->fSeekNTuple, &ntupleOnDisk, ntupleOnDisk.GetSize());
1218 memcpy(fFileSimple.fHeaderBlock + fFileSimple.fControlBlock->fSeekNTuple + ntupleOnDisk.GetSize(), &checksum,
1219 sizeof(checksum));
1220 fFileSimple.Flush();
1221 return;
1222 }
1223
1224 WriteTFileNTupleKey();
1225 WriteTFileKeysList();
1226 WriteTFileStreamerInfo();
1227 WriteTFileFreeList();
1228
1229 // Update header and TFile record
1230 memcpy(fFileSimple.fHeaderBlock, &fFileSimple.fControlBlock->fHeader, fFileSimple.fControlBlock->fHeader.GetSize());
1231 R__ASSERT(fFileSimple.fControlBlock->fSeekFileRecord + fFileSimple.fControlBlock->fFileRecord.GetSize() <
1232 RFileSimple::kHeaderBlockSize);
1233 memcpy(fFileSimple.fHeaderBlock + fFileSimple.fControlBlock->fSeekFileRecord,
1234 &fFileSimple.fControlBlock->fFileRecord, fFileSimple.fControlBlock->fFileRecord.GetSize());
1235
1236 fFileSimple.Flush();
1237}
1238
1239std::uint64_t ROOT::Experimental::Internal::RNTupleFileWriter::WriteBlob(const void *data, size_t nbytes, size_t len)
1240{
1241 auto writeKey = [this](const void *payload, size_t nBytes, size_t length) {
1242 std::uint64_t offset = ReserveBlob(nBytes, length);
1243 WriteIntoReservedBlob(payload, nBytes, offset);
1244 return offset;
1245 };
1246
1247 const std::uint64_t maxKeySize = fNTupleAnchor.fMaxKeySize;
1248 R__ASSERT(maxKeySize > 0);
1249 // We don't need the object length except for seeing compression ratios in TFile::Map()
1250 // Make sure that the on-disk object length fits into the TKey header.
1251 if (static_cast<std::uint64_t>(len) > static_cast<std::uint64_t>(std::numeric_limits<std::uint32_t>::max()))
1252 len = nbytes;
1253
1254 if (nbytes <= maxKeySize) {
1255 // Fast path: only write 1 key.
1256 return writeKey(data, nbytes, len);
1257 }
1258
1259 /**
1260 * Writing a key bigger than the max allowed size. In this case we split the payload
1261 * into multiple keys, reserving the end of the first key payload for pointers to the
1262 * next ones. E.g. if a key needs to be split into 3 chunks, the first chunk will have
1263 * the format:
1264 * +--------------------+
1265 * | |
1266 * | Data |
1267 * |--------------------|
1268 * | pointer to chunk 2 |
1269 * | pointer to chunk 3 |
1270 * +--------------------+
1271 */
1272 const size_t nChunks = ComputeNumChunks(nbytes, maxKeySize);
1273 const size_t nbytesChunkOffsets = (nChunks - 1) * sizeof(std::uint64_t);
1274 const size_t nbytesFirstChunk = maxKeySize - nbytesChunkOffsets;
1275 // Skip writing the first chunk, it will be written last (in the file) below.
1276
1277 const uint8_t *chunkData = reinterpret_cast<const uint8_t *>(data) + nbytesFirstChunk;
1278 size_t remainingBytes = nbytes - nbytesFirstChunk;
1279
1280 const auto chunkOffsetsToWrite = MakeUninitArray<std::uint64_t>(nChunks - 1);
1281 std::uint64_t chunkOffsetIdx = 0;
1282
1283 do {
1284 const size_t bytesNextChunk = std::min<size_t>(remainingBytes, maxKeySize);
1285 const std::uint64_t offset = writeKey(chunkData, bytesNextChunk, bytesNextChunk);
1286
1287 RNTupleSerializer::SerializeUInt64(offset, &chunkOffsetsToWrite[chunkOffsetIdx]);
1288 ++chunkOffsetIdx;
1289
1290 remainingBytes -= bytesNextChunk;
1291 chunkData += bytesNextChunk;
1292
1293 } while (remainingBytes > 0);
1294
1295 // Write the first key, with part of the data and the pointers to (logically) following keys appended.
1296 const std::uint64_t firstOffset = ReserveBlob(maxKeySize, maxKeySize);
1297 WriteIntoReservedBlob(data, nbytesFirstChunk, firstOffset);
1298 const std::uint64_t chunkOffsetsOffset = firstOffset + nbytesFirstChunk;
1299 WriteIntoReservedBlob(chunkOffsetsToWrite.get(), nbytesChunkOffsets, chunkOffsetsOffset);
1300
1301 return firstOffset;
1302}
1303
1305 unsigned char keyBuffer[kBlobKeyLen])
1306{
1307 // ReserveBlob cannot be used to reserve a multi-key blob
1308 R__ASSERT(nbytes <= fNTupleAnchor.GetMaxKeySize());
1309
1310 std::uint64_t offset;
1311 if (fFileSimple) {
1312 if (fIsBare) {
1313 offset = fFileSimple.fKeyOffset;
1314 fFileSimple.fKeyOffset += nbytes;
1315 } else {
1316 offset = fFileSimple.ReserveBlobKey(nbytes, len, keyBuffer);
1317 }
1318 } else {
1319 offset = fFileProper.ReserveBlobKey(nbytes, len, keyBuffer);
1320 }
1321 return offset;
1322}
1323
1325 std::int64_t offset)
1326{
1327 if (fFileSimple) {
1328 fFileSimple.Write(buffer, nbytes, offset);
1329 } else {
1330 fFileProper.Write(buffer, nbytes, offset);
1331 }
1332}
1333
1334std::uint64_t
1336{
1337 auto offset = WriteBlob(data, nbytes, lenHeader);
1338 fNTupleAnchor.fLenHeader = lenHeader;
1339 fNTupleAnchor.fNBytesHeader = nbytes;
1340 fNTupleAnchor.fSeekHeader = offset;
1341 return offset;
1342}
1343
1344std::uint64_t
1346{
1347 auto offset = WriteBlob(data, nbytes, lenFooter);
1348 fNTupleAnchor.fLenFooter = lenFooter;
1349 fNTupleAnchor.fNBytesFooter = nbytes;
1350 fNTupleAnchor.fSeekFooter = offset;
1351 return offset;
1352}
1353
1355{
1356 RBareFileHeader bareHeader;
1357 bareHeader.fCompress = defaultCompression;
1358 fFileSimple.Write(&bareHeader, sizeof(bareHeader), 0);
1359 RTFString ntupleName{fNTupleName};
1360 fFileSimple.Write(&ntupleName, ntupleName.GetSize());
1361
1362 // Write zero-initialized ntuple to reserve the space; will be overwritten on commit
1363 RTFNTuple ntupleOnDisk;
1364 fFileSimple.fControlBlock->fSeekNTuple = fFileSimple.fFilePos;
1365 fFileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize());
1366 std::uint64_t checksum = 0;
1367 fFileSimple.Write(&checksum, sizeof(checksum));
1368 fFileSimple.fKeyOffset = fFileSimple.fFilePos;
1369}
1370
1372{
1373 // The streamer info record is a TList of TStreamerInfo object. We cannot use
1374 // RNTupleSerializer::SerializeStreamerInfos because that uses TBufferIO::WriteObject.
1375 // This would prepend the streamed TList with self-decription information.
1376 // The streamer info record is just the streamed TList.
1377
1378 TList streamerInfoList;
1379 for (auto [_, info] : fStreamerInfoMap) {
1380 streamerInfoList.Add(info);
1381 }
1382
1383 // We will stream the list with a TBufferFile. When reading the streamer info records back,
1384 // the read buffer includes the key and the streamed list. Therefore, we need to start streaming
1385 // with an offset of the key length. Otherwise, the offset for referencing duplicate objects in the
1386 // buffer will point to the wrong places.
1387
1388 // Figure out key length
1389 RTFString strTList{"TList"};
1390 RTFString strStreamerInfo{"StreamerInfo"};
1391 RTFString strStreamerTitle{"Doubly linked list"};
1392 fFileSimple.fControlBlock->fHeader.SetSeekInfo(fFileSimple.fKeyOffset);
1393 auto keyLen = RTFKey(fFileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, strTList, strStreamerInfo,
1394 strStreamerTitle, 0)
1395 .fKeyLen;
1396
1397 TBufferFile buffer(TBuffer::kWrite, keyLen + 1);
1398 buffer.SetBufferOffset(keyLen);
1399 streamerInfoList.Streamer(buffer);
1400 assert(buffer.Length() > keyLen);
1401 const auto bufPayload = buffer.Buffer() + keyLen;
1402 const auto lenPayload = buffer.Length() - keyLen;
1403
1404 RNTupleCompressor compressor;
1405 auto zipStreamerInfos = MakeUninitArray<unsigned char>(lenPayload);
1406 auto szZipStreamerInfos = compressor.Zip(bufPayload, lenPayload, 1, zipStreamerInfos.get());
1407
1408 fFileSimple.WriteKey(zipStreamerInfos.get(), szZipStreamerInfos, lenPayload,
1409 fFileSimple.fControlBlock->fHeader.GetSeekInfo(), RTFHeader::kBEGIN, "TList", "StreamerInfo",
1410 "Doubly linked list");
1411 fFileSimple.fControlBlock->fHeader.SetNbytesInfo(fFileSimple.fFilePos -
1412 fFileSimple.fControlBlock->fHeader.GetSeekInfo());
1413}
1414
1416{
1417 RTFString strEmpty;
1418 RTFString strRNTupleClass{"ROOT::RNTuple"};
1419 RTFString strRNTupleName{fNTupleName};
1420 RTFString strFileName{fFileName};
1421
1422 RTFKey keyRNTuple(fFileSimple.fControlBlock->fSeekNTuple, RTFHeader::kBEGIN, strRNTupleClass, strRNTupleName,
1423 strEmpty, RTFNTuple::GetSizePlusChecksum());
1424
1425 fFileSimple.fControlBlock->fFileRecord.SetSeekKeys(fFileSimple.fKeyOffset);
1426 RTFKeyList keyList{1};
1427 RTFKey keyKeyList(fFileSimple.fControlBlock->fFileRecord.GetSeekKeys(), RTFHeader::kBEGIN, strEmpty, strFileName,
1428 strEmpty, keyList.GetSize() + keyRNTuple.fKeyLen);
1429 fFileSimple.Write(&keyKeyList, keyKeyList.GetHeaderSize(), fFileSimple.fControlBlock->fFileRecord.GetSeekKeys());
1430 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1431 fFileSimple.Write(&strFileName, strFileName.GetSize());
1432 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1433 fFileSimple.Write(&keyList, keyList.GetSize());
1434 fFileSimple.Write(&keyRNTuple, keyRNTuple.GetHeaderSize());
1435 // Write class name, object name, and title for this key.
1436 fFileSimple.Write(&strRNTupleClass, strRNTupleClass.GetSize());
1437 fFileSimple.Write(&strRNTupleName, strRNTupleName.GetSize());
1438 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1439 fFileSimple.fControlBlock->fFileRecord.fNBytesKeys =
1440 fFileSimple.fFilePos - fFileSimple.fControlBlock->fFileRecord.GetSeekKeys();
1441 fFileSimple.fKeyOffset = fFileSimple.fFilePos;
1442}
1443
1445{
1446 fFileSimple.fControlBlock->fHeader.SetSeekFree(fFileSimple.fKeyOffset);
1447 RTFString strEmpty;
1448 RTFString strFileName{fFileName};
1449 RTFFreeEntry freeEntry;
1450 RTFKey keyFreeList(fFileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, strEmpty, strFileName,
1451 strEmpty, freeEntry.GetSize());
1452 std::uint64_t firstFree = fFileSimple.fControlBlock->fHeader.GetSeekFree() + keyFreeList.GetSize();
1453 freeEntry.Set(firstFree, std::max(2000000000ULL, ((firstFree / 1000000000ULL) + 1) * 1000000000ULL));
1454 fFileSimple.WriteKey(&freeEntry, freeEntry.GetSize(), freeEntry.GetSize(),
1455 fFileSimple.fControlBlock->fHeader.GetSeekFree(), RTFHeader::kBEGIN, "", fFileName, "");
1456 fFileSimple.fControlBlock->fHeader.SetNbytesFree(fFileSimple.fFilePos -
1457 fFileSimple.fControlBlock->fHeader.GetSeekFree());
1458 fFileSimple.fControlBlock->fHeader.SetEnd(fFileSimple.fFilePos);
1459}
1460
1462{
1463 RTFString strRNTupleClass{"ROOT::RNTuple"};
1464 RTFString strRNTupleName{fNTupleName};
1465 RTFString strEmpty;
1466
1467 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1468 RUInt64BE checksum{XXH3_64bits(ntupleOnDisk.GetPtrCkData(), ntupleOnDisk.GetSizeCkData())};
1469 fFileSimple.fControlBlock->fSeekNTuple = fFileSimple.fKeyOffset;
1470
1471 char keyBuf[RTFNTuple::GetSizePlusChecksum()];
1472
1473 // concatenate the RNTuple anchor with its checksum
1474 memcpy(keyBuf, &ntupleOnDisk, sizeof(RTFNTuple));
1475 memcpy(keyBuf + sizeof(RTFNTuple), &checksum, sizeof(checksum));
1476
1477 fFileSimple.WriteKey(keyBuf, sizeof(keyBuf), sizeof(keyBuf), fFileSimple.fControlBlock->fSeekNTuple,
1478 RTFHeader::kBEGIN, "ROOT::RNTuple", fNTupleName, "");
1479}
1480
1482{
1483 RTFString strTFile{"TFile"};
1484 RTFString strFileName{fFileName};
1485 RTFString strEmpty;
1486
1487 fFileSimple.fControlBlock->fHeader = RTFHeader(defaultCompression);
1488
1489 RTFUUID uuid;
1490
1491 // First record of the file: the TFile object at offset kBEGIN (= 100)
1492 RTFKey keyRoot(RTFHeader::kBEGIN, 0, strTFile, strFileName, strEmpty,
1493 sizeof(RTFDirectory) + strFileName.GetSize() + strEmpty.GetSize() + uuid.GetSize());
1494 std::uint32_t nbytesName = keyRoot.fKeyLen + strFileName.GetSize() + 1;
1495 fFileSimple.fControlBlock->fFileRecord.fNBytesName = nbytesName;
1496 fFileSimple.fControlBlock->fHeader.SetNbytesName(nbytesName);
1497
1498 fFileSimple.Write(&keyRoot, keyRoot.GetHeaderSize(), RTFHeader::kBEGIN);
1499 // Write class name, object name, and title for the TFile key.
1500 fFileSimple.Write(&strTFile, strTFile.GetSize());
1501 fFileSimple.Write(&strFileName, strFileName.GetSize());
1502 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1503 // Write the name and title of the TNamed preceding the TFile entry.
1504 fFileSimple.Write(&strFileName, strFileName.GetSize());
1505 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1506 // Will be overwritten on commit
1507 fFileSimple.fControlBlock->fSeekFileRecord = fFileSimple.fFilePos;
1508 fFileSimple.Write(&fFileSimple.fControlBlock->fFileRecord, fFileSimple.fControlBlock->fFileRecord.GetSize());
1509 fFileSimple.Write(&uuid, uuid.GetSize());
1510
1511 // Padding bytes to allow the TFile record to grow for a big file
1512 RUInt32BE padding{0};
1513 for (int i = 0; i < 3; ++i)
1514 fFileSimple.Write(&padding, sizeof(padding));
1515 fFileSimple.fKeyOffset = fFileSimple.fFilePos;
1516}
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:290
static void PrepareBlobKey(std::int64_t offset, size_t nbytes, size_t len, unsigned char buffer[kBlobKeyLen])
Prepare a blob key in the provided buffer, which must provide space for kBlobKeyLen bytes.
static size_t ComputeNumChunks(size_t nbytes, size_t maxChunkSize)
static constexpr auto kBlobKeyLen
#define ROOT_VERSION_CODE
Definition RVersion.hxx:24
#define ClassDefInlineOverride(name, id)
Definition Rtypes.h:358
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
const Int_t kBEGIN
Definition TFile.cxx:180
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void value
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
char name[80]
Definition TGX11.cxx:110
Binding & operator=(OUT(*fun)(void))
void ReadBuffer(char *&buffer) override
T1 fFirst
Definition X11Events.mm:86
#define _(A, B)
Definition cfortran.h:108
The RKeyBlob writes an invisible key into a TFile.
void Reserve(size_t nbytes, std::uint64_t *seekKey)
Register a new key for a data record of size nbytes.
std::uint64_t SearchInDirectory(std::uint64_t &offsetDir, std::string_view keyName, std::string_view typeName)
Searches for a key with the given name and type in the key index of the directory starting at offsetD...
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
RResult< RNTuple > GetNTupleProper(std::string_view ntuplePath)
Used when the file turns out to be a TFile container.
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
Helper class to compress data blocks in the ROOT compression frame format.
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
Helper class to uncompress data blocks in the ROOT compression frame format.
static void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Write RNTuple data blocks in a TFile or a bare file container.
RNTupleFileWriter(std::string_view name, std::uint64_t maxKeySize)
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
void WriteTFileKeysList()
Write the TList with the RNTuple key.
void UpdateStreamerInfos(const RNTupleSerializer::StreamerInfoMap_t &streamerInfos)
Ensures that the streamer info records passed as argument are written to the file.
void Commit()
Writes the RNTuple key to the file so that the header and footer keys can be found.
RFileSimple fFileSimple
For simple use cases, survives without libRIO dependency.
void WriteTFileNTupleKey()
The only key that will be visible in file->ls()
void WriteTFileFreeList()
Last record in the file.
EContainerFormat
For testing purposes, RNTuple data can be written into a bare file container instead of a ROOT file.
std::uint64_t ReserveBlob(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves a new record as an RBlob key in the file.
static std::unique_ptr< RNTupleFileWriter > Recreate(std::string_view ntupleName, std::string_view path, EContainerFormat containerFormat, const RNTupleWriteOptions &options)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
static std::unique_ptr< RNTupleFileWriter > Append(std::string_view ntupleName, TDirectory &fileOrDirectory, std::uint64_t maxKeySize)
The directory parameter can also be a TFile object (TFile inherits from TDirectory).
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the header and TFile object.
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
void WriteTFileStreamerInfo()
Write the compressed streamer info record with the description of the RNTuple class.
static constexpr std::size_t kBlobKeyLen
The key length of a blob. It is always a big key (version > 1000) with class name RBlob.
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
RNTuple fNTupleAnchor
Header and footer location of the ntuple, written on Commit()
void WriteIntoReservedBlob(const void *buffer, size_t nbytes, std::int64_t offset)
Write into a reserved record; the caller is responsible for making sure that the written byte range i...
RNTupleSerializer::StreamerInfoMap_t fStreamerInfoMap
Set of streamer info records that should be written to the file.
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
Base class for all ROOT issued exceptions.
Definition RError.hxx:78
Common user-tunable settings for storing ntuples.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:194
The RRawFile provides read-only access to local and remote files.
Definition RRawFile.hxx:43
Representation of an RNTuple data set in a ROOT file.
Definition RNTuple.hxx:69
std::uint64_t GetLenFooter() const
Definition RNTuple.hxx:128
std::uint64_t GetSeekHeader() const
Definition RNTuple.hxx:122
std::uint16_t GetVersionMajor() const
Definition RNTuple.hxx:118
std::uint64_t GetNBytesFooter() const
Definition RNTuple.hxx:127
std::uint64_t GetNBytesHeader() const
Definition RNTuple.hxx:123
std::uint64_t fMaxKeySize
The maximum size for a TKey payload. Payloads bigger than this size will be written as multiple blobs...
Definition RNTuple.hxx:109
std::uint16_t GetVersionMinor() const
Definition RNTuple.hxx:119
std::uint16_t GetVersionPatch() const
Definition RNTuple.hxx:120
std::uint64_t GetMaxKeySize() const
Definition RNTuple.hxx:129
std::uint64_t GetLenHeader() const
Definition RNTuple.hxx:124
static TClass * Class()
std::uint64_t GetSeekFooter() const
Definition RNTuple.hxx:126
std::uint16_t GetVersionEpoch() const
Definition RNTuple.hxx:117
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
void TagStreamerInfo(TVirtualStreamerInfo *info) override
Mark the classindex of the current file as using this TStreamerInfo.
void SetParent(TObject *parent)
Set parent owning this buffer.
Definition TBuffer.cxx:270
@ kWrite
Definition TBuffer.h:73
void SetBufferOffset(Int_t offset=0)
Definition TBuffer.h:93
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
TVirtualStreamerInfo * GetStreamerInfo(Int_t version=0, Bool_t isTransient=kFALSE) const
returns a pointer to the TVirtualStreamerInfo object for version If the object does not exist,...
Definition TClass.cxx:4668
Describe directory structure in memory.
Definition TDirectory.h:45
virtual TFile * GetFile() const
Definition TDirectory.h:220
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
Definition TFile.h:53
Bool_t IsBinary() const
Definition TFile.h:259
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition TKey.h:28
Int_t Sizeof() const override
Return the size in bytes of the key header structure.
Definition TKey.cxx:1342
Int_t fVersion
Key version identifier.
Definition TKey.h:39
Short_t fKeylen
Number of bytes for the key itself.
Definition TKey.h:43
Long64_t fSeekKey
Location of object on file.
Definition TKey.h:45
virtual void Create(Int_t nbytes, TFile *f=nullptr)
Create a TKey object of specified size.
Definition TKey.cxx:460
TString fClassName
Object Class name.
Definition TKey.h:47
A doubly linked list.
Definition TList.h:38
void Streamer(TBuffer &) override
Stream all objects in the collection to or from the I/O buffer.
Definition TList.cxx:1189
void Add(TObject *obj) override
Definition TList.h:81
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition TUUID.h:42
void FillBuffer(char *&buffer)
Stream UUID into output buffer.
Definition TUUID.cxx:275
#define Swap(a, b)
Definition geom.c:201
RNTuple CreateAnchor(std::uint16_t versionEpoch, std::uint16_t versionMajor, std::uint16_t versionMinor, std::uint16_t versionPatch, std::uint64_t seekHeader, std::uint64_t nbytesHeader, std::uint64_t lenHeader, std::uint64_t seekFooter, std::uint64_t nbytesFooter, std::uint64_t lenFooter, std::uint64_t maxKeySize)
Definition RNTuple.cxx:53
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Helper templated class for swapping bytes; specializations for N={2,4,8} are provided below.
Definition Byteswap.h:124
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
std::uint64_t ReserveBlobKey(size_t nbytes, size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
std::unique_ptr< ROOT::Experimental::Internal::RTFileControlBlock > fControlBlock
Keeps track of TFile control structures, which need to be updated on committing the data set.
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
std::uint64_t ReserveBlobKey(std::size_t nbytes, std::size_t len, unsigned char keyBuffer[kBlobKeyLen]=nullptr)
Reserves an RBlob opaque key as data record and returns the offset of the record.
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
If a TFile container is written by a C stream (simple file), on dataset commit, the file header and t...
auto * tt
Definition textangle.C:16