Logo ROOT  
Reference Guide
RMiniFile.cxx
Go to the documentation of this file.
1/// \file RMiniFile.cxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-12-22
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#include <ROOT/RConfig.hxx>
17
18#include "ROOT/RMiniFile.hxx"
19
20#include <ROOT/RRawFile.hxx>
21#include <ROOT/RNTupleZip.hxx>
22
23#include <TError.h>
24#include <TFile.h>
25#include <TKey.h>
26
27#include <algorithm>
28#include <cstdio>
29#include <cstring>
30#include <iostream>
31#include <string>
32#include <utility>
33#include <chrono>
34
35namespace {
36
37// The following types are used to read and write the TFile binary format
38
39/// Big-endian 16-bit unsigned integer
40class RUInt16BE {
41private:
42 std::uint16_t fValBE = 0;
43 static std::uint16_t Swap(std::uint16_t val) {
44 return (val & 0x00FF) << 8 | (val & 0xFF00) >> 8;
45 }
46public:
47 RUInt16BE() = default;
48 explicit RUInt16BE(const std::uint16_t val) : fValBE(Swap(val)) {}
49 operator std::uint16_t() const {
50 return Swap(fValBE);
51 }
52 RUInt16BE& operator =(const std::uint16_t val) {
53 fValBE = Swap(val);
54 return *this;
55 }
56};
57
58/// Big-endian 32-bit unsigned integer
59class RUInt32BE {
60private:
61 std::uint32_t fValBE = 0;
62 static std::uint32_t Swap(std::uint32_t val) {
63 auto x = (val & 0x0000FFFF) << 16 | (val & 0xFFFF0000) >> 16;
64 return (x & 0x00FF00FF) << 8 | (x & 0xFF00FF00) >> 8;
65 }
66public:
67 RUInt32BE() = default;
68 explicit RUInt32BE(const std::uint32_t val) : fValBE(Swap(val)) {}
69 operator std::uint32_t() const {
70 return Swap(fValBE);
71 }
72 RUInt32BE& operator =(const std::uint32_t val) {
73 fValBE = Swap(val);
74 return *this;
75 }
76};
77
78/// Big-endian 32-bit signed integer
79class RInt32BE {
80private:
81 std::int32_t fValBE = 0;
82 static std::int32_t Swap(std::int32_t val) {
83 auto x = (val & 0x0000FFFF) << 16 | (val & 0xFFFF0000) >> 16;
84 return (x & 0x00FF00FF) << 8 | (x & 0xFF00FF00) >> 8;
85 }
86public:
87 RInt32BE() = default;
88 explicit RInt32BE(const std::int32_t val) : fValBE(Swap(val)) {}
89 operator std::int32_t() const {
90 return Swap(fValBE);
91 }
92 RInt32BE& operator =(const std::int32_t val) {
93 fValBE = Swap(val);
94 return *this;
95 }
96};
97
98/// Big-endian 64-bit unsigned integer
99class RUInt64BE {
100private:
101 std::uint64_t fValBE = 0;
102 static std::uint64_t Swap(std::uint64_t val) {
103 auto x = (val & 0x00000000FFFFFFFF) << 32 | (val & 0xFFFFFFFF00000000) >> 32;
104 x = (x & 0x0000FFFF0000FFFF) << 16 | (x & 0xFFFF0000FFFF0000) >> 16;
105 return (x & 0x00FF00FF00FF00FF) << 8 | (x & 0xFF00FF00FF00FF00) >> 8;
106 }
107public:
108 RUInt64BE() = default;
109 explicit RUInt64BE(const std::uint64_t val) : fValBE(Swap(val)) {}
110 operator std::uint64_t() const {
111 return Swap(fValBE);
112 }
113 RUInt64BE& operator =(const std::uint64_t val) {
114 fValBE = Swap(val);
115 return *this;
116 }
117};
118
119/// Composition of class RNTuple as being interpreted by TFile
120constexpr std::int32_t ChecksumRNTupleClass() {
121 const char ident[] = "ROOT::Experimental::RNTuple"
122 "fVersion"
123 "unsigned int"
124 "fSize"
125 "unsigned int"
126 "fSeekHeader"
127 "unsigned long"
128 "fNBytesHeader"
129 "unsigned int"
130 "fLenHeader"
131 "unsigned int"
132 "fSeekFooter"
133 "unsigned long"
134 "fNBytesFooter"
135 "unsigned int"
136 "fLenFooter"
137 "unsigned int"
138 "fReserved"
139 "unsigned long";
140 std::int32_t id = 0;
141 for (unsigned i = 0; i < (sizeof(ident) - 1); i++)
142 id = static_cast<std::int32_t>(static_cast<std::int64_t>(id) * 3 + ident[i]);
143 return id;
144}
145
146
147#pragma pack(push, 1)
148/// A name (type, identifies, ...) in the TFile binary format
149struct RTFString {
150 char fLName{0};
151 char fData[255];
152 RTFString() = default;
153 RTFString(const std::string &str) {
154 R__ASSERT(str.length() < 256);
155 fLName = str.length();
156 memcpy(fData, str.data(), fLName);
157 }
158 char GetSize() const { return 1 + fLName; }
159};
160
161/// The timestamp format used in TFile; the default constructor initializes with the current time
162struct RTFDatetime {
163 RUInt32BE fDatetime;
164 RTFDatetime() {
165 auto now = std::chrono::system_clock::now();
166 auto tt = std::chrono::system_clock::to_time_t(now);
167 auto tm = *localtime(&tt);
168 fDatetime = (tm.tm_year + 1900 - 1995) << 26 | (tm.tm_mon + 1) << 22 | tm.tm_mday << 17 |
169 tm.tm_hour << 12 | tm.tm_min << 6 | tm.tm_sec;
170 }
171 explicit RTFDatetime(RUInt32BE val) : fDatetime(val) {}
172};
173
174/// The key part of a TFile record excluding the class, object, and title names
175struct RTFKey {
176 RInt32BE fNbytes{0};
177 RUInt16BE fVersion{4};
178 RUInt32BE fObjLen{0};
179 RTFDatetime fDatetime;
180 RUInt16BE fKeyLen{0};
181 RUInt16BE fCycle{1};
182 union {
183 struct {
184 RUInt32BE fSeekKey{0};
185 RUInt32BE fSeekPdir{0};
186 } fInfoShort;
187 struct {
188 RUInt64BE fSeekKey{0};
189 RUInt64BE fSeekPdir{0};
190 } fInfoLong;
191 };
192
193 std::uint32_t fKeyHeaderSize{18 + sizeof(fInfoShort)}; // not part of serialization
194
195 RTFKey() : fInfoShort() {}
196 RTFKey(std::uint64_t seekKey, std::uint64_t seekPdir,
197 const RTFString &clName, const RTFString &objName, const RTFString &titleName,
198 std::uint32_t szObjInMem, std::uint32_t szObjOnDisk = 0)
199 {
200 fObjLen = szObjInMem;
201 if ((seekKey > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) ||
202 (seekPdir > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())))
203 {
204 fKeyHeaderSize = 18 + sizeof(fInfoLong);
205 fKeyLen = fKeyHeaderSize + clName.GetSize() + objName.GetSize() + titleName.GetSize();
206 fInfoLong.fSeekKey = seekKey;
207 fInfoLong.fSeekPdir = seekPdir;
208 fVersion = fVersion + 1000;
209 } else {
210 fKeyHeaderSize = 18 + sizeof(fInfoShort);
211 fKeyLen = fKeyHeaderSize + clName.GetSize() + objName.GetSize() + titleName.GetSize();
212 fInfoShort.fSeekKey = seekKey;
213 fInfoShort.fSeekPdir = seekPdir;
214 }
215 fNbytes = fKeyLen + ((szObjOnDisk == 0) ? szObjInMem : szObjOnDisk);
216 }
217
218 std::uint32_t GetSize() const {
219 // Negative size indicates a gap in the file
220 if (fNbytes < 0)
221 return -fNbytes;
222 return fNbytes;
223 }
224
225 std::uint32_t GetHeaderSize() const {
226 if (fVersion >= 1000)
227 return 18 + sizeof(fInfoLong);
228 return 18 + sizeof(fInfoShort);
229 }
230
231 std::uint64_t GetSeekKey() const {
232 if (fVersion >= 1000)
233 return fInfoLong.fSeekKey;
234 return fInfoShort.fSeekKey;
235 }
236};
237
238/// The TFile global header
239struct RTFHeader {
240 char fMagic[4]{ 'r', 'o', 'o', 't' };
241 RUInt32BE fVersion{(ROOT_VERSION_CODE >> 16)*10000 +
242 ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
243 (ROOT_VERSION_CODE & 0xFF)};
244 RUInt32BE fBEGIN{100};
245 union {
246 struct {
247 RUInt32BE fEND{0};
248 RUInt32BE fSeekFree{0};
249 RUInt32BE fNbytesFree{0};
250 RUInt32BE fNfree{1};
251 RUInt32BE fNbytesName{0};
252 unsigned char fUnits{4};
253 RUInt32BE fCompress{0};
254 RUInt32BE fSeekInfo{0};
255 RUInt32BE fNbytesInfo{0};
256 } fInfoShort;
257 struct {
258 RUInt64BE fEND{0};
259 RUInt64BE fSeekFree{0};
260 RUInt32BE fNbytesFree{0};
261 RUInt32BE fNfree{1};
262 RUInt32BE fNbytesName{0};
263 unsigned char fUnits{8};
264 RUInt32BE fCompress{0};
265 RUInt64BE fSeekInfo{0};
266 RUInt32BE fNbytesInfo{0};
267 } fInfoLong;
268 };
269
270 RTFHeader() : fInfoShort() {}
271 RTFHeader(int compression) : fInfoShort() {
272 fInfoShort.fCompress = compression;
273 }
274
275 void SetBigFile() {
276 if (fVersion >= 1000000)
277 return;
278
279 std::uint32_t end = fInfoShort.fEND;
280 std::uint32_t seekFree = fInfoShort.fSeekFree;
281 std::uint32_t nbytesFree = fInfoShort.fNbytesFree;
282 std::uint32_t nFree = fInfoShort.fNfree;
283 std::uint32_t nbytesName = fInfoShort.fNbytesName;
284 std::uint32_t compress = fInfoShort.fCompress;
285 std::uint32_t seekInfo = fInfoShort.fSeekInfo;
286 std::uint32_t nbytesInfo = fInfoShort.fNbytesInfo;
287 fInfoLong.fEND = end;
288 fInfoLong.fSeekFree = seekFree;
289 fInfoLong.fNbytesFree = nbytesFree;
290 fInfoLong.fNfree = nFree;
291 fInfoLong.fNbytesName = nbytesName;
292 fInfoLong.fUnits = 8;
293 fInfoLong.fCompress = compress;
294 fInfoLong.fSeekInfo = seekInfo;
295 fInfoLong.fNbytesInfo = nbytesInfo;
296 fVersion = fVersion + 1000000;
297 }
298
299 bool IsBigFile(std::uint64_t offset = 0) const {
300 return (fVersion >= 1000000) || (offset > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max()));
301 }
302
303 std::uint32_t GetSize() const {
304 std::uint32_t sizeHead = 4 + sizeof(fVersion) + sizeof(fBEGIN);
305 if (IsBigFile()) return sizeHead + sizeof(fInfoLong);
306 return sizeHead + sizeof(fInfoShort);
307 }
308
309 std::uint64_t GetEnd() const {
310 if (IsBigFile()) return fInfoLong.fEND;
311 return fInfoShort.fEND;
312 }
313
314 void SetEnd(std::uint64_t value) {
315 if (IsBigFile(value)) {
316 SetBigFile();
317 fInfoLong.fEND = value;
318 } else {
319 fInfoShort.fEND = value;
320 }
321 }
322
323 std::uint64_t GetSeekFree() const {
324 if (IsBigFile()) return fInfoLong.fSeekFree;
325 return fInfoShort.fSeekFree;
326 }
327
328 void SetSeekFree(std::uint64_t value) {
329 if (IsBigFile(value)) {
330 SetBigFile();
331 fInfoLong.fSeekFree = value;
332 } else {
333 fInfoShort.fSeekFree = value;
334 }
335 }
336
337 void SetNbytesFree(std::uint32_t value) {
338 if (IsBigFile()) {
339 fInfoLong.fNbytesFree = value;
340 } else {
341 fInfoShort.fNbytesFree = value;
342 }
343 }
344
345 void SetNbytesName(std::uint32_t value) {
346 if (IsBigFile()) {
347 fInfoLong.fNbytesName = value;
348 } else {
349 fInfoShort.fNbytesName = value;
350 }
351 }
352
353 std::uint64_t GetSeekInfo() const {
354 if (IsBigFile()) return fInfoLong.fSeekInfo;
355 return fInfoShort.fSeekInfo;
356 }
357
358 void SetSeekInfo(std::uint64_t value) {
359 if (IsBigFile(value)) {
360 SetBigFile();
361 fInfoLong.fSeekInfo = value;
362 } else {
363 fInfoShort.fSeekInfo = value;
364 }
365 }
366
367 void SetNbytesInfo(std::uint32_t value) {
368 if (IsBigFile()) {
369 fInfoLong.fNbytesInfo = value;
370 } else {
371 fInfoShort.fNbytesInfo = value;
372 }
373 }
374
375 void SetCompression(std::uint32_t value) {
376 if (IsBigFile()) {
377 fInfoLong.fCompress = value;
378 } else {
379 fInfoShort.fCompress = value;
380 }
381 }
382};
383
384
385/// A reference to an unused byte-range in a TFile
386struct RTFFreeEntry {
387 RUInt16BE fVersion{1};
388 union {
389 struct {
390 RUInt32BE fFirst{0};
391 RUInt32BE fLast{0};
392 } fInfoShort;
393 struct {
394 RUInt64BE fFirst{0};
395 RUInt64BE fLast{0};
396 } fInfoLong;
397 };
398
399 RTFFreeEntry() : fInfoShort() {}
400 void Set(std::uint64_t first, std::uint64_t last) {
401 if (last > static_cast<unsigned int>(std::numeric_limits<std::int32_t>::max())) {
402 fVersion = fVersion + 1000;
403 fInfoLong.fFirst = first;
404 fInfoLong.fLast = last;
405 } else {
406 fInfoShort.fFirst = first;
407 fInfoShort.fLast = last;
408 }
409 }
410 std::uint32_t GetSize() { return (fVersion >= 1000) ? 18 : 10; }
411};
412
413/// Streamer info for TObject
414struct RTFObject {
415 RUInt16BE fVersion{1};
416 RUInt32BE fUniqueID{0}; // unused
417 RUInt32BE fBits;
418 explicit RTFObject(std::uint32_t bits) : fBits(bits) {}
419};
420
421/// Streamer info for data member RNTuple::fVersion
422struct RTFStreamerElementVersion {
423 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementVersion) - sizeof(RUInt32BE))};
424 RUInt16BE fVersion{4};
425
426 RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 10)};
427 RUInt16BE fVersionNamed{1};
428 RTFObject fObjectNamed{0x02000000 | 0x01000000};
429 char fLName = 8;
430 char fName[8]{ 'f', 'V', 'e', 'r', 's', 'i', 'o', 'n' };
431 char fLTitle = 0;
432
433 RUInt32BE fType{13};
434 RUInt32BE fSize{4};
435 RUInt32BE fArrLength{0};
436 RUInt32BE fArrDim{0};
437 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
438 char fLTypeName = 12;
439 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
440};
441
442/// Streamer info for data member RNTuple::fSize
443struct RTFStreamerElementSize {
444 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSize) - sizeof(RUInt32BE))};
445 RUInt16BE fVersion{4};
446
447 RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 7)};
448 RUInt16BE fVersionNamed{1};
449 RTFObject fObjectNamed{0x02000000 | 0x01000000};
450 char fLName = 5;
451 char fName[5]{ 'f', 'S', 'i', 'z', 'e' };
452 char fLTitle = 0;
453
454 RUInt32BE fType{13};
455 RUInt32BE fSize{4};
456 RUInt32BE fArrLength{0};
457 RUInt32BE fArrDim{0};
458 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
459 char fLTypeName = 12;
460 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
461};
462
463/// Streamer info for data member RNTuple::fSeekHeader
464struct RTFStreamerElementSeekHeader {
465 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekHeader) - sizeof(RUInt32BE))};
466 RUInt16BE fVersion{4};
467
468 RUInt32BE fByteCountNamed{0x40000000 |
469 (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)};
470 RUInt16BE fVersionNamed{1};
471 RTFObject fObjectNamed{0x02000000 | 0x01000000};
472 char fLName = 11;
473 char fName[11]{ 'f', 'S', 'e', 'e', 'k', 'H', 'e', 'a', 'd', 'e', 'r' };
474 char fLTitle = 0;
475
476 RUInt32BE fType{14};
477 RUInt32BE fSize{8};
478 RUInt32BE fArrLength{0};
479 RUInt32BE fArrDim{0};
480 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
481 char fLTypeName = 13;
482 char fTypeName[13]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g' };
483};
484
485/// Streamer info for data member RNTuple::fNbytesHeader
486struct RTFStreamerElementNBytesHeader {
487 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesHeader) - sizeof(RUInt32BE))};
488 RUInt16BE fVersion{4};
489
490 RUInt32BE fByteCountNamed{0x40000000 |
491 (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
492 RUInt16BE fVersionNamed{1};
493 RTFObject fObjectNamed{0x02000000 | 0x01000000};
494 char fLName = 13;
495 char fName[13]{ 'f', 'N', 'B', 'y', 't', 'e', 's', 'H', 'e', 'a', 'd', 'e', 'r' };
496 char fLTitle = 0;
497
498 RUInt32BE fType{13};
499 RUInt32BE fSize{4};
500 RUInt32BE fArrLength{0};
501 RUInt32BE fArrDim{0};
502 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
503 char fLTypeName = 12;
504 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
505};
506
507/// Streamer info for data member RNTuple::fLenHeader
508struct RTFStreamerElementLenHeader {
509 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenHeader) - sizeof(RUInt32BE))};
510 RUInt16BE fVersion{4};
511
512 RUInt32BE fByteCountNamed{0x40000000 |
513 (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)};
514 RUInt16BE fVersionNamed{1};
515 RTFObject fObjectNamed{0x02000000 | 0x01000000};
516 char fLName = 10;
517 char fName[10]{ 'f', 'L', 'e', 'n', 'H', 'e', 'a', 'd', 'e', 'r' };
518 char fLTitle = 0;
519
520 RUInt32BE fType{13};
521 RUInt32BE fSize{4};
522 RUInt32BE fArrLength{0};
523 RUInt32BE fArrDim{0};
524 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
525 char fLTypeName = 12;
526 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
527};
528
529/// Streamer info for data member RNTuple::fSeekFooter
530struct RTFStreamerElementSeekFooter {
531 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementSeekFooter) - sizeof(RUInt32BE))};
532 RUInt16BE fVersion{4};
533
534 RUInt32BE fByteCountNamed{0x40000000 |
535 (sizeof(RUInt16BE) + sizeof(RTFObject) + 13)};
536 RUInt16BE fVersionNamed{1};
537 RTFObject fObjectNamed{0x02000000 | 0x01000000};
538 char fLName = 11;
539 char fName[11]{ 'f', 'S', 'e', 'e', 'k', 'F', 'o', 'o', 't', 'e', 'r' };
540 char fLTitle = 0;
541
542 RUInt32BE fType{14};
543 RUInt32BE fSize{8};
544 RUInt32BE fArrLength{0};
545 RUInt32BE fArrDim{0};
546 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
547 char fLTypeName = 13;
548 char fTypeName[13]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g' };
549};
550
551/// Streamer info for data member RNTuple::fNbytesFooter
552struct RTFStreamerElementNBytesFooter {
553 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementNBytesFooter) - sizeof(RUInt32BE))};
554 RUInt16BE fVersion{4};
555
556 RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 15)};
557 RUInt16BE fVersionNamed{1};
558 RTFObject fObjectNamed{0x02000000 | 0x01000000};
559 char fLName = 13;
560 char fName[13]{ 'f', 'N', 'B', 'y', 't', 'e', 's', 'F', 'o', 'o', 't', 'e', 'r' };
561 char fLTitle = 0;
562
563 RUInt32BE fType{13};
564 RUInt32BE fSize{4};
565 RUInt32BE fArrLength{0};
566 RUInt32BE fArrDim{0};
567 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
568 char fLTypeName = 12;
569 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
570};
571
572/// Streamer info for data member RNTuple::fLenFooter
573struct RTFStreamerElementLenFooter {
574 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementLenFooter) - sizeof(RUInt32BE))};
575 RUInt16BE fVersion{4};
576
577 RUInt32BE fByteCountNamed{0x40000000 | (sizeof(RUInt16BE) + sizeof(RTFObject) + 12)};
578 RUInt16BE fVersionNamed{1};
579 RTFObject fObjectNamed{0x02000000 | 0x01000000};
580 char fLName = 10;
581 char fName[10]{ 'f', 'L', 'e', 'n', 'F', 'o', 'o', 't', 'e', 'r' };
582 char fLTitle = 0;
583
584 RUInt32BE fType{13};
585 RUInt32BE fSize{4};
586 RUInt32BE fArrLength{0};
587 RUInt32BE fArrDim{0};
588 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
589 char fLTypeName = 12;
590 char fTypeName[12]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'i', 'n', 't' };
591};
592
593/// Streamer info for data member RNTuple::fReserved
594struct RTFStreamerElementReserved {
595 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerElementReserved) - sizeof(RUInt32BE))};
596 RUInt16BE fVersion{4};
597
598 RUInt32BE fByteCountNamed{0x40000000 |
599 (sizeof(RUInt16BE) + sizeof(RTFObject) + 11)};
600 RUInt16BE fVersionNamed{1};
601 RTFObject fObjectNamed{0x02000000 | 0x01000000};
602 char fLName = 9;
603 char fName[9]{ 'f', 'R', 'e', 's', 'e', 'r', 'v', 'e', 'd' };
604 char fLTitle = 0;
605
606 RUInt32BE fType{14};
607 RUInt32BE fSize{8};
608 RUInt32BE fArrLength{0};
609 RUInt32BE fArrDim{0};
610 char fMaxIndex[20]{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
611 char fLTypeName = 13;
612 char fTypeName[13]{ 'u', 'n', 's', 'i', 'g', 'n', 'e', 'd', ' ', 'l', 'o', 'n', 'g' };
613};
614
615/// Streamer info frame for data member RNTuple::fVersion
616struct RTFStreamerVersion {
617 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerVersion) - sizeof(RUInt32BE))};
618 RUInt32BE fNewClassTag{0xffffffff};
619 char fClassName[19]{'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'B', 'a', 's', 'i', 'c', 'T', 'y', 'p', 'e', '\0'};
620 RUInt32BE fByteCountRemaining{0x40000000 |
621 (sizeof(RTFStreamerVersion) - 2 * sizeof(RUInt32BE) - 19 /* strlen(fClassName) + 1 */ - sizeof(RUInt32BE))};
622 RUInt16BE fVersion{2};
623 RTFStreamerElementVersion fStreamerElementVersion;
624};
625
626/// Streamer info frame for data member RNTuple::fSize
627struct RTFStreamerSize {
628 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSize) - sizeof(RUInt32BE))};
629 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
630 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSize) - 3 * sizeof(RUInt32BE))};
631 RUInt16BE fVersion{2};
632 RTFStreamerElementSize fStreamerElementSize;
633};
634
635/// Streamer info frame for data member RNTuple::fSeekHeader
636struct RTFStreamerSeekHeader {
637 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekHeader) - sizeof(RUInt32BE))};
638 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
639 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekHeader) - 3 * sizeof(RUInt32BE))};
640 RUInt16BE fVersion{2};
641 RTFStreamerElementSeekHeader fStreamerElementSeekHeader;
642};
643
644/// Streamer info frame for data member RNTuple::fNbytesHeader
645struct RTFStreamerNBytesHeader {
646 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - sizeof(RUInt32BE))};
647 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
648 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesHeader) - 3 * sizeof(RUInt32BE))};
649 RUInt16BE fVersion{2};
650 RTFStreamerElementNBytesHeader fStreamerElementNBytesHeader;
651};
652
653/// Streamer info frame for data member RNTuple::fLenHeader
654struct RTFStreamerLenHeader {
655 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenHeader) - sizeof(RUInt32BE))};
656 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
657 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenHeader) - 3 * sizeof(RUInt32BE))};
658 RUInt16BE fVersion{2};
659 RTFStreamerElementLenHeader fStreamerElementLenHeader;
660};
661
662/// Streamer info frame for data member RNTuple::fSeekFooter
663struct RTFStreamerSeekFooter {
664 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerSeekFooter) - sizeof(RUInt32BE))};
665 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
666 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerSeekFooter) - 3 * sizeof(RUInt32BE))};
667 RUInt16BE fVersion{2};
668 RTFStreamerElementSeekFooter fStreamerElementSeekFooter;
669};
670
671/// Streamer info frame for data member RNTuple::fNBytesFooter
672struct RTFStreamerNBytesFooter {
673 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - sizeof(RUInt32BE))};
674 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
675 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerNBytesFooter) - 3 * sizeof(RUInt32BE))};
676 RUInt16BE fVersion{2};
677 RTFStreamerElementNBytesFooter fStreamerElementNBytesFooter;
678};
679
680/// Streamer info frame for data member RNTuple::fLenFooter
681struct RTFStreamerLenFooter {
682 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerLenFooter) - sizeof(RUInt32BE))};
683 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
684 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerLenFooter) - 3 * sizeof(RUInt32BE))};
685 RUInt16BE fVersion{2};
686 RTFStreamerElementLenFooter fStreamerElementLenFooter;
687};
688
689/// Streamer info frame for data member RNTuple::fReserved
690struct RTFStreamerReserved {
691 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerReserved) - sizeof(RUInt32BE))};
692 RUInt32BE fClassTag{0x80000000}; // Fix-up after construction, or'd with 0x80000000
693 RUInt32BE fByteCountRemaining{0x40000000 | (sizeof(RTFStreamerReserved) - 3 * sizeof(RUInt32BE))};
694 RUInt16BE fVersion{2};
695 RTFStreamerElementReserved fStreamerElementReserved;
696};
697
698/// Streamer info for class RNTuple
699struct RTFStreamerInfoObject {
700 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoObject) - sizeof(fByteCount))};
701 RUInt32BE fNewClassTag{0xffffffff};
702 char fClassName[14]{ 'T', 'S', 't', 'r', 'e', 'a', 'm', 'e', 'r', 'I', 'n', 'f', 'o', '\0' };
703 RUInt32BE fByteCountRemaining{0x40000000 |
704 (sizeof(RTFStreamerInfoObject) - 2 * sizeof(RUInt32BE) - 14 - sizeof(RUInt32BE))};
705 RUInt16BE fVersion{9};
706
707 RUInt32BE fByteCountNamed{0x40000000 |
708 (sizeof(RUInt16BE) + sizeof(RTFObject) + 29 /* strlen("ROOT::Experimental::RNTuple") + 2 */)};
709 RUInt16BE fVersionNamed{1};
710 RTFObject fObjectNamed{0x02000000 | 0x01000000 | 0x00010000};
711 char fLName = 27;
712 char fName[27]{ 'R', 'O', 'O', 'T', ':', ':',
713 'E', 'x', 'p', 'e', 'r', 'i', 'm', 'e', 'n', 't', 'a', 'l', ':', ':',
714 'R', 'N', 'T', 'u', 'p', 'l', 'e'};
715 char fLTitle = 0;
716
717 RInt32BE fChecksum{ChecksumRNTupleClass()};
718 RUInt32BE fVersionRNTuple{1};
719
720 RUInt32BE fByteCountObjArr{0x40000000 |
721 (sizeof(RUInt32BE) + 10 /* strlen(TObjArray) + 1 */ + sizeof(RUInt32BE) +
722 sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2*sizeof(RUInt32BE) +
723 sizeof(fStreamers))};
724 RUInt32BE fNewClassTagObjArray{0xffffffff};
725 char fClassNameObjArray[10]{'T', 'O', 'b', 'j', 'A', 'r', 'r', 'a', 'y', '\0'};
726 RUInt32BE fByteCountObjArrRemaining{0x40000000 |
727 (sizeof(RUInt16BE) + sizeof(RTFObject) + 1 + 2*sizeof(RUInt32BE) +
728 sizeof(fStreamers))};
729 RUInt16BE fVersionObjArr{3};
730 RTFObject fObjectObjArr{0x02000000};
731 char fNameObjArr{0};
732
733 RUInt32BE fNObjects{9};
734 RUInt32BE fLowerBound{0};
735
736 struct {
737 RTFStreamerVersion fStreamerVersion;
738 RTFStreamerSize fStreamerSize;
739 RTFStreamerSeekHeader fStreamerSeekHeader;
740 RTFStreamerNBytesHeader fStreamerNBytesHeader;
741 RTFStreamerLenHeader fStreamerLenHeader;
742 RTFStreamerSeekFooter fStreamerSeekFooter;
743 RTFStreamerNBytesFooter fStreamerNBytesFooter;
744 RTFStreamerLenFooter fStreamerLenFooter;
745 RTFStreamerReserved fStreamerReserved;
746 } fStreamers;
747};
748
749/// The list of streamer info objects, for a new ntuple contains only the RNTuple class
750struct RTFStreamerInfoList {
751 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFStreamerInfoList) - sizeof(fByteCount))};
752 RUInt16BE fVersion{5};
753 RTFObject fObject{0x02000000};
754 char fName{0};
755 RUInt32BE fNObjects{1};
756 RTFStreamerInfoObject fStreamerInfo;
757 char fEnd{0};
758
759 std::uint32_t GetSize() const { return sizeof(RTFStreamerInfoList); }
760};
761
762/// The header of the directory key index
763struct RTFKeyList {
764 RUInt32BE fNKeys;
765 std::uint32_t GetSize() const { return sizeof(RTFKeyList); }
766 explicit RTFKeyList(std::uint32_t nKeys) : fNKeys(nKeys) {}
767};
768
769/// A streamed TFile object
770struct RTFFile {
771 char fModified{0};
772 char fWritable{1};
773 RTFDatetime fDateC;
774 RTFDatetime fDateM;
775 RUInt32BE fNBytesKeys{0};
776 RUInt32BE fNBytesName{0};
777 // The version of the key has to tell whether offsets are 32bit or 64bit long
778 union {
779 struct {
780 RUInt32BE fSeekDir{100};
781 RUInt32BE fSeekParent{0};
782 RUInt32BE fSeekKeys{0};
783 } fInfoShort;
784 struct {
785 RUInt64BE fSeekDir{100};
786 RUInt64BE fSeekParent{0};
787 RUInt64BE fSeekKeys{0};
788 } fInfoLong;
789 };
790
791 RTFFile() : fInfoShort() {}
792
793 std::uint32_t GetSize(std::uint32_t versionKey = 0) const {
794 if (versionKey >= 1000)
795 return 18 + sizeof(fInfoLong);
796 return 18 + sizeof(fInfoShort);
797 }
798
799 std::uint64_t GetSeekKeys(std::uint32_t versionKey = 0) const {
800 if (versionKey >= 1000)
801 return fInfoLong.fSeekKeys;
802 return fInfoShort.fSeekKeys;
803 }
804};
805
806/// A streamed RNTuple class
807struct RTFNTuple {
808 RUInt32BE fByteCount{0x40000000 | (sizeof(RTFNTuple) - sizeof(fByteCount))};
809 RUInt16BE fVersionClass{0};
810 RInt32BE fChecksum{ChecksumRNTupleClass()};
811 RUInt32BE fVersionInternal{0};
812 RUInt32BE fSize{sizeof(ROOT::Experimental::RNTuple)};
813 RUInt64BE fSeekHeader{0};
814 RUInt32BE fNBytesHeader{0};
815 RUInt32BE fLenHeader{0};
816 RUInt64BE fSeekFooter{0};
817 RUInt32BE fNBytesFooter{0};
818 RUInt32BE fLenFooter{0};
819 RUInt64BE fReserved{0};
820
821 RTFNTuple() = default;
822 explicit RTFNTuple(const ROOT::Experimental::RNTuple &inMemoryNTuple) {
823 fVersionInternal = inMemoryNTuple.fVersion;
824 fSize = inMemoryNTuple.fSize;
825 fSeekHeader = inMemoryNTuple.fSeekHeader;
826 fNBytesHeader = inMemoryNTuple.fNBytesHeader;
827 fLenHeader = inMemoryNTuple.fLenHeader;
828 fSeekFooter = inMemoryNTuple.fSeekFooter;
829 fNBytesFooter = inMemoryNTuple.fNBytesFooter;
830 fLenFooter = inMemoryNTuple.fLenFooter;
831 fReserved = inMemoryNTuple.fReserved;
832 }
833 std::uint32_t GetSize() const { return sizeof(RTFNTuple); }
834 ROOT::Experimental::RNTuple ToRNTuple() const {
836 ntuple.fVersion = fVersionInternal;
837 ntuple.fSize = fSize;
838 ntuple.fSeekHeader = fSeekHeader;
839 ntuple.fNBytesHeader = fNBytesHeader;
840 ntuple.fLenHeader = fLenHeader;
841 ntuple.fSeekFooter = fSeekFooter;
842 ntuple.fNBytesFooter = fNBytesFooter;
843 ntuple.fLenFooter = fLenFooter;
844 ntuple.fReserved = fReserved;
845 return ntuple;
846 }
847 void Print(std::ostream& output) {
848 output << "RTFNTuple {\n";
849 output << " fByteCount: " << fByteCount << ",\n";
850 output << " fVersionClass: " << fVersionClass << ",\n";
851 output << " fChecksum: " << fChecksum << ",\n";
852 output << " fVersionInternal: " << fVersionInternal << ",\n";
853 output << " fSize: " << fSize << ",\n";
854 output << " fSeekHeader: " << fSeekHeader << ",\n";
855 output << " fNBytesHeader: " << fNBytesHeader << ",\n";
856 output << " fLenHeader: " << fLenHeader << ",\n";
857 output << " fSeekFooter: " << fSeekFooter << ",\n";
858 output << " fNBytesFooter: " << fNBytesFooter << ",\n";
859 output << " fLenFooter: " << fLenFooter << ",\n";
860 output << " fReserved: " << fReserved << ",\n";
861 output << "}";
862 }
863};
864
865/// The bare file global header
866struct RBareFileHeader {
867 char fMagic[7]{ 'r', 'n', 't', 'u', 'p', 'l', 'e' };
868 RUInt32BE fRootVersion{(ROOT_VERSION_CODE >> 16) * 10000 +
869 ((ROOT_VERSION_CODE & 0xFF00) >> 8) * 100 +
870 (ROOT_VERSION_CODE & 0xFF)};
871 RUInt32BE fFormatVersion{1};
872 RUInt32BE fCompress{0};
873 RTFNTuple fNTuple;
874 // followed by the ntuple name
875};
876#pragma pack(pop)
877
878/// The artifical class name shown for opaque RNTuple keys (see TBasket)
879static constexpr char const *kBlobClassName = "RBlob";
880/// The class name of the RNTuple anchor
881static constexpr char const *kNTupleClassName = "ROOT::Experimental::RNTuple";
882
883/// The RKeyBlob writes an invsisble key into a TFile. That is, a key that is not indexed in the list of keys,
884/// like a TBasket.
885class RKeyBlob : public TKey {
886public:
887 explicit RKeyBlob(TFile *file) : TKey(file) {
888 fClassName = kBlobClassName;
889 fKeylen += strlen(kBlobClassName);
890 }
891
892 /// Register a new key for a data record of size nbytes
893 void Reserve(size_t nbytes, std::uint64_t *seekKey)
894 {
895 Create(nbytes);
896 *seekKey = fSeekKey;
897 }
898};
899
900} // anonymous namespace
901
902
903namespace ROOT {
904namespace Experimental {
905namespace Internal {
906/// If a TFile container is written by a C stream (simple file), on dataset commit, the file header needs to be updated
907/// and the RNTuple object needs to be written at the offset that was reserved on file creation
909 RTFHeader fHeader;
910 std::uint32_t fSeekNTuple{0};
911};
912} // namespace ROOT
913} // namespace Experimental
914} // namespace Internal
915
916
918 : fRawFile(rawFile)
919{
920}
921
924{
925 char ident[4];
926 ReadBuffer(ident, 4, 0);
927 if (std::string(ident, 4) == "root")
928 return GetNTupleProper(ntupleName);
929 fIsBare = true;
930 return GetNTupleBare(ntupleName);
931}
932
933
936{
937 RTFHeader fileHeader;
938 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
939
940 RTFKey key;
941 RTFString name;
942 ReadBuffer(&key, sizeof(key), fileHeader.fBEGIN);
943 auto offset = fileHeader.fBEGIN + key.fKeyLen;
944 ReadBuffer(&name, 1, offset);
945 offset += name.GetSize();
946 ReadBuffer(&name, 1, offset);
947 offset += name.GetSize();
948 RTFFile file;
949 ReadBuffer(&file, sizeof(file), offset);
950
951 RUInt32BE nKeys;
952 offset = file.GetSeekKeys(key.fVersion);
953 ReadBuffer(&key, sizeof(key), offset);
954 offset += key.fKeyLen;
955 ReadBuffer(&nKeys, sizeof(nKeys), offset);
956 offset += sizeof(nKeys);
957 bool found = false;
958 for (unsigned int i = 0; i < nKeys; ++i) {
959 ReadBuffer(&key, sizeof(key), offset);
960 auto offsetNextKey = offset + key.fKeyLen;
961
962 offset += key.GetHeaderSize();
963 ReadBuffer(&name, 1, offset);
964 ReadBuffer(&name, name.GetSize(), offset);
965 if (std::string_view(name.fData, name.fLName) != kNTupleClassName) {
966 offset = offsetNextKey;
967 continue;
968 }
969 offset += name.GetSize();
970 ReadBuffer(&name, 1, offset);
971 ReadBuffer(&name, name.GetSize(), offset);
972 if (std::string_view(name.fData, name.fLName) == ntupleName) {
973 found = true;
974 break;
975 }
976 offset = offsetNextKey;
977 }
978 if (!found) {
979 return R__FAIL("no RNTuple named '" + std::string(ntupleName)
980 + "' in file '" + fRawFile->GetUrl() + "'");
981 }
982
983 ReadBuffer(&key, sizeof(key), key.GetSeekKey());
984 offset = key.GetSeekKey() + key.fKeyLen;
985 RTFNTuple ntuple;
986 ReadBuffer(&ntuple, sizeof(ntuple), offset);
987 return ntuple.ToRNTuple();
988}
989
992{
993 RBareFileHeader fileHeader;
994 ReadBuffer(&fileHeader, sizeof(fileHeader), 0);
995 RTFString name;
996 auto offset = sizeof(fileHeader);
997 ReadBuffer(&name, 1, offset);
998 ReadBuffer(&name, name.GetSize(), offset);
999 std::string_view foundName(name.fData, name.fLName);
1000 if (foundName != ntupleName) {
1001 return R__FAIL("expected RNTuple named '" + std::string(ntupleName)
1002 + "' but instead found '" + std::string(foundName)
1003 + "' in file '" + fRawFile->GetUrl() + "'");
1004 }
1005 offset += name.GetSize();
1006
1007 RTFNTuple ntuple;
1008 ReadBuffer(&ntuple, sizeof(ntuple), offset);
1009 return ntuple.ToRNTuple();
1010}
1011
1012
1013void ROOT::Experimental::Internal::RMiniFileReader::ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
1014{
1015 auto nread = fRawFile->ReadAt(buffer, nbytes, offset);
1016 R__ASSERT(nread == nbytes);
1017}
1018
1019
1020////////////////////////////////////////////////////////////////////////////////
1021
1022
1024{
1025 if (fFile)
1026 fclose(fFile);
1027}
1028
1029
1031 const void *buffer, size_t nbytes, std::int64_t offset)
1032{
1033 R__ASSERT(fFile);
1034 size_t retval;
1035 if ((offset >= 0) && (static_cast<std::uint64_t>(offset) != fFilePos)) {
1036#ifdef R__SEEK64
1037 retval = fseeko64(fFile, offset, SEEK_SET);
1038#else
1039 retval = fseek(fFile, offset, SEEK_SET);
1040#endif
1041 R__ASSERT(retval == 0);
1042 fFilePos = offset;
1043 }
1044 retval = fwrite(buffer, 1, nbytes, fFile);
1045 R__ASSERT(retval == nbytes);
1046 fFilePos += nbytes;
1047}
1048
1049
1051 const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset,
1052 std::uint64_t directoryOffset,
1053 const std::string &className,
1054 const std::string &objectName,
1055 const std::string &title)
1056{
1057 if (offset < 0)
1058 offset = fFilePos;
1059 RTFString strClass{className};
1060 RTFString strObject{objectName};
1061 RTFString strTitle{title};
1062
1063 RTFKey key(offset, directoryOffset, strClass, strObject, strTitle, len, nbytes);
1064 Write(&key, key.fKeyHeaderSize, offset);
1065 Write(&strClass, strClass.GetSize());
1066 Write(&strObject, strObject.GetSize());
1067 Write(&strTitle, strTitle.GetSize());
1068 auto offsetData = fFilePos;
1069 if (buffer)
1070 Write(buffer, nbytes);
1071
1072 return offsetData;
1073}
1074
1075
1076////////////////////////////////////////////////////////////////////////////////
1077
1078
1080 const void *buffer, size_t nbytes, std::int64_t offset)
1081{
1082 R__ASSERT(fFile);
1083 fFile->Seek(offset);
1084 bool rv = fFile->WriteBuffer((char *)(buffer), nbytes);
1085 R__ASSERT(!rv);
1086}
1087
1088
1090 const void *buffer, size_t nbytes, size_t len)
1091{
1092 std::uint64_t offsetKey;
1093 RKeyBlob keyBlob(fFile);
1094 keyBlob.Reserve(nbytes, &offsetKey);
1095
1096 auto offset = offsetKey;
1097 RTFString strClass{kBlobClassName};
1098 RTFString strObject;
1099 RTFString strTitle;
1100 RTFKey keyHeader(offset, offset, strClass, strObject, strTitle, len, nbytes);
1101
1102 Write(&keyHeader, keyHeader.fKeyHeaderSize, offset);
1103 offset += keyHeader.fKeyHeaderSize;
1104 Write(&strClass, strClass.GetSize(), offset);
1105 offset += strClass.GetSize();
1106 Write(&strObject, strObject.GetSize(), offset);
1107 offset += strObject.GetSize();
1108 Write(&strTitle, strTitle.GetSize(), offset);
1109 offset += strTitle.GetSize();
1110 auto offsetData = offset;
1111 Write(buffer, nbytes, offset);
1112
1113 return offsetData;
1114}
1115
1116
1117////////////////////////////////////////////////////////////////////////////////
1118
1119
1121 : fNTupleName(name)
1122{
1123 fFileSimple.fControlBlock = std::make_unique<ROOT::Experimental::Internal::RTFileControlBlock>();
1124}
1125
1126
1128{
1129}
1130
1131
1133 std::string_view ntupleName, std::string_view path, int defaultCompression, ENTupleContainerFormat containerFormat)
1134{
1135 std::string fileName(path);
1136 size_t idxDirSep = fileName.find_last_of("\\/");
1137 if (idxDirSep != std::string::npos) {
1138 fileName.erase(0, idxDirSep + 1);
1139 }
1140#ifdef R__SEEK64
1141 FILE *fileStream = fopen64(std::string(path.data(), path.size()).c_str(), "wb");
1142#else
1143 FILE *fileStream = fopen(std::string(path.data(), path.size()).c_str(), "wb");
1144#endif
1145 R__ASSERT(fileStream);
1146
1147 auto writer = new RNTupleFileWriter(ntupleName);
1148 writer->fFileSimple.fFile = fileStream;
1149 writer->fFileName = fileName;
1150
1151 switch (containerFormat) {
1153 writer->WriteTFileSkeleton(defaultCompression);
1154 break;
1156 writer->fIsBare = true;
1157 writer->WriteBareFileSkeleton(defaultCompression);
1158 break;
1159 default:
1160 R__ASSERT(false && "Internal error: unhandled container format");
1161 }
1162
1163 return writer;
1164}
1165
1166
1168 std::string_view ntupleName, std::string_view path, std::unique_ptr<TFile> &file)
1169{
1170 file = std::unique_ptr<TFile>(TFile::Open(std::string(path.data(), path.size()).c_str(), "RECREATE"));
1171 R__ASSERT(file && !file->IsZombie());
1172
1173 auto writer = new RNTupleFileWriter(ntupleName);
1174 writer->fFileProper.fFile = file.get();
1175 return writer;
1176}
1177
1178
1180 std::string_view ntupleName, TFile &file)
1181{
1182 auto writer = new RNTupleFileWriter(ntupleName);
1183 writer->fFileProper.fFile = &file;
1184 return writer;
1185}
1186
1187
1189{
1190 if (fFileProper) {
1191 // Easy case, the ROOT file header and the RNTuple streaming is taken care of by TFile
1192 fFileProper.fFile->WriteObject(&fNTupleAnchor, fNTupleName.c_str());
1193 fFileProper.fFile->Write();
1194 return;
1195 }
1196
1197 // Writing by C file stream: prepare the container format header and stream the RNTuple anchor object
1198 R__ASSERT(fFileSimple);
1199 RTFNTuple ntupleOnDisk(fNTupleAnchor);
1200
1201 if (fIsBare) {
1202 fFileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize(), fFileSimple.fControlBlock->fSeekNTuple);
1203 fflush(fFileSimple.fFile);
1204 return;
1205 }
1206
1207 fFileSimple.fControlBlock->fHeader.SetSeekFree(fFileSimple.fFilePos);
1208 RTFString strEmpty;
1209 RTFFreeEntry freeEntry;
1210 RTFKey keyFreeList(fFileSimple.fControlBlock->fHeader.GetSeekFree(), 100,
1211 strEmpty, strEmpty, strEmpty, freeEntry.GetSize());
1212 std::uint64_t firstFree = fFileSimple.fControlBlock->fHeader.GetSeekFree() + keyFreeList.GetSize();
1213 freeEntry.Set(firstFree, std::max(2000000000ULL, ((firstFree / 1000000000ULL) + 1) * 1000000000ULL));
1214 fFileSimple.WriteKey(
1215 &freeEntry, freeEntry.GetSize(), freeEntry.GetSize(), fFileSimple.fControlBlock->fHeader.GetSeekFree(),
1216 100, "", "", "");
1217 fFileSimple.fControlBlock->fHeader.SetNbytesFree(fFileSimple.fFilePos -
1218 fFileSimple.fControlBlock->fHeader.GetSeekFree());
1219 fFileSimple.fControlBlock->fHeader.SetEnd(fFileSimple.fFilePos);
1220
1221 auto szNTuple = ntupleOnDisk.GetSize();
1222 fFileSimple.WriteKey(&ntupleOnDisk, szNTuple, szNTuple, fFileSimple.fControlBlock->fSeekNTuple, 100,
1223 kNTupleClassName, fNTupleName);
1224
1225 fFileSimple.Write(&fFileSimple.fControlBlock->fHeader, fFileSimple.fControlBlock->fHeader.GetSize(), 0);
1226 fflush(fFileSimple.fFile);
1227}
1228
1229
1230std::uint64_t ROOT::Experimental::Internal::RNTupleFileWriter::WriteBlob(const void *data, size_t nbytes, size_t len)
1231{
1232 std::uint64_t offset;
1233 if (fFileSimple) {
1234 if (fIsBare) {
1235 offset = fFileSimple.fFilePos;
1236 fFileSimple.Write(data, nbytes);
1237 } else {
1238 offset = fFileSimple.WriteKey(data, nbytes, len, -1, 100, kBlobClassName);
1239 }
1240 } else {
1241 offset = fFileProper.WriteKey(data, nbytes, len);
1242 }
1243 return offset;
1244}
1245
1246
1248 const void *data, size_t nbytes, size_t lenHeader)
1249{
1250 auto offset = WriteBlob(data, nbytes, lenHeader);
1251 fNTupleAnchor.fLenHeader = lenHeader;
1252 fNTupleAnchor.fNBytesHeader = nbytes;
1253 fNTupleAnchor.fSeekHeader = offset;
1254 return offset;
1255}
1256
1257
1259 const void *data, size_t nbytes, size_t lenFooter)
1260{
1261 auto offset = WriteBlob(data, nbytes, lenFooter);
1262 fNTupleAnchor.fLenFooter = lenFooter;
1263 fNTupleAnchor.fNBytesFooter = nbytes;
1264 fNTupleAnchor.fSeekFooter = offset;
1265 return offset;
1266}
1267
1268
1270{
1271 RBareFileHeader bareHeader;
1272 bareHeader.fCompress = defaultCompression;
1273 fFileSimple.Write(&bareHeader, sizeof(bareHeader), 0);
1274 RTFString ntupleName{fNTupleName};
1275 fFileSimple.Write(&ntupleName, ntupleName.GetSize());
1276
1277 // Write zero-initialized ntuple to reserve the space; will be overwritten on commit
1278 RTFNTuple ntupleOnDisk;
1279 fFileSimple.fControlBlock->fSeekNTuple = fFileSimple.fFilePos;
1280 fFileSimple.Write(&ntupleOnDisk, ntupleOnDisk.GetSize());
1281}
1282
1283
1285{
1286 RTFString strTFile{"TFile"};
1287 RTFString strFileName{fFileName};
1288 RTFString strTList{"TList"};
1289 RTFString strStreamerInfo{"StreamerInfo"};
1290 RTFString strStreamerTitle{"Doubly linked list"};
1291 RTFString strRNTupleClass{"ROOT::Experimental::RNTuple"};
1292 RTFString strRNTupleName{fNTupleName};
1293 RTFString strEmpty;
1294
1295 fFileSimple.fControlBlock->fHeader = RTFHeader(defaultCompression);
1296
1297 // First record of the file: the TFile object at offset 100
1298 RTFFile fileRoot;
1299 RTFKey keyRoot(100, 0, strTFile, strFileName, strEmpty,
1300 fileRoot.GetSize() + strFileName.GetSize() + strEmpty.GetSize());
1301 std::uint32_t nbytesName = keyRoot.fKeyLen + strFileName.GetSize() + 1;
1302 fileRoot.fNBytesName = nbytesName;
1303 fFileSimple.fControlBlock->fHeader.SetNbytesName(nbytesName);
1304
1305 // Second record: the compressed StreamerInfo with the description of the RNTuple class.
1306 // This record usually comes at the end, but in this context we create a file with a single RNTuple only,
1307 // so we know beforehand all the types.
1308 fFileSimple.fControlBlock->fHeader.SetSeekInfo(100 + keyRoot.GetSize());
1309 RTFKey keyStreamerInfo(
1310 fFileSimple.fControlBlock->fHeader.GetSeekInfo(), 100, strTList, strStreamerInfo, strStreamerTitle, 0);
1311 RTFStreamerInfoList streamerInfo;
1312 auto classTagOffset = keyStreamerInfo.fKeyLen +
1313 offsetof(struct RTFStreamerInfoList, fStreamerInfo) +
1314 offsetof(struct RTFStreamerInfoObject, fStreamers) +
1315 offsetof(struct RTFStreamerVersion, fNewClassTag) + 2;
1316 streamerInfo.fStreamerInfo.fStreamers.fStreamerSize.fClassTag = 0x80000000 | classTagOffset;
1317 streamerInfo.fStreamerInfo.fStreamers.fStreamerSeekHeader.fClassTag = 0x80000000 | classTagOffset;
1318 streamerInfo.fStreamerInfo.fStreamers.fStreamerNBytesHeader.fClassTag = 0x80000000 | classTagOffset;
1319 streamerInfo.fStreamerInfo.fStreamers.fStreamerLenHeader.fClassTag = 0x80000000 | classTagOffset;
1320 streamerInfo.fStreamerInfo.fStreamers.fStreamerSeekFooter.fClassTag = 0x80000000 | classTagOffset;
1321 streamerInfo.fStreamerInfo.fStreamers.fStreamerNBytesFooter.fClassTag = 0x80000000 | classTagOffset;
1322 streamerInfo.fStreamerInfo.fStreamers.fStreamerLenFooter.fClassTag = 0x80000000 | classTagOffset;
1323 streamerInfo.fStreamerInfo.fStreamers.fStreamerReserved.fClassTag = 0x80000000 | classTagOffset;
1324 Detail::RNTupleCompressor compressor;
1325 auto szStreamerInfo = compressor.Zip(&streamerInfo, streamerInfo.GetSize(), 1);
1326 fFileSimple.WriteKey(compressor.GetZipBuffer(), szStreamerInfo, streamerInfo.GetSize(),
1327 fFileSimple.fControlBlock->fHeader.GetSeekInfo(), 100,
1328 "TList", "StreamerInfo", "Doubly linked list");
1329 fFileSimple.fControlBlock->fHeader.SetNbytesInfo(
1330 fFileSimple.fFilePos - fFileSimple.fControlBlock->fHeader.GetSeekInfo());
1331
1332 // Reserve the space for the RNTuple record, which will be written on commit
1333 RTFNTuple ntupleOnDisk;
1334 fFileSimple.fControlBlock->fSeekNTuple = fFileSimple.fFilePos;
1335 RTFKey keyRNTuple(fFileSimple.fControlBlock->fSeekNTuple, 100, strRNTupleClass, strRNTupleName, strEmpty,
1336 ntupleOnDisk.GetSize());
1337 fFileSimple.WriteKey(&ntupleOnDisk, ntupleOnDisk.GetSize(), ntupleOnDisk.GetSize(),
1338 fFileSimple.fControlBlock->fSeekNTuple, 100, "ROOT::Experimental::RNTUple", fNTupleName, "");
1339
1340 // The key index of the root TFile object, containing for the time being only the RNTuple key
1341 fileRoot.fInfoShort.fSeekKeys = fFileSimple.fFilePos;
1342 RTFKeyList keyList{1};
1343 RTFKey keyKeyList(fileRoot.GetSeekKeys(), 100, strEmpty, strEmpty, strEmpty, keyList.GetSize() + keyRNTuple.fKeyLen);
1344 fFileSimple.Write(&keyKeyList, keyKeyList.fKeyHeaderSize, fileRoot.GetSeekKeys());
1345 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1346 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1347 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1348 fFileSimple.Write(&keyList, keyList.GetSize());
1349 fFileSimple.Write(&keyRNTuple, keyRNTuple.fKeyHeaderSize);
1350 fFileSimple.Write(&strRNTupleClass, strRNTupleClass.GetSize());
1351 fFileSimple.Write(&strRNTupleName, strRNTupleName.GetSize());
1352 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1353 fileRoot.fNBytesKeys = fFileSimple.fFilePos - fileRoot.GetSeekKeys();
1354
1355 auto tail = fFileSimple.fFilePos;
1356 fFileSimple.Write(&keyRoot, keyRoot.fKeyHeaderSize, 100);
1357 fFileSimple.Write(&strTFile, strTFile.GetSize());
1358 fFileSimple.Write(&strFileName, strFileName.GetSize());
1359 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1360 fFileSimple.Write(&strFileName, strFileName.GetSize());
1361 fFileSimple.Write(&strEmpty, strEmpty.GetSize());
1362 fFileSimple.Write(&fileRoot, fileRoot.GetSize());
1363 fFileSimple.fFilePos = tail;
1364#ifdef R__SEEK64
1365 auto retval = fseeko64(fFileSimple.fFile, tail, SEEK_SET);
1366#else
1367 auto retval = fseek(fFileSimple.fFile, tail, SEEK_SET);
1368#endif
1369 R__ASSERT(retval == 0);
1370 fFileSimple.fFilePos = tail;
1371}
size_t fSize
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition: RError.hxx:291
#define ROOT_VERSION_CODE
Definition: RVersion.h:21
#define R__ASSERT(e)
Definition: TError.h:118
XFontStruct * id
Definition: TGX11.cxx:109
char name[80]
Definition: TGX11.cxx:110
Binding & operator=(OUT(*fun)(void))
virtual void ReadBuffer(char *&buffer)
T1 fFirst
Definition: X11Events.mm:86
Helper class to compress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:40
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
Definition: RNTupleZip.hxx:59
RResult< RNTuple > GetNTuple(std::string_view ntupleName)
Extracts header and footer location for the RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:923
RResult< RNTuple > GetNTupleProper(std::string_view ntupleName)
Used when the file turns out to be a TFile container.
Definition: RMiniFile.cxx:935
RResult< RNTuple > GetNTupleBare(std::string_view ntupleName)
Used when the file container turns out to be a bare file.
Definition: RMiniFile.cxx:991
void ReadBuffer(void *buffer, size_t nbytes, std::uint64_t offset)
Reads a given byte range from the file into the provided memory buffer.
Definition: RMiniFile.cxx:1013
Write RNTuple data blocks in a TFile or a bare file container.
Definition: RMiniFile.hxx:142
std::uint64_t WriteBlob(const void *data, size_t nbytes, size_t len)
Writes a new record as an RBlob key into the file.
Definition: RMiniFile.cxx:1230
std::uint64_t WriteNTupleFooter(const void *data, size_t nbytes, size_t lenFooter)
Writes the compressed footer and registeres its location; lenFooter is the size of the uncompressed f...
Definition: RMiniFile.cxx:1258
void Commit()
Writes the RNTuple key to the file so that the header and footer keys can be found.
Definition: RMiniFile.cxx:1188
RFileSimple fFileSimple
For simple use cases, survives without libRIO dependency.
Definition: RMiniFile.hxx:184
static RNTupleFileWriter * Append(std::string_view ntupleName, TFile &file)
Add a new RNTuple identified by ntupleName to the existing TFile.
Definition: RMiniFile.cxx:1179
static RNTupleFileWriter * Recreate(std::string_view ntupleName, std::string_view path, int defaultCompression, ENTupleContainerFormat containerFormat)
Create or truncate the local file given by path with the new empty RNTuple identified by ntupleName.
Definition: RMiniFile.cxx:1132
void WriteTFileSkeleton(int defaultCompression)
For a TFile container written by a C file stream, write the records that constitute an empty file.
Definition: RMiniFile.cxx:1284
void WriteBareFileSkeleton(int defaultCompression)
For a bare file, which is necessarily written by a C file stream, write file header.
Definition: RMiniFile.cxx:1269
std::uint64_t WriteNTupleHeader(const void *data, size_t nbytes, size_t lenHeader)
Writes the compressed header and registeres its location; lenHeader is the size of the uncompressed h...
Definition: RMiniFile.cxx:1247
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition: RError.hxx:195
The RRawFile provides read-only access to local and remote files.
Definition: RRawFile.hxx:43
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format.
Definition: TFile.h:54
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition: TFile.cxx:4011
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:28
virtual void Create(Int_t nbytes, TFile *f=0)
Create a TKey object of specified size.
Definition: TKey.cxx:459
Long64_t fSeekKey
Location of object on file.
Definition: TKey.h:45
struct void * fTypeName
Definition: cppyy.h:9
#define Swap(a, b)
Definition: geom.c:201
Double_t x[n]
Definition: legend1.C:17
basic_string_view< char > string_view
void Print(std::ostream &os, const OptionType &opt)
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Definition: file.py:1
Definition: first.py:1
Definition: writer.py:1
void Write(const void *buffer, size_t nbytes, std::int64_t offset)
Low-level writing using a TFile.
Definition: RMiniFile.cxx:1079
std::uint64_t WriteKey(const void *buffer, size_t nbytes, size_t len)
Writes an RBlob opaque key with the provided buffer as data record and returns the offset of the reco...
Definition: RMiniFile.cxx:1089
std::unique_ptr< ROOT::Experimental::Internal::RTFileControlBlock > fControlBlock
Keeps track of TFile control structures, which need to be updated on committing the data set.
Definition: RMiniFile.hxx:159
void Write(const void *buffer, size_t nbytes, std::int64_t offset=-1)
Writes bytes in the open stream, either at fFilePos or at the given offset.
Definition: RMiniFile.cxx:1030
std::uint64_t WriteKey(const void *buffer, std::size_t nbytes, std::size_t len, std::int64_t offset=-1, std::uint64_t directoryOffset=100, const std::string &className="", const std::string &objectName="", const std::string &title="")
Writes a TKey including the data record, given by buffer, into fFile; returns the file offset to the ...
Definition: RMiniFile.cxx:1050
If a TFile container is written by a C stream (simple file), on dataset commit, the file header needs...
Definition: RMiniFile.cxx:908
Entry point for an RNTuple in a ROOT file.
Definition: RMiniFile.hxx:55
std::uint64_t fSeekFooter
The file offset of the footer excluding the TKey part.
Definition: RMiniFile.hxx:67
std::uint32_t fLenHeader
The size of the uncompressed ntuple header.
Definition: RMiniFile.hxx:65
std::uint64_t fReserved
Currently unused, reserved for later use.
Definition: RMiniFile.hxx:73
std::uint32_t fVersion
Allows for evolving the struct in future versions.
Definition: RMiniFile.hxx:57
std::uint32_t fNBytesHeader
The size of the compressed ntuple header.
Definition: RMiniFile.hxx:63
std::uint32_t fSize
Allows for skipping the struct.
Definition: RMiniFile.hxx:59
std::uint32_t fNBytesFooter
The size of the compressed ntuple footer.
Definition: RMiniFile.hxx:69
std::uint32_t fLenFooter
The size of the uncompressed ntuple footer.
Definition: RMiniFile.hxx:71
std::uint64_t fSeekHeader
The file offset of the header excluding the TKey part.
Definition: RMiniFile.hxx:61
auto * tt
Definition: textangle.C:16
static void output(int code)
Definition: gifencode.c:226