Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleSerialize.cxx
Go to the documentation of this file.
1/// \file RNTupleSerialize.cxx
2/// \ingroup NTuple
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \author Javier Lopez-Gomez <javier.lopez.gomez@cern.ch>
5/// \date 2021-08-02
6
7/*************************************************************************
8 * Copyright (C) 1995-2021, Rene Brun and Fons Rademakers. *
9 * All rights reserved. *
10 * *
11 * For the licensing terms see $ROOTSYS/LICENSE. *
12 * For the list of contributors see $ROOTSYS/README/CREDITS. *
13 *************************************************************************/
14
16#include <ROOT/RError.hxx>
19#include <ROOT/RNTupleUtil.hxx>
20
21#include <RVersion.h>
22#include <TBufferFile.h>
23#include <TClass.h>
24#include <TList.h>
25#include <TStreamerInfo.h>
27#include <xxhash.h>
28
29#include <cassert>
30#include <cmath>
31#include <cstring> // for memcpy
32#include <deque>
33#include <functional>
34#include <limits>
35#include <set>
36#include <unordered_map>
37
44
45namespace {
46using RNTupleSerializer = ROOT::Internal::RNTupleSerializer;
47
51{
52
53 auto base = reinterpret_cast<unsigned char *>(buffer);
54 auto pos = base;
55 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
56
57 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
58
59 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetFieldVersion(), *where);
60 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeVersion(), *where);
61 pos += RNTupleSerializer::SerializeUInt32(onDiskParentId, *where);
62 if (auto res = RNTupleSerializer::SerializeFieldStructure(fieldDesc.GetStructure(), *where)) {
63 pos += res.Unwrap();
64 } else {
65 return R__FORWARD_ERROR(res);
66 }
67
68 std::uint16_t flags = 0;
69 if (fieldDesc.GetNRepetitions() > 0)
70 flags |= RNTupleSerializer::kFlagRepetitiveField;
71 if (fieldDesc.IsProjectedField())
72 flags |= RNTupleSerializer::kFlagProjectedField;
73 if (fieldDesc.GetTypeChecksum().has_value())
74 flags |= RNTupleSerializer::kFlagHasTypeChecksum;
75 pos += RNTupleSerializer::SerializeUInt16(flags, *where);
76
77 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldName(), *where);
78 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeName(), *where);
79 pos += RNTupleSerializer::SerializeString(fieldDesc.GetTypeAlias(), *where);
80 pos += RNTupleSerializer::SerializeString(fieldDesc.GetFieldDescription(), *where);
81
82 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
83 pos += RNTupleSerializer::SerializeUInt64(fieldDesc.GetNRepetitions(), *where);
84 }
85 if (flags & RNTupleSerializer::kFlagProjectedField) {
86 pos += RNTupleSerializer::SerializeUInt32(onDiskProjectionSourceId, *where);
87 }
88 if (flags & RNTupleSerializer::kFlagHasTypeChecksum) {
89 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.GetTypeChecksum().value(), *where);
90 }
91
92 auto size = pos - base;
93 RNTupleSerializer::SerializeFramePostscript(base, size);
94
95 return size;
96}
97
98// clang-format off
99/// Serialize, in order, fields enumerated in `fieldList` to `buffer`. `firstOnDiskId` specifies the on-disk ID for the
100/// first element in the `fieldList` sequence. Before calling this function `RContext::MapSchema()` should have been
101/// called on `context` in order to map in-memory field IDs to their on-disk counterpart.
102/// \return The number of bytes written to the output buffer; if `buffer` is `nullptr` no data is serialized and the
103/// required buffer size is returned
104// clang-format on
106SerializeFieldList(const ROOT::RNTupleDescriptor &desc, std::span<const ROOT::DescriptorId_t> fieldList,
107 std::size_t firstOnDiskId, const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer)
108{
109 auto base = reinterpret_cast<unsigned char *>(buffer);
110 auto pos = base;
111 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
112
113 auto fieldZeroId = desc.GetFieldZeroId();
115 for (auto fieldId : fieldList) {
116 const auto &f = desc.GetFieldDescriptor(fieldId);
117 auto onDiskParentId =
118 (f.GetParentId() == fieldZeroId) ? onDiskFieldId : context.GetOnDiskFieldId(f.GetParentId());
120 f.IsProjectedField() ? context.GetOnDiskFieldId(f.GetProjectionSourceId()) : ROOT::kInvalidDescriptorId;
122 pos += res.Unwrap();
123 } else {
124 return R__FORWARD_ERROR(res);
125 }
127 }
128
129 return pos - base;
130}
131
134{
135 using ENTupleStructure = ROOT::ENTupleStructure;
136
137 auto base = reinterpret_cast<const unsigned char *>(buffer);
138 auto bytes = base;
139 std::uint64_t frameSize;
140 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
141 if (auto res = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize)) {
142 bytes += res.Unwrap();
143 } else {
144 return R__FORWARD_ERROR(res);
145 }
146
147 std::uint32_t fieldVersion;
148 std::uint32_t typeVersion;
149 std::uint32_t parentId;
150 // initialize properly for call to SerializeFieldStructure()
151 ENTupleStructure structure{ENTupleStructure::kLeaf};
152 std::uint16_t flags;
153 std::uint32_t result;
154 if (auto res = RNTupleSerializer::SerializeFieldStructure(structure, nullptr)) {
155 result = res.Unwrap();
156 } else {
157 return R__FORWARD_ERROR(res);
158 }
159 if (fnFrameSizeLeft() < 3 * sizeof(std::uint32_t) + result + sizeof(std::uint16_t)) {
160 return R__FAIL("field record frame too short");
161 }
162 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldVersion);
163 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
164 bytes += RNTupleSerializer::DeserializeUInt32(bytes, parentId);
165 if (auto res = RNTupleSerializer::DeserializeFieldStructure(bytes, structure)) {
166 bytes += res.Unwrap();
167 } else {
168 return R__FORWARD_ERROR(res);
169 }
170 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
171 fieldDesc.FieldVersion(fieldVersion).TypeVersion(typeVersion).ParentId(parentId).Structure(structure);
172
173 std::string fieldName;
174 std::string typeName;
175 std::string aliasName;
176 std::string description;
177 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), fieldName)) {
178 bytes += res.Unwrap();
179 } else {
180 return R__FORWARD_ERROR(res);
181 }
182 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName)) {
183 bytes += res.Unwrap();
184 } else {
185 return R__FORWARD_ERROR(res);
186 }
187 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), aliasName)) {
188 bytes += res.Unwrap();
189 } else {
190 return R__FORWARD_ERROR(res);
191 }
192 if (auto res = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), description)) {
193 bytes += res.Unwrap();
194 } else {
195 return R__FORWARD_ERROR(res);
196 }
197 fieldDesc.FieldName(fieldName).TypeName(typeName).TypeAlias(aliasName).FieldDescription(description);
198
199 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
200 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
201 return R__FAIL("field record frame too short");
202 std::uint64_t nRepetitions;
203 bytes += RNTupleSerializer::DeserializeUInt64(bytes, nRepetitions);
204 fieldDesc.NRepetitions(nRepetitions);
205 }
206
207 if (flags & RNTupleSerializer::kFlagProjectedField) {
208 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
209 return R__FAIL("field record frame too short");
210 std::uint32_t projectionSourceId;
211 bytes += RNTupleSerializer::DeserializeUInt32(bytes, projectionSourceId);
212 fieldDesc.ProjectionSourceId(projectionSourceId);
213 }
214
215 if (flags & RNTupleSerializer::kFlagHasTypeChecksum) {
216 if (fnFrameSizeLeft() < sizeof(std::uint32_t))
217 return R__FAIL("field record frame too short");
218 std::uint32_t typeChecksum;
219 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeChecksum);
220 fieldDesc.TypeChecksum(typeChecksum);
221 }
222
223 return frameSize;
224}
225
228 void *buffer)
229{
230 R__ASSERT(!columnDesc.IsAliasColumn());
231
232 auto base = reinterpret_cast<unsigned char *>(buffer);
233 auto pos = base;
234 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
235
236 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
237
238 if (auto res = RNTupleSerializer::SerializeColumnType(columnDesc.GetType(), *where)) {
239 pos += res.Unwrap();
240 } else {
241 return R__FORWARD_ERROR(res);
242 }
243 pos += RNTupleSerializer::SerializeUInt16(columnDesc.GetBitsOnStorage(), *where);
244 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(columnDesc.GetFieldId()), *where);
245 std::uint16_t flags = 0;
246 if (columnDesc.IsDeferredColumn())
247 flags |= RNTupleSerializer::kFlagDeferredColumn;
248 if (columnDesc.GetValueRange().has_value())
249 flags |= RNTupleSerializer::kFlagHasValueRange;
250 std::int64_t firstElementIdx = columnDesc.GetFirstElementIndex();
251 if (columnDesc.IsSuppressedDeferredColumn())
253 pos += RNTupleSerializer::SerializeUInt16(flags, *where);
254 pos += RNTupleSerializer::SerializeUInt16(columnDesc.GetRepresentationIndex(), *where);
255 if (flags & RNTupleSerializer::kFlagDeferredColumn)
256 pos += RNTupleSerializer::SerializeInt64(firstElementIdx, *where);
257 if (flags & RNTupleSerializer::kFlagHasValueRange) {
258 auto [min, max] = *columnDesc.GetValueRange();
259 std::uint64_t intMin, intMax;
260 static_assert(sizeof(min) == sizeof(intMin) && sizeof(max) == sizeof(intMax));
261 memcpy(&intMin, &min, sizeof(min));
262 memcpy(&intMax, &max, sizeof(max));
263 pos += RNTupleSerializer::SerializeUInt64(intMin, *where);
264 pos += RNTupleSerializer::SerializeUInt64(intMax, *where);
265 }
266
267 if (auto res = RNTupleSerializer::SerializeFramePostscript(buffer ? base : nullptr, pos - base)) {
268 pos += res.Unwrap();
269 } else {
270 return R__FORWARD_ERROR(res);
271 }
272
273 return pos - base;
274}
275
277 std::span<const ROOT::DescriptorId_t> fieldList,
279 void *buffer, bool forHeaderExtension)
280{
281 auto base = reinterpret_cast<unsigned char *>(buffer);
282 auto pos = base;
283 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
284
285 const auto *xHeader = !forHeaderExtension ? desc.GetHeaderExtension() : nullptr;
286
287 for (auto parentId : fieldList) {
288 // If we're serializing the non-extended header and we already have a header extension (which may happen if
289 // we load an RNTuple for incremental merging), we need to skip all the extended fields, as they need to be
290 // written in the header extension, not in the regular header.
291 if (xHeader && xHeader->ContainsField(parentId))
292 continue;
293
294 for (const auto &c : desc.GetColumnIterable(parentId)) {
295 if (c.IsAliasColumn() || (xHeader && xHeader->ContainsExtendedColumnRepresentation(c.GetLogicalId())))
296 continue;
297
298 if (auto res = SerializePhysicalColumn(c, context, *where)) {
299 pos += res.Unwrap();
300 } else {
301 return R__FORWARD_ERROR(res);
302 }
303 }
304 }
305
306 return pos - base;
307}
308
311{
313
314 auto base = reinterpret_cast<const unsigned char *>(buffer);
315 auto bytes = base;
316 std::uint64_t frameSize;
317 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
318 if (auto res = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize)) {
319 bytes += res.Unwrap();
320 } else {
321 return R__FORWARD_ERROR(res);
322 }
323
324 // Initialize properly for SerializeColumnType
325 ENTupleColumnType type{ENTupleColumnType::kIndex32};
326 std::uint16_t bitsOnStorage;
327 std::uint32_t fieldId;
328 std::uint16_t flags;
329 std::uint16_t representationIndex;
330 std::int64_t firstElementIdx = 0;
331 if (fnFrameSizeLeft() < RNTupleSerializer::SerializeColumnType(type, nullptr).Unwrap() + sizeof(std::uint16_t) +
332 2 * sizeof(std::uint32_t)) {
333 return R__FAIL("column record frame too short");
334 }
335 if (auto res = RNTupleSerializer::DeserializeColumnType(bytes, type)) {
336 bytes += res.Unwrap();
337 } else {
338 return R__FORWARD_ERROR(res);
339 }
340 bytes += RNTupleSerializer::DeserializeUInt16(bytes, bitsOnStorage);
341 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
342 bytes += RNTupleSerializer::DeserializeUInt16(bytes, flags);
343 bytes += RNTupleSerializer::DeserializeUInt16(bytes, representationIndex);
344 if (flags & RNTupleSerializer::kFlagDeferredColumn) {
345 if (fnFrameSizeLeft() < sizeof(std::uint64_t))
346 return R__FAIL("column record frame too short");
347 bytes += RNTupleSerializer::DeserializeInt64(bytes, firstElementIdx);
348 }
349 if (flags & RNTupleSerializer::kFlagHasValueRange) {
350 if (fnFrameSizeLeft() < 2 * sizeof(std::uint64_t))
351 return R__FAIL("field record frame too short");
352 std::uint64_t minInt, maxInt;
353 bytes += RNTupleSerializer::DeserializeUInt64(bytes, minInt);
354 bytes += RNTupleSerializer::DeserializeUInt64(bytes, maxInt);
355 double min, max;
356 memcpy(&min, &minInt, sizeof(min));
357 memcpy(&max, &maxInt, sizeof(max));
358 columnDesc.ValueRange(min, max);
359 }
360
361 columnDesc.FieldId(fieldId).BitsOnStorage(bitsOnStorage).Type(type).RepresentationIndex(representationIndex);
362 columnDesc.FirstElementIndex(std::abs(firstElementIdx));
363 if (firstElementIdx < 0)
364 columnDesc.SetSuppressedDeferred();
365
366 return frameSize;
367}
368
370{
371 auto base = reinterpret_cast<unsigned char *>(buffer);
372 auto pos = base;
373 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
374
375 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
376
377 if (auto res = RNTupleSerializer::SerializeExtraTypeInfoId(desc.GetContentId(), *where)) {
378 pos += res.Unwrap();
379 } else {
380 return R__FORWARD_ERROR(res);
381 }
382 pos += RNTupleSerializer::SerializeUInt32(desc.GetTypeVersion(), *where);
383 pos += RNTupleSerializer::SerializeString(desc.GetTypeName(), *where);
384 pos += RNTupleSerializer::SerializeString(desc.GetContent(), *where);
385
386 auto size = pos - base;
387 RNTupleSerializer::SerializeFramePostscript(base, size);
388
389 return size;
390}
391
393{
394 auto base = reinterpret_cast<unsigned char *>(buffer);
395 auto pos = base;
396 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
397
398 for (const auto &extraTypeInfoDesc : ntplDesc.GetExtraTypeInfoIterable()) {
400 pos += res.Unwrap();
401 } else {
402 return R__FORWARD_ERROR(res);
403 }
404 }
405
406 return pos - base;
407}
408
409ROOT::RResult<std::uint32_t> DeserializeExtraTypeInfo(const void *buffer, std::uint64_t bufSize,
411{
413
414 auto base = reinterpret_cast<const unsigned char *>(buffer);
415 auto bytes = base;
416 std::uint64_t frameSize;
417 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
418 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
419 if (!result)
420 return R__FORWARD_ERROR(result);
421 bytes += result.Unwrap();
422
423 EExtraTypeInfoIds contentId{EExtraTypeInfoIds::kInvalid};
424 std::uint32_t typeVersion;
425 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
426 return R__FAIL("extra type info record frame too short");
427 }
428 result = RNTupleSerializer::DeserializeExtraTypeInfoId(bytes, contentId);
429 if (!result)
430 return R__FORWARD_ERROR(result);
431 bytes += result.Unwrap();
432 bytes += RNTupleSerializer::DeserializeUInt32(bytes, typeVersion);
433
434 std::string typeName;
435 std::string content;
436 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), typeName).Unwrap();
437 if (!result)
438 return R__FORWARD_ERROR(result);
439 bytes += result.Unwrap();
440 result = RNTupleSerializer::DeserializeString(bytes, fnFrameSizeLeft(), content).Unwrap();
441 if (!result)
442 return R__FORWARD_ERROR(result);
443 bytes += result.Unwrap();
444
446
447 return frameSize;
448}
449
450std::uint32_t SerializeLocatorPayloadLarge(const ROOT::RNTupleLocator &locator, unsigned char *buffer)
451{
452 if (buffer) {
453 RNTupleSerializer::SerializeUInt64(locator.GetNBytesOnStorage(), buffer);
454 RNTupleSerializer::SerializeUInt64(locator.GetPosition<std::uint64_t>(), buffer + sizeof(std::uint64_t));
455 }
456 return sizeof(std::uint64_t) + sizeof(std::uint64_t);
457}
458
459void DeserializeLocatorPayloadLarge(const unsigned char *buffer, ROOT::RNTupleLocator &locator)
460{
461 std::uint64_t nBytesOnStorage;
462 std::uint64_t position;
463 RNTupleSerializer::DeserializeUInt64(buffer, nBytesOnStorage);
464 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint64_t), position);
465 locator.SetNBytesOnStorage(nBytesOnStorage);
466 locator.SetPosition(position);
467}
468
469std::uint32_t SerializeLocatorPayloadObject64(const ROOT::RNTupleLocator &locator, unsigned char *buffer)
470{
471 const auto &data = locator.GetPosition<ROOT::RNTupleLocatorObject64>();
472 const uint32_t sizeofNBytesOnStorage = (locator.GetNBytesOnStorage() > std::numeric_limits<std::uint32_t>::max())
473 ? sizeof(std::uint64_t)
474 : sizeof(std::uint32_t);
475 if (buffer) {
476 if (sizeofNBytesOnStorage == sizeof(std::uint32_t)) {
477 RNTupleSerializer::SerializeUInt32(locator.GetNBytesOnStorage(), buffer);
478 } else {
479 RNTupleSerializer::SerializeUInt64(locator.GetNBytesOnStorage(), buffer);
480 }
481 RNTupleSerializer::SerializeUInt64(data.GetLocation(), buffer + sizeofNBytesOnStorage);
482 }
483 return sizeofNBytesOnStorage + sizeof(std::uint64_t);
484}
485
486ROOT::RResult<void> DeserializeLocatorPayloadObject64(const unsigned char *buffer, std::uint32_t sizeofLocatorPayload,
488{
489 std::uint64_t location;
490 if (sizeofLocatorPayload == 12) {
491 std::uint32_t nBytesOnStorage;
492 RNTupleSerializer::DeserializeUInt32(buffer, nBytesOnStorage);
493 locator.SetNBytesOnStorage(nBytesOnStorage);
494 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint32_t), location);
495 } else if (sizeofLocatorPayload == 16) {
496 std::uint64_t nBytesOnStorage;
497 RNTupleSerializer::DeserializeUInt64(buffer, nBytesOnStorage);
498 locator.SetNBytesOnStorage(nBytesOnStorage);
499 RNTupleSerializer::DeserializeUInt64(buffer + sizeof(std::uint64_t), location);
500 } else {
501 return R__FAIL("invalid DAOS locator payload size: " + std::to_string(sizeofLocatorPayload));
502 }
503 locator.SetPosition(ROOT::RNTupleLocatorObject64{location});
505}
506
508 const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer)
509{
510 R__ASSERT(columnDesc.IsAliasColumn());
511
512 auto base = reinterpret_cast<unsigned char *>(buffer);
513 auto pos = base;
514 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
515
516 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
517
518 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskColumnId(columnDesc.GetPhysicalId()), *where);
519 pos += RNTupleSerializer::SerializeUInt32(context.GetOnDiskFieldId(columnDesc.GetFieldId()), *where);
520
521 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? base : nullptr, pos - base).Unwrap();
522
523 return pos - base;
524}
525
527 std::span<const ROOT::DescriptorId_t> fieldList,
528 const ROOT::Internal::RNTupleSerializer::RContext &context, void *buffer,
530{
531 auto base = reinterpret_cast<unsigned char *>(buffer);
532 auto pos = base;
533 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
534
535 const auto *xHeader = !forHeaderExtension ? desc.GetHeaderExtension() : nullptr;
536
537 for (auto parentId : fieldList) {
538 if (xHeader && xHeader->ContainsField(parentId))
539 continue;
540
541 for (const auto &c : desc.GetColumnIterable(parentId)) {
542 if (!c.IsAliasColumn() || (xHeader && xHeader->ContainsExtendedColumnRepresentation(c.GetLogicalId())))
543 continue;
544
545 pos += SerializeAliasColumn(c, context, *where);
546 }
547 }
548
549 return pos - base;
550}
551
552ROOT::RResult<std::uint32_t> DeserializeAliasColumn(const void *buffer, std::uint64_t bufSize,
553 std::uint32_t &physicalColumnId, std::uint32_t &fieldId)
554{
555 auto base = reinterpret_cast<const unsigned char *>(buffer);
556 auto bytes = base;
557 std::uint64_t frameSize;
558 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
559 auto result = RNTupleSerializer::DeserializeFrameHeader(bytes, bufSize, frameSize);
560 if (!result)
561 return R__FORWARD_ERROR(result);
562 bytes += result.Unwrap();
563
564 if (fnFrameSizeLeft() < 2 * sizeof(std::uint32_t)) {
565 return R__FAIL("alias column record frame too short");
566 }
567
568 bytes += RNTupleSerializer::DeserializeUInt32(bytes, physicalColumnId);
569 bytes += RNTupleSerializer::DeserializeUInt32(bytes, fieldId);
570
571 return frameSize;
572}
573
574} // anonymous namespace
575
576std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeXxHash3(const unsigned char *data, std::uint64_t length,
577 std::uint64_t &xxhash3, void *buffer)
578{
579 if (buffer != nullptr) {
581 SerializeUInt64(xxhash3, buffer);
582 }
583 return 8;
584}
585
587 std::uint64_t &xxhash3)
588{
590 DeserializeUInt64(data + length, xxhash3);
591 if (xxhash3 != checksumReal)
592 return R__FAIL("XxHash-3 checksum mismatch");
593 return RResult<void>::Success();
594}
595
597{
598 std::uint64_t xxhash3;
599 return R__FORWARD_RESULT(VerifyXxHash3(data, length, xxhash3));
600}
601
602std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt16(std::int16_t val, void *buffer)
603{
604 if (buffer != nullptr) {
605 auto bytes = reinterpret_cast<unsigned char *>(buffer);
606 bytes[0] = (val & 0x00FF);
607 bytes[1] = (val & 0xFF00) >> 8;
608 }
609 return 2;
610}
611
612std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt16(const void *buffer, std::int16_t &val)
613{
614 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
615 val = std::int16_t(bytes[0]) + (std::int16_t(bytes[1]) << 8);
616 return 2;
617}
618
619std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt16(std::uint16_t val, void *buffer)
620{
621 return SerializeInt16(val, buffer);
622}
623
624std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt16(const void *buffer, std::uint16_t &val)
625{
626 return DeserializeInt16(buffer, *reinterpret_cast<std::int16_t *>(&val));
627}
628
629std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt32(std::int32_t val, void *buffer)
630{
631 if (buffer != nullptr) {
632 auto bytes = reinterpret_cast<unsigned char *>(buffer);
633 bytes[0] = (val & 0x000000FF);
634 bytes[1] = (val & 0x0000FF00) >> 8;
635 bytes[2] = (val & 0x00FF0000) >> 16;
636 bytes[3] = (val & 0xFF000000) >> 24;
637 }
638 return 4;
639}
640
641std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt32(const void *buffer, std::int32_t &val)
642{
643 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
644 val = std::int32_t(bytes[0]) + (std::int32_t(bytes[1]) << 8) + (std::int32_t(bytes[2]) << 16) +
645 (std::int32_t(bytes[3]) << 24);
646 return 4;
647}
648
649std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt32(std::uint32_t val, void *buffer)
650{
651 return SerializeInt32(val, buffer);
652}
653
654std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt32(const void *buffer, std::uint32_t &val)
655{
656 return DeserializeInt32(buffer, *reinterpret_cast<std::int32_t *>(&val));
657}
658
659std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeInt64(std::int64_t val, void *buffer)
660{
661 if (buffer != nullptr) {
662 auto bytes = reinterpret_cast<unsigned char *>(buffer);
663 bytes[0] = (val & 0x00000000000000FF);
664 bytes[1] = (val & 0x000000000000FF00) >> 8;
665 bytes[2] = (val & 0x0000000000FF0000) >> 16;
666 bytes[3] = (val & 0x00000000FF000000) >> 24;
667 bytes[4] = (val & 0x000000FF00000000) >> 32;
668 bytes[5] = (val & 0x0000FF0000000000) >> 40;
669 bytes[6] = (val & 0x00FF000000000000) >> 48;
670 bytes[7] = (val & 0xFF00000000000000) >> 56;
671 }
672 return 8;
673}
674
675std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeInt64(const void *buffer, std::int64_t &val)
676{
677 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
678 val = std::int64_t(bytes[0]) + (std::int64_t(bytes[1]) << 8) + (std::int64_t(bytes[2]) << 16) +
679 (std::int64_t(bytes[3]) << 24) + (std::int64_t(bytes[4]) << 32) + (std::int64_t(bytes[5]) << 40) +
680 (std::int64_t(bytes[6]) << 48) + (std::int64_t(bytes[7]) << 56);
681 return 8;
682}
683
684std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeUInt64(std::uint64_t val, void *buffer)
685{
686 return SerializeInt64(val, buffer);
687}
688
689std::uint32_t ROOT::Internal::RNTupleSerializer::DeserializeUInt64(const void *buffer, std::uint64_t &val)
690{
691 return DeserializeInt64(buffer, *reinterpret_cast<std::int64_t *>(&val));
692}
693
694std::uint32_t ROOT::Internal::RNTupleSerializer::SerializeString(const std::string &val, void *buffer)
695{
696 if (buffer) {
697 auto pos = reinterpret_cast<unsigned char *>(buffer);
698 pos += SerializeUInt32(val.length(), pos);
699 memcpy(pos, val.data(), val.length());
700 }
701 return sizeof(std::uint32_t) + val.length();
702}
703
705ROOT::Internal::RNTupleSerializer::DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
706{
707 if (bufSize < sizeof(std::uint32_t))
708 return R__FAIL("string buffer too short");
709 bufSize -= sizeof(std::uint32_t);
710
711 auto base = reinterpret_cast<const unsigned char *>(buffer);
712 auto bytes = base;
713 std::uint32_t length;
714 bytes += DeserializeUInt32(buffer, length);
715 if (bufSize < length)
716 return R__FAIL("string buffer too short");
717
718 val.resize(length);
719 memcpy(&val[0], bytes, length);
720 return sizeof(std::uint32_t) + length;
721}
722
725{
726 switch (type) {
727 case ENTupleColumnType::kBit: return SerializeUInt16(0x00, buffer);
728 case ENTupleColumnType::kByte: return SerializeUInt16(0x01, buffer);
729 case ENTupleColumnType::kChar: return SerializeUInt16(0x02, buffer);
730 case ENTupleColumnType::kInt8: return SerializeUInt16(0x03, buffer);
731 case ENTupleColumnType::kUInt8: return SerializeUInt16(0x04, buffer);
732 case ENTupleColumnType::kInt16: return SerializeUInt16(0x05, buffer);
733 case ENTupleColumnType::kUInt16: return SerializeUInt16(0x06, buffer);
734 case ENTupleColumnType::kInt32: return SerializeUInt16(0x07, buffer);
735 case ENTupleColumnType::kUInt32: return SerializeUInt16(0x08, buffer);
736 case ENTupleColumnType::kInt64: return SerializeUInt16(0x09, buffer);
737 case ENTupleColumnType::kUInt64: return SerializeUInt16(0x0A, buffer);
738 case ENTupleColumnType::kReal16: return SerializeUInt16(0x0B, buffer);
739 case ENTupleColumnType::kReal32: return SerializeUInt16(0x0C, buffer);
740 case ENTupleColumnType::kReal64: return SerializeUInt16(0x0D, buffer);
741 case ENTupleColumnType::kIndex32: return SerializeUInt16(0x0E, buffer);
742 case ENTupleColumnType::kIndex64: return SerializeUInt16(0x0F, buffer);
743 case ENTupleColumnType::kSwitch: return SerializeUInt16(0x10, buffer);
744 case ENTupleColumnType::kSplitInt16: return SerializeUInt16(0x11, buffer);
745 case ENTupleColumnType::kSplitUInt16: return SerializeUInt16(0x12, buffer);
746 case ENTupleColumnType::kSplitInt32: return SerializeUInt16(0x13, buffer);
747 case ENTupleColumnType::kSplitUInt32: return SerializeUInt16(0x14, buffer);
748 case ENTupleColumnType::kSplitInt64: return SerializeUInt16(0x15, buffer);
749 case ENTupleColumnType::kSplitUInt64: return SerializeUInt16(0x16, buffer);
750 case ENTupleColumnType::kSplitReal32: return SerializeUInt16(0x18, buffer);
751 case ENTupleColumnType::kSplitReal64: return SerializeUInt16(0x19, buffer);
752 case ENTupleColumnType::kSplitIndex32: return SerializeUInt16(0x1A, buffer);
753 case ENTupleColumnType::kSplitIndex64: return SerializeUInt16(0x1B, buffer);
754 case ENTupleColumnType::kReal32Trunc: return SerializeUInt16(0x1C, buffer);
755 case ENTupleColumnType::kReal32Quant: return SerializeUInt16(0x1D, buffer);
756 default:
758 return SerializeUInt16(0x99, buffer);
759 return R__FAIL("unexpected column type");
760 }
761}
762
765{
766 std::uint16_t onDiskType;
767 auto result = DeserializeUInt16(buffer, onDiskType);
768
769 switch (onDiskType) {
770 case 0x00: type = ENTupleColumnType::kBit; break;
771 case 0x01: type = ENTupleColumnType::kByte; break;
772 case 0x02: type = ENTupleColumnType::kChar; break;
773 case 0x03: type = ENTupleColumnType::kInt8; break;
774 case 0x04: type = ENTupleColumnType::kUInt8; break;
775 case 0x05: type = ENTupleColumnType::kInt16; break;
776 case 0x06: type = ENTupleColumnType::kUInt16; break;
777 case 0x07: type = ENTupleColumnType::kInt32; break;
778 case 0x08: type = ENTupleColumnType::kUInt32; break;
779 case 0x09: type = ENTupleColumnType::kInt64; break;
780 case 0x0A: type = ENTupleColumnType::kUInt64; break;
781 case 0x0B: type = ENTupleColumnType::kReal16; break;
782 case 0x0C: type = ENTupleColumnType::kReal32; break;
783 case 0x0D: type = ENTupleColumnType::kReal64; break;
784 case 0x0E: type = ENTupleColumnType::kIndex32; break;
785 case 0x0F: type = ENTupleColumnType::kIndex64; break;
786 case 0x10: type = ENTupleColumnType::kSwitch; break;
787 case 0x11: type = ENTupleColumnType::kSplitInt16; break;
788 case 0x12: type = ENTupleColumnType::kSplitUInt16; break;
789 case 0x13: type = ENTupleColumnType::kSplitInt32; break;
790 case 0x14: type = ENTupleColumnType::kSplitUInt32; break;
791 case 0x15: type = ENTupleColumnType::kSplitInt64; break;
792 case 0x16: type = ENTupleColumnType::kSplitUInt64; break;
793 case 0x18: type = ENTupleColumnType::kSplitReal32; break;
794 case 0x19: type = ENTupleColumnType::kSplitReal64; break;
795 case 0x1A: type = ENTupleColumnType::kSplitIndex32; break;
796 case 0x1B: type = ENTupleColumnType::kSplitIndex64; break;
797 case 0x1C: type = ENTupleColumnType::kReal32Trunc; break;
798 case 0x1D: type = ENTupleColumnType::kReal32Quant; break;
799 // case 0x99 => kTestFutureColumnType missing on purpose
800 default:
801 // may be a column type introduced by a future version
803 break;
804 }
805 return result;
806}
807
810{
812 switch (structure) {
813 case ENTupleStructure::kLeaf: return SerializeUInt16(0x00, buffer);
814 case ENTupleStructure::kCollection: return SerializeUInt16(0x01, buffer);
815 case ENTupleStructure::kRecord: return SerializeUInt16(0x02, buffer);
816 case ENTupleStructure::kVariant: return SerializeUInt16(0x03, buffer);
817 case ENTupleStructure::kStreamer: return SerializeUInt16(0x04, buffer);
818 default:
820 return SerializeUInt16(0x99, buffer);
821 return R__FAIL("unexpected field structure type");
822 }
823}
824
827{
829 std::uint16_t onDiskValue;
830 auto result = DeserializeUInt16(buffer, onDiskValue);
831 switch (onDiskValue) {
832 case 0x00: structure = ENTupleStructure::kLeaf; break;
833 case 0x01: structure = ENTupleStructure::kCollection; break;
834 case 0x02: structure = ENTupleStructure::kRecord; break;
835 case 0x03: structure = ENTupleStructure::kVariant; break;
836 case 0x04: structure = ENTupleStructure::kStreamer; break;
837 // case 0x99 => kTestFutureFieldStructure intentionally missing
838 default: structure = ENTupleStructure::kUnknown;
839 }
840 return result;
841}
842
845{
846 switch (id) {
847 case ROOT::EExtraTypeInfoIds::kStreamerInfo: return SerializeUInt32(0x00, buffer);
848 default: return R__FAIL("unexpected extra type info id");
849 }
850}
851
854{
855 std::uint32_t onDiskValue;
856 auto result = DeserializeUInt32(buffer, onDiskValue);
857 switch (onDiskValue) {
858 case 0x00: id = ROOT::EExtraTypeInfoIds::kStreamerInfo; break;
859 default:
861 R__LOG_DEBUG(0, ROOT::Internal::NTupleLog()) << "Unknown extra type info id: " << onDiskValue;
862 }
863 return result;
864}
865
867{
868 auto base = reinterpret_cast<unsigned char *>(buffer);
869 auto pos = base;
870 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
871
872 pos += SerializeUInt64(envelopeType, *where);
873 // The 48bits size information is filled in the postscript
874 return pos - base;
875}
876
878 std::uint64_t size,
879 std::uint64_t &xxhash3)
880{
881 if (size < sizeof(std::uint64_t))
882 return R__FAIL("envelope size too small");
883 if (size >= static_cast<uint64_t>(1) << 48)
884 return R__FAIL("envelope size too big");
885 if (envelope) {
886 std::uint64_t typeAndSize;
887 DeserializeUInt64(envelope, typeAndSize);
888 typeAndSize |= (size + 8) << 16;
889 SerializeUInt64(typeAndSize, envelope);
890 }
891 return SerializeXxHash3(envelope, size, xxhash3, envelope ? (envelope + size) : nullptr);
892}
893
896{
897 std::uint64_t xxhash3;
898 return R__FORWARD_RESULT(SerializeEnvelopePostscript(envelope, size, xxhash3));
899}
900
903 std::uint16_t expectedType, std::uint64_t &xxhash3)
904{
905 const std::uint64_t minEnvelopeSize = sizeof(std::uint64_t) + sizeof(std::uint64_t);
907 return R__FAIL("invalid envelope buffer, too short");
908
909 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
910 auto base = bytes;
911
912 std::uint64_t typeAndSize;
913 bytes += DeserializeUInt64(bytes, typeAndSize);
914
915 std::uint16_t envelopeType = typeAndSize & 0xFFFF;
916 if (envelopeType != expectedType) {
917 return R__FAIL("envelope type mismatch: expected " + std::to_string(expectedType) + ", found " +
918 std::to_string(envelopeType));
919 }
920
921 std::uint64_t envelopeSize = typeAndSize >> 16;
922 if (bufSize < envelopeSize)
923 return R__FAIL("envelope buffer size too small");
925 return R__FAIL("invalid envelope, too short");
926
927 auto result = VerifyXxHash3(base, envelopeSize - 8, xxhash3);
928 if (!result)
929 return R__FORWARD_ERROR(result);
930
931 return sizeof(typeAndSize);
932}
933
935 std::uint64_t bufSize,
936 std::uint16_t expectedType)
937{
938 std::uint64_t xxhash3;
939 return R__FORWARD_RESULT(DeserializeEnvelope(buffer, bufSize, expectedType, xxhash3));
940}
941
943{
944 // Marker: multiply the final size with 1
945 return SerializeInt64(1, buffer);
946}
947
949{
950 auto base = reinterpret_cast<unsigned char *>(buffer);
951 auto pos = base;
952 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
953
954 // Marker: multiply the final size with -1
955 pos += SerializeInt64(-1, *where);
956 pos += SerializeUInt32(nitems, *where);
957 return pos - base;
958}
959
962{
963 auto preambleSize = sizeof(std::int64_t);
964 if (size < preambleSize)
965 return R__FAIL("frame too short: " + std::to_string(size));
966 if (frame) {
967 std::int64_t marker;
968 DeserializeInt64(frame, marker);
969 if ((marker < 0) && (size < (sizeof(std::uint32_t) + preambleSize)))
970 return R__FAIL("frame too short: " + std::to_string(size));
971 SerializeInt64(marker * static_cast<int64_t>(size), frame);
972 }
973 return 0;
974}
975
978 std::uint64_t &frameSize, std::uint32_t &nitems)
979{
980 std::uint64_t minSize = sizeof(std::int64_t);
981 if (bufSize < minSize)
982 return R__FAIL("frame too short");
983
984 std::int64_t *ssize = reinterpret_cast<std::int64_t *>(&frameSize);
985 DeserializeInt64(buffer, *ssize);
986
987 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
988 bytes += minSize;
989
990 if (*ssize >= 0) {
991 // Record frame
992 nitems = 1;
993 } else {
994 // List frame
995 minSize += sizeof(std::uint32_t);
996 if (bufSize < minSize)
997 return R__FAIL("frame too short");
998 bytes += DeserializeUInt32(bytes, nitems);
999 *ssize = -(*ssize);
1000 }
1001
1002 if (frameSize < minSize)
1003 return R__FAIL("corrupt frame size");
1004 if (bufSize < frameSize)
1005 return R__FAIL("frame too short");
1006
1007 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1008}
1009
1011 std::uint64_t bufSize,
1012 std::uint64_t &frameSize)
1013{
1014 std::uint32_t nitems;
1015 return R__FORWARD_RESULT(DeserializeFrameHeader(buffer, bufSize, frameSize, nitems));
1016}
1017
1019ROOT::Internal::RNTupleSerializer::SerializeFeatureFlags(const std::vector<std::uint64_t> &flags, void *buffer)
1020{
1021 if (flags.empty())
1022 return SerializeUInt64(0, buffer);
1023
1024 if (buffer) {
1025 auto bytes = reinterpret_cast<unsigned char *>(buffer);
1026
1027 for (unsigned i = 0; i < flags.size(); ++i) {
1028 if (flags[i] & 0x8000000000000000)
1029 return R__FAIL("feature flag out of bounds");
1030
1031 // The MSb indicates that another Int64 follows; set this bit to 1 for all except the last element
1032 if (i == (flags.size() - 1))
1033 SerializeUInt64(flags[i], bytes);
1034 else
1035 bytes += SerializeUInt64(flags[i] | 0x8000000000000000, bytes);
1036 }
1037 }
1038 return (flags.size() * sizeof(std::int64_t));
1039}
1040
1043 std::vector<std::uint64_t> &flags)
1044{
1045 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1046
1047 flags.clear();
1048 std::uint64_t f;
1049 do {
1050 if (bufSize < sizeof(std::uint64_t))
1051 return R__FAIL("feature flag buffer too short");
1052 bytes += DeserializeUInt64(bytes, f);
1053 bufSize -= sizeof(std::uint64_t);
1054 flags.emplace_back(f & ~0x8000000000000000);
1055 } while (f & 0x8000000000000000);
1056
1057 return (flags.size() * sizeof(std::uint64_t));
1058}
1059
1062{
1064 return R__FAIL("locator is not serializable");
1065
1066 std::uint32_t size = 0;
1067 if ((locator.GetType() == RNTupleLocator::kTypeFile) &&
1068 (locator.GetNBytesOnStorage() <= std::numeric_limits<std::int32_t>::max())) {
1069 size += SerializeUInt32(locator.GetNBytesOnStorage(), buffer);
1070 size += SerializeUInt64(locator.GetPosition<std::uint64_t>(),
1071 buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
1072 return size;
1073 }
1074
1075 std::uint8_t locatorType = 0;
1076 auto payloadp = buffer ? reinterpret_cast<unsigned char *>(buffer) + sizeof(std::int32_t) : nullptr;
1077 switch (locator.GetType()) {
1080 locatorType = 0x01;
1081 break;
1084 locatorType = 0x02;
1085 break;
1086 default:
1087 if (locator.GetType() == ROOT::Internal::kTestLocatorType) {
1088 // For the testing locator, use the same payload as Object64. We're not gonna really read it back anyway.
1090 locatorType = 0x7e;
1091 } else {
1092 return R__FAIL("locator has unknown type");
1093 }
1094 }
1095 std::int32_t head = sizeof(std::int32_t) + size;
1096 head |= locator.GetReserved() << 16;
1097 head |= static_cast<int>(locatorType & 0x7F) << 24;
1098 head = -head;
1099 size += RNTupleSerializer::SerializeInt32(head, buffer);
1100 return size;
1101}
1102
1104 std::uint64_t bufSize,
1106{
1107 if (bufSize < sizeof(std::int32_t))
1108 return R__FAIL("too short locator");
1109
1110 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1111 std::int32_t head;
1112
1113 bytes += DeserializeInt32(bytes, head);
1114 bufSize -= sizeof(std::int32_t);
1115 if (head < 0) {
1116 head = -head;
1117 const int type = head >> 24;
1118 const std::uint32_t payloadSize = (static_cast<std::uint32_t>(head) & 0x0000FFFF) - sizeof(std::int32_t);
1119 if (bufSize < payloadSize)
1120 return R__FAIL("too short locator");
1121
1122 locator.SetReserved(static_cast<std::uint32_t>(head >> 16) & 0xFF);
1123 switch (type) {
1124 case 0x01:
1127 break;
1128 case 0x02:
1131 break;
1132 default: locator.SetType(RNTupleLocator::kTypeUnknown);
1133 }
1134 bytes += payloadSize;
1135 } else {
1136 if (bufSize < sizeof(std::uint64_t))
1137 return R__FAIL("too short locator");
1138 std::uint64_t offset;
1139 bytes += DeserializeUInt64(bytes, offset);
1141 locator.SetNBytesOnStorage(head);
1142 locator.SetPosition(offset);
1143 }
1144
1145 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1146}
1147
1150{
1151 auto size = SerializeUInt64(envelopeLink.fLength, buffer);
1152 auto res =
1153 SerializeLocator(envelopeLink.fLocator, buffer ? reinterpret_cast<unsigned char *>(buffer) + size : nullptr);
1154 if (res)
1155 size += res.Unwrap();
1156 else
1157 return R__FORWARD_ERROR(res);
1158 return size;
1159}
1160
1162 std::uint64_t bufSize,
1164{
1165 if (bufSize < sizeof(std::int64_t))
1166 return R__FAIL("too short envelope link");
1167
1168 auto bytes = reinterpret_cast<const unsigned char *>(buffer);
1169 bytes += DeserializeUInt64(bytes, envelopeLink.fLength);
1170 bufSize -= sizeof(std::uint64_t);
1171 if (auto res = DeserializeLocator(bytes, bufSize, envelopeLink.fLocator)) {
1172 bytes += res.Unwrap();
1173 } else {
1174 return R__FORWARD_ERROR(res);
1175 }
1176 return bytes - reinterpret_cast<const unsigned char *>(buffer);
1177}
1178
1181{
1182 if (clusterSummary.fNEntries >= (static_cast<std::uint64_t>(1) << 56)) {
1183 return R__FAIL("number of entries in cluster exceeds maximum of 2^56");
1184 }
1185
1186 auto base = reinterpret_cast<unsigned char *>(buffer);
1187 auto pos = base;
1188 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1189
1190 auto frame = pos;
1191 pos += SerializeRecordFramePreamble(*where);
1192 pos += SerializeUInt64(clusterSummary.fFirstEntry, *where);
1193 const std::uint64_t nEntriesAndFlags =
1194 (static_cast<std::uint64_t>(clusterSummary.fFlags) << 56) | clusterSummary.fNEntries;
1195 pos += SerializeUInt64(nEntriesAndFlags, *where);
1196
1197 auto size = pos - frame;
1198 if (auto res = SerializeFramePostscript(frame, size)) {
1199 pos += res.Unwrap();
1200 } else {
1201 return R__FORWARD_ERROR(res);
1202 }
1203 return size;
1204}
1205
1209{
1210 auto base = reinterpret_cast<const unsigned char *>(buffer);
1211 auto bytes = base;
1212 std::uint64_t frameSize;
1213 if (auto res = DeserializeFrameHeader(bytes, bufSize, frameSize)) {
1214 bytes += res.Unwrap();
1215 } else {
1216 return R__FORWARD_ERROR(res);
1217 }
1218
1219 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
1220 if (fnFrameSizeLeft() < 2 * sizeof(std::uint64_t))
1221 return R__FAIL("too short cluster summary");
1222
1223 bytes += DeserializeUInt64(bytes, clusterSummary.fFirstEntry);
1224 std::uint64_t nEntriesAndFlags;
1225 bytes += DeserializeUInt64(bytes, nEntriesAndFlags);
1226
1227 const std::uint64_t nEntries = (nEntriesAndFlags << 8) >> 8;
1228 const std::uint8_t flags = nEntriesAndFlags >> 56;
1229
1230 if (flags & 0x01) {
1231 return R__FAIL("sharded cluster flag set in cluster summary; sharded clusters are currently unsupported.");
1232 }
1233
1234 clusterSummary.fNEntries = nEntries;
1235 clusterSummary.fFlags = flags;
1236
1237 return frameSize;
1238}
1239
1242{
1243 auto base = reinterpret_cast<unsigned char *>(buffer);
1244 auto pos = base;
1245 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1246
1247 auto frame = pos;
1248 pos += SerializeRecordFramePreamble(*where);
1249 pos += SerializeUInt64(clusterGroup.fMinEntry, *where);
1250 pos += SerializeUInt64(clusterGroup.fEntrySpan, *where);
1251 pos += SerializeUInt32(clusterGroup.fNClusters, *where);
1252 if (auto res = SerializeEnvelopeLink(clusterGroup.fPageListEnvelopeLink, *where)) {
1253 pos += res.Unwrap();
1254 } else {
1255 return R__FORWARD_ERROR(res);
1256 }
1257 auto size = pos - frame;
1258 if (auto res = SerializeFramePostscript(frame, size)) {
1259 return size;
1260 } else {
1261 return R__FORWARD_ERROR(res);
1262 }
1263}
1264
1266 std::uint64_t bufSize,
1268{
1269 auto base = reinterpret_cast<const unsigned char *>(buffer);
1270 auto bytes = base;
1271
1272 std::uint64_t frameSize;
1273 if (auto res = DeserializeFrameHeader(bytes, bufSize, frameSize)) {
1274 bytes += res.Unwrap();
1275 } else {
1276 return R__FORWARD_ERROR(res);
1277 }
1278
1279 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - base); };
1280 if (fnFrameSizeLeft() < sizeof(std::uint32_t) + 2 * sizeof(std::uint64_t))
1281 return R__FAIL("too short cluster group");
1282
1283 bytes += DeserializeUInt64(bytes, clusterGroup.fMinEntry);
1284 bytes += DeserializeUInt64(bytes, clusterGroup.fEntrySpan);
1285 bytes += DeserializeUInt32(bytes, clusterGroup.fNClusters);
1286 if (auto res = DeserializeEnvelopeLink(bytes, fnFrameSizeLeft(), clusterGroup.fPageListEnvelopeLink)) {
1287 bytes += res.Unwrap();
1288 } else {
1289 return R__FORWARD_ERROR(res);
1290 }
1291
1292 return frameSize;
1293}
1294
1296 bool forHeaderExtension)
1297{
1298 auto fieldZeroId = desc.GetFieldZeroId();
1299 auto depthFirstTraversal = [&](std::span<ROOT::DescriptorId_t> fieldTrees, auto doForEachField) {
1300 std::deque<ROOT::DescriptorId_t> idQueue{fieldTrees.begin(), fieldTrees.end()};
1301 while (!idQueue.empty()) {
1302 auto fieldId = idQueue.front();
1303 idQueue.pop_front();
1304 // Field zero has no physical representation nor columns of its own; recurse over its subfields only
1305 if (fieldId != fieldZeroId)
1307 unsigned i = 0;
1308 for (const auto &f : desc.GetFieldIterable(fieldId))
1309 idQueue.insert(idQueue.begin() + i++, f.GetId());
1310 }
1311 };
1312
1313 R__ASSERT(desc.GetNFields() > 0); // we must have at least a zero field
1314
1315 std::vector<ROOT::DescriptorId_t> fieldTrees;
1316 if (!forHeaderExtension) {
1317 fieldTrees.emplace_back(fieldZeroId);
1318 } else if (auto xHeader = desc.GetHeaderExtension()) {
1319 fieldTrees = xHeader->GetTopLevelFields(desc);
1320 }
1323 for (const auto &c : desc.GetColumnIterable(fieldId)) {
1324 if (!c.IsAliasColumn()) {
1325 MapPhysicalColumnId(c.GetPhysicalId());
1326 }
1327 }
1328 });
1329
1330 if (forHeaderExtension) {
1331 // Create physical IDs for column representations that extend fields of the regular header.
1332 // First the physical columns then the alias columns.
1333 for (auto memId : desc.GetHeaderExtension()->GetExtendedColumnRepresentations()) {
1334 const auto &columnDesc = desc.GetColumnDescriptor(memId);
1335 if (!columnDesc.IsAliasColumn()) {
1336 MapPhysicalColumnId(columnDesc.GetPhysicalId());
1337 }
1338 }
1339 }
1340}
1341
1344 const RContext &context, bool forHeaderExtension)
1345{
1346 auto base = reinterpret_cast<unsigned char *>(buffer);
1347 auto pos = base;
1348 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1349
1350 std::size_t nFields = 0, nColumns = 0, nAliasColumns = 0, fieldListOffset = 0;
1351 // Columns in the extension header that are attached to a field of the regular header
1352 std::vector<std::reference_wrapper<const ROOT::RColumnDescriptor>> extraColumns;
1353 if (forHeaderExtension) {
1354 // A call to `RNTupleDescriptorBuilder::BeginHeaderExtension()` is not strictly required after serializing the
1355 // header, which may happen, e.g., in unit tests. Ensure an empty schema extension is serialized in this case
1356 if (auto xHeader = desc.GetHeaderExtension()) {
1357 nFields = xHeader->GetNFields();
1358 nColumns = xHeader->GetNPhysicalColumns();
1359 nAliasColumns = xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns();
1360 fieldListOffset = desc.GetNFields() - nFields - 1;
1361
1362 extraColumns.reserve(xHeader->GetExtendedColumnRepresentations().size());
1363 for (auto columnId : xHeader->GetExtendedColumnRepresentations()) {
1364 extraColumns.emplace_back(desc.GetColumnDescriptor(columnId));
1365 }
1366 }
1367 } else {
1368 if (auto xHeader = desc.GetHeaderExtension()) {
1369 nFields = desc.GetNFields() - xHeader->GetNFields() - 1;
1370 nColumns = desc.GetNPhysicalColumns() - xHeader->GetNPhysicalColumns();
1372 (xHeader->GetNLogicalColumns() - xHeader->GetNPhysicalColumns());
1373 } else {
1374 nFields = desc.GetNFields() - 1;
1377 }
1378 }
1379 const auto nExtraTypeInfos = desc.GetNExtraTypeInfos();
1380 const auto &onDiskFields = context.GetOnDiskFieldList();
1382 std::span<const ROOT::DescriptorId_t> fieldList{onDiskFields.data() + fieldListOffset, nFields};
1383
1384 auto frame = pos;
1385 pos += SerializeListFramePreamble(nFields, *where);
1386 if (auto res = SerializeFieldList(desc, fieldList, /*firstOnDiskId=*/fieldListOffset, context, *where)) {
1387 pos += res.Unwrap();
1388 } else {
1389 return R__FORWARD_ERROR(res);
1390 }
1391 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1392 pos += res.Unwrap();
1393 } else {
1394 return R__FORWARD_ERROR(res);
1395 }
1396
1397 frame = pos;
1398 pos += SerializeListFramePreamble(nColumns, *where);
1399 if (auto res = SerializeColumnsOfFields(desc, fieldList, context, *where, forHeaderExtension)) {
1400 pos += res.Unwrap();
1401 } else {
1402 return R__FORWARD_ERROR(res);
1403 }
1404 for (const auto &c : extraColumns) {
1405 if (!c.get().IsAliasColumn()) {
1406 if (auto res = SerializePhysicalColumn(c.get(), context, *where)) {
1407 pos += res.Unwrap();
1408 } else {
1409 return R__FORWARD_ERROR(res);
1410 }
1411 }
1412 }
1413 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1414 pos += res.Unwrap();
1415 } else {
1416 return R__FORWARD_ERROR(res);
1417 }
1418
1419 frame = pos;
1420 pos += SerializeListFramePreamble(nAliasColumns, *where);
1422 for (const auto &c : extraColumns) {
1423 if (c.get().IsAliasColumn()) {
1424 pos += SerializeAliasColumn(c.get(), context, *where);
1425 }
1426 }
1427 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1428 pos += res.Unwrap();
1429 } else {
1430 return R__FORWARD_ERROR(res);
1431 }
1432
1433 frame = pos;
1434 // We only serialize the extra type info list in the header extension.
1435 if (forHeaderExtension) {
1436 pos += SerializeListFramePreamble(nExtraTypeInfos, *where);
1437 if (auto res = SerializeExtraTypeInfoList(desc, *where)) {
1438 pos += res.Unwrap();
1439 } else {
1440 return R__FORWARD_ERROR(res);
1441 }
1442 } else {
1443 pos += SerializeListFramePreamble(0, *where);
1444 }
1445 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1446 pos += res.Unwrap();
1447 } else {
1448 return R__FORWARD_ERROR(res);
1449 }
1450
1451 return static_cast<std::uint32_t>(pos - base);
1452}
1453
1457{
1458 auto base = reinterpret_cast<const unsigned char *>(buffer);
1459 auto bytes = base;
1460 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1461
1462 std::uint64_t frameSize;
1463 auto frame = bytes;
1464 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1465
1466 std::uint32_t nFields;
1467 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nFields)) {
1468 bytes += res.Unwrap();
1469 } else {
1470 return R__FORWARD_ERROR(res);
1471 }
1472 // The zero field is always added before `DeserializeSchemaDescription()` is called
1473 const std::uint32_t fieldIdRangeBegin = descBuilder.GetDescriptor().GetNFields() - 1;
1474 for (unsigned i = 0; i < nFields; ++i) {
1475 std::uint32_t fieldId = fieldIdRangeBegin + i;
1477 if (auto res = DeserializeField(bytes, fnFrameSizeLeft(), fieldBuilder)) {
1478 bytes += res.Unwrap();
1479 } else {
1480 return R__FORWARD_ERROR(res);
1481 }
1482 if (fieldId == fieldBuilder.GetParentId())
1483 fieldBuilder.ParentId(kZeroFieldId);
1484 auto fieldDesc = fieldBuilder.FieldId(fieldId).MakeDescriptor();
1485 if (!fieldDesc)
1487 const auto parentId = fieldDesc.Inspect().GetParentId();
1488 const auto projectionSourceId = fieldDesc.Inspect().GetProjectionSourceId();
1489 descBuilder.AddField(fieldDesc.Unwrap());
1490 auto resVoid = descBuilder.AddFieldLink(parentId, fieldId);
1491 if (!resVoid)
1492 return R__FORWARD_ERROR(resVoid);
1494 resVoid = descBuilder.AddFieldProjection(projectionSourceId, fieldId);
1495 if (!resVoid)
1496 return R__FORWARD_ERROR(resVoid);
1497 }
1498 }
1499 bytes = frame + frameSize;
1500
1501 // As columns are added in order of representation index and column index, determine the column index
1502 // for the currently deserialized column from the columns already added.
1504 std::uint16_t representationIndex) -> std::uint32_t {
1505 const auto &existingColumns = descBuilder.GetDescriptor().GetFieldDescriptor(fieldId).GetLogicalColumnIds();
1506 if (existingColumns.empty())
1507 return 0;
1508 const auto &lastColumnDesc = descBuilder.GetDescriptor().GetColumnDescriptor(existingColumns.back());
1509 return (representationIndex == lastColumnDesc.GetRepresentationIndex()) ? (lastColumnDesc.GetIndex() + 1) : 0;
1510 };
1511
1512 std::uint32_t nColumns;
1513 frame = bytes;
1514 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nColumns)) {
1515 bytes += res.Unwrap();
1516 } else {
1517 return R__FORWARD_ERROR(res);
1518 }
1519
1520 if (descBuilder.GetDescriptor().GetNLogicalColumns() > descBuilder.GetDescriptor().GetNPhysicalColumns())
1521 descBuilder.ShiftAliasColumns(nColumns);
1522
1523 const std::uint32_t columnIdRangeBegin = descBuilder.GetDescriptor().GetNPhysicalColumns();
1524 for (unsigned i = 0; i < nColumns; ++i) {
1525 std::uint32_t columnId = columnIdRangeBegin + i;
1528 bytes += res.Unwrap();
1529 } else {
1530 return R__FORWARD_ERROR(res);
1531 }
1532
1533 columnBuilder.Index(fnNextColumnIndex(columnBuilder.GetFieldId(), columnBuilder.GetRepresentationIndex()));
1534 columnBuilder.LogicalColumnId(columnId);
1535 columnBuilder.PhysicalColumnId(columnId);
1536 auto columnDesc = columnBuilder.MakeDescriptor();
1537 if (!columnDesc)
1539 auto resVoid = descBuilder.AddColumn(columnDesc.Unwrap());
1540 if (!resVoid)
1541 return R__FORWARD_ERROR(resVoid);
1542 }
1543 bytes = frame + frameSize;
1544
1545 std::uint32_t nAliasColumns;
1546 frame = bytes;
1547 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nAliasColumns)) {
1548 bytes += res.Unwrap();
1549 } else {
1550 return R__FORWARD_ERROR(res);
1551 }
1552 const std::uint32_t aliasColumnIdRangeBegin = descBuilder.GetDescriptor().GetNLogicalColumns();
1553 for (unsigned i = 0; i < nAliasColumns; ++i) {
1554 std::uint32_t physicalId;
1555 std::uint32_t fieldId;
1557 bytes += res.Unwrap();
1558 } else {
1559 return R__FORWARD_ERROR(res);
1560 }
1561
1563 columnBuilder.LogicalColumnId(aliasColumnIdRangeBegin + i).PhysicalColumnId(physicalId).FieldId(fieldId);
1564 const auto &physicalColumnDesc = descBuilder.GetDescriptor().GetColumnDescriptor(physicalId);
1565 columnBuilder.BitsOnStorage(physicalColumnDesc.GetBitsOnStorage());
1566 columnBuilder.ValueRange(physicalColumnDesc.GetValueRange());
1567 columnBuilder.Type(physicalColumnDesc.GetType());
1568 columnBuilder.RepresentationIndex(physicalColumnDesc.GetRepresentationIndex());
1569 columnBuilder.Index(fnNextColumnIndex(columnBuilder.GetFieldId(), columnBuilder.GetRepresentationIndex()));
1570
1571 auto aliasColumnDesc = columnBuilder.MakeDescriptor();
1572 if (!aliasColumnDesc)
1574 auto resVoid = descBuilder.AddColumn(aliasColumnDesc.Unwrap());
1575 if (!resVoid)
1576 return R__FORWARD_ERROR(resVoid);
1577 }
1578 bytes = frame + frameSize;
1579
1580 std::uint32_t nExtraTypeInfos;
1581 frame = bytes;
1582 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nExtraTypeInfos)) {
1583 bytes += res.Unwrap();
1584 } else {
1585 return R__FORWARD_ERROR(res);
1586 }
1587 for (unsigned i = 0; i < nExtraTypeInfos; ++i) {
1590 bytes += res.Unwrap();
1591 } else {
1592 return R__FORWARD_ERROR(res);
1593 }
1594
1595 auto extraTypeInfoDesc = extraTypeInfoBuilder.MoveDescriptor();
1596 // We ignore unknown extra type information
1598 descBuilder.AddExtraTypeInfo(extraTypeInfoDesc.Unwrap());
1599 }
1600 bytes = frame + frameSize;
1601
1602 return bytes - base;
1603}
1604
1607{
1608 RContext context;
1609
1610 auto base = reinterpret_cast<unsigned char *>(buffer);
1611 auto pos = base;
1612 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1613
1614 pos += SerializeEnvelopePreamble(kEnvelopeTypeHeader, *where);
1615 // So far we don't make use of feature flags
1616 if (auto res = SerializeFeatureFlags(desc.GetFeatureFlags(), *where)) {
1617 pos += res.Unwrap();
1618 } else {
1619 return R__FORWARD_ERROR(res);
1620 }
1621 pos += SerializeString(desc.GetName(), *where);
1622 pos += SerializeString(desc.GetDescription(), *where);
1623 pos += SerializeString(std::string("ROOT v") + ROOT_RELEASE, *where);
1624
1625 context.MapSchema(desc, /*forHeaderExtension=*/false);
1626
1627 if (auto res = SerializeSchemaDescription(*where, desc, context)) {
1628 pos += res.Unwrap();
1629 } else {
1630 return R__FORWARD_ERROR(res);
1631 }
1632
1633 std::uint64_t size = pos - base;
1634 std::uint64_t xxhash3 = 0;
1635 if (auto res = SerializeEnvelopePostscript(base, size, xxhash3)) {
1636 size += res.Unwrap();
1637 } else {
1638 return R__FORWARD_ERROR(res);
1639 }
1640
1641 context.SetHeaderSize(size);
1642 context.SetHeaderXxHash3(xxhash3);
1643 return context;
1644}
1645
1648 std::span<ROOT::DescriptorId_t> physClusterIDs,
1649 const RContext &context)
1650{
1651 auto base = reinterpret_cast<unsigned char *>(buffer);
1652 auto pos = base;
1653 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1654
1655 pos += SerializeEnvelopePreamble(kEnvelopeTypePageList, *where);
1656
1657 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1658
1659 // Cluster summaries
1660 const auto nClusters = physClusterIDs.size();
1661 auto clusterSummaryFrame = pos;
1662 pos += SerializeListFramePreamble(nClusters, *where);
1663 for (auto clusterId : physClusterIDs) {
1664 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1665 RClusterSummary summary{clusterDesc.GetFirstEntryIndex(), clusterDesc.GetNEntries(), 0};
1666 if (auto res = SerializeClusterSummary(summary, *where)) {
1667 pos += res.Unwrap();
1668 } else {
1669 return R__FORWARD_ERROR(res);
1670 }
1671 }
1672 if (auto res = SerializeFramePostscript(buffer ? clusterSummaryFrame : nullptr, pos - clusterSummaryFrame)) {
1673 pos += res.Unwrap();
1674 } else {
1675 return R__FORWARD_ERROR(res);
1676 }
1677
1678 // Page locations
1679 auto topMostFrame = pos;
1680 pos += SerializeListFramePreamble(nClusters, *where);
1681
1682 for (auto clusterId : physClusterIDs) {
1683 const auto &clusterDesc = desc.GetClusterDescriptor(context.GetMemClusterId(clusterId));
1684 // Get an ordered set of physical column ids
1685 std::set<ROOT::DescriptorId_t> onDiskColumnIds;
1686 for (const auto &columnRange : clusterDesc.GetColumnRangeIterable())
1687 onDiskColumnIds.insert(context.GetOnDiskColumnId(columnRange.GetPhysicalColumnId()));
1688
1689 auto outerFrame = pos;
1690 pos += SerializeListFramePreamble(onDiskColumnIds.size(), *where);
1691 for (auto onDiskId : onDiskColumnIds) {
1692 auto memId = context.GetMemColumnId(onDiskId);
1693 const auto &columnRange = clusterDesc.GetColumnRange(memId);
1694
1695 auto innerFrame = pos;
1696 if (columnRange.IsSuppressed()) {
1697 // Empty page range
1698 pos += SerializeListFramePreamble(0, *where);
1699 pos += SerializeInt64(kSuppressedColumnMarker, *where);
1700 } else {
1701 const auto &pageRange = clusterDesc.GetPageRange(memId);
1702 pos += SerializeListFramePreamble(pageRange.GetPageInfos().size(), *where);
1703
1704 for (const auto &pi : pageRange.GetPageInfos()) {
1705 std::int32_t nElements =
1706 pi.HasChecksum() ? -static_cast<std::int32_t>(pi.GetNElements()) : pi.GetNElements();
1707 pos += SerializeUInt32(nElements, *where);
1708 if (auto res = SerializeLocator(pi.GetLocator(), *where)) {
1709 pos += res.Unwrap();
1710 } else {
1711 return R__FORWARD_ERROR(res);
1712 }
1713 }
1714 pos += SerializeInt64(columnRange.GetFirstElementIndex(), *where);
1715 pos += SerializeUInt32(columnRange.GetCompressionSettings().value(), *where);
1716 }
1717
1718 if (auto res = SerializeFramePostscript(buffer ? innerFrame : nullptr, pos - innerFrame)) {
1719 pos += res.Unwrap();
1720 } else {
1721 return R__FORWARD_ERROR(res);
1722 }
1723 }
1724 if (auto res = SerializeFramePostscript(buffer ? outerFrame : nullptr, pos - outerFrame)) {
1725 pos += res.Unwrap();
1726 } else {
1727 return R__FORWARD_ERROR(res);
1728 }
1729 }
1730
1731 if (auto res = SerializeFramePostscript(buffer ? topMostFrame : nullptr, pos - topMostFrame)) {
1732 pos += res.Unwrap();
1733 } else {
1734 return R__FORWARD_ERROR(res);
1735 }
1736 std::uint64_t size = pos - base;
1737 if (auto res = SerializeEnvelopePostscript(base, size)) {
1738 size += res.Unwrap();
1739 } else {
1740 return R__FORWARD_ERROR(res);
1741 }
1742 return size;
1743}
1744
1746 const ROOT::RNTupleDescriptor &desc,
1747 const RContext &context)
1748{
1749 auto base = reinterpret_cast<unsigned char *>(buffer);
1750 auto pos = base;
1751 void **where = (buffer == nullptr) ? &buffer : reinterpret_cast<void **>(&pos);
1752
1753 pos += SerializeEnvelopePreamble(kEnvelopeTypeFooter, *where);
1754
1755 // So far we don't make use of footer feature flags
1756 if (auto res = SerializeFeatureFlags(std::vector<std::uint64_t>(), *where)) {
1757 pos += res.Unwrap();
1758 } else {
1759 return R__FORWARD_ERROR(res);
1760 }
1761 pos += SerializeUInt64(context.GetHeaderXxHash3(), *where);
1762
1763 // Schema extension, i.e. incremental changes with respect to the header
1764 auto frame = pos;
1765 pos += SerializeRecordFramePreamble(*where);
1766 if (auto res = SerializeSchemaDescription(*where, desc, context, /*forHeaderExtension=*/true)) {
1767 pos += res.Unwrap();
1768 } else {
1769 return R__FORWARD_ERROR(res);
1770 }
1771 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1772 pos += res.Unwrap();
1773 } else {
1774 return R__FORWARD_ERROR(res);
1775 }
1776
1777 // Cluster groups
1778 frame = pos;
1779 const auto nClusterGroups = desc.GetNClusterGroups();
1780 pos += SerializeListFramePreamble(nClusterGroups, *where);
1781 for (unsigned int i = 0; i < nClusterGroups; ++i) {
1782 const auto &cgDesc = desc.GetClusterGroupDescriptor(context.GetMemClusterGroupId(i));
1784 clusterGroup.fMinEntry = cgDesc.GetMinEntry();
1785 clusterGroup.fEntrySpan = cgDesc.GetEntrySpan();
1786 clusterGroup.fNClusters = cgDesc.GetNClusters();
1787 clusterGroup.fPageListEnvelopeLink.fLength = cgDesc.GetPageListLength();
1788 clusterGroup.fPageListEnvelopeLink.fLocator = cgDesc.GetPageListLocator();
1789 if (auto res = SerializeClusterGroup(clusterGroup, *where)) {
1790 pos += res.Unwrap();
1791 } else {
1792 return R__FORWARD_ERROR(res);
1793 }
1794 }
1795 if (auto res = SerializeFramePostscript(buffer ? frame : nullptr, pos - frame)) {
1796 pos += res.Unwrap();
1797 } else {
1798 return R__FORWARD_ERROR(res);
1799 }
1800
1801 std::uint32_t size = pos - base;
1802 if (auto res = SerializeEnvelopePostscript(base, size)) {
1803 size += res.Unwrap();
1804 } else {
1805 return R__FORWARD_ERROR(res);
1806 }
1807 return size;
1808}
1809
1812{
1813 auto base = reinterpret_cast<const unsigned char *>(buffer);
1814 auto bytes = base;
1815 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1816
1817 std::uint64_t xxhash3{0};
1818 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeHeader, xxhash3)) {
1819 bytes += res.Unwrap();
1820 } else {
1821 return R__FORWARD_ERROR(res);
1822 }
1823 descBuilder.SetOnDiskHeaderXxHash3(xxhash3);
1824
1825 std::vector<std::uint64_t> featureFlags;
1826 if (auto res = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags)) {
1827 bytes += res.Unwrap();
1828 } else {
1829 return R__FORWARD_ERROR(res);
1830 }
1831 for (std::size_t i = 0; i < featureFlags.size(); ++i) {
1832 if (!featureFlags[i])
1833 continue;
1834 unsigned int bit = 0;
1835 while (!(featureFlags[i] & (static_cast<uint64_t>(1) << bit)))
1836 bit++;
1837 return R__FAIL("unsupported format feature: " + std::to_string(i * 64 + bit));
1838 }
1839
1840 std::string name;
1841 std::string description;
1842 std::string writer;
1843 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), name)) {
1844 bytes += res.Unwrap();
1845 } else {
1846 return R__FORWARD_ERROR(res);
1847 }
1848 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), description)) {
1849 bytes += res.Unwrap();
1850 } else {
1851 return R__FORWARD_ERROR(res);
1852 }
1853 if (auto res = DeserializeString(bytes, fnBufSizeLeft(), writer)) {
1854 bytes += res.Unwrap();
1855 } else {
1856 return R__FORWARD_ERROR(res);
1857 }
1858 descBuilder.SetNTuple(name, description);
1859
1860 // Zero field
1862 .FieldId(kZeroFieldId)
1864 .MakeDescriptor()
1865 .Unwrap());
1866 if (auto res = DeserializeSchemaDescription(bytes, fnBufSizeLeft(), descBuilder)) {
1867 return RResult<void>::Success();
1868 } else {
1869 return R__FORWARD_ERROR(res);
1870 }
1871}
1872
1875{
1876 auto base = reinterpret_cast<const unsigned char *>(buffer);
1877 auto bytes = base;
1878 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1879 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypeFooter)) {
1880 bytes += res.Unwrap();
1881 } else {
1882 return R__FORWARD_ERROR(res);
1883 }
1884
1885 std::vector<std::uint64_t> featureFlags;
1886 if (auto res = DeserializeFeatureFlags(bytes, fnBufSizeLeft(), featureFlags)) {
1887 bytes += res.Unwrap();
1888 } else {
1889 return R__FORWARD_ERROR(res);
1890 }
1891 for (auto f : featureFlags) {
1892 if (f)
1893 R__LOG_WARNING(ROOT::Internal::NTupleLog()) << "Unsupported feature flag! " << f;
1894 }
1895
1896 std::uint64_t xxhash3{0};
1897 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1898 return R__FAIL("footer too short");
1899 bytes += DeserializeUInt64(bytes, xxhash3);
1900 if (xxhash3 != descBuilder.GetDescriptor().GetOnDiskHeaderXxHash3())
1901 return R__FAIL("XxHash-3 mismatch between header and footer");
1902
1903 std::uint64_t frameSize;
1904 auto frame = bytes;
1905 auto fnFrameSizeLeft = [&]() { return frameSize - (bytes - frame); };
1906
1907 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize)) {
1908 bytes += res.Unwrap();
1909 } else {
1910 return R__FORWARD_ERROR(res);
1911 }
1912 if (fnFrameSizeLeft() > 0) {
1913 descBuilder.BeginHeaderExtension();
1914 if (auto res = DeserializeSchemaDescription(bytes, fnFrameSizeLeft(), descBuilder); !res) {
1915 return R__FORWARD_ERROR(res);
1916 }
1917 }
1918 bytes = frame + frameSize;
1919
1920 std::uint32_t nClusterGroups;
1921 frame = bytes;
1922 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), frameSize, nClusterGroups)) {
1923 bytes += res.Unwrap();
1924 } else {
1925 return R__FORWARD_ERROR(res);
1926 }
1927 for (std::uint32_t groupId = 0; groupId < nClusterGroups; ++groupId) {
1929 if (auto res = DeserializeClusterGroup(bytes, fnFrameSizeLeft(), clusterGroup)) {
1930 bytes += res.Unwrap();
1931 } else {
1932 return R__FORWARD_ERROR(res);
1933 }
1934
1935 descBuilder.AddToOnDiskFooterSize(clusterGroup.fPageListEnvelopeLink.fLocator.GetNBytesOnStorage());
1937 clusterGroupBuilder.ClusterGroupId(groupId)
1938 .PageListLocator(clusterGroup.fPageListEnvelopeLink.fLocator)
1939 .PageListLength(clusterGroup.fPageListEnvelopeLink.fLength)
1940 .MinEntry(clusterGroup.fMinEntry)
1941 .EntrySpan(clusterGroup.fEntrySpan)
1942 .NClusters(clusterGroup.fNClusters);
1943 descBuilder.AddClusterGroup(clusterGroupBuilder.MoveDescriptor().Unwrap());
1944 }
1945 bytes = frame + frameSize;
1946
1947 return RResult<void>::Success();
1948}
1949
1953 const ROOT::RNTupleDescriptor &desc)
1954{
1955 auto base = reinterpret_cast<const unsigned char *>(buffer);
1956 auto bytes = base;
1957 auto fnBufSizeLeft = [&]() { return bufSize - (bytes - base); };
1958
1959 if (auto res = DeserializeEnvelope(bytes, fnBufSizeLeft(), kEnvelopeTypePageList)) {
1960 bytes += res.Unwrap();
1961 } else {
1962 return R__FORWARD_ERROR(res);
1963 }
1964
1965 std::uint64_t xxhash3{0};
1966 if (fnBufSizeLeft() < static_cast<int>(sizeof(std::uint64_t)))
1967 return R__FAIL("page list too short");
1968 bytes += DeserializeUInt64(bytes, xxhash3);
1969 if (xxhash3 != desc.GetOnDiskHeaderXxHash3())
1970 return R__FAIL("XxHash-3 mismatch between header and page list");
1971
1972 std::vector<RClusterDescriptorBuilder> clusterBuilders;
1974 for (ROOT::DescriptorId_t i = 0; i < clusterGroupId; ++i) {
1975 firstClusterId = firstClusterId + desc.GetClusterGroupDescriptor(i).GetNClusters();
1976 }
1977
1978 std::uint64_t clusterSummaryFrameSize;
1981
1982 std::uint32_t nClusterSummaries;
1983 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), clusterSummaryFrameSize, nClusterSummaries)) {
1984 bytes += res.Unwrap();
1985 } else {
1986 return R__FORWARD_ERROR(res);
1987 }
1990 if (auto res = DeserializeClusterSummary(bytes, fnClusterSummaryFrameSizeLeft(), clusterSummary)) {
1991 bytes += res.Unwrap();
1992 } else {
1993 return R__FORWARD_ERROR(res);
1994 }
1995
1998 clusterBuilders.emplace_back(std::move(builder));
1999 }
2001
2002 std::uint64_t topMostFrameSize;
2003 auto topMostFrame = bytes;
2004 auto fnTopMostFrameSizeLeft = [&]() { return topMostFrameSize - (bytes - topMostFrame); };
2005
2006 std::uint32_t nClusters;
2007 if (auto res = DeserializeFrameHeader(bytes, fnBufSizeLeft(), topMostFrameSize, nClusters)) {
2008 bytes += res.Unwrap();
2009 } else {
2010 return R__FORWARD_ERROR(res);
2011 }
2012
2014 return R__FAIL("mismatch between number of clusters and number of cluster summaries");
2015
2016 for (std::uint32_t i = 0; i < nClusters; ++i) {
2017 std::uint64_t outerFrameSize;
2018 auto outerFrame = bytes;
2019 auto fnOuterFrameSizeLeft = [&]() { return outerFrameSize - (bytes - outerFrame); };
2020
2021 std::uint32_t nColumns;
2022 if (auto res = DeserializeFrameHeader(bytes, fnTopMostFrameSizeLeft(), outerFrameSize, nColumns)) {
2023 bytes += res.Unwrap();
2024 } else {
2025 return R__FORWARD_ERROR(res);
2026 }
2027
2028 for (std::uint32_t j = 0; j < nColumns; ++j) {
2029 std::uint64_t innerFrameSize;
2030 auto innerFrame = bytes;
2031 auto fnInnerFrameSizeLeft = [&]() { return innerFrameSize - (bytes - innerFrame); };
2032
2033 std::uint32_t nPages;
2034 if (auto res = DeserializeFrameHeader(bytes, fnOuterFrameSizeLeft(), innerFrameSize, nPages)) {
2035 bytes += res.Unwrap();
2036 } else {
2037 return R__FORWARD_ERROR(res);
2038 }
2039
2041 pageRange.SetPhysicalColumnId(j);
2042 for (std::uint32_t k = 0; k < nPages; ++k) {
2043 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
2044 return R__FAIL("inner frame too short");
2045 std::int32_t nElements;
2046 bool hasChecksum = false;
2048 bytes += DeserializeInt32(bytes, nElements);
2049 if (nElements < 0) {
2051 hasChecksum = true;
2052 }
2053 if (auto res = DeserializeLocator(bytes, fnInnerFrameSizeLeft(), locator)) {
2054 bytes += res.Unwrap();
2055 } else {
2056 return R__FORWARD_ERROR(res);
2057 }
2058 pageRange.GetPageInfos().push_back({static_cast<std::uint32_t>(nElements), locator, hasChecksum});
2059 }
2060
2061 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::int64_t)))
2062 return R__FAIL("page list frame too short");
2063 std::int64_t columnOffset;
2064 bytes += DeserializeInt64(bytes, columnOffset);
2065 if (columnOffset < 0) {
2066 if (nPages > 0)
2067 return R__FAIL("unexpected non-empty page list");
2068 clusterBuilders[i].MarkSuppressedColumnRange(j);
2069 } else {
2070 if (fnInnerFrameSizeLeft() < static_cast<int>(sizeof(std::uint32_t)))
2071 return R__FAIL("page list frame too short");
2072 std::uint32_t compressionSettings;
2073 bytes += DeserializeUInt32(bytes, compressionSettings);
2075 }
2076
2078 } // loop over columns
2079
2081 } // loop over clusters
2082
2083 return clusterBuilders;
2084}
2085
2090{
2092 if (!clusterBuildersRes)
2094
2095 auto clusterBuilders = clusterBuildersRes.Unwrap();
2096
2097 std::vector<ROOT::RClusterDescriptor> clusters;
2098 clusters.reserve(clusterBuilders.size());
2099
2100 // Conditionally fixup the clusters depending on the attach purpose
2101 switch (mode) {
2102 case EDescriptorDeserializeMode::kForReading:
2103 for (auto &builder : clusterBuilders) {
2104 if (auto res = builder.CommitSuppressedColumnRanges(desc); !res)
2105 return R__FORWARD_RESULT(res);
2106 builder.AddExtendedColumnRanges(desc);
2107 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2108 }
2109 break;
2110 case EDescriptorDeserializeMode::kForWriting:
2111 for (auto &builder : clusterBuilders) {
2112 if (auto res = builder.CommitSuppressedColumnRanges(desc); !res)
2113 return R__FORWARD_RESULT(res);
2114 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2115 }
2116 break;
2117 case EDescriptorDeserializeMode::kRaw:
2118 for (auto &builder : clusterBuilders)
2119 clusters.emplace_back(builder.MoveDescriptor().Unwrap());
2120 break;
2121 }
2122
2124
2125 return RResult<void>::Success();
2126}
2127
2129{
2131 for (auto si : infos) {
2132 assert(si.first == si.second->GetNumber());
2133 streamerInfos.Add(si.second);
2134 }
2136 buffer.WriteObject(&streamerInfos);
2137 assert(buffer.Length() > 0);
2138 return std::string{buffer.Buffer(), static_cast<UInt_t>(buffer.Length())};
2139}
2140
2143{
2145
2146 TBufferFile buffer(TBuffer::kRead, extraTypeInfoContent.length(), const_cast<char *>(extraTypeInfoContent.data()),
2147 false /* adopt */);
2148 auto infoList = reinterpret_cast<TList *>(buffer.ReadObject(TList::Class()));
2149
2150 TObjLink *lnk = infoList->FirstLink();
2151 while (lnk) {
2152 auto info = reinterpret_cast<TStreamerInfo *>(lnk->GetObject());
2153 info->BuildCheck();
2154 infoMap[info->GetNumber()] = info->GetClass()->GetStreamerInfo(info->GetClassVersion());
2155 assert(info->GetNumber() == infoMap[info->GetNumber()]->GetNumber());
2156 lnk = lnk->Next();
2157 }
2158
2159 delete infoList;
2160
2161 return infoMap;
2162}
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
Definition RError.hxx:303
#define R__FORWARD_RESULT(res)
Short-hand to return an RResult<T> value from a subroutine to the calling stack frame.
Definition RError.hxx:301
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
Definition RError.hxx:299
#define R__LOG_WARNING(...)
Definition RLogger.hxx:358
#define R__LOG_DEBUG(DEBUGLEVEL,...)
Definition RLogger.hxx:360
#define f(i)
Definition RSha256.hxx:104
#define c(i)
Definition RSha256.hxx:101
#define ROOT_RELEASE
Definition RVersion.hxx:44
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
ROOT::Detail::TRangeCast< T, true > TRangeDynCast
TRangeDynCast is an adapter class that allows the typed iteration through a TCollection.
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t nitems
Option_t Option_t TPoint TPoint const char mode
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition TGX11.cxx:110
The available trivial, native content types of a column.
A helper class for piece-wise construction of an RClusterDescriptor.
RClusterDescriptorBuilder & NEntries(std::uint64_t nEntries)
RClusterDescriptorBuilder & ClusterId(ROOT::DescriptorId_t clusterId)
RClusterDescriptorBuilder & FirstEntryIndex(std::uint64_t firstEntryIndex)
A helper class for piece-wise construction of an RClusterGroupDescriptor.
A helper class for piece-wise construction of an RColumnDescriptor.
A helper class for piece-wise construction of an RExtraTypeInfoDescriptor.
RExtraTypeInfoDescriptorBuilder & ContentId(EExtraTypeInfoIds contentId)
RExtraTypeInfoDescriptorBuilder & TypeName(const std::string &typeName)
RExtraTypeInfoDescriptorBuilder & Content(const std::string &content)
RExtraTypeInfoDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
A helper class for piece-wise construction of an RFieldDescriptor.
A helper class for piece-wise construction of an RNTupleDescriptor.
The serialization context is used for the piecewise serialization of a descriptor.
ROOT::DescriptorId_t GetOnDiskFieldId(ROOT::DescriptorId_t memId) const
ROOT::DescriptorId_t GetMemColumnId(ROOT::DescriptorId_t onDiskId) const
ROOT::DescriptorId_t GetMemClusterGroupId(ROOT::DescriptorId_t onDiskId) const
ROOT::DescriptorId_t GetOnDiskColumnId(ROOT::DescriptorId_t memId) const
void MapSchema(const RNTupleDescriptor &desc, bool forHeaderExtension)
Map in-memory field and column IDs to their on-disk counterparts.
ROOT::DescriptorId_t GetMemClusterId(ROOT::DescriptorId_t onDiskId) const
const std::vector< ROOT::DescriptorId_t > & GetOnDiskFieldList() const
Return a vector containing the in-memory field ID for each on-disk counterpart, in order,...
A helper class for serializing and deserialization of the RNTuple binary format.
static std::uint32_t SerializeXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3, void *buffer)
Writes a XxHash-3 64bit checksum of the byte range given by data and length.
static RResult< std::vector< ROOT::Internal::RClusterDescriptorBuilder > > DeserializePageListRaw(const void *buffer, std::uint64_t bufSize, ROOT::DescriptorId_t clusterGroupId, const RNTupleDescriptor &desc)
static RResult< std::uint32_t > SerializeSchemaDescription(void *buffer, const RNTupleDescriptor &desc, const RContext &context, bool forHeaderExtension=false)
Serialize the schema description in desc into buffer.
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint64_t bufSize, std::string &val)
static RResult< std::uint32_t > SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer)
static std::uint32_t SerializeInt32(std::int32_t val, void *buffer)
static RResult< std::uint32_t > DeserializeEnvelopeLink(const void *buffer, std::uint64_t bufSize, REnvelopeLink &envelopeLink)
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
static RResult< std::uint32_t > SerializeFieldStructure(ROOT::ENTupleStructure structure, void *buffer)
While we could just interpret the enums as ints, we make the translation explicit in order to avoid a...
static RResult< std::uint32_t > SerializeEnvelopePostscript(unsigned char *envelope, std::uint64_t size)
static RResult< std::uint32_t > SerializeFeatureFlags(const std::vector< std::uint64_t > &flags, void *buffer)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static RResult< std::uint32_t > DeserializeFrameHeader(const void *buffer, std::uint64_t bufSize, std::uint64_t &frameSize, std::uint32_t &nitems)
static RResult< std::uint32_t > DeserializeEnvelope(const void *buffer, std::uint64_t bufSize, std::uint16_t expectedType)
static RResult< std::uint32_t > SerializeColumnType(ROOT::ENTupleColumnType type, void *buffer)
static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer)
static std::uint32_t SerializeInt16(std::int16_t val, void *buffer)
static RResult< std::uint32_t > SerializeFramePostscript(void *frame, std::uint64_t size)
static RResult< std::uint32_t > DeserializeClusterGroup(const void *buffer, std::uint64_t bufSize, RClusterGroup &clusterGroup)
static RResult< std::uint32_t > DeserializeLocator(const void *buffer, std::uint64_t bufSize, RNTupleLocator &locator)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static RResult< void > DeserializePageList(const void *buffer, std::uint64_t bufSize, ROOT::DescriptorId_t clusterGroupId, RNTupleDescriptor &desc, EDescriptorDeserializeMode mode)
static RResult< void > DeserializeFooter(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val)
static std::uint32_t SerializeEnvelopePreamble(std::uint16_t envelopeType, void *buffer)
static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
std::map< Int_t, TVirtualStreamerInfo * > StreamerInfoMap_t
static RResult< std::uint32_t > SerializeExtraTypeInfoId(ROOT::EExtraTypeInfoIds id, void *buffer)
static RResult< StreamerInfoMap_t > DeserializeStreamerInfos(const std::string &extraTypeInfoContent)
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > VerifyXxHash3(const unsigned char *data, std::uint64_t length, std::uint64_t &xxhash3)
Expects an xxhash3 checksum in the 8 bytes following data + length and verifies it.
static RResult< std::uint32_t > SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer)
static RResult< std::uint32_t > DeserializeColumnType(const void *buffer, ROOT::ENTupleColumnType &type)
static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val)
static RResult< std::uint32_t > DeserializeExtraTypeInfoId(const void *buffer, ROOT::EExtraTypeInfoIds &id)
static RResult< std::uint32_t > DeserializeClusterSummary(const void *buffer, std::uint64_t bufSize, RClusterSummary &clusterSummary)
static RResult< std::uint32_t > SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer)
static std::uint32_t SerializeRecordFramePreamble(void *buffer)
static RResult< std::uint32_t > SerializePageList(void *buffer, const RNTupleDescriptor &desc, std::span< ROOT::DescriptorId_t > physClusterIDs, const RContext &context)
static RResult< std::uint32_t > DeserializeFieldStructure(const void *buffer, ROOT::ENTupleStructure &structure)
static std::uint32_t SerializeInt64(std::int64_t val, void *buffer)
static RResult< void > DeserializeHeader(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static RResult< std::uint32_t > SerializeFooter(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static RResult< std::uint32_t > SerializeLocator(const RNTupleLocator &locator, void *buffer)
static RResult< std::uint32_t > DeserializeSchemaDescription(const void *buffer, std::uint64_t bufSize, ROOT::Internal::RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static RResult< std::uint32_t > DeserializeFeatureFlags(const void *buffer, std::uint64_t bufSize, std::vector< std::uint64_t > &flags)
static RResult< RContext > SerializeHeader(void *buffer, const RNTupleDescriptor &desc)
static std::string SerializeStreamerInfos(const StreamerInfoMap_t &infos)
Records the partition of data into pages for a particular column in a particular cluster.
Metadata stored for every column of an RNTuple.
Field specific extra type information from the header / extenstion header.
const std::string & GetContent() const
const std::string & GetTypeName() const
EExtraTypeInfoIds GetContentId() const
Metadata stored for every field of an RNTuple.
The on-storage metadata of an RNTuple.
const RClusterGroupDescriptor & GetClusterGroupDescriptor(ROOT::DescriptorId_t clusterGroupId) const
const RColumnDescriptor & GetColumnDescriptor(ROOT::DescriptorId_t columnId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
const RFieldDescriptor & GetFieldDescriptor(ROOT::DescriptorId_t fieldId) const
std::size_t GetNExtraTypeInfos() const
RColumnDescriptorIterable GetColumnIterable() const
const std::string & GetName() const
std::vector< std::uint64_t > GetFeatureFlags() const
ROOT::DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level RNTuple data fields.
std::size_t GetNPhysicalColumns() const
const RHeaderExtension * GetHeaderExtension() const
Return header extension information; if the descriptor does not have a header extension,...
const RClusterDescriptor & GetClusterDescriptor(ROOT::DescriptorId_t clusterId) const
std::uint64_t GetOnDiskHeaderXxHash3() const
std::size_t GetNFields() const
RResult< void > AddClusterGroupDetails(ROOT::DescriptorId_t clusterGroupId, std::vector< RClusterDescriptor > &clusterDescs)
Methods to load and drop cluster group details (cluster IDs and page locations)
std::size_t GetNLogicalColumns() const
std::size_t GetNClusterGroups() const
const std::string & GetDescription() const
RNTupleLocator payload that is common for object stores using 64bit location information.
Generic information about the physical location of data.
const_iterator begin() const
const_iterator end() const
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
Definition RError.hxx:197
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition TBufferFile.h:47
TObject * ReadObject(const TClass *cl) override
Read object from I/O buffer.
void WriteObject(const TObject *obj, Bool_t cacheReuse=kTRUE) override
Write object to I/O buffer.
@ kWrite
Definition TBuffer.h:73
@ kRead
Definition TBuffer.h:73
Int_t Length() const
Definition TBuffer.h:100
char * Buffer() const
Definition TBuffer.h:96
A doubly linked list.
Definition TList.h:38
static TClass * Class()
Describes a persistent version of a class.
constexpr ROOT::ENTupleStructure kTestFutureFieldStructure
ROOT::RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
constexpr ENTupleColumnType kTestFutureColumnType
constexpr RNTupleLocator::ELocatorType kTestLocatorType
EExtraTypeInfoIds
Used in RExtraTypeInfoDescriptor.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
constexpr DescriptorId_t kInvalidDescriptorId
ENTupleStructure
The fields in the ntuple model tree can carry different structural information about the type system.
ENTupleColumnType