28#include <unordered_map>
36std::uint32_t SerializeFieldV1(
41 auto base =
reinterpret_cast<unsigned char *
>(buffer);
43 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
45 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
47 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.
GetFieldVersion(), *where);
48 pos += RNTupleSerializer::SerializeUInt32(fieldDesc.
GetTypeVersion(), *where);
49 pos += RNTupleSerializer::SerializeUInt32(physParentId, *where);
50 pos += RNTupleSerializer::SerializeFieldStructure(fieldDesc.
GetStructure(), *where);
52 pos += RNTupleSerializer::SerializeUInt16(RNTupleSerializer::kFlagRepetitiveField, *where);
53 pos += RNTupleSerializer::SerializeUInt64(fieldDesc.
GetNRepetitions(), *where);
55 pos += RNTupleSerializer::SerializeUInt16(0, *where);
57 pos += RNTupleSerializer::SerializeString(fieldDesc.
GetFieldName(), *where);
58 pos += RNTupleSerializer::SerializeString(fieldDesc.
GetTypeName(), *where);
59 pos += RNTupleSerializer::SerializeString(
"" , *where);
62 auto size = pos - base;
63 RNTupleSerializer::SerializeFramePostscript(base,
size);
69std::uint32_t SerializeFieldTree(
74 auto base =
reinterpret_cast<unsigned char *
>(buffer);
76 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
78 std::deque<ROOT::Experimental::DescriptorId_t> idQueue{desc.
GetFieldZeroId()};
80 while (!idQueue.empty()) {
81 auto parentId = idQueue.front();
87 pos += SerializeFieldV1(
f, physParentId, *where);
88 idQueue.push_back(
f.GetId());
97 std::uint32_t bufSize,
103 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
105 std::uint32_t frameSize;
106 auto fnFrameSizeLeft = [&]() {
return frameSize -
static_cast<std::uint32_t
>(
bytes - base); };
107 auto result = RNTupleSerializer::DeserializeFrameHeader(
bytes, bufSize, frameSize);
112 std::uint32_t fieldVersion;
113 std::uint32_t typeVersion;
114 std::uint32_t parentId;
118 if (fnFrameSizeLeft() < 3 *
sizeof(std::uint32_t) +
119 RNTupleSerializer::SerializeFieldStructure(structure,
nullptr) +
120 sizeof(std::uint16_t))
122 return R__FAIL(
"field record frame too short");
124 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fieldVersion);
125 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, typeVersion);
126 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, parentId);
127 auto res16 = RNTupleSerializer::DeserializeFieldStructure(
bytes, structure);
130 bytes += res16.Unwrap();
131 bytes += RNTupleSerializer::DeserializeUInt16(
bytes, flags);
134 if (flags & RNTupleSerializer::kFlagRepetitiveField) {
135 if (fnFrameSizeLeft() <
sizeof(std::uint64_t))
136 return R__FAIL(
"field record frame too short");
137 std::uint64_t nRepetitions;
138 bytes += RNTupleSerializer::DeserializeUInt64(
bytes, nRepetitions);
142 std::string fieldName;
143 std::string typeName;
144 std::string aliasName;
145 std::string description;
146 result = RNTupleSerializer::DeserializeString(
bytes, fnFrameSizeLeft(), fieldName).Unwrap();
150 result = RNTupleSerializer::DeserializeString(
bytes, fnFrameSizeLeft(), typeName).Unwrap();
154 result = RNTupleSerializer::DeserializeString(
bytes, fnFrameSizeLeft(), aliasName).Unwrap();
158 result = RNTupleSerializer::DeserializeString(
bytes, fnFrameSizeLeft(), description).Unwrap();
167std::uint32_t SerializeColumnListV1(
175 auto base =
reinterpret_cast<unsigned char *
>(buffer);
177 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
179 std::deque<ROOT::Experimental::DescriptorId_t> idQueue{desc.
GetFieldZeroId()};
181 while (!idQueue.empty()) {
182 auto parentId = idQueue.front();
187 pos += RNTupleSerializer::SerializeRecordFramePreamble(*where);
189 auto type =
c.GetModel().GetType();
190 pos += RNTupleSerializer::SerializeColumnType(
type, *where);
191 pos += RNTupleSerializer::SerializeUInt16(RColumnElementBase::GetBitsOnStorage(
type), *where);
192 pos += RNTupleSerializer::SerializeUInt32(context.
GetPhysFieldId(
c.GetFieldId()), *where);
193 std::uint32_t flags = 0;
195 if (
c.GetModel().GetIsSorted())
196 flags |= RNTupleSerializer::kFlagSortAscColumn;
199 flags |= RNTupleSerializer::kFlagNonNegativeColumn;
200 pos += RNTupleSerializer::SerializeUInt32(flags, *where);
202 pos += RNTupleSerializer::SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
208 idQueue.push_back(
f.GetId());
216 std::uint32_t bufSize,
222 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
224 std::uint32_t frameSize;
225 auto fnFrameSizeLeft = [&]() {
return frameSize -
static_cast<std::uint32_t
>(
bytes - base); };
226 auto result = RNTupleSerializer::DeserializeFrameHeader(
bytes, bufSize, frameSize);
233 std::uint16_t bitsOnStorage;
234 std::uint32_t fieldId;
236 if (fnFrameSizeLeft() < RNTupleSerializer::SerializeColumnType(
type,
nullptr) +
237 sizeof(std::uint16_t) + 2 *
sizeof(std::uint32_t))
239 return R__FAIL(
"column record frame too short");
241 auto res16 = RNTupleSerializer::DeserializeColumnType(
bytes,
type);
244 bytes += res16.Unwrap();
245 bytes += RNTupleSerializer::DeserializeUInt16(
bytes, bitsOnStorage);
246 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, fieldId);
247 bytes += RNTupleSerializer::DeserializeUInt32(
bytes, flags);
250 return R__FAIL(
"column element size mismatch");
252 const bool isSorted = (flags & (RNTupleSerializer::kFlagSortAscColumn | RNTupleSerializer::kFlagSortDesColumn));
262 const unsigned char *
data, std::uint32_t
length, std::uint32_t &crc32,
void *buffer)
264 if (buffer !=
nullptr) {
265 crc32 = R__crc32(0,
nullptr, 0);
273 const unsigned char *
data, std::uint32_t
length, std::uint32_t &crc32)
275 auto checksumReal = R__crc32(0,
nullptr, 0);
276 checksumReal = R__crc32(checksumReal,
data,
length);
278 if (crc32 != checksumReal)
279 return R__FAIL(
"CRC32 checksum mismatch");
285 const unsigned char *
data, std::uint32_t
length)
294 if (buffer !=
nullptr) {
295 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
296 bytes[0] = (val & 0x00FF);
297 bytes[1] = (val & 0xFF00) >> 8;
304 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
305 val = std::int16_t(
bytes[0]) + (std::int16_t(
bytes[1]) << 8);
311 return SerializeInt16(val, buffer);
316 return DeserializeInt16(buffer, *
reinterpret_cast<std::int16_t *
>(&val));
321 if (buffer !=
nullptr) {
322 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
323 bytes[0] = (val & 0x000000FF);
324 bytes[1] = (val & 0x0000FF00) >> 8;
325 bytes[2] = (val & 0x00FF0000) >> 16;
326 bytes[3] = (val & 0xFF000000) >> 24;
333 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
334 val = std::int32_t(
bytes[0]) + (std::int32_t(
bytes[1]) << 8) +
335 (std::int32_t(
bytes[2]) << 16) + (std::int32_t(
bytes[3]) << 24);
341 return SerializeInt32(val, buffer);
346 return DeserializeInt32(buffer, *
reinterpret_cast<std::int32_t *
>(&val));
351 if (buffer !=
nullptr) {
352 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
353 bytes[0] = (val & 0x00000000000000FF);
354 bytes[1] = (val & 0x000000000000FF00) >> 8;
355 bytes[2] = (val & 0x0000000000FF0000) >> 16;
356 bytes[3] = (val & 0x00000000FF000000) >> 24;
357 bytes[4] = (val & 0x000000FF00000000) >> 32;
358 bytes[5] = (val & 0x0000FF0000000000) >> 40;
359 bytes[6] = (val & 0x00FF000000000000) >> 48;
360 bytes[7] = (val & 0xFF00000000000000) >> 56;
367 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
368 val = std::int64_t(
bytes[0]) + (std::int64_t(
bytes[1]) << 8) +
369 (std::int64_t(
bytes[2]) << 16) + (std::int64_t(
bytes[3]) << 24) +
370 (std::int64_t(
bytes[4]) << 32) + (std::int64_t(
bytes[5]) << 40) +
371 (std::int64_t(
bytes[6]) << 48) + (std::int64_t(
bytes[7]) << 56);
377 return SerializeInt64(val, buffer);
382 return DeserializeInt64(buffer, *
reinterpret_cast<std::int64_t *
>(&val));
388 auto pos =
reinterpret_cast<unsigned char *
>(buffer);
389 pos += SerializeUInt32(val.length(), pos);
390 memcpy(pos, val.data(), val.length());
392 return sizeof(std::uint32_t) + val.length();
396 const void *buffer, std::uint32_t bufSize, std::string &val)
398 if (bufSize <
sizeof(std::uint32_t))
399 return R__FAIL(
"string buffer too short");
400 bufSize -=
sizeof(std::uint32_t);
402 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
407 return R__FAIL(
"string buffer too short");
411 return sizeof(std::uint32_t) +
length;
421 return SerializeUInt16(0x02, buffer);
423 return SerializeUInt16(0x03, buffer);
425 return SerializeUInt16(0x04, buffer);
427 return SerializeUInt16(0x05, buffer);
429 return SerializeUInt16(0x06, buffer);
431 return SerializeUInt16(0x07, buffer);
433 return SerializeUInt16(0x08, buffer);
435 return SerializeUInt16(0x09, buffer);
437 return SerializeUInt16(0x0A, buffer);
439 return SerializeUInt16(0x0B, buffer);
441 return SerializeUInt16(0x0C, buffer);
443 return SerializeUInt16(0x0D, buffer);
454 std::uint16_t onDiskType;
455 auto result = DeserializeUInt16(buffer, onDiskType);
456 switch (onDiskType) {
494 return R__FAIL(
"unexpected on-disk column type");
506 return SerializeUInt16(0x00, buffer);
508 return SerializeUInt16(0x01, buffer);
510 return SerializeUInt16(0x02, buffer);
512 return SerializeUInt16(0x03, buffer);
514 return SerializeUInt16(0x04, buffer);
525 std::uint16_t onDiskValue;
526 auto result = DeserializeUInt16(buffer, onDiskValue);
527 switch (onDiskValue) {
544 return R__FAIL(
"unexpected on-disk field structure value");
554 auto base =
reinterpret_cast<unsigned char *
>(buffer);
556 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
558 pos += SerializeUInt16(kEnvelopeCurrentVersion, *where);
559 pos += SerializeUInt16(kEnvelopeMinVersion, *where);
565 const unsigned char *envelope, std::uint32_t
size, std::uint32_t &crc32,
void *buffer)
567 return SerializeCRC32(envelope,
size, crc32, buffer);
571 const unsigned char *envelope, std::uint32_t
size,
void *buffer)
574 return SerializeEnvelopePostscript(envelope,
size, crc32, buffer);
580 const void *buffer, std::uint32_t bufSize, std::uint32_t &crc32)
582 if (bufSize < (2 *
sizeof(std::uint16_t) +
sizeof(std::uint32_t)))
583 return R__FAIL(
"invalid envelope, too short");
585 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
588 std::uint16_t protocolVersionAtWrite;
589 std::uint16_t protocolVersionMinRequired;
590 bytes += DeserializeUInt16(
bytes, protocolVersionAtWrite);
592 if (protocolVersionAtWrite < 1)
593 return R__FAIL(
"The RNTuple format is too old (version 0)");
595 bytes += DeserializeUInt16(
bytes, protocolVersionMinRequired);
596 if (protocolVersionMinRequired > kEnvelopeCurrentVersion) {
597 return R__FAIL(std::string(
"The RNTuple format is too new (version ") +
598 std::to_string(protocolVersionMinRequired) +
")");
602 auto result = VerifyCRC32(base, bufSize - 4, crc32);
606 return sizeof(protocolVersionAtWrite) +
sizeof(protocolVersionMinRequired);
611 const void *buffer, std::uint32_t bufSize)
621 return SerializeInt32(1, buffer);
626 std::uint32_t
nitems,
void *buffer)
631 auto base =
reinterpret_cast<unsigned char *
>(buffer);
633 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
637 pos += SerializeUInt32(
nitems, *where);
642 void *frame, std::int32_t
size)
646 if (
size <
static_cast<std::int32_t
>(
sizeof(std::int32_t)))
650 DeserializeInt32(frame, marker);
651 if ((marker < 0) && (
size <
static_cast<std::int32_t
>(2 *
sizeof(std::int32_t))))
654 SerializeInt32(marker *
size, frame);
660 const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize, std::uint32_t &
nitems)
662 if (bufSize <
sizeof(std::int32_t))
663 return R__FAIL(
"frame too short");
665 std::int32_t *ssize =
reinterpret_cast<std::int32_t *
>(&frameSize);
666 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
671 if (frameSize <
sizeof(std::int32_t))
672 return R__FAIL(
"corrupt record frame size");
675 if (bufSize < 2 *
sizeof(std::int32_t))
676 return R__FAIL(
"frame too short");
680 if (frameSize < 2 *
sizeof(std::int32_t))
681 return R__FAIL(
"corrupt list frame size");
684 if (bufSize < frameSize)
685 return R__FAIL(
"frame too short");
687 return bytes -
reinterpret_cast<const unsigned char *
>(buffer);
691 const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize)
698 const std::vector<std::int64_t> &flags,
void *buffer)
701 return SerializeInt64(0, buffer);
704 auto bytes =
reinterpret_cast<unsigned char *
>(buffer);
706 for (
unsigned i = 0; i < flags.size(); ++i) {
711 if (i == (flags.size() - 1))
712 SerializeInt64(flags[i],
bytes);
714 bytes += SerializeInt64(flags[i] | 0x8000000000000000,
bytes);
717 return (flags.size() *
sizeof(std::int64_t));
721 const void *buffer, std::uint32_t bufSize, std::vector<std::int64_t> &flags)
723 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
728 if (bufSize <
sizeof(std::int64_t))
729 return R__FAIL(
"feature flag buffer too short");
731 bufSize -=
sizeof(std::int64_t);
732 flags.emplace_back(
f & ~0x8000000000000000);
735 return (flags.size() *
sizeof(std::int64_t));
741 std::uint32_t
size = 0;
742 if (!locator.
fUrl.empty()) {
743 if (locator.
fUrl.length() >= (1 << 24))
745 std::int32_t head = locator.
fUrl.length();
748 size += SerializeInt32(head, buffer);
750 memcpy(
reinterpret_cast<unsigned char *
>(buffer) +
size, locator.
fUrl.data(), locator.
fUrl.length());
758 size += SerializeUInt64(locator.
fPosition, buffer ?
reinterpret_cast<unsigned char *
>(buffer) +
size :
nullptr);
763 const void *buffer, std::uint32_t bufSize,
RNTupleLocator &locator)
765 if (bufSize <
sizeof(std::int32_t))
766 return R__FAIL(
"too short locator");
768 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
772 bufSize -=
sizeof(std::int32_t);
775 int type = head >> 24;
777 return R__FAIL(
"unsupported locator type: " + std::to_string(
type));
778 std::uint32_t locatorSize =
static_cast<std::uint32_t
>(head) & 0x00FFFFFF;
779 if (bufSize < locatorSize)
780 return R__FAIL(
"too short locator");
783 locator.
fUrl.resize(locatorSize);
784 memcpy(&locator.
fUrl[0],
bytes, locatorSize);
785 bytes += locatorSize;
787 if (bufSize <
sizeof(std::uint64_t))
788 return R__FAIL(
"too short locator");
791 locator.
fUrl.clear();
796 return bytes -
reinterpret_cast<const unsigned char *
>(buffer);
804 buffer ?
reinterpret_cast<unsigned char *
>(buffer) +
size :
nullptr);
809 const void *buffer, std::uint32_t bufSize,
REnvelopeLink &envelopeLink)
811 if (bufSize <
sizeof(std::int32_t))
812 return R__FAIL(
"too short envelope link");
814 auto bytes =
reinterpret_cast<const unsigned char *
>(buffer);
816 bufSize -=
sizeof(std::uint32_t);
821 return bytes -
reinterpret_cast<const unsigned char *
>(buffer);
828 auto base =
reinterpret_cast<unsigned char *
>(buffer);
830 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
833 pos += SerializeRecordFramePreamble(*where);
834 pos += SerializeUInt64(clusterSummary.
fFirstEntry, *where);
836 pos += SerializeInt64(-
static_cast<int64_t
>(clusterSummary.
fNEntries), *where);
839 pos += SerializeInt64(
static_cast<int64_t
>(clusterSummary.
fNEntries), *where);
841 auto size = pos - frame;
842 pos += SerializeFramePostscript(frame,
size);
848 const void *buffer, std::uint32_t bufSize,
RClusterSummary &clusterSummary)
850 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
852 std::uint32_t frameSize;
853 auto result = DeserializeFrameHeader(
bytes, bufSize, frameSize);
858 auto fnBufSizeLeft = [&]() {
return frameSize -
static_cast<std::uint32_t
>(
bytes - base); };
859 if (fnBufSizeLeft() < 2 *
sizeof(std::uint64_t))
860 return R__FAIL(
"too short cluster summary");
863 std::int64_t nEntries;
867 if (fnBufSizeLeft() <
sizeof(std::uint32_t))
868 return R__FAIL(
"too short cluster summary");
870 std::uint32_t columnGroupID;
871 bytes += DeserializeUInt32(
bytes, columnGroupID);
885 auto base =
reinterpret_cast<unsigned char *
>(buffer);
887 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
890 pos += SerializeRecordFramePreamble(*where);
891 pos += SerializeUInt32(clusterGroup.
fNClusters, *where);
893 auto size = pos - frame;
894 pos += SerializeFramePostscript(frame,
size);
900 const void *buffer, std::uint32_t bufSize,
RClusterGroup &clusterGroup)
902 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
905 std::uint32_t frameSize;
906 auto result = DeserializeFrameHeader(
bytes, bufSize, frameSize);
911 auto fnFrameSizeLeft = [&]() {
return frameSize -
static_cast<std::uint32_t
>(
bytes - base); };
912 if (fnFrameSizeLeft() <
sizeof(std::uint32_t))
913 return R__FAIL(
"too short cluster group");
930 auto base =
reinterpret_cast<unsigned char *
>(buffer);
932 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
934 pos += SerializeEnvelopePreamble(*where);
936 pos += SerializeFeatureFlags(std::vector<std::int64_t>(), *where);
937 pos += SerializeUInt32(kReleaseCandidateTag, *where);
938 pos += SerializeString(desc.
GetName(), *where);
940 pos += SerializeString(std::string(
"ROOT v") +
ROOT_RELEASE, *where);
944 pos += SerializeListFramePreamble(desc.
GetNFields() - 1, *where);
945 pos += SerializeFieldTree(desc, context, *where);
946 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
949 pos += SerializeListFramePreamble(desc.
GetNColumns(), *where);
950 pos += SerializeColumnListV1(desc, context, *where);
951 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
955 pos += SerializeListFramePreamble(0, *where);
956 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
960 pos += SerializeListFramePreamble(0, *where);
961 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
963 std::uint32_t
size = pos - base;
964 std::uint32_t crc32 = 0;
965 size += SerializeEnvelopePostscript(base,
size, crc32, *where);
975 auto base =
reinterpret_cast<unsigned char *
>(buffer);
977 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
979 pos += SerializeEnvelopePreamble(*where);
980 auto topMostFrame = pos;
981 pos += SerializeListFramePreamble(physClusterIDs.size(), *where);
983 for (
auto clusterId : physClusterIDs) {
986 std::set<DescriptorId_t> physColumnIds;
987 for (
auto column : clusterDesc.GetColumnIds())
990 auto outerFrame = pos;
991 pos += SerializeListFramePreamble(physColumnIds.size(), *where);
992 for (
auto physId : physColumnIds) {
994 const auto &columnRange = clusterDesc.GetColumnRange(memId);
995 const auto &pageRange = clusterDesc.GetPageRange(memId);
997 auto innerFrame = pos;
998 pos += SerializeListFramePreamble(pageRange.fPageInfos.size(), *where);
1000 for (
const auto &
pi : pageRange.fPageInfos) {
1001 pos += SerializeUInt32(
pi.fNElements, *where);
1002 pos += SerializeLocator(
pi.fLocator, *where);
1004 pos += SerializeUInt64(columnRange.fFirstElementIndex, *where);
1005 pos += SerializeUInt32(columnRange.fCompressionSettings, *where);
1007 pos += SerializeFramePostscript(buffer ? innerFrame :
nullptr, pos - innerFrame);
1009 pos += SerializeFramePostscript(buffer ? outerFrame :
nullptr, pos - outerFrame);
1012 pos += SerializeFramePostscript(buffer ? topMostFrame :
nullptr, pos - topMostFrame);
1013 std::uint32_t
size = pos - base;
1014 size += SerializeEnvelopePostscript(base,
size, *where);
1021 auto base =
reinterpret_cast<unsigned char *
>(buffer);
1023 void** where = (buffer ==
nullptr) ? &buffer :
reinterpret_cast<void**
>(&pos);
1025 pos += SerializeEnvelopePreamble(*where);
1028 pos += SerializeFeatureFlags(std::vector<std::int64_t>(), *where);
1033 pos += SerializeListFramePreamble(0, *where);
1034 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
1038 pos += SerializeListFramePreamble(0, *where);
1039 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
1043 unsigned int nClusters = 0;
1045 nClusters += cgDesc.GetNClusters();
1047 pos += SerializeListFramePreamble(nClusters, *where);
1048 for (
unsigned int i = 0; i < nClusterGroups; ++i) {
1051 const auto &clusterIds = cgDesc.GetClusterIds();
1052 for (
unsigned int j = 0; j < nClustersInGroup; ++j) {
1054 RClusterSummary summary{clusterDesc.GetFirstEntryIndex(), clusterDesc.GetNEntries(), -1};
1055 pos += SerializeClusterSummary(summary, *where);
1058 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
1062 pos += SerializeListFramePreamble(nClusterGroups, *where);
1063 for (
unsigned int i = 0; i < nClusterGroups; ++i) {
1066 clusterGroup.
fNClusters = cgDesc.GetNClusters();
1069 pos += SerializeClusterGroup(clusterGroup, *where);
1071 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
1075 pos += SerializeListFramePreamble(0, *where);
1076 pos += SerializeFramePostscript(buffer ? frame :
nullptr, pos - frame);
1078 std::uint32_t
size = pos - base;
1079 size += SerializeEnvelopePostscript(base,
size, *where);
1086 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
1088 auto fnBufSizeLeft = [&]() {
return bufSize - (
bytes - base); };
1091 std::uint32_t crc32{0};
1092 result = DeserializeEnvelope(
bytes, fnBufSizeLeft(), crc32);
1098 std::vector<std::int64_t> featureFlags;
1099 result = DeserializeFeatureFlags(
bytes, fnBufSizeLeft(), featureFlags);
1103 for (
auto f: featureFlags) {
1108 std::uint32_t rcTag;
1109 if (fnBufSizeLeft() <
static_cast<int>(
sizeof(std::uint32_t)))
1110 return R__FAIL(
"header too short");
1117 std::string description;
1123 result = DeserializeString(
bytes, fnBufSizeLeft(), description);
1133 std::uint32_t frameSize;
1135 auto fnFrameSizeLeft = [&]() {
return frameSize - (
bytes - frame); };
1137 std::uint32_t nFields;
1138 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nFields);
1144 .FieldId(kZeroFieldId)
1148 for (std::uint32_t fieldId = 0; fieldId < nFields; ++fieldId) {
1150 result = DeserializeFieldV1(
bytes, fnFrameSizeLeft(), fieldBuilder);
1155 fieldBuilder.
ParentId(kZeroFieldId);
1159 auto parentId = fieldDesc.Inspect().GetParentId();
1160 descBuilder.
AddField(fieldDesc.Unwrap());
1161 auto resVoid = descBuilder.
AddFieldLink(parentId, fieldId);
1165 bytes = frame + frameSize;
1167 std::uint32_t nColumns;
1169 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nColumns);
1173 std::unordered_map<DescriptorId_t, std::uint32_t> maxIndexes;
1174 for (std::uint32_t columnId = 0; columnId < nColumns; ++columnId) {
1176 result = DeserializeColumnV1(
bytes, fnFrameSizeLeft(), columnBuilder);
1181 std::uint32_t idx = 0;
1182 const auto fieldId = columnBuilder.
GetFieldId();
1183 auto maxIdx = maxIndexes.find(fieldId);
1184 if (maxIdx != maxIndexes.end())
1185 idx = maxIdx->second + 1;
1186 maxIndexes[fieldId] = idx;
1191 auto resVoid = descBuilder.
AddColumn(columnDesc.Unwrap());
1195 bytes = frame + frameSize;
1197 std::uint32_t nAliasColumns;
1199 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nAliasColumns);
1203 if (nAliasColumns > 0)
1206 std::uint32_t nTypeInfo;
1208 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nTypeInfo);
1222 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
1224 auto fnBufSizeLeft = [&]() {
return bufSize - (
bytes - base); };
1227 result = DeserializeEnvelope(
bytes, fnBufSizeLeft());
1232 std::vector<std::int64_t> featureFlags;
1233 result = DeserializeFeatureFlags(
bytes, fnBufSizeLeft(), featureFlags);
1237 for (
auto f: featureFlags) {
1242 std::uint32_t crc32{0};
1243 if (fnBufSizeLeft() <
static_cast<int>(
sizeof(std::uint32_t)))
1244 return R__FAIL(
"footer too short");
1247 return R__FAIL(
"CRC32 mismatch between header and footer");
1249 std::uint32_t frameSize;
1251 auto fnFrameSizeLeft = [&]() {
return frameSize - (
bytes - frame); };
1253 std::uint32_t nXHeaders;
1254 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nXHeaders);
1259 bytes = frame + frameSize;
1261 std::uint32_t nColumnGroups;
1263 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nColumnGroups);
1266 if (nColumnGroups > 0)
1267 return R__FAIL(
"sharded clusters are still unsupported");
1268 bytes = frame + frameSize;
1270 std::uint32_t nClusterSummaries;
1272 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nClusterSummaries);
1276 for (std::uint32_t clusterId = 0; clusterId < nClusterSummaries; ++clusterId) {
1278 result = DeserializeClusterSummary(
bytes, fnFrameSizeLeft(), clusterSummary);
1283 return R__FAIL(
"sharded clusters are still unsupported");
1286 bytes = frame + frameSize;
1288 std::uint32_t nClusterGroups;
1290 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nClusterGroups);
1294 std::uint64_t clusterId = 0;
1295 for (std::uint32_t groupId = 0; groupId < nClusterGroups; ++groupId) {
1297 result = DeserializeClusterGroup(
bytes, fnFrameSizeLeft(), clusterGroup);
1307 for (std::uint64_t i = 0; i < clusterGroup.
fNClusters; ++i)
1308 clusterGroupBuilder.
AddCluster(clusterId + i);
1312 bytes = frame + frameSize;
1314 std::uint32_t nMDBlocks;
1316 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), frameSize, nMDBlocks);
1321 bytes = frame + frameSize;
1328 const void *buffer, std::uint32_t bufSize, std::vector<RClusterDescriptorBuilder> &clusters)
1330 auto base =
reinterpret_cast<const unsigned char *
>(buffer);
1332 auto fnBufSizeLeft = [&]() {
return bufSize - (
bytes - base); };
1335 result = DeserializeEnvelope(
bytes, fnBufSizeLeft());
1340 std::uint32_t topMostFrameSize;
1341 auto topMostFrame =
bytes;
1342 auto fnTopMostFrameSizeLeft = [&]() {
return topMostFrameSize - (
bytes - topMostFrame); };
1344 std::uint32_t nClusters;
1345 result = DeserializeFrameHeader(
bytes, fnBufSizeLeft(), topMostFrameSize, nClusters);
1350 if (nClusters != clusters.size())
1351 return R__FAIL(
"mismatch of page list and cluster summaries");
1353 for (std::uint32_t i = 0; i < nClusters; ++i) {
1354 std::uint32_t outerFrameSize;
1355 auto outerFrame =
bytes;
1356 auto fnOuterFrameSizeLeft = [&]() {
return outerFrameSize - (
bytes - outerFrame); };
1358 std::uint32_t nColumns;
1359 result = DeserializeFrameHeader(
bytes, fnTopMostFrameSizeLeft(), outerFrameSize, nColumns);
1364 for (std::uint32_t j = 0; j < nColumns; ++j) {
1365 std::uint32_t innerFrameSize;
1366 auto innerFrame =
bytes;
1367 auto fnInnerFrameSizeLeft = [&]() {
return innerFrameSize - (
bytes - innerFrame); };
1369 std::uint32_t nPages;
1370 result = DeserializeFrameHeader(
bytes, fnOuterFrameSizeLeft(), innerFrameSize, nPages);
1377 for (std::uint32_t k = 0; k < nPages; ++k) {
1378 if (fnInnerFrameSizeLeft() <
static_cast<int>(
sizeof(std::uint32_t)))
1379 return R__FAIL(
"inner frame too short");
1380 std::uint32_t nElements;
1382 bytes += DeserializeUInt32(
bytes, nElements);
1383 result = DeserializeLocator(
bytes, fnInnerFrameSizeLeft(), locator);
1390 if (fnInnerFrameSizeLeft() <
static_cast<int>(
sizeof(std::uint32_t) +
sizeof(std::uint64_t)))
1391 return R__FAIL(
"page list frame too short");
1392 std::uint64_t columnOffset;
1393 bytes += DeserializeUInt64(
bytes, columnOffset);
1394 std::uint32_t compressionSettings;
1395 bytes += DeserializeUInt32(
bytes, compressionSettings);
1397 clusters[i].CommitColumnRange(j, columnOffset, compressionSettings, pageRange);
1398 bytes = innerFrame + innerFrameSize;
1401 bytes = outerFrame + outerFrameSize;
#define R__FORWARD_ERROR(res)
Short-hand to return an RResult<T> in an error state (i.e. after checking)
#define R__FORWARD_RESULT(res)
Short-hand to return an RResult<T> value from a subroutine to the calling stack frame.
#define R__FAIL(msg)
Short-hand to return an RResult<T> in an error state; the RError is implicitly converted into RResult...
#define R__LOG_WARNING(...)
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h length
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t nitems
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t bytes
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
virtual std::size_t GetBitsOnStorage() const
The available trivial, native content types of a column.
The serialization context is used for the piecewise serialization of a descriptor.
void SetHeaderCRC32(std::uint32_t crc32)
void SetHeaderSize(std::uint32_t size)
std::uint32_t GetHeaderCRC32() const
DescriptorId_t MapColumnId(DescriptorId_t memId)
DescriptorId_t MapFieldId(DescriptorId_t memId)
DescriptorId_t GetMemClusterId(DescriptorId_t physId) const
DescriptorId_t GetPhysColumnId(DescriptorId_t memId) const
DescriptorId_t GetPhysFieldId(DescriptorId_t memId) const
DescriptorId_t GetMemClusterGroupId(DescriptorId_t physId) const
DescriptorId_t GetMemColumnId(DescriptorId_t physId) const
A helper class for serializing and deserialization of the RNTuple binary format.
static RResult< std::uint32_t > DeserializeString(const void *buffer, std::uint32_t bufSize, std::string &val)
static std::uint32_t SerializeFeatureFlags(const std::vector< std::int64_t > &flags, void *buffer)
static std::uint32_t SerializePageListV1(void *buffer, const RNTupleDescriptor &desc, std::span< DescriptorId_t > physClusterIDs, const RContext &context)
static std::uint32_t SerializeListFramePreamble(std::uint32_t nitems, void *buffer)
static RResult< std::uint32_t > DeserializeLocator(const void *buffer, std::uint32_t bufSize, RNTupleLocator &locator)
static std::uint32_t SerializeCRC32(const unsigned char *data, std::uint32_t length, std::uint32_t &crc32, void *buffer)
Writes a CRC32 checksum of the byte range given by data and length.
static std::uint16_t SerializeColumnType(ROOT::Experimental::EColumnType type, void *buffer)
static std::uint32_t DeserializeUInt16(const void *buffer, std::uint16_t &val)
static RResult< void > DeserializePageListV1(const void *buffer, std::uint32_t bufSize, std::vector< RClusterDescriptorBuilder > &clusters)
static std::uint32_t SerializeString(const std::string &val, void *buffer)
static RResult< std::uint32_t > DeserializeEnvelope(const void *buffer, std::uint32_t bufSize)
static std::uint32_t DeserializeUInt32(const void *buffer, std::uint32_t &val)
static std::uint32_t SerializeUInt64(std::uint64_t val, void *buffer)
static std::uint32_t DeserializeInt16(const void *buffer, std::int16_t &val)
static std::uint32_t SerializeClusterSummary(const RClusterSummary &clusterSummary, void *buffer)
static RContext SerializeHeaderV1(void *buffer, const RNTupleDescriptor &desc)
static RResult< void > DeserializeFooterV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static std::uint32_t SerializeInt16(std::int16_t val, void *buffer)
static std::uint32_t SerializeLocator(const RNTupleLocator &locator, void *buffer)
static std::uint32_t SerializeInt32(std::int32_t val, void *buffer)
static std::uint32_t SerializeEnvelopePreamble(void *buffer)
Currently all enevelopes have the same version number (1).
static RResult< std::uint16_t > DeserializeColumnType(const void *buffer, ROOT::Experimental::EColumnType &type)
static RResult< std::uint32_t > DeserializeClusterGroup(const void *buffer, std::uint32_t bufSize, RClusterGroup &clusterGroup)
static std::uint32_t DeserializeUInt64(const void *buffer, std::uint64_t &val)
static std::uint32_t DeserializeInt32(const void *buffer, std::int32_t &val)
static std::uint32_t DeserializeInt64(const void *buffer, std::int64_t &val)
static RResult< std::uint32_t > DeserializeEnvelopeLink(const void *buffer, std::uint32_t bufSize, REnvelopeLink &envelopeLink)
static std::uint32_t SerializeEnvelopePostscript(const unsigned char *envelope, std::uint32_t size, void *buffer)
static RResult< std::uint16_t > DeserializeFieldStructure(const void *buffer, ROOT::Experimental::ENTupleStructure &structure)
static std::uint32_t SerializeEnvelopeLink(const REnvelopeLink &envelopeLink, void *buffer)
static std::uint32_t SerializeRecordFramePreamble(void *buffer)
static std::uint32_t SerializeUInt16(std::uint16_t val, void *buffer)
static RResult< std::uint32_t > DeserializeFrameHeader(const void *buffer, std::uint32_t bufSize, std::uint32_t &frameSize, std::uint32_t &nitems)
static RResult< std::uint32_t > DeserializeFeatureFlags(const void *buffer, std::uint32_t bufSize, std::vector< std::int64_t > &flags)
static std::uint32_t SerializeClusterGroup(const RClusterGroup &clusterGroup, void *buffer)
static std::uint32_t SerializeFramePostscript(void *frame, std::int32_t size)
static std::uint32_t SerializeFooterV1(void *buffer, const RNTupleDescriptor &desc, const RContext &context)
static std::uint32_t SerializeInt64(std::int64_t val, void *buffer)
static RResult< void > VerifyCRC32(const unsigned char *data, std::uint32_t length, std::uint32_t &crc32)
Expects a CRC32 checksum in the 4 bytes following data + length and verifies it.
static std::uint16_t SerializeFieldStructure(ROOT::Experimental::ENTupleStructure structure, void *buffer)
While we could just interpret the enums as ints, we make the translation explicit in order to avoid a...
static std::uint32_t SerializeUInt32(std::uint32_t val, void *buffer)
static RResult< void > DeserializeHeaderV1(const void *buffer, std::uint32_t bufSize, RNTupleDescriptorBuilder &descBuilder)
static RResult< std::uint32_t > DeserializeClusterSummary(const void *buffer, std::uint32_t bufSize, RClusterSummary &clusterSummary)
A helper class for piece-wise construction of an RClusterGroupDescriptor.
RClusterGroupDescriptorBuilder & ClusterGroupId(DescriptorId_t clusterGroupId)
void AddCluster(DescriptorId_t clusterId)
RClusterGroupDescriptorBuilder & PageListLength(std::uint32_t pageListLength)
RClusterGroupDescriptorBuilder & PageListLocator(const RNTupleLocator &pageListLocator)
std::uint64_t GetNClusters() const
A helper class for piece-wise construction of an RColumnDescriptor.
RColumnDescriptorBuilder & Model(const RColumnModel &model)
RResult< RColumnDescriptor > MakeDescriptor() const
Attempt to make a column descriptor.
RColumnDescriptorBuilder & FieldId(DescriptorId_t fieldId)
DescriptorId_t GetFieldId() const
RColumnDescriptorBuilder & Index(std::uint32_t index)
RColumnDescriptorBuilder & ColumnId(DescriptorId_t columnId)
Base class for all ROOT issued exceptions.
A helper class for piece-wise construction of an RFieldDescriptor.
RFieldDescriptorBuilder & FieldName(const std::string &fieldName)
RFieldDescriptorBuilder & NRepetitions(std::uint64_t nRepetitions)
DescriptorId_t GetParentId() const
RFieldDescriptorBuilder & Structure(const ENTupleStructure &structure)
RResult< RFieldDescriptor > MakeDescriptor() const
Attempt to make a field descriptor.
RFieldDescriptorBuilder & TypeName(const std::string &typeName)
RFieldDescriptorBuilder & TypeVersion(std::uint32_t typeVersion)
RFieldDescriptorBuilder & ParentId(DescriptorId_t id)
RFieldDescriptorBuilder & FieldDescription(const std::string &fieldDescription)
RFieldDescriptorBuilder & FieldVersion(std::uint32_t fieldVersion)
RFieldDescriptorBuilder & FieldId(DescriptorId_t fieldId)
Meta-data stored for every field of an ntuple.
std::uint32_t GetTypeVersion() const
std::string GetFieldName() const
std::uint32_t GetFieldVersion() const
std::string GetFieldDescription() const
std::string GetTypeName() const
std::uint64_t GetNRepetitions() const
ENTupleStructure GetStructure() const
A helper class for piece-wise construction of an RNTupleDescriptor.
void AddColumn(DescriptorId_t columnId, DescriptorId_t fieldId, const RColumnModel &model, std::uint32_t index)
std::uint32_t GetHeaderCRC32() const
void AddToOnDiskFooterSize(std::uint64_t size)
The real footer size also include the page list envelopes.
RResult< void > AddClusterSummary(DescriptorId_t clusterId, std::uint64_t firstEntry, std::uint64_t nEntries)
RResult< void > AddFieldLink(DescriptorId_t fieldId, DescriptorId_t linkId)
void SetHeaderCRC32(std::uint32_t crc32)
void SetNTuple(const std::string_view name, const std::string_view description)
void AddClusterGroup(RClusterGroupDescriptorBuilder &&clusterGroup)
void AddField(const RFieldDescriptor &fieldDesc)
The on-storage meta-data of an ntuple.
RClusterGroupDescriptorIterable GetClusterGroupIterable() const
std::string GetDescription() const
std::string GetName() const
DescriptorId_t GetFieldZeroId() const
Returns the logical parent of all top-level NTuple data fields.
RColumnDescriptorIterable GetColumnIterable(const RFieldDescriptor &fieldDesc) const
const RClusterDescriptor & GetClusterDescriptor(DescriptorId_t clusterId) const
RFieldDescriptorIterable GetFieldIterable(const RFieldDescriptor &fieldDesc) const
std::size_t GetNFields() const
std::size_t GetNClusterGroups() const
std::size_t GetNColumns() const
const RClusterGroupDescriptor & GetClusterGroupDescriptor(DescriptorId_t clusterGroupId) const
RResult<void> has no data member and no Inspect() method but instead a Success() factory method.
The class is used as a return type for operations that can fail; wraps a value of type T or an RError...
RLogChannel & NTupleLog()
Log channel for RNTuple diagnostics.
RClusterSize ClusterSize_t
ENTupleStructure
The fields in the ntuple model tree can carry different structural information about the type system.
std::uint64_t DescriptorId_t
Distriniguishes elements of the same type within a descriptor, e.g. different fields.
static constexpr double pi
REnvelopeLink fPageListEnvelopeLink
std::uint64_t fFirstEntry
std::int32_t fColumnGroupID
-1 for "all columns"
std::uint32_t fUnzippedSize
Generic information about the physical location of data.
std::uint32_t fBytesOnStorage