Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
RNTupleZip.hxx
Go to the documentation of this file.
1/// \file ROOT/RNTupleZip.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-11-21
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RNTupleZip
17#define ROOT7_RNTupleZip
18
19#include <RZip.h>
20#include <TError.h>
21
22#include <algorithm>
23#include <array>
24#include <cstring>
25#include <functional>
26#include <memory>
27#include <utility>
28
29namespace ROOT {
30namespace Experimental {
31namespace Internal {
32
33// clang-format off
34/**
35\class ROOT::Experimental::Internal::RNTupleCompressor
36\ingroup NTuple
37\brief Helper class to compress data blocks in the ROOT compression frame format
38*/
39// clang-format on
41private:
42 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
43 std::unique_ptr<Buffer_t> fZipBuffer;
44
45public:
46 /// Data might be overwritten, if a zipped block in the middle of a large input data stream
47 /// turns out to be uncompressible
48 using Writer_t = std::function<void(const void *buffer, size_t nbytes, size_t offset)>;
49 static Writer_t MakeMemCopyWriter(unsigned char *dest)
50 {
51 return [=](const void *b, size_t n, size_t o) { memcpy(dest + o, b, n); };
52 }
53 static constexpr size_t kMaxSingleBlock = kMAXZIPBUF;
54
55 RNTupleCompressor() : fZipBuffer(std::make_unique<Buffer_t>()) {}
56 RNTupleCompressor(const RNTupleCompressor &other) = delete;
60
61 /// Returns the size of the compressed data. Data is compressed in 16MB (kMAXZIPBUF) blocks and written
62 /// piecewise using the provided writer
63 size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter) {
64 R__ASSERT(from != nullptr);
65
66 auto cxLevel = compression % 100;
67 if (cxLevel == 0) {
68 fnWriter(from, nbytes, 0);
69 return nbytes;
70 }
71
72 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
73 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
74 char *source = const_cast<char *>(static_cast<const char *>(from));
75 int szTarget = kMAXZIPBUF;
76 char *target = reinterpret_cast<char *>(fZipBuffer->data());
77 int szOutBlock = 0;
78 int szRemaining = nbytes;
79 size_t szZipData = 0;
80 for (unsigned int i = 0; i < nZipBlocks; ++i) {
81 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
82 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
83 R__ASSERT(szOutBlock >= 0);
84 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
85 // Uncompressible block, we have to store the entire input data stream uncompressed
86 fnWriter(from, nbytes, 0);
87 return nbytes;
88 }
89
90 fnWriter(target, szOutBlock, szZipData);
91 szZipData += szOutBlock;
92 source += szSource;
93 szRemaining -= szSource;
94 }
95 R__ASSERT(szRemaining == 0);
96 R__ASSERT(szZipData < nbytes);
97 return szZipData;
98 }
99
100 /// Returns the size of the compressed data block. The data is written into the zip buffer.
101 /// This works only for small input buffer up to 16MB (kMAXZIPBUF)
102 size_t Zip(const void *from, size_t nbytes, int compression) {
103 R__ASSERT(from != nullptr);
104 R__ASSERT(nbytes <= kMAXZIPBUF);
105
106 auto cxLevel = compression % 100;
107 if (cxLevel == 0) {
108 memcpy(fZipBuffer->data(), from, nbytes);
109 return nbytes;
110 }
111
112 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
113 int szSource = nbytes;
114 char *source = const_cast<char *>(static_cast<const char *>(from));
115 int szTarget = nbytes;
116 char *target = reinterpret_cast<char *>(fZipBuffer->data());
117 int szOut = 0;
118 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOut, cxAlgorithm);
119 R__ASSERT(szOut >= 0);
120 if ((szOut > 0) && (static_cast<unsigned int>(szOut) < nbytes))
121 return szOut;
122
123 memcpy(fZipBuffer->data(), from, nbytes);
124 return nbytes;
125 }
126
127 /// Returns the size of the compressed data, written into the provided output buffer.
128 static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to) {
129 R__ASSERT(from != nullptr);
130 R__ASSERT(to != nullptr);
131 auto cxLevel = compression % 100;
132 if (cxLevel == 0) {
133 memcpy(to, from, nbytes);
134 return nbytes;
135 }
136
137 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
138 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
139 char *source = const_cast<char *>(static_cast<const char *>(from));
140 int szTarget = nbytes;
141 char *target = reinterpret_cast<char *>(to);
142 int szOutBlock = 0;
143 int szRemaining = nbytes;
144 size_t szZipData = 0;
145 for (unsigned int i = 0; i < nZipBlocks; ++i) {
146 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
147 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
148 R__ASSERT(szOutBlock >= 0);
149 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
150 // Uncompressible block, we have to store the entire input data stream uncompressed
151 memcpy(to, from, nbytes);
152 return nbytes;
153 }
154
155 szZipData += szOutBlock;
156 source += szSource;
157 target += szOutBlock;
158 szRemaining -= szSource;
159 }
160 R__ASSERT(szRemaining == 0);
161 R__ASSERT(szZipData < nbytes);
162 return szZipData;
163 }
164
165 void *GetZipBuffer() { return fZipBuffer->data(); }
166};
167
168
169// clang-format off
170/**
171\class ROOT::Experimental::Internal::RNTupleDecompressor
172\ingroup NTuple
173\brief Helper class to uncompress data blocks in the ROOT compression frame format
174*/
175// clang-format on
177private:
178 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
179 std::unique_ptr<Buffer_t> fUnzipBuffer;
180
181public:
182 RNTupleDecompressor() : fUnzipBuffer(std::make_unique<Buffer_t>()) {}
187
188 /**
189 * The nbytes parameter provides the size ls of the from buffer. The dataLen gives the size of the uncompressed data.
190 * The block is uncompressed iff nbytes == dataLen.
191 */
192 void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to) {
193 if (dataLen == nbytes) {
194 memcpy(to, from, nbytes);
195 return;
196 }
197 R__ASSERT(dataLen > nbytes);
198
199 unsigned char *source = const_cast<unsigned char *>(static_cast<const unsigned char *>(from));
200 unsigned char *target = static_cast<unsigned char *>(to);
201 int szRemaining = dataLen;
202 do {
203 int szSource;
204 int szTarget;
205 int retval = R__unzip_header(&szSource, source, &szTarget);
206 R__ASSERT(retval == 0);
207 R__ASSERT(szSource > 0);
208 R__ASSERT(szTarget > szSource);
209 R__ASSERT(static_cast<unsigned int>(szSource) <= nbytes);
210 R__ASSERT(static_cast<unsigned int>(szTarget) <= dataLen);
211
212 int unzipBytes = 0;
213 R__unzip(&szSource, source, &szTarget, target, &unzipBytes);
214 R__ASSERT(unzipBytes == szTarget);
215
216 target += szTarget;
217 source += szSource;
218 szRemaining -= unzipBytes;
219 } while (szRemaining > 0);
220 R__ASSERT(szRemaining == 0);
221 }
222
223 /**
224 * In-place decompression via unzip buffer
225 */
226 void Unzip(void *fromto, size_t nbytes, size_t dataLen) {
227 R__ASSERT(dataLen <= kMAXZIPBUF);
228 Unzip(fromto, nbytes, dataLen, fUnzipBuffer->data());
229 memcpy(fromto, fUnzipBuffer->data(), dataLen);
230 }
231};
232
233} // namespace Internal
234} // namespace Experimental
235} // namespace ROOT
236
237#endif
#define b(i)
Definition RSha256.hxx:100
#define R__ASSERT(e)
Checks condition e and reports a fatal error if it's false.
Definition TError.h:125
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t dest
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
Helper class to compress data blocks in the ROOT compression frame format.
size_t Zip(const void *from, size_t nbytes, int compression)
Returns the size of the compressed data block.
RNTupleCompressor & operator=(const RNTupleCompressor &other)=delete
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
std::function< void(const void *buffer, size_t nbytes, size_t offset)> Writer_t
Data might be overwritten, if a zipped block in the middle of a large input data stream turns out to ...
static Writer_t MakeMemCopyWriter(unsigned char *dest)
RNTupleCompressor(const RNTupleCompressor &other)=delete
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
RNTupleCompressor(RNTupleCompressor &&other)=default
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Helper class to uncompress data blocks in the ROOT compression frame format.
RNTupleDecompressor(const RNTupleDecompressor &other)=delete
void Unzip(void *fromto, size_t nbytes, size_t dataLen)
In-place decompression via unzip buffer.
RNTupleDecompressor & operator=(const RNTupleDecompressor &other)=delete
std::array< unsigned char, kMAXZIPBUF > Buffer_t
RNTupleDecompressor(RNTupleDecompressor &&other)=default
void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
const Int_t n
Definition legend1.C:16
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition Compression.h:85