Logo ROOT  
Reference Guide
RNTupleZip.hxx
Go to the documentation of this file.
1/// \file ROOT/RNTupleZip.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-11-21
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RNTupleZip
17#define ROOT7_RNTupleZip
18
19#include <RZip.h>
20#include <TError.h>
21
22#include <algorithm>
23#include <array>
24#include <cstring>
25#include <functional>
26#include <memory>
27#include <utility>
28
29namespace ROOT {
30namespace Experimental {
31namespace Detail {
32
33// clang-format off
34/**
35\class ROOT::Experimental::Detail::RNTupleCompressor
36\ingroup NTuple
37\brief Helper class to compress data blocks in the ROOT compression frame format
38*/
39// clang-format on
41private:
42 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
43 std::unique_ptr<Buffer_t> fZipBuffer;
44
45public:
46 /// Data might be overwritten, if a zipped block in the middle of a large input data stream
47 /// turns out to be uncompressible
48 using Writer_t = std::function<void(const void *buffer, size_t nbytes, size_t offset)>;
49 static constexpr size_t kMaxSingleBlock = kMAXZIPBUF;
50
51 RNTupleCompressor() : fZipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
52 RNTupleCompressor(const RNTupleCompressor &other) = delete;
56
57 /// Returns the size of the compressed data. Data is compressed in 16MB (kMAXZIPBUF) blocks and written
58 /// piecewise using the provided writer
59 size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter) {
60 R__ASSERT(from != nullptr);
61
62 auto cxLevel = compression % 100;
63 if (cxLevel == 0) {
64 fnWriter(from, nbytes, 0);
65 return nbytes;
66 }
67
68 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
69 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
70 char *source = const_cast<char *>(static_cast<const char *>(from));
71 int szTarget = kMAXZIPBUF;
72 char *target = reinterpret_cast<char *>(fZipBuffer->data());
73 int szOutBlock = 0;
74 int szRemaining = nbytes;
75 size_t szZipData = 0;
76 for (unsigned int i = 0; i < nZipBlocks; ++i) {
77 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
78 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
79 R__ASSERT(szOutBlock >= 0);
80 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
81 // Uncompressible block, we have to store the entire input data stream uncompressed
82 fnWriter(from, nbytes, 0);
83 return nbytes;
84 }
85
86 fnWriter(target, szOutBlock, szZipData);
87 szZipData += szOutBlock;
88 source += szSource;
89 szRemaining -= szSource;
90 }
91 R__ASSERT(szRemaining == 0);
92 R__ASSERT(szZipData < nbytes);
93 return szZipData;
94 }
95
96 /// Returns the size of the compressed data block. The data is written into the zip buffer.
97 /// This works only for small input buffer up to 16MB (kMAXZIPBUF)
98 size_t Zip(const void *from, size_t nbytes, int compression) {
99 R__ASSERT(from != nullptr);
100 R__ASSERT(nbytes <= kMAXZIPBUF);
101
102 auto cxLevel = compression % 100;
103 if (cxLevel == 0) {
104 memcpy(fZipBuffer->data(), from, nbytes);
105 return nbytes;
106 }
107
108 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
109 int szSource = nbytes;
110 char *source = const_cast<char *>(static_cast<const char *>(from));
111 int szTarget = nbytes;
112 char *target = reinterpret_cast<char *>(fZipBuffer->data());
113 int szOut = 0;
114 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOut, cxAlgorithm);
115 R__ASSERT(szOut >= 0);
116 if ((szOut > 0) && (static_cast<unsigned int>(szOut) < nbytes))
117 return szOut;
118
119 memcpy(fZipBuffer->data(), from, nbytes);
120 return nbytes;
121 }
122
123 /// Returns the size of the compressed data, written into the provided output buffer.
124 static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to) {
125 R__ASSERT(from != nullptr);
126 R__ASSERT(to != nullptr);
127 auto cxLevel = compression % 100;
128 if (cxLevel == 0) {
129 memcpy(to, from, nbytes);
130 return nbytes;
131 }
132
133 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
134 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
135 char *source = const_cast<char *>(static_cast<const char *>(from));
136 int szTarget = nbytes;
137 char *target = reinterpret_cast<char *>(to);
138 int szOutBlock = 0;
139 int szRemaining = nbytes;
140 size_t szZipData = 0;
141 for (unsigned int i = 0; i < nZipBlocks; ++i) {
142 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
143 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
144 R__ASSERT(szOutBlock >= 0);
145 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
146 // Uncompressible block, we have to store the entire input data stream uncompressed
147 memcpy(to, from, nbytes);
148 return nbytes;
149 }
150
151 szZipData += szOutBlock;
152 source += szSource;
153 szRemaining -= szSource;
154 }
155 R__ASSERT(szRemaining == 0);
156 R__ASSERT(szZipData < nbytes);
157 return szZipData;
158 }
159
160 void *GetZipBuffer() { return fZipBuffer->data(); }
161};
162
163
164// clang-format off
165/**
166\class ROOT::Experimental::Detail::RNTupleDecompressor
167\ingroup NTuple
168\brief Helper class to uncompress data blocks in the ROOT compression frame format
169*/
170// clang-format on
172private:
173 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
174 std::unique_ptr<Buffer_t> fUnzipBuffer;
175
176public:
177 RNTupleDecompressor() : fUnzipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
182
183 /**
184 * The nbytes parameter provides the size ls of the from buffer. The dataLen gives the size of the uncompressed data.
185 * The block is uncompressed iff nbytes == dataLen.
186 */
187 void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to) {
188 if (dataLen == nbytes) {
189 memcpy(to, from, nbytes);
190 return;
191 }
192 R__ASSERT(dataLen > nbytes);
193
194 unsigned char *source = const_cast<unsigned char *>(static_cast<const unsigned char *>(from));
195 unsigned char *target = static_cast<unsigned char *>(to);
196 int szRemaining = dataLen;
197 do {
198 int szSource;
199 int szTarget;
200 int retval = R__unzip_header(&szSource, source, &szTarget);
201 R__ASSERT(retval == 0);
202 R__ASSERT(szSource > 0);
203 R__ASSERT(szTarget > szSource);
204 R__ASSERT(static_cast<unsigned int>(szSource) <= nbytes);
205 R__ASSERT(static_cast<unsigned int>(szTarget) <= dataLen);
206
207 int unzipBytes = 0;
208 R__unzip(&szSource, source, &szTarget, target, &unzipBytes);
209 R__ASSERT(unzipBytes == szTarget);
210
211 target += szTarget;
212 source += szSource;
213 szRemaining -= unzipBytes;
214 } while (szRemaining > 0);
215 R__ASSERT(szRemaining == 0);
216 }
217
218 /**
219 * In-place decompression via unzip buffer
220 */
221 void Unzip(void *fromto, size_t nbytes, size_t dataLen) {
222 R__ASSERT(dataLen <= kMAXZIPBUF);
223 Unzip(fromto, nbytes, dataLen, fUnzipBuffer->data());
224 memcpy(fromto, fUnzipBuffer->data(), dataLen);
225 }
226};
227
228} // namespace Detail
229} // namespace Experimental
230} // namespace ROOT
231
232#endif
typedef void(GLAPIENTRYP _GLUfuncptr)(void)
#define R__ASSERT(e)
Definition: TError.h:118
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
Helper class to compress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:40
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:42
std::unique_ptr< Buffer_t > fZipBuffer
Definition: RNTupleZip.hxx:43
RNTupleCompressor(RNTupleCompressor &&other)=default
std::function< void(const void *buffer, size_t nbytes, size_t offset)> Writer_t
Data might be overwritten, if a zipped block in the middle of a large input data stream turns out to ...
Definition: RNTupleZip.hxx:48
size_t Zip(const void *from, size_t nbytes, int compression)
Returns the size of the compressed data block.
Definition: RNTupleZip.hxx:98
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
Definition: RNTupleZip.hxx:124
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
Definition: RNTupleZip.hxx:59
RNTupleCompressor(const RNTupleCompressor &other)=delete
RNTupleCompressor & operator=(const RNTupleCompressor &other)=delete
Helper class to uncompress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:171
RNTupleDecompressor & operator=(const RNTupleDecompressor &other)=delete
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:173
void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Definition: RNTupleZip.hxx:187
void Unzip(void *fromto, size_t nbytes, size_t dataLen)
In-place decompression via unzip buffer.
Definition: RNTupleZip.hxx:221
RNTupleDecompressor(RNTupleDecompressor &&other)=default
RNTupleDecompressor(const RNTupleDecompressor &other)=delete
void function(const Char_t *name_, T fun, const Char_t *docstring=0)
Definition: RExports.h:150
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition: Compression.h:83