Logo ROOT  
Reference Guide
RNTupleZip.hxx
Go to the documentation of this file.
1/// \file ROOT/RNTupleZip.hxx
2/// \ingroup NTuple ROOT7
3/// \author Jakob Blomer <jblomer@cern.ch>
4/// \date 2019-11-21
5/// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6/// is welcome!
7
8/*************************************************************************
9 * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10 * All rights reserved. *
11 * *
12 * For the licensing terms see $ROOTSYS/LICENSE. *
13 * For the list of contributors see $ROOTSYS/README/CREDITS. *
14 *************************************************************************/
15
16#ifndef ROOT7_RNTupleZip
17#define ROOT7_RNTupleZip
18
19#include <RZip.h>
20#include <TError.h>
21
22#include <algorithm>
23#include <array>
24#include <cstring>
25#include <functional>
26#include <memory>
27#include <utility>
28
29namespace ROOT {
30namespace Experimental {
31namespace Detail {
32
33// clang-format off
34/**
35\class ROOT::Experimental::Detail::RNTupleCompressor
36\ingroup NTuple
37\brief Helper class to compress data blocks in the ROOT compression frame format
38*/
39// clang-format on
41private:
42 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
43 std::unique_ptr<Buffer_t> fZipBuffer;
44
45public:
46 /// Data might be overwritten, if a zipped block in the middle of a large input data stream
47 /// turns out to be uncompressible
48 using Writer_t = std::function<void(const void *buffer, size_t nbytes, size_t offset)>;
49 static Writer_t MakeMemCopyWriter(unsigned char *dest)
50 {
51 return [=](const void *b, size_t n, size_t o) { memcpy(dest + o, b, n); };
52 }
53 static constexpr size_t kMaxSingleBlock = kMAXZIPBUF;
54
55 RNTupleCompressor() : fZipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
56 RNTupleCompressor(const RNTupleCompressor &other) = delete;
60
61 /// Returns the size of the compressed data. Data is compressed in 16MB (kMAXZIPBUF) blocks and written
62 /// piecewise using the provided writer
63 size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter) {
64 R__ASSERT(from != nullptr);
65
66 auto cxLevel = compression % 100;
67 if (cxLevel == 0) {
68 fnWriter(from, nbytes, 0);
69 return nbytes;
70 }
71
72 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
73 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
74 char *source = const_cast<char *>(static_cast<const char *>(from));
75 int szTarget = kMAXZIPBUF;
76 char *target = reinterpret_cast<char *>(fZipBuffer->data());
77 int szOutBlock = 0;
78 int szRemaining = nbytes;
79 size_t szZipData = 0;
80 for (unsigned int i = 0; i < nZipBlocks; ++i) {
81 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
82 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
83 R__ASSERT(szOutBlock >= 0);
84 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
85 // Uncompressible block, we have to store the entire input data stream uncompressed
86 fnWriter(from, nbytes, 0);
87 return nbytes;
88 }
89
90 fnWriter(target, szOutBlock, szZipData);
91 szZipData += szOutBlock;
92 source += szSource;
93 szRemaining -= szSource;
94 }
95 R__ASSERT(szRemaining == 0);
96 R__ASSERT(szZipData < nbytes);
97 return szZipData;
98 }
99
100 /// Returns the size of the compressed data block. The data is written into the zip buffer.
101 /// This works only for small input buffer up to 16MB (kMAXZIPBUF)
102 size_t Zip(const void *from, size_t nbytes, int compression) {
103 R__ASSERT(from != nullptr);
104 R__ASSERT(nbytes <= kMAXZIPBUF);
105
106 auto cxLevel = compression % 100;
107 if (cxLevel == 0) {
108 memcpy(fZipBuffer->data(), from, nbytes);
109 return nbytes;
110 }
111
112 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
113 int szSource = nbytes;
114 char *source = const_cast<char *>(static_cast<const char *>(from));
115 int szTarget = nbytes;
116 char *target = reinterpret_cast<char *>(fZipBuffer->data());
117 int szOut = 0;
118 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOut, cxAlgorithm);
119 R__ASSERT(szOut >= 0);
120 if ((szOut > 0) && (static_cast<unsigned int>(szOut) < nbytes))
121 return szOut;
122
123 memcpy(fZipBuffer->data(), from, nbytes);
124 return nbytes;
125 }
126
127 /// Returns the size of the compressed data, written into the provided output buffer.
128 static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to) {
129 R__ASSERT(from != nullptr);
130 R__ASSERT(to != nullptr);
131 auto cxLevel = compression % 100;
132 if (cxLevel == 0) {
133 memcpy(to, from, nbytes);
134 return nbytes;
135 }
136
137 auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
138 unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
139 char *source = const_cast<char *>(static_cast<const char *>(from));
140 int szTarget = nbytes;
141 char *target = reinterpret_cast<char *>(to);
142 int szOutBlock = 0;
143 int szRemaining = nbytes;
144 size_t szZipData = 0;
145 for (unsigned int i = 0; i < nZipBlocks; ++i) {
146 int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
147 R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
148 R__ASSERT(szOutBlock >= 0);
149 if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
150 // Uncompressible block, we have to store the entire input data stream uncompressed
151 memcpy(to, from, nbytes);
152 return nbytes;
153 }
154
155 szZipData += szOutBlock;
156 source += szSource;
157 szRemaining -= szSource;
158 }
159 R__ASSERT(szRemaining == 0);
160 R__ASSERT(szZipData < nbytes);
161 return szZipData;
162 }
163
164 void *GetZipBuffer() { return fZipBuffer->data(); }
165};
166
167
168// clang-format off
169/**
170\class ROOT::Experimental::Detail::RNTupleDecompressor
171\ingroup NTuple
172\brief Helper class to uncompress data blocks in the ROOT compression frame format
173*/
174// clang-format on
176private:
177 using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
178 std::unique_ptr<Buffer_t> fUnzipBuffer;
179
180public:
181 RNTupleDecompressor() : fUnzipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
186
187 /**
188 * The nbytes parameter provides the size ls of the from buffer. The dataLen gives the size of the uncompressed data.
189 * The block is uncompressed iff nbytes == dataLen.
190 */
191 void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to) {
192 if (dataLen == nbytes) {
193 memcpy(to, from, nbytes);
194 return;
195 }
196 R__ASSERT(dataLen > nbytes);
197
198 unsigned char *source = const_cast<unsigned char *>(static_cast<const unsigned char *>(from));
199 unsigned char *target = static_cast<unsigned char *>(to);
200 int szRemaining = dataLen;
201 do {
202 int szSource;
203 int szTarget;
204 int retval = R__unzip_header(&szSource, source, &szTarget);
205 R__ASSERT(retval == 0);
206 R__ASSERT(szSource > 0);
207 R__ASSERT(szTarget > szSource);
208 R__ASSERT(static_cast<unsigned int>(szSource) <= nbytes);
209 R__ASSERT(static_cast<unsigned int>(szTarget) <= dataLen);
210
211 int unzipBytes = 0;
212 R__unzip(&szSource, source, &szTarget, target, &unzipBytes);
213 R__ASSERT(unzipBytes == szTarget);
214
215 target += szTarget;
216 source += szSource;
217 szRemaining -= unzipBytes;
218 } while (szRemaining > 0);
219 R__ASSERT(szRemaining == 0);
220 }
221
222 /**
223 * In-place decompression via unzip buffer
224 */
225 void Unzip(void *fromto, size_t nbytes, size_t dataLen) {
226 R__ASSERT(dataLen <= kMAXZIPBUF);
227 Unzip(fromto, nbytes, dataLen, fUnzipBuffer->data());
228 memcpy(fromto, fUnzipBuffer->data(), dataLen);
229 }
230};
231
232} // namespace Detail
233} // namespace Experimental
234} // namespace ROOT
235
236#endif
#define R__ASSERT(e)
Definition: TError.h:118
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t dest
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h offset
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t b
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
Helper class to compress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:40
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:42
std::unique_ptr< Buffer_t > fZipBuffer
Definition: RNTupleZip.hxx:43
RNTupleCompressor(RNTupleCompressor &&other)=default
static Writer_t MakeMemCopyWriter(unsigned char *dest)
Definition: RNTupleZip.hxx:49
std::function< void(const void *buffer, size_t nbytes, size_t offset)> Writer_t
Data might be overwritten, if a zipped block in the middle of a large input data stream turns out to ...
Definition: RNTupleZip.hxx:48
size_t Zip(const void *from, size_t nbytes, int compression)
Returns the size of the compressed data block.
Definition: RNTupleZip.hxx:102
static std::size_t Zip(const void *from, std::size_t nbytes, int compression, void *to)
Returns the size of the compressed data, written into the provided output buffer.
Definition: RNTupleZip.hxx:128
size_t Zip(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
Definition: RNTupleZip.hxx:63
RNTupleCompressor(const RNTupleCompressor &other)=delete
RNTupleCompressor & operator=(const RNTupleCompressor &other)=delete
Helper class to uncompress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:175
RNTupleDecompressor & operator=(const RNTupleDecompressor &other)=delete
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:177
void Unzip(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Definition: RNTupleZip.hxx:191
void Unzip(void *fromto, size_t nbytes, size_t dataLen)
In-place decompression via unzip buffer.
Definition: RNTupleZip.hxx:225
RNTupleDecompressor(RNTupleDecompressor &&other)=default
RNTupleDecompressor(const RNTupleDecompressor &other)=delete
const Int_t n
Definition: legend1.C:16
void(off) SmallVectorTemplateBase< T
void function(const Char_t *name_, T fun, const Char_t *docstring=0)
Definition: RExports.h:167
This file contains a specialised ROOT message handler to test for diagnostic in unit tests.
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition: Compression.h:83