Logo ROOT  
Reference Guide
RNTupleZip.hxx
Go to the documentation of this file.
1 /// \file ROOT/RNTupleZip.hxx
2 /// \ingroup NTuple ROOT7
3 /// \author Jakob Blomer <jblomer@cern.ch>
4 /// \date 2019-11-21
5 /// \warning This is part of the ROOT 7 prototype! It will change without notice. It might trigger earthquakes. Feedback
6 /// is welcome!
7 
8 /*************************************************************************
9  * Copyright (C) 1995-2019, Rene Brun and Fons Rademakers. *
10  * All rights reserved. *
11  * *
12  * For the licensing terms see $ROOTSYS/LICENSE. *
13  * For the list of contributors see $ROOTSYS/README/CREDITS. *
14  *************************************************************************/
15 
16 #ifndef ROOT7_RNTupleZip
17 #define ROOT7_RNTupleZip
18 
19 #include <RZip.h>
20 #include <TError.h>
21 
22 #include <algorithm>
23 #include <array>
24 #include <cstring>
25 #include <functional>
26 #include <memory>
27 #include <utility>
28 
29 namespace ROOT {
30 namespace Experimental {
31 namespace Detail {
32 
33 // clang-format off
34 /**
35 \class ROOT::Experimental::Detail::RNTupleCompressor
36 \ingroup NTuple
37 \brief Helper class to compress data blocks in the ROOT compression frame format
38 */
39 // clang-format on
41 private:
42  using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
43  std::unique_ptr<Buffer_t> fZipBuffer;
44 
45 public:
46  /// Data might be overwritten, if a zipped block in the middle of a large input data stream
47  /// turns out to be uncompressible
48  using Writer_t = std::function<void(const void *buffer, size_t nbytes, size_t offset)>;
49  static constexpr size_t kMaxSingleBlock = kMAXZIPBUF;
50 
51  RNTupleCompressor() : fZipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
52  RNTupleCompressor(const RNTupleCompressor &other) = delete;
56 
57  /// Returns the size of the compressed data. Data is compressed in 16MB blocks and written
58  /// piecewise using the provided writer
59  size_t operator() (const void *from, size_t nbytes, int compression, Writer_t fnWriter) {
60  R__ASSERT(from != nullptr);
61 
62  auto cxLevel = compression % 100;
63  if (cxLevel == 0) {
64  fnWriter(from, nbytes, 0);
65  return nbytes;
66  }
67 
68  auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
69  unsigned int nZipBlocks = 1 + (nbytes - 1) / kMAXZIPBUF;
70  char *source = const_cast<char *>(static_cast<const char *>(from));
71  int szTarget = kMAXZIPBUF;
72  char *target = reinterpret_cast<char *>(fZipBuffer->data());
73  int szOutBlock = 0;
74  int szRemaining = nbytes;
75  size_t szZipData = 0;
76  for (unsigned int i = 0; i < nZipBlocks; ++i) {
77  int szSource = std::min(static_cast<int>(kMAXZIPBUF), szRemaining);
78  R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOutBlock, cxAlgorithm);
79  R__ASSERT(szOutBlock >= 0);
80  if ((szOutBlock == 0) || (szOutBlock >= szSource)) {
81  // Uncompressible block, we have to store the entire input data stream uncompressed
82  fnWriter(from, nbytes, 0);
83  return nbytes;
84  }
85 
86  fnWriter(target, szOutBlock, szZipData);
87  szZipData += szOutBlock;
88  source += szSource;
89  szRemaining -= szSource;
90  }
91  R__ASSERT(szRemaining == 0);
92  R__ASSERT(szZipData < nbytes);
93  return szZipData;
94  }
95 
96  /// Returns the size of the compressed data block. The data is written into the zip buffer.
97  /// This works only for small input buffer up to 16MB
98  size_t operator() (const void *from, size_t nbytes, int compression) {
99  R__ASSERT(from != nullptr);
100  R__ASSERT(nbytes <= kMAXZIPBUF);
101 
102  auto cxLevel = compression % 100;
103  if (cxLevel == 0) {
104  memcpy(fZipBuffer->data(), from, nbytes);
105  return nbytes;
106  }
107 
108  auto cxAlgorithm = static_cast<ROOT::RCompressionSetting::EAlgorithm::EValues>(compression / 100);
109  int szSource = nbytes;
110  char *source = const_cast<char *>(static_cast<const char *>(from));
111  int szTarget = nbytes;
112  char *target = reinterpret_cast<char *>(fZipBuffer->data());
113  int szOut = 0;
114  R__zipMultipleAlgorithm(cxLevel, &szSource, source, &szTarget, target, &szOut, cxAlgorithm);
115  R__ASSERT(szOut >= 0);
116  if ((szOut > 0) && (static_cast<unsigned int>(szOut) < nbytes))
117  return szOut;
118 
119  memcpy(fZipBuffer->data(), from, nbytes);
120  return nbytes;
121  }
122 
123  const void *GetZipBuffer() { return fZipBuffer->data(); }
124 };
125 
126 
127 // clang-format off
128 /**
129 \class ROOT::Experimental::Detail::RNTupleDecompressor
130 \ingroup NTuple
131 \brief Helper class to uncompress data blocks in the ROOT compression frame format
132 */
133 // clang-format on
135 private:
136  using Buffer_t = std::array<unsigned char, kMAXZIPBUF>;
137  std::unique_ptr<Buffer_t> fUnzipBuffer;
138 
139 public:
140  RNTupleDecompressor() : fUnzipBuffer(std::unique_ptr<Buffer_t>(new Buffer_t())) {}
141  RNTupleDecompressor(const RNTupleDecompressor &other) = delete;
145 
146  /**
147  * The nbytes parameter provides the size ls of the from buffer. The dataLen gives the size of the uncompressed data.
148  * The block is uncompressed iff nbytes == dataLen.
149  */
150  void operator() (const void *from, size_t nbytes, size_t dataLen, void *to) {
151  if (dataLen == nbytes) {
152  memcpy(to, from, nbytes);
153  return;
154  }
155  R__ASSERT(dataLen > nbytes);
156 
157  unsigned char *source = const_cast<unsigned char *>(static_cast<const unsigned char *>(from));
158  unsigned char *target = static_cast<unsigned char *>(to);
159  int szRemaining = dataLen;
160  do {
161  int szSource;
162  int szTarget;
163  int retval = R__unzip_header(&szSource, source, &szTarget);
164  R__ASSERT(retval == 0);
165  R__ASSERT(szSource > 0);
166  R__ASSERT(szTarget > szSource);
167  R__ASSERT(static_cast<unsigned int>(szSource) <= nbytes);
168  R__ASSERT(static_cast<unsigned int>(szTarget) <= dataLen);
169 
170  int unzipBytes = 0;
171  R__unzip(&szSource, source, &szTarget, target, &unzipBytes);
172  R__ASSERT(unzipBytes == szTarget);
173 
174  target += szTarget;
175  source += szSource;
176  szRemaining -= unzipBytes;
177  } while (szRemaining > 0);
178  R__ASSERT(szRemaining == 0);
179  }
180 
181  /**
182  * In-place decompression via unzip buffer
183  */
184  void operator() (void *fromto, size_t nbytes, size_t dataLen) {
185  R__ASSERT(dataLen <= kMAXZIPBUF);
186  operator()(fromto, nbytes, dataLen, fUnzipBuffer->data());
187  memcpy(fromto, fUnzipBuffer->data(), dataLen);
188  }
189 };
190 
191 } // namespace Detail
192 } // namespace Experimental
193 } // namespace ROOT
194 
195 #endif
ROOT::Experimental::Detail::RNTupleCompressor::Buffer_t
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:42
ROOT::Experimental::Detail::RNTupleDecompressor::fUnzipBuffer
std::unique_ptr< Buffer_t > fUnzipBuffer
Definition: RNTupleZip.hxx:137
ROOT::Experimental::Detail::RNTupleCompressor::kMaxSingleBlock
static constexpr size_t kMaxSingleBlock
Definition: RNTupleZip.hxx:49
ROOT::Experimental::Detail::RNTupleCompressor::RNTupleCompressor
RNTupleCompressor(const RNTupleCompressor &other)=delete
ROOT::Experimental::Detail::RNTupleCompressor::operator=
RNTupleCompressor & operator=(const RNTupleCompressor &other)=delete
R__unzip
void R__unzip(Int_t *nin, UChar_t *bufin, Int_t *lout, char *bufout, Int_t *nout)
ROOT::RCompressionSetting::EAlgorithm::EValues
EValues
Note: this is only temporarily a struct and will become a enum class hence the name.
Definition: Compression.h:83
ROOT::Experimental::Detail::RNTupleCompressor::operator()
size_t operator()(const void *from, size_t nbytes, int compression, Writer_t fnWriter)
Returns the size of the compressed data.
Definition: RNTupleZip.hxx:59
ROOT::Experimental::Detail::RNTupleDecompressor
Helper class to uncompress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:134
R__unzip_header
int R__unzip_header(Int_t *nin, UChar_t *bufin, Int_t *lout)
ROOT::Experimental::Detail::RNTupleDecompressor::operator=
RNTupleDecompressor & operator=(const RNTupleDecompressor &other)=delete
ROOT::Experimental::Detail::RNTupleDecompressor::RNTupleDecompressor
RNTupleDecompressor()
Definition: RNTupleZip.hxx:140
ROOT::Experimental::Detail::RNTupleDecompressor::operator()
void operator()(const void *from, size_t nbytes, size_t dataLen, void *to)
The nbytes parameter provides the size ls of the from buffer.
Definition: RNTupleZip.hxx:150
ROOT::R::function
void function(const Char_t *name_, T fun, const Char_t *docstring=0)
Definition: RExports.h:151
ROOT::Experimental::Detail::RNTupleCompressor::Writer_t
std::function< void(const void *buffer, size_t nbytes, size_t offset)> Writer_t
Data might be overwritten, if a zipped block in the middle of a large input data stream turns out to ...
Definition: RNTupleZip.hxx:48
ROOT::Experimental::Detail::RNTupleCompressor::GetZipBuffer
const void * GetZipBuffer()
Definition: RNTupleZip.hxx:123
void
typedef void((*Func_t)())
ROOT::Experimental::Detail::RNTupleCompressor::fZipBuffer
std::unique_ptr< Buffer_t > fZipBuffer
Definition: RNTupleZip.hxx:43
R__ASSERT
#define R__ASSERT(e)
Definition: TError.h:120
ROOT::Experimental::Detail::RNTupleCompressor::RNTupleCompressor
RNTupleCompressor(RNTupleCompressor &&other)=default
ROOT::Experimental::Detail::RNTupleCompressor::RNTupleCompressor
RNTupleCompressor()
Definition: RNTupleZip.hxx:51
ROOT::Experimental::Detail::RNTupleDecompressor::RNTupleDecompressor
RNTupleDecompressor(RNTupleDecompressor &&other)=default
ROOT
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Definition: EExecutionPolicy.hxx:4
ROOT::Experimental::Detail::RNTupleCompressor
Helper class to compress data blocks in the ROOT compression frame format.
Definition: RNTupleZip.hxx:40
ROOT::Experimental::Detail::RNTupleDecompressor::RNTupleDecompressor
RNTupleDecompressor(const RNTupleDecompressor &other)=delete
ROOT::Experimental::Detail::RNTupleDecompressor::Buffer_t
std::array< unsigned char, kMAXZIPBUF > Buffer_t
Definition: RNTupleZip.hxx:136
TError.h