Logo ROOT  
Reference Guide
TBufferMerger.hxx
Go to the documentation of this file.
1// @(#)root/io:$Id$
2// Author: Philippe Canal, Witold Pokorski, and Guilherme Amadio
3
4/*************************************************************************
5 * Copyright (C) 1995-2017, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12#ifndef ROOT_TBufferMerger
13#define ROOT_TBufferMerger
14
15#include "TFileMerger.h"
16#include "TMemFile.h"
17
18#include <functional>
19#include <memory>
20#include <mutex>
21#include <queue>
22
23namespace ROOT {
24namespace Experimental {
25
26class TBufferMergerFile;
27
28/**
29 * \class TBufferMerger TBufferMerger.hxx
30 * \ingroup IO
31 *
32 * TBufferMerger is a class to facilitate writing data in
33 * parallel from multiple threads, while writing to a single
34 * output file. Its purpose is similar to TParallelMergingFile,
35 * but instead of using processes that connect to a network
36 * socket, TBufferMerger uses threads that each write to a
37 * TBufferMergerFile, which in turn push data into a queue
38 * managed by the TBufferMerger.
39 */
40
42public:
43 /** Constructor
44 * @param name Output file name
45 * @param option Output file creation options
46 * @param compression Output file compression level
47 */
49
50 /** Constructor
51 * @param output Output \c TFile
52 */
53 TBufferMerger(std::unique_ptr<TFile> output);
54
55 /** Destructor */
56 virtual ~TBufferMerger();
57
58 /** Returns a TBufferMergerFile to which data can be written.
59 * At the end, all TBufferMergerFiles get merged into the output file.
60 * The user is responsible to "cd" into the file to associate objects
61 * such as histograms or trees to it.
62 *
63 * After the creation of this file, the user must reset the kMustCleanup
64 * bit on any objects attached to it and take care of their deletion, as
65 * there is a possibility that a race condition will happen that causes
66 * a crash if ROOT manages these objects.
67 */
68 std::shared_ptr<TBufferMergerFile> GetFile();
69
70 /** Returns the number of buffers currently in the queue. */
71 size_t GetQueueSize() const;
72
73 /** Returns the number of bytes currently buffered (i.e. in the queue). */
74 size_t GetBuffered() const
75 {
76 return fBuffered;
77 }
78
79 /** Returns the current value of the auto save setting in bytes (default = 0). */
80 size_t GetAutoSave() const;
81
82 /** Returns the current merge options. */
83 const char* GetMergeOptions();
84
85 /** By default, TBufferMerger will call TFileMerger::PartialMerge() for each
86 * buffer pushed onto its merge queue. This function lets the user change
87 * this behaviour by telling TBufferMerger to accumulate at least size
88 * bytes in memory before performing a partial merge and flushing to disk.
89 * This can be useful to avoid an excessive amount of work to happen in the
90 * output thread, as the number of TTree headers (which require compression)
91 * written to disk can be reduced.
92 */
93 void SetAutoSave(size_t size);
94
95 /** Sets the merge options. SetMergeOptions("fast") will disable
96 * recompression of input data into the output if they have different
97 * compression settings.
98 * @param options TFileMerger/TFileMergeInfo merge options
99 */
100 void SetMergeOptions(const TString& options);
101
102 /** Indicates that any TTree objects in the file should be skipped
103 * and thus that steps that are specific to TTree can be skipped */
105 {
106 fMerger.SetNotrees(notrees);
107 }
108
109 /** Returns whether the the file has been marked as not containing any TTree objects
110 * and thus that steps that are specific to TTree can be skipped */
112 {
113 return fMerger.GetNotrees();
114 }
115
116 /** Indicates that the temporary keys (corresponding to the object held by the directories
117 * of the TMemFile) should be compressed or not. Those object are stored in the TMemFile
118 * (and thus possibly compressed) when a thread push its data forward (by calling
119 * TBufferMergerFile::Write) and the queue is being processed by another.
120 * Once the TMemFile is picked (by any thread to be merged), *after* taking the
121 * TBufferMerger::fMergeMutex, those object are read back (and thus possibly uncompressed)
122 * and then used by merging.
123 * In order word, the compression of those objects/keys is only usefull to reduce the size
124 * in memory (of the TMemFile) and does not affect (at all) the compression factor of the end
125 * result.
126 */
127 void SetCompressTemporaryKeys(Bool_t request_compression = true)
128 {
129 fCompressTemporaryKeys = request_compression;
130 }
131
132 /** Returns whether to compressed the TKey in the TMemFile for the object held by
133 * the TDirectories. See TBufferMerger::SetCompressTemporaryKeys for more details.
134 */
136 {
138 }
139
140 friend class TBufferMergerFile;
141
142private:
143 /** TBufferMerger has no default constructor */
145
146 /** TBufferMerger has no copy constructor */
148
149 /** TBufferMerger has no copy operator */
151
152 void Init(std::unique_ptr<TFile>);
153
154 void MergeImpl();
155
156 void Merge();
157 void Push(TBufferFile *buffer);
158 bool TryMerge(TBufferMergerFile *memfile);
159
160 bool fCompressTemporaryKeys{false}; //< Enable compression of the TKeys in the TMemFile (save memory at the expense of time, end result is unchanged)
161 size_t fAutoSave{0}; //< AutoSave only every fAutoSave bytes
162 std::atomic<size_t> fBuffered{0}; //< Number of bytes currently buffered
163 TFileMerger fMerger{false, false}; //< TFileMerger used to merge all buffers
164 std::mutex fMergeMutex; //< Mutex used to lock fMerger
165 mutable std::mutex fQueueMutex; //< Mutex used to lock fQueue
166 std::queue<TBufferFile *> fQueue; //< Queue to which data is pushed and merged
167 std::vector<std::weak_ptr<TBufferMergerFile>> fAttachedFiles; //< Attached files
168};
169
170/**
171 * \class TBufferMerger TBufferMerger.hxx
172 * \ingroup IO
173 *
174 * A TBufferMergerFile is similar to a TMemFile, but when data
175 * is written to it, it is appended to the TBufferMerger queue.
176 * The TBufferMerger merges all data into the output file on disk.
177 */
178
180private:
181 TBufferMerger &fMerger; //< TBufferMerger this file is attached to
182
183 /** Constructor. Can only be called by TBufferMerger.
184 * @param m Merger this file is attached to. */
186
187 /** TBufferMergerFile has no default constructor. */
189
190 /** TBufferMergerFile has no copy constructor. */
192
193 /** TBufferMergerFile has no copy operator */
195
196 friend class TBufferMerger;
197
198public:
199 /** Destructor */
201
202 using TMemFile::Write;
203
204 /** Write data into a TBufferFile and append it to TBufferMerger.
205 * @param name Name
206 * @param opt Options
207 * @param bufsize Buffer size
208 * This function must be called before the TBufferMergerFile gets destroyed,
209 * or no data is appended to the TBufferMerger.
210 */
211 virtual Int_t Write(const char *name = nullptr, Int_t opt = 0, Int_t bufsize = 0) override;
212
214};
215
216} // namespace Experimental
217} // namespace ROOT
218
219#endif
int Int_t
Definition: RtypesCore.h:43
const Bool_t kFALSE
Definition: RtypesCore.h:90
bool Bool_t
Definition: RtypesCore.h:61
const char Option_t
Definition: RtypesCore.h:64
char name[80]
Definition: TGX11.cxx:109
TBufferMergerFile()
TBufferMergerFile has no default constructor.
virtual Int_t Write(const char *name=nullptr, Int_t opt=0, Int_t bufsize=0) override
Write data into a TBufferFile and append it to TBufferMerger.
TBufferMergerFile(const TBufferMergerFile &)
TBufferMergerFile has no copy constructor.
TBufferMergerFile & operator=(const TBufferMergerFile &)
TBufferMergerFile has no copy operator.
ClassDefOverride(TBufferMergerFile, 0)
TBufferMerger is a class to facilitate writing data in parallel from multiple threads,...
const char * GetMergeOptions()
Returns the current merge options.
bool TryMerge(TBufferMergerFile *memfile)
void Push(TBufferFile *buffer)
std::shared_ptr< TBufferMergerFile > GetFile()
Returns a TBufferMergerFile to which data can be written.
void SetCompressTemporaryKeys(Bool_t request_compression=true)
Indicates that the temporary keys (corresponding to the object held by the directories of the TMemFil...
std::queue< TBufferFile * > fQueue
size_t GetQueueSize() const
Returns the number of buffers currently in the queue.
Bool_t GetNotrees() const
Returns whether the the file has been marked as not containing any TTree objects and thus that steps ...
void SetMergeOptions(const TString &options)
Sets the merge options.
void Init(std::unique_ptr< TFile >)
virtual ~TBufferMerger()
Destructor.
size_t GetAutoSave() const
Returns the current value of the auto save setting in bytes (default = 0).
void SetAutoSave(size_t size)
By default, TBufferMerger will call TFileMerger::PartialMerge() for each buffer pushed onto its merge...
void SetNotrees(Bool_t notrees=kFALSE)
Indicates that any TTree objects in the file should be skipped and thus that steps that are specific ...
TBufferMerger()
TBufferMerger has no default constructor.
TBufferMerger & operator=(const TBufferMerger &)
TBufferMerger has no copy operator.
TBufferMerger(const TBufferMerger &)
TBufferMerger has no copy constructor.
Bool_t GetCompressTemporaryKeys() const
Returns whether to compressed the TKey in the TMemFile for the object held by the TDirectories.
size_t GetBuffered() const
Returns the number of bytes currently buffered (i.e.
std::vector< std::weak_ptr< TBufferMergerFile > > fAttachedFiles
The concrete implementation of TBuffer for writing/reading to/from a ROOT file or socket.
Definition: TBufferFile.h:46
This class provides file copy and merging services.
Definition: TFileMerger.h:32
Bool_t GetNotrees() const
Definition: TFileMerger.h:125
virtual void SetNotrees(Bool_t notrees=kFALSE)
Definition: TFileMerger.h:126
Int_t Write(const char *name=nullptr, Int_t opt=0, Int_t bufsiz=0) override
Write memory objects to this file.
Definition: TFile.cxx:2297
A TMemFile is like a normal TFile except that it reads and writes only from memory.
Definition: TMemFile.h:19
Basic string class.
Definition: TString.h:131
tbb::task_arena is an alias of tbb::interface7::task_arena, which doesn't allow to forward declare tb...
Definition: StringConv.hxx:21
@ kUseCompiledDefault
Use the compile-time default setting.
Definition: Compression.h:50
auto * m
Definition: textangle.C:8
static void output(int code)
Definition: gifencode.c:226