Logo ROOT  
Reference Guide
TTreeCacheUnzip.h
Go to the documentation of this file.
1 // Authors: Rene Brun 04/06/2006
2 // Leandro Franco 10/04/2008
3 // Fabrizio Furano (CERN) Aug 2009
4 
5 /*************************************************************************
6  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
7  * All rights reserved. *
8  * *
9  * For the licensing terms see $ROOTSYS/LICENSE. *
10  * For the list of contributors see $ROOTSYS/README/CREDITS. *
11  *************************************************************************/
12 
13 #ifndef ROOT_TTreeCacheUnzip
14 #define ROOT_TTreeCacheUnzip
15 
16 #include "Bytes.h"
17 #include "TTreeCache.h"
18 #include <atomic>
19 #include <memory>
20 #include <vector>
21 
22 class TBasket;
23 class TBranch;
24 class TMutex;
25 class TTree;
26 
27 #ifdef R__USE_IMT
28 namespace ROOT {
29 namespace Experimental {
30 class TTaskGroup;
31 }
32 }
33 #endif
34 
35 class TTreeCacheUnzip : public TTreeCache {
36 
37 public:
38  // We have three possibilities for the unzipping mode:
39  // enable, disable and force
41 
42  // Unzipping states for a basket:
44 
45 protected:
46  // Unzipping state for baskets
47  struct UnzipState {
48  // Note: we cannot use std::unique_ptr<std::unique_ptr<char[]>[]> or vector of unique_ptr
49  // for fUnzipChunks since std::unique_ptr is not copy constructable.
50  // However, in future upgrade we cannot use make_vector in C++14.
51  std::unique_ptr<char[]> *fUnzipChunks; ///<! [fNseek] Individual unzipped chunks. Their summed size is kept under control.
52  std::vector<Int_t> fUnzipLen; ///<! [fNseek] Length of the unzipped buffers
53  std::atomic<Byte_t> *fUnzipStatus; ///<! [fNSeek]
54 
56  fUnzipChunks = nullptr;
57  fUnzipStatus = nullptr;
58  }
60  if (fUnzipChunks) delete [] fUnzipChunks;
61  if (fUnzipStatus) delete [] fUnzipStatus;
62  }
63  void Clear(Int_t size);
64  Bool_t IsUntouched(Int_t index) const;
65  Bool_t IsProgress(Int_t index) const;
66  Bool_t IsFinished(Int_t index) const;
67  Bool_t IsUnzipped(Int_t index) const;
68  void Reset(Int_t oldSize, Int_t newSize);
69  void SetUntouched(Int_t index);
70  void SetProgress(Int_t index);
71  void SetFinished(Int_t index);
72  void SetMissed(Int_t index);
73  void SetUnzipped(Int_t index, char* buf, Int_t len);
74  Bool_t TryUnzipping(Int_t index);
75  };
76 
77  typedef struct UnzipState UnzipState_t;
79 
80  // Members for paral. managing
84  Bool_t fParallel; ///< Indicate if we want to activate the parallelism (for this instance)
85 
86  std::unique_ptr<TMutex> fIOMutex;
87 
88  static TTreeCacheUnzip::EParUnzipMode fgParallel; ///< Indicate if we want to activate the parallelism
89 
90  // IMT TTaskGroup Manager
91 #ifdef R__USE_IMT
92  std::unique_ptr<ROOT::Experimental::TTaskGroup> fUnzipTaskGroup;
93 #endif
94 
95  // Unzipping related members
96  Int_t fNseekMax; ///<! fNseek can change so we need to know its max size
97  Int_t fUnzipGroupSize; ///<! Min accumulated size of a group of baskets ready to be unzipped by a IMT task
98  Long64_t fUnzipBufferSize; ///<! Max Size for the ready unzipped blocks (default is 2*fBufferSize)
99 
100  static Double_t fgRelBuffSize; ///< This is the percentage of the TTreeCacheUnzip that will be used
101 
102  // Members use to keep statistics
103  Int_t fNFound; ///<! number of blocks that were found in the cache
104  Int_t fNMissed; ///<! number of blocks that were not found in the cache and were unzipped
105  Int_t fNStalls; ///<! number of hits which caused a stall
106  Int_t fNUnzip; ///<! number of blocks that were unzipped
107 
108 private:
109  TTreeCacheUnzip(const TTreeCacheUnzip &); //this class cannot be copied
111 
112  char *fCompBuffer;
114 
115  // Private methods
116  void Init();
117 
118 public:
119  TTreeCacheUnzip();
120  TTreeCacheUnzip(TTree *tree, Int_t buffersize=0);
121  virtual ~TTreeCacheUnzip();
122 
123  virtual Int_t AddBranch(TBranch *b, Bool_t subbranches = kFALSE);
124  virtual Int_t AddBranch(const char *branch, Bool_t subbranches = kFALSE);
125  Bool_t FillBuffer();
126  virtual Int_t ReadBufferExt(char *buf, Long64_t pos, Int_t len, Int_t &loc);
127  void SetEntryRange(Long64_t emin, Long64_t emax);
128  virtual void StopLearningPhase();
129  void UpdateBranches(TTree *tree);
130 
131  // Methods related to the thread
133  static Bool_t IsParallelUnzip();
135 
136  // Unzipping related methods
137 #ifdef R__USE_IMT
138  Int_t CreateTasks();
139 #endif
140  Int_t GetRecordHeader(char *buf, Int_t maxbytes, Int_t &nbytes, Int_t &objlen, Int_t &keylen);
141  virtual Int_t GetUnzipBuffer(char **buf, Long64_t pos, Int_t len, Bool_t *free);
143  virtual void ResetCache();
144  virtual Int_t SetBufferSize(Int_t buffersize);
145  void SetUnzipBufferSize(Long64_t bufferSize);
146  void SetUnzipGroupSize(Int_t groupSize) { fUnzipGroupSize = groupSize; }
147  static void SetUnzipRelBufferSize(Float_t relbufferSize);
148  Int_t UnzipBuffer(char **dest, char *src);
149  Int_t UnzipCache(Int_t index);
150 
151  // Methods to get stats
152  Int_t GetNUnzip() { return fNUnzip; }
153  Int_t GetNMissed(){ return fNMissed; }
154  Int_t GetNFound() { return fNFound; }
155 
156  void Print(Option_t* option = "") const;
157 
158  // static members
159  ClassDef(TTreeCacheUnzip,0) //Specialization of TTreeCache for parallel unzipping
160 };
161 
162 #endif
TTreeCacheUnzip::fgRelBuffSize
static Double_t fgRelBuffSize
This is the percentage of the TTreeCacheUnzip that will be used.
Definition: TTreeCacheUnzip.h:100
TTreeCacheUnzip::~TTreeCacheUnzip
virtual ~TTreeCacheUnzip()
Destructor. (in general called by the TFile destructor)
Definition: TTreeCacheUnzip.cxx:234
TTreeCacheUnzip::kEnable
@ kEnable
Definition: TTreeCacheUnzip.h:40
TTreeCacheUnzip::fUnzipBufferSize
Long64_t fUnzipBufferSize
! Max Size for the ready unzipped blocks (default is 2*fBufferSize)
Definition: TTreeCacheUnzip.h:98
TTreeCacheUnzip::UnzipState::SetFinished
void SetFinished(Int_t index)
Set cache as finished.
Definition: TTreeCacheUnzip.cxx:125
dest
#define dest(otri, vertexptr)
Definition: triangle.c:1040
tree
Definition: tree.py:1
TTreeCacheUnzip::UnzipState::fUnzipChunks
std::unique_ptr< char[]> * fUnzipChunks
! [fNseek] Individual unzipped chunks. Their summed size is kept under control.
Definition: TTreeCacheUnzip.h:51
TTreeCacheUnzip::fParallel
Bool_t fParallel
Indicate if we want to activate the parallelism (for this instance)
Definition: TTreeCacheUnzip.h:84
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TTree
Definition: TTree.h:79
TTreeCacheUnzip::SetEntryRange
void SetEntryRange(Long64_t emin, Long64_t emax)
Set the minimum and maximum entry number to be processed this information helps to optimize the numbe...
Definition: TTreeCacheUnzip.cxx:374
TTreeCacheUnzip::AddBranch
virtual Int_t AddBranch(TBranch *b, Bool_t subbranches=kFALSE)
Add a branch to the list of branches to be stored in the cache this function is called by TBranch::Ge...
Definition: TTreeCacheUnzip.cxx:247
TTreeCacheUnzip::fAsyncReading
Bool_t fAsyncReading
Definition: TTreeCacheUnzip.h:81
Float_t
float Float_t
Definition: RtypesCore.h:57
TTreeCacheUnzip::kDisable
@ kDisable
Definition: TTreeCacheUnzip.h:40
TTreeCacheUnzip::SetParallelUnzip
static Int_t SetParallelUnzip(TTreeCacheUnzip::EParUnzipMode option=TTreeCacheUnzip::kEnable)
Static function that (de)activates multithreading unzipping.
Definition: TTreeCacheUnzip.cxx:435
Int_t
int Int_t
Definition: RtypesCore.h:45
TTreeCacheUnzip::UnzipBuffer
Int_t UnzipBuffer(char **dest, char *src)
Unzips a ROOT specific buffer...
Definition: TTreeCacheUnzip.cxx:850
TTreeCacheUnzip::GetUnzipBuffer
virtual Int_t GetUnzipBuffer(char **buf, Long64_t pos, Int_t len, Bool_t *free)
We try to read a buffer that has already been unzipped Returns -1 in case of read failure,...
Definition: TTreeCacheUnzip.cxx:665
TTreeCacheUnzip::SetUnzipRelBufferSize
static void SetUnzipRelBufferSize(Float_t relbufferSize)
static function: Sets the unzip relatibe buffer size
Definition: TTreeCacheUnzip.cxx:827
TTreeCacheUnzip::UnzipState::SetUntouched
void SetUntouched(Int_t index)
TTreeCacheUnzip::UnzipCache
Int_t UnzipCache(Int_t index)
This inflates a basket in the cache.
Definition: TTreeCacheUnzip.cxx:525
TTreeCacheUnzip
Definition: TTreeCacheUnzip.h:35
TTreeCacheUnzip::fNMissed
Int_t fNMissed
! number of blocks that were not found in the cache and were unzipped
Definition: TTreeCacheUnzip.h:104
TTreeCacheUnzip::fNseekMax
Int_t fNseekMax
! fNseek can change so we need to know its max size
Definition: TTreeCacheUnzip.h:96
TTreeCacheUnzip::fNUnzip
Int_t fNUnzip
! number of blocks that were unzipped
Definition: TTreeCacheUnzip.h:106
TTreeCacheUnzip::UnzipState::IsUnzipped
Bool_t IsUnzipped(Int_t index) const
Check if the basket is unzipped already.
Definition: TTreeCacheUnzip.cxx:84
TTreeCacheUnzip::Print
void Print(Option_t *option="") const
Print cache statistics.
Definition: TTreeCacheUnzip.cxx:944
b
#define b(i)
Definition: RSha256.hxx:118
TTreeCacheUnzip::operator=
TTreeCacheUnzip & operator=(const TTreeCacheUnzip &)
TTreeCacheUnzip::EUnzipState
EUnzipState
Definition: TTreeCacheUnzip.h:43
bool
TTreeCacheUnzip::UnzipState::Clear
void Clear(Int_t size)
Clear all baskets' state arrays.
Definition: TTreeCacheUnzip.cxx:52
TTreeCacheUnzip::fCompBuffer
char * fCompBuffer
Definition: TTreeCacheUnzip.h:112
TTreeCacheUnzip::UnzipState::fUnzipLen
std::vector< Int_t > fUnzipLen
! [fNseek] Length of the unzipped buffers
Definition: TTreeCacheUnzip.h:52
TTreeCacheUnzip::GetNUnzip
Int_t GetNUnzip()
Definition: TTreeCacheUnzip.h:152
TTreeCacheUnzip::fCompBufferSize
Int_t fCompBufferSize
Definition: TTreeCacheUnzip.h:113
TTreeCacheUnzip::ResetCache
virtual void ResetCache()
This will delete the list of buffers that are in the unzipping cache and will reset certain values in...
Definition: TTreeCacheUnzip.cxx:500
TTreeCacheUnzip::fNStalls
Int_t fNStalls
! number of hits which caused a stall
Definition: TTreeCacheUnzip.h:105
TBranch
Definition: TBranch.h:89
TTreeCacheUnzip::StopLearningPhase
virtual void StopLearningPhase()
It's the same as TTreeCache::StopLearningPhase but we guarantee that we start the unzipping just afte...
Definition: TTreeCacheUnzip.cxx:383
TTreeCacheUnzip::UnzipState::IsProgress
Bool_t IsProgress(Int_t index) const
Definition: TTreeCacheUnzip.cxx:70
TTreeCacheUnzip::SetBufferSize
virtual Int_t SetBufferSize(Int_t buffersize)
Change the underlying buffer size of the cache.
Definition: TTreeCacheUnzip.cxx:358
TTreeCacheUnzip::fCycle
Int_t fCycle
Definition: TTreeCacheUnzip.h:83
TTreeCacheUnzip::fIOMutex
std::unique_ptr< TMutex > fIOMutex
Definition: TTreeCacheUnzip.h:86
Option_t
const typedef char Option_t
Definition: RtypesCore.h:66
TTreeCacheUnzip::fgParallel
static TTreeCacheUnzip::EParUnzipMode fgParallel
Indicate if we want to activate the parallelism.
Definition: TTreeCacheUnzip.h:88
TTreeCache.h
TTreeCacheUnzip::UnzipState::SetProgress
void SetProgress(Int_t index)
TTreeCacheUnzip::FillBuffer
Bool_t FillBuffer()
Fill the cache buffer with the branches in the cache.
Definition: TTreeCacheUnzip.cxx:266
TTreeCacheUnzip::GetNFound
Int_t GetNFound()
Definition: TTreeCacheUnzip.h:154
TTreeCacheUnzip::EParUnzipMode
EParUnzipMode
Definition: TTreeCacheUnzip.h:40
TTreeCacheUnzip::UnzipState::fUnzipStatus
std::atomic< Byte_t > * fUnzipStatus
! [fNSeek]
Definition: TTreeCacheUnzip.h:53
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TTreeCacheUnzip::fNFound
Int_t fNFound
! number of blocks that were found in the cache
Definition: TTreeCacheUnzip.h:103
TTreeCacheUnzip::UnzipState
Definition: TTreeCacheUnzip.h:47
TTreeCacheUnzip::UnzipState::SetMissed
void SetMissed(Int_t index)
Definition: TTreeCacheUnzip.cxx:133
TTreeCacheUnzip::UnzipState::IsUntouched
Bool_t IsUntouched(Int_t index) const
Definition: TTreeCacheUnzip.cxx:64
TTreeCacheUnzip::IsParallelUnzip
static Bool_t IsParallelUnzip()
Static function that tells wether the multithreading unzipping is activated.
Definition: TTreeCacheUnzip.cxx:414
TTreeCacheUnzip::UnzipState::TryUnzipping
Bool_t TryUnzipping(Int_t index)
Start unzipping the basket if it is untouched yet.
Definition: TTreeCacheUnzip.cxx:150
TTreeCacheUnzip::GetRecordHeader
Int_t GetRecordHeader(char *buf, Int_t maxbytes, Int_t &nbytes, Int_t &objlen, Int_t &keylen)
Read the logical record header from the buffer buf.
Definition: TTreeCacheUnzip.cxx:468
TTreeCacheUnzip::fUnzipTaskGroup
std::unique_ptr< ROOT::Experimental::TTaskGroup > fUnzipTaskGroup
Definition: TTreeCacheUnzip.h:92
TTreeCacheUnzip::UnzipState::UnzipState
UnzipState()
Definition: TTreeCacheUnzip.h:55
TTreeCacheUnzip::GetParallelUnzip
static EParUnzipMode GetParallelUnzip()
Static function that returns the parallel option (to indicate an additional thread)
Definition: TTreeCacheUnzip.cxx:406
TTreeCacheUnzip::UnzipState::IsFinished
Bool_t IsFinished(Int_t index) const
Definition: TTreeCacheUnzip.cxx:76
TBasket
Definition: TBasket.h:34
TTreeCacheUnzip::GetNMissed
Int_t GetNMissed()
Definition: TTreeCacheUnzip.h:153
TTreeCache
A cache to speed-up the reading of ROOT datasets.
Definition: TTreeCache.h:32
TTreeCacheUnzip::kFinished
@ kFinished
Definition: TTreeCacheUnzip.h:43
Double_t
double Double_t
Definition: RtypesCore.h:59
TTreeCacheUnzip::SetUnzipGroupSize
void SetUnzipGroupSize(Int_t groupSize)
Definition: TTreeCacheUnzip.h:146
TTreeCacheUnzip::fUnzipGroupSize
Int_t fUnzipGroupSize
! Min accumulated size of a group of baskets ready to be unzipped by a IMT task
Definition: TTreeCacheUnzip.h:97
TTreeCacheUnzip::GetUnzipGroupSize
Int_t GetUnzipGroupSize()
Definition: TTreeCacheUnzip.h:142
TTreeCacheUnzip::UpdateBranches
void UpdateBranches(TTree *tree)
update pointer to current Tree and recompute pointers to the branches in the cache
Definition: TTreeCacheUnzip.cxx:391
TTreeCacheUnzip::UnzipState::SetUnzipped
void SetUnzipped(Int_t index, char *buf, Int_t len)
Definition: TTreeCacheUnzip.cxx:140
ClassDef
#define ClassDef(name, id)
Definition: Rtypes.h:325
Bytes.h
TTreeCacheUnzip::TTreeCacheUnzip
TTreeCacheUnzip()
Definition: TTreeCacheUnzip.cxx:158
TTreeCacheUnzip::fUnzipState
UnzipState_t fUnzipState
Definition: TTreeCacheUnzip.h:78
TTreeCacheUnzip::ReadBufferExt
virtual Int_t ReadBufferExt(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Definition: TTreeCacheUnzip.cxx:958
TTreeCacheUnzip::Init
void Init()
Initialization procedure common to all the constructors.
Definition: TTreeCacheUnzip.cxx:195
TTreeCacheUnzip::CreateTasks
Int_t CreateTasks()
We create a TTaskGroup and asynchronously maps each group of baskets(> 100 kB in total) to a task.
Definition: TTreeCacheUnzip.cxx:610
TTreeCacheUnzip::UnzipState::Reset
void Reset(Int_t oldSize, Int_t newSize)
Reset all baskets' state arrays.
Definition: TTreeCacheUnzip.cxx:94
free
#define free
Definition: civetweb.c:1539
TMutex
Definition: TMutex.h:30
TTreeCacheUnzip::kUntouched
@ kUntouched
Definition: TTreeCacheUnzip.h:43
TTreeCacheUnzip::kForce
@ kForce
Definition: TTreeCacheUnzip.h:40
ROOT
VSD Structures.
Definition: StringConv.hxx:21
TTreeCacheUnzip::fEmpty
Bool_t fEmpty
Definition: TTreeCacheUnzip.h:82
TTreeCacheUnzip::UnzipState::~UnzipState
~UnzipState()
Definition: TTreeCacheUnzip.h:59
TTreeCacheUnzip::kProgress
@ kProgress
Definition: TTreeCacheUnzip.h:43
TTreeCacheUnzip::SetUnzipBufferSize
void SetUnzipBufferSize(Long64_t bufferSize)
Sets the size for the unzipping cache...
Definition: TTreeCacheUnzip.cxx:836
int