Logo ROOT   6.19/01
Reference Guide
TTreeCache.h
Go to the documentation of this file.
1 // @(#)root/tree:$Id$
2 // Author: Rene Brun 04/06/2006
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 #ifndef ROOT_TTreeCache
13 #define ROOT_TTreeCache
14 
15 
16 //////////////////////////////////////////////////////////////////////////
17 // //
18 // TTreeCache //
19 // //
20 // Specialization of TFileCacheRead for a TTree //
21 // //
22 //////////////////////////////////////////////////////////////////////////
23 
24 #include "TFileCacheRead.h"
25 
26 #include <cstdint>
27 #include <memory>
28 #include <utility>
29 #include <vector>
30 
31 class TTree;
32 class TBranch;
33 class TObjArray;
34 
35 class TTreeCache : public TFileCacheRead {
36 
37 public:
39 
40 protected:
41  Long64_t fEntryMin{0}; ///<! first entry in the cache
42  Long64_t fEntryMax{1}; ///<! last entry in the cache
43  Long64_t fEntryCurrent{-1}; ///<! current lowest entry number in the cache
44  Long64_t fEntryNext{-1}; ///<! next entry number where cache must be filled
45  Long64_t fCurrentClusterStart{-1}; ///<! Start of the cluster(s) where the current content was picked out
46  Long64_t fNextClusterStart{-1}; ///<! End+1 of the cluster(s) where the current content was picked out
47  Int_t fNbranches{0}; ///<! Number of branches in the cache
48  Int_t fNReadOk{0}; ///< Number of blocks read and found in the cache
49  Int_t fNMissReadOk{0}; ///< Number of blocks read, not found in the primary cache, and found in the secondary cache.
50  Int_t fNReadMiss{0}; ///< Number of blocks read and not found in the cache
51  Int_t fNMissReadMiss{0}; ///< Number of blocks read and not found in either cache.
52  Int_t fNReadPref{0}; ///< Number of blocks that were prefetched
53  Int_t fNMissReadPref{0}; ///< Number of blocks read into the secondary ("miss") cache.
54  TObjArray *fBranches{nullptr}; ///<! List of branches to be stored in the cache
55  TList *fBrNames{nullptr}; ///<! list of branch names in the cache
56  TTree *fTree{nullptr}; ///<! pointer to the current Tree
57  Bool_t fIsLearning{kTRUE}; ///<! true if cache is in learning mode
58  Bool_t fIsManual{kFALSE}; ///<! true if cache is StopLearningPhase was used
59  Bool_t fFirstBuffer{kTRUE}; ///<! true if first buffer is used for prefetching
60  Bool_t fOneTime{kFALSE}; ///<! used in the learning phase
61  Bool_t fReverseRead{kFALSE}; ///<! reading in reverse mode
62  Int_t fFillTimes{0}; ///<! how many times we can fill the current buffer
63  Bool_t fFirstTime{kTRUE}; ///<! save the fact that we processes the first entry
64  Long64_t fFirstEntry{-1}; ///<! save the value of the first entry
65  Bool_t fReadDirectionSet{kFALSE}; ///<! read direction established
66  Bool_t fEnabled{kTRUE}; ///<! cache enabled for cached reading
67  EPrefillType fPrefillType; ///< Whether a pre-filling is enabled (and if applicable which type)
68  static Int_t fgLearnEntries; ///< number of entries used for learning mode
69  Bool_t fAutoCreated{kFALSE}; ///<! true if cache was automatically created
70 
71  Bool_t fLearnPrefilling{kFALSE}; ///<! true if we are in the process of executing LearnPrefill
72 
73  // These members hold cached data for missed branches when miss optimization
74  // is enabled. Pointers are only initialized if the miss cache is enabled.
75  Bool_t fOptimizeMisses{kFALSE}; ///<! true if we should optimize cache misses.
76  Long64_t fFirstMiss{-1}; ///<! set to the event # of the first miss.
77  Long64_t fLastMiss{-1}; ///<! set to the event # of the last miss.
78 
79  // Representation of a positioned buffer IO.
80  // {0,0} designates the uninitialized - or invalid - buffer.
81  struct IOPos {
82  IOPos(Long64_t pos, Int_t len) : fPos(pos), fLen(len) {}
83 
84  Long64_t fPos{0}; //! Position in file of cache entry.
85  Int_t fLen{0}; //! Length of cache entry.
86  };
87 
88  struct MissCache {
89  struct Entry {
90  Entry(IOPos io) : fIO(io) {}
91 
93  ULong64_t fIndex{0}; ///<! Location in fData corresponding to this entry.
94  friend bool operator<(const Entry &a, const Entry &b) { return a.fIO.fPos < b.fIO.fPos; }
95  };
96  std::vector<Entry> fEntries; ///<! Description of buffers in the miss cache.
97  std::vector<TBranch *> fBranches; ///<! list of branches that we read on misses.
98  std::vector<char> fData; ///<! Actual data in the cache.
99 
100  void clear()
101  {
102  fEntries.clear();
103  fBranches.clear();
104  fData.clear();
105  }
106  };
107 
108  std::unique_ptr<MissCache> fMissCache; ///<! Cache contents for misses
109 
110 private:
111  TTreeCache(const TTreeCache &) = delete; ///< this class cannot be copied
112  TTreeCache &operator=(const TTreeCache &) = delete;
113 
114  // These are all functions related to the "miss cache": this attempts to
115  // optimize ROOT's behavior when the TTreeCache has a cache miss. In this
116  // case, we try to read several branches for the event with the miss.
117  //
118  // The miss cache is more CPU-intensive than the rest of the TTreeCache code;
119  // for local work (i.e., laptop with SSD), this CPU cost may outweight the
120  // benefit.
121  Bool_t CheckMissCache(char *buf, Long64_t pos,
122  int len); ///< Check the miss cache for a particular buffer, fetching if deemed necessary.
123  Bool_t FillMissCache(); ///< Fill the miss cache from the current set of active branches.
124  Bool_t CalculateMissCache(); ///< Calculate the appropriate miss cache to fetch; helper function for FillMissCache
125  IOPos FindBranchBasketPos(TBranch &, Long64_t entry); ///< Given a branch and an entry, determine the file location
126  ///< (offset / size) of the corresponding basket.
127  TBranch *CalculateMissEntries(Long64_t, int, bool); ///< Given an file read, try to determine the corresponding branch.
128  Bool_t ProcessMiss(Long64_t pos, int len); ///<! Given a file read not in the miss cache, handle (possibly) loading the data.
129 
130 public:
131 
132  TTreeCache();
133  TTreeCache(TTree *tree, Int_t buffersize=0);
134  virtual ~TTreeCache();
135  virtual Int_t AddBranch(TBranch *b, Bool_t subgbranches = kFALSE);
136  virtual Int_t AddBranch(const char *branch, Bool_t subbranches = kFALSE);
137  virtual Int_t DropBranch(TBranch *b, Bool_t subbranches = kFALSE);
138  virtual Int_t DropBranch(const char *branch, Bool_t subbranches = kFALSE);
139  virtual void Disable() {fEnabled = kFALSE;}
140  virtual void Enable() {fEnabled = kTRUE;}
142  const TObjArray *GetCachedBranches() const { return fBranches; }
144  Double_t GetEfficiency() const;
145  Double_t GetEfficiencyRel() const;
146  virtual Int_t GetEntryMin() const {return fEntryMin;}
147  virtual Int_t GetEntryMax() const {return fEntryMax;}
148  static Int_t GetLearnEntries();
149  virtual EPrefillType GetLearnPrefill() const {return fPrefillType;}
150  Double_t GetMissEfficiency() const;
152  TTree *GetTree() const {return fTree;}
154  virtual Bool_t IsEnabled() const {return fEnabled;}
155  virtual Bool_t IsLearning() const {return fIsLearning;}
156 
157  virtual Bool_t FillBuffer();
158  virtual Int_t LearnBranch(TBranch *b, Bool_t subgbranches = kFALSE);
159  virtual void LearnPrefill();
160 
161  virtual void Print(Option_t *option="") const;
162  virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len);
163  virtual Int_t ReadBufferNormal(char *buf, Long64_t pos, Int_t len);
164  virtual Int_t ReadBufferPrefetch(char *buf, Long64_t pos, Int_t len);
165  virtual void ResetCache();
166  void ResetMissCache(); // Reset the miss cache.
167  void SetAutoCreated(Bool_t val) {fAutoCreated = val;}
168  virtual Int_t SetBufferSize(Int_t buffersize);
169  virtual void SetEntryRange(Long64_t emin, Long64_t emax);
172  static void SetLearnEntries(Int_t n = 10);
173  void SetOptimizeMisses(Bool_t opt);
174  void StartLearningPhase();
175  virtual void StopLearningPhase();
176  virtual void UpdateBranches(TTree *tree);
177 
178  ClassDef(TTreeCache,3) //Specialization of TFileCacheRead for a TTree
179 };
180 
181 #endif
Bool_t CheckMissCache(char *buf, Long64_t pos, int len)
Check the miss cache for a particular buffer, fetching if deemed necessary.
Definition: TTreeCache.cxx:913
std::vector< char > fData
! Actual data in the cache.
Definition: TTreeCache.h:98
Long64_t fEntryMax
! last entry in the cache
Definition: TTreeCache.h:42
An array of TObjects.
Definition: TObjArray.h:37
Double_t GetMissEfficiencyRel() const
Relative efficiency of the &#39;miss cache&#39; - ratio of the reads found in cache to the number of reads so...
Int_t fNReadOk
Number of blocks read and found in the cache.
Definition: TTreeCache.h:48
long long Long64_t
Definition: RtypesCore.h:69
const TObjArray * GetCachedBranches() const
Definition: TTreeCache.h:142
virtual Int_t LearnBranch(TBranch *b, Bool_t subgbranches=kFALSE)
Add a branch discovered by actual usage to the list of branches to be stored in the cache this functi...
Definition: TTreeCache.cxx:342
std::vector< Entry > fEntries
! Description of buffers in the miss cache.
Definition: TTreeCache.h:96
TObjArray * fBranches
! List of branches to be stored in the cache
Definition: TTreeCache.h:54
A cache when reading files over the network.
const char Option_t
Definition: RtypesCore.h:62
Long64_t fLastMiss
! set to the event # of the last miss.
Definition: TTreeCache.h:77
Bool_t fOneTime
! used in the learning phase
Definition: TTreeCache.h:60
Int_t fNMissReadMiss
Number of blocks read and not found in either cache.
Definition: TTreeCache.h:51
virtual EPrefillType GetLearnPrefill() const
Definition: TTreeCache.h:149
virtual void SetLearnPrefill(EPrefillType type=kNoPrefill)
Set whether the learning period is started with a prefilling of the cache and which type of prefillin...
A cache to speed-up the reading of ROOT datasets.
Definition: TTreeCache.h:35
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:48
Long64_t fEntryMin
! first entry in the cache
Definition: TTreeCache.h:41
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
virtual void StopLearningPhase()
This is the counterpart of StartLearningPhase() and can be used to stop the learning phase...
virtual void LearnPrefill()
Perform an initial prefetch, attempting to read as much of the learning phase baskets for all branche...
void SetAutoCreated(Bool_t val)
Definition: TTreeCache.h:167
Bool_t fLearnPrefilling
! true if we are in the process of executing LearnPrefill
Definition: TTreeCache.h:71
void StartLearningPhase()
The name should be enough to explain the method.
Bool_t CalculateMissCache()
Calculate the appropriate miss cache to fetch; helper function for FillMissCache. ...
virtual Int_t AddBranch(TBranch *b, Bool_t subgbranches=kFALSE)
Add a branch to the list of branches to be stored in the cache this function is called by the user vi...
Definition: TTreeCache.cxx:368
Bool_t fAutoCreated
! true if cache was automatically created
Definition: TTreeCache.h:69
Int_t fNMissReadPref
Number of blocks read into the secondary ("miss") cache.
Definition: TTreeCache.h:53
Long64_t fNextClusterStart
! End+1 of the cluster(s) where the current content was picked out
Definition: TTreeCache.h:46
virtual Int_t ReadBufferNormal(char *buf, Long64_t pos, Int_t len)
Old method ReadBuffer before the addition of the prefetch mechanism.
Bool_t FillMissCache()
Fill the miss cache from the current set of active branches.
IOPos FindBranchBasketPos(TBranch &, Long64_t entry)
Given a branch and an entry, determine the file location (offset / size) of the corresponding basket...
Definition: TTreeCache.cxx:708
#define ClassDef(name, id)
Definition: Rtypes.h:326
Bool_t IsAutoCreated() const
Definition: TTreeCache.h:153
Bool_t fOptimizeMisses
! true if we should optimize cache misses.
Definition: TTreeCache.h:75
Bool_t GetOptimizeMisses() const
Definition: TTreeCache.h:141
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos if the request is in the list of prefetched blocks read from fBuffer...
virtual void ResetCache()
This will simply clear the cache.
Bool_t fEnabled
! cache enabled for cached reading
Definition: TTreeCache.h:66
TBranch * CalculateMissEntries(Long64_t, int, bool)
Given an file read, try to determine the corresponding branch.
Definition: TTreeCache.cxx:782
virtual void Enable()
Definition: TTreeCache.h:140
Double_t GetEfficiencyRel() const
This will indicate a sort of relative efficiency...
A doubly linked list.
Definition: TList.h:44
TTree * fTree
! pointer to the current Tree
Definition: TTreeCache.h:56
EPrefillType GetConfiguredPrefillType() const
Return the desired prefill type from the environment or resource variable.
virtual void SetEntryRange(Long64_t emin, Long64_t emax)
Set the minimum and maximum entry number to be processed this information helps to optimize the numbe...
virtual Int_t DropBranch(TBranch *b, Bool_t subbranches=kFALSE)
Remove a branch to the list of branches to be stored in the cache this function is called by TBranch:...
Definition: TTreeCache.cxx:533
auto * a
Definition: textangle.C:12
virtual Bool_t IsEnabled() const
Definition: TTreeCache.h:154
TTree * GetTree() const
Definition: TTreeCache.h:152
virtual Int_t ReadBufferPrefetch(char *buf, Long64_t pos, Int_t len)
Used to read a chunk from a block previously fetched.
virtual ~TTreeCache()
Destructor. (in general called by the TFile destructor)
Definition: TTreeCache.cxx:324
Bool_t fReadDirectionSet
! read direction established
Definition: TTreeCache.h:65
Int_t fNReadMiss
Number of blocks read and not found in the cache.
Definition: TTreeCache.h:50
static void SetLearnEntries(Int_t n=10)
Static function to set the number of entries to be used in learning mode The default value for n is 1...
std::unique_ptr< MissCache > fMissCache
! Cache contents for misses
Definition: TTreeCache.h:108
Long64_t fCurrentClusterStart
! Start of the cluster(s) where the current content was picked out
Definition: TTreeCache.h:45
virtual Bool_t FillBuffer()
Fill the cache buffer with the branches in the cache.
Int_t fNReadPref
Number of blocks that were prefetched.
Definition: TTreeCache.h:52
const Bool_t kFALSE
Definition: RtypesCore.h:88
void SetOptimizeMisses(Bool_t opt)
Start of methods for the miss cache.
Definition: TTreeCache.cxx:675
friend bool operator<(const Entry &a, const Entry &b)
Definition: TTreeCache.h:94
Bool_t fReverseRead
! reading in reverse mode
Definition: TTreeCache.h:61
Bool_t fFirstTime
! save the fact that we processes the first entry
Definition: TTreeCache.h:63
Long64_t fEntryCurrent
! current lowest entry number in the cache
Definition: TTreeCache.h:43
TTreeCache()
Default Constructor.
Definition: TTreeCache.cxx:305
virtual void Print(Option_t *option="") const
Print cache statistics.
double Double_t
Definition: RtypesCore.h:55
Bool_t fIsLearning
! true if cache is in learning mode
Definition: TTreeCache.h:57
virtual Int_t GetEntryMax() const
Definition: TTreeCache.h:147
int type
Definition: TGX11.cxx:120
unsigned long long ULong64_t
Definition: RtypesCore.h:70
void ResetMissCache()
Reset all the miss cache training.
Definition: TTreeCache.cxx:689
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Overload to make sure that the object specific.
IOPos(Long64_t pos, Int_t len)
Definition: TTreeCache.h:82
Int_t fNMissReadOk
Number of blocks read, not found in the primary cache, and found in the secondary cache...
Definition: TTreeCache.h:49
EPrefillType fPrefillType
Whether a pre-filling is enabled (and if applicable which type)
Definition: TTreeCache.h:67
Long64_t fFirstMiss
! set to the event # of the first miss.
Definition: TTreeCache.h:76
Int_t fFillTimes
! how many times we can fill the current buffer
Definition: TTreeCache.h:62
virtual Int_t SetBufferSize(Int_t buffersize)
Change the underlying buffer size of the cache.
virtual Bool_t IsLearning() const
Definition: TTreeCache.h:155
Int_t fNbranches
! Number of branches in the cache
Definition: TTreeCache.h:47
Bool_t ProcessMiss(Long64_t pos, int len)
! Given a file read not in the miss cache, handle (possibly) loading the data.
Definition: TTreeCache.cxx:855
Long64_t fFirstEntry
! save the value of the first entry
Definition: TTreeCache.h:64
Definition: file.py:1
Int_t fLen
Position in file of cache entry.
Definition: TTreeCache.h:85
TList * fBrNames
! list of branch names in the cache
Definition: TTreeCache.h:55
you should not use this method at all Int_t Int_t Double_t Double_t Double_t Int_t Double_t Double_t Double_t Double_t b
Definition: TRolke.cxx:630
Bool_t fFirstBuffer
! true if first buffer is used for prefetching
Definition: TTreeCache.h:59
static Int_t fgLearnEntries
number of entries used for learning mode
Definition: TTreeCache.h:68
Double_t GetMissEfficiency() const
The total efficiency of the &#39;miss cache&#39; - defined as the ratio of blocks found in the cache versus t...
Definition: tree.py:1
A TTree represents a columnar dataset.
Definition: TTree.h:72
Bool_t fIsManual
! true if cache is StopLearningPhase was used
Definition: TTreeCache.h:58
A TTree is a list of TBranches.
Definition: TBranch.h:90
Long64_t fEntryNext
! next entry number where cache must be filled
Definition: TTreeCache.h:44
std::vector< TBranch * > fBranches
! list of branches that we read on misses.
Definition: TTreeCache.h:97
virtual Int_t GetEntryMin() const
Definition: TTreeCache.h:146
virtual void UpdateBranches(TTree *tree)
Update pointer to current Tree and recompute pointers to the branches in the cache.
const Bool_t kTRUE
Definition: RtypesCore.h:87
TTreeCache & operator=(const TTreeCache &)=delete
const Int_t n
Definition: legend1.C:16
Double_t GetEfficiency() const
Give the total efficiency of the primary cache...
static Int_t GetLearnEntries()
Static function returning the number of entries used to train the cache see SetLearnEntries.
virtual void Disable()
Definition: TTreeCache.h:139
ULong64_t fIndex
! Location in fData corresponding to this entry.
Definition: TTreeCache.h:93
ECacheAction
TTreeCache flushing semantics.
Definition: TFile.h:65