Logo ROOT   master
Reference Guide
TTreeCache.h
Go to the documentation of this file.
1 // @(#)root/tree:$Id$
2 // Author: Rene Brun 04/06/2006
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 #ifndef ROOT_TTreeCache
13 #define ROOT_TTreeCache
14 
15 
16 //////////////////////////////////////////////////////////////////////////
17 // //
18 // TTreeCache //
19 // //
20 // Specialization of TFileCacheRead for a TTree //
21 // //
22 //////////////////////////////////////////////////////////////////////////
23 
24 #include "TFileCacheRead.h"
25 
26 #include <vector>
27 
28 class TTree;
29 class TBranch;
30 class TObjArray;
31 
32 class TTreeCache : public TFileCacheRead {
33 
34 public:
36 
37 protected:
38  Long64_t fEntryMin{0}; ///<! first entry in the cache
39  Long64_t fEntryMax{1}; ///<! last entry in the cache
40  Long64_t fEntryCurrent{-1}; ///<! current lowest entry number in the cache
41  Long64_t fEntryNext{-1}; ///<! next entry number where cache must be filled
42  Long64_t fCurrentClusterStart{-1}; ///<! Start of the cluster(s) where the current content was picked out
43  Long64_t fNextClusterStart{-1}; ///<! End+1 of the cluster(s) where the current content was picked out
44  Int_t fNbranches{0}; ///<! Number of branches in the cache
45  Int_t fNReadOk{0}; ///< Number of blocks read and found in the cache
46  Int_t fNMissReadOk{0}; ///< Number of blocks read, not found in the primary cache, and found in the secondary cache.
47  Int_t fNReadMiss{0}; ///< Number of blocks read and not found in the cache
48  Int_t fNMissReadMiss{0}; ///< Number of blocks read and not found in either cache.
49  Int_t fNReadPref{0}; ///< Number of blocks that were prefetched
50  Int_t fNMissReadPref{0}; ///< Number of blocks read into the secondary ("miss") cache.
51  TObjArray *fBranches{nullptr}; ///<! List of branches to be stored in the cache
52  TList *fBrNames{nullptr}; ///<! list of branch names in the cache
53  TTree *fTree{nullptr}; ///<! pointer to the current Tree
54  Bool_t fIsLearning{kTRUE}; ///<! true if cache is in learning mode
55  Bool_t fIsManual{kFALSE}; ///<! true if cache is StopLearningPhase was used
56  Bool_t fFirstBuffer{kTRUE}; ///<! true if first buffer is used for prefetching
57  Bool_t fOneTime{kFALSE}; ///<! used in the learning phase
58  Bool_t fReverseRead{kFALSE}; ///<! reading in reverse mode
59  Int_t fFillTimes{0}; ///<! how many times we can fill the current buffer
60  Bool_t fFirstTime{kTRUE}; ///<! save the fact that we processes the first entry
61  Long64_t fFirstEntry{-1}; ///<! save the value of the first entry
62  Bool_t fReadDirectionSet{kFALSE}; ///<! read direction established
63  Bool_t fEnabled{kTRUE}; ///<! cache enabled for cached reading
64  EPrefillType fPrefillType; ///< Whether a pre-filling is enabled (and if applicable which type)
65  static Int_t fgLearnEntries; ///< number of entries used for learning mode
66  Bool_t fAutoCreated{kFALSE}; ///<! true if cache was automatically created
67 
68  Bool_t fLearnPrefilling{kFALSE}; ///<! true if we are in the process of executing LearnPrefill
69 
70  // These members hold cached data for missed branches when miss optimization
71  // is enabled. Pointers are only initialized if the miss cache is enabled.
72  Bool_t fOptimizeMisses{kFALSE}; ///<! true if we should optimize cache misses.
73  Long64_t fFirstMiss{-1}; ///<! set to the event # of the first miss.
74  Long64_t fLastMiss{-1}; ///<! set to the event # of the last miss.
75 
76  // Representation of a positioned buffer IO.
77  // {0,0} designates the uninitialized - or invalid - buffer.
78  struct IOPos {
79  IOPos(Long64_t pos, Int_t len) : fPos(pos), fLen(len) {}
80 
81  Long64_t fPos{0}; //! Position in file of cache entry.
82  Int_t fLen{0}; //! Length of cache entry.
83  };
84 
85  struct MissCache {
86  struct Entry {
87  Entry(IOPos io) : fIO(io) {}
88 
90  ULong64_t fIndex{0}; ///<! Location in fData corresponding to this entry.
91  friend bool operator<(const Entry &a, const Entry &b) { return a.fIO.fPos < b.fIO.fPos; }
92  };
93  std::vector<Entry> fEntries; ///<! Description of buffers in the miss cache.
94  std::vector<TBranch *> fBranches; ///<! list of branches that we read on misses.
95  std::vector<char> fData; ///<! Actual data in the cache.
96 
97  void clear()
98  {
99  fEntries.clear();
100  fBranches.clear();
101  fData.clear();
102  }
103  };
104 
105  std::unique_ptr<MissCache> fMissCache; ///<! Cache contents for misses
106 
107 private:
108  TTreeCache(const TTreeCache &) = delete; ///< this class cannot be copied
109  TTreeCache &operator=(const TTreeCache &) = delete;
110 
111  // These are all functions related to the "miss cache": this attempts to
112  // optimize ROOT's behavior when the TTreeCache has a cache miss. In this
113  // case, we try to read several branches for the event with the miss.
114  //
115  // The miss cache is more CPU-intensive than the rest of the TTreeCache code;
116  // for local work (i.e., laptop with SSD), this CPU cost may outweight the
117  // benefit.
118  Bool_t CheckMissCache(char *buf, Long64_t pos,
119  int len); ///< Check the miss cache for a particular buffer, fetching if deemed necessary.
120  Bool_t FillMissCache(); ///< Fill the miss cache from the current set of active branches.
121  Bool_t CalculateMissCache(); ///< Calculate the appropriate miss cache to fetch; helper function for FillMissCache
122  IOPos FindBranchBasketPos(TBranch &, Long64_t entry); ///< Given a branch and an entry, determine the file location
123  ///< (offset / size) of the corresponding basket.
124  TBranch *CalculateMissEntries(Long64_t, int, bool); ///< Given an file read, try to determine the corresponding branch.
125  Bool_t ProcessMiss(Long64_t pos, int len); ///<! Given a file read not in the miss cache, handle (possibly) loading the data.
126 
127 public:
128 
129  TTreeCache();
130  TTreeCache(TTree *tree, Int_t buffersize=0);
131  virtual ~TTreeCache();
132  virtual Int_t AddBranch(TBranch *b, Bool_t subgbranches = kFALSE);
133  virtual Int_t AddBranch(const char *branch, Bool_t subbranches = kFALSE);
134  virtual Int_t DropBranch(TBranch *b, Bool_t subbranches = kFALSE);
135  virtual Int_t DropBranch(const char *branch, Bool_t subbranches = kFALSE);
136  virtual void Disable() {fEnabled = kFALSE;}
137  virtual void Enable() {fEnabled = kTRUE;}
139  const TObjArray *GetCachedBranches() const { return fBranches; }
141  Double_t GetEfficiency() const;
142  Double_t GetEfficiencyRel() const;
143  virtual Int_t GetEntryMin() const {return fEntryMin;}
144  virtual Int_t GetEntryMax() const {return fEntryMax;}
145  static Int_t GetLearnEntries();
146  virtual EPrefillType GetLearnPrefill() const {return fPrefillType;}
147  Double_t GetMissEfficiency() const;
149  TTree *GetTree() const {return fTree;}
151  virtual Bool_t IsEnabled() const {return fEnabled;}
152  virtual Bool_t IsLearning() const {return fIsLearning;}
153 
154  virtual Bool_t FillBuffer();
155  virtual Int_t LearnBranch(TBranch *b, Bool_t subgbranches = kFALSE);
156  virtual void LearnPrefill();
157 
158  virtual void Print(Option_t *option="") const;
159  virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len);
160  virtual Int_t ReadBufferNormal(char *buf, Long64_t pos, Int_t len);
161  virtual Int_t ReadBufferPrefetch(char *buf, Long64_t pos, Int_t len);
162  virtual void ResetCache();
163  void ResetMissCache(); // Reset the miss cache.
164  void SetAutoCreated(Bool_t val) {fAutoCreated = val;}
165  virtual Int_t SetBufferSize(Int_t buffersize);
166  virtual void SetEntryRange(Long64_t emin, Long64_t emax);
169  static void SetLearnEntries(Int_t n = 10);
170  void SetOptimizeMisses(Bool_t opt);
171  void StartLearningPhase();
172  virtual void StopLearningPhase();
173  virtual void UpdateBranches(TTree *tree);
174 
175  ClassDef(TTreeCache,3) //Specialization of TFileCacheRead for a TTree
176 };
177 
178 #endif
Bool_t CheckMissCache(char *buf, Long64_t pos, int len)
Check the miss cache for a particular buffer, fetching if deemed necessary.
Definition: TTreeCache.cxx:913
std::vector< char > fData
! Actual data in the cache.
Definition: TTreeCache.h:95
Long64_t fEntryMax
! last entry in the cache
Definition: TTreeCache.h:39
An array of TObjects.
Definition: TObjArray.h:37
Double_t GetMissEfficiencyRel() const
Relative efficiency of the &#39;miss cache&#39; - ratio of the reads found in cache to the number of reads so...
Int_t fNReadOk
Number of blocks read and found in the cache.
Definition: TTreeCache.h:45
long long Long64_t
Definition: RtypesCore.h:71
const TObjArray * GetCachedBranches() const
Definition: TTreeCache.h:139
virtual Int_t LearnBranch(TBranch *b, Bool_t subgbranches=kFALSE)
Add a branch discovered by actual usage to the list of branches to be stored in the cache this functi...
Definition: TTreeCache.cxx:342
std::vector< Entry > fEntries
! Description of buffers in the miss cache.
Definition: TTreeCache.h:93
TObjArray * fBranches
! List of branches to be stored in the cache
Definition: TTreeCache.h:51
A cache when reading files over the network.
const char Option_t
Definition: RtypesCore.h:64
Long64_t fLastMiss
! set to the event # of the last miss.
Definition: TTreeCache.h:74
Bool_t fOneTime
! used in the learning phase
Definition: TTreeCache.h:57
Int_t fNMissReadMiss
Number of blocks read and not found in either cache.
Definition: TTreeCache.h:48
virtual EPrefillType GetLearnPrefill() const
Definition: TTreeCache.h:146
virtual void SetLearnPrefill(EPrefillType type=kNoPrefill)
Set whether the learning period is started with a prefilling of the cache and which type of prefillin...
A cache to speed-up the reading of ROOT datasets.
Definition: TTreeCache.h:32
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:53
Long64_t fEntryMin
! first entry in the cache
Definition: TTreeCache.h:38
int Int_t
Definition: RtypesCore.h:43
bool Bool_t
Definition: RtypesCore.h:61
virtual void StopLearningPhase()
This is the counterpart of StartLearningPhase() and can be used to stop the learning phase...
virtual void LearnPrefill()
Perform an initial prefetch, attempting to read as much of the learning phase baskets for all branche...
void SetAutoCreated(Bool_t val)
Definition: TTreeCache.h:164
Bool_t fLearnPrefilling
! true if we are in the process of executing LearnPrefill
Definition: TTreeCache.h:68
void StartLearningPhase()
The name should be enough to explain the method.
Bool_t CalculateMissCache()
Calculate the appropriate miss cache to fetch; helper function for FillMissCache. ...
virtual Int_t AddBranch(TBranch *b, Bool_t subgbranches=kFALSE)
Add a branch to the list of branches to be stored in the cache this function is called by the user vi...
Definition: TTreeCache.cxx:368
Bool_t fAutoCreated
! true if cache was automatically created
Definition: TTreeCache.h:66
Int_t fNMissReadPref
Number of blocks read into the secondary ("miss") cache.
Definition: TTreeCache.h:50
Long64_t fNextClusterStart
! End+1 of the cluster(s) where the current content was picked out
Definition: TTreeCache.h:43
virtual Int_t ReadBufferNormal(char *buf, Long64_t pos, Int_t len)
Old method ReadBuffer before the addition of the prefetch mechanism.
Bool_t FillMissCache()
Fill the miss cache from the current set of active branches.
IOPos FindBranchBasketPos(TBranch &, Long64_t entry)
Given a branch and an entry, determine the file location (offset / size) of the corresponding basket...
Definition: TTreeCache.cxx:708
#define ClassDef(name, id)
Definition: Rtypes.h:322
Bool_t IsAutoCreated() const
Definition: TTreeCache.h:150
Bool_t fOptimizeMisses
! true if we should optimize cache misses.
Definition: TTreeCache.h:72
Bool_t GetOptimizeMisses() const
Definition: TTreeCache.h:138
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos if the request is in the list of prefetched blocks read from fBuffer...
virtual void ResetCache()
This will simply clear the cache.
Bool_t fEnabled
! cache enabled for cached reading
Definition: TTreeCache.h:63
TBranch * CalculateMissEntries(Long64_t, int, bool)
Given an file read, try to determine the corresponding branch.
Definition: TTreeCache.cxx:782
virtual void Enable()
Definition: TTreeCache.h:137
Double_t GetEfficiencyRel() const
This will indicate a sort of relative efficiency...
A doubly linked list.
Definition: TList.h:44
TTree * fTree
! pointer to the current Tree
Definition: TTreeCache.h:53
EPrefillType GetConfiguredPrefillType() const
Return the desired prefill type from the environment or resource variable.
virtual void SetEntryRange(Long64_t emin, Long64_t emax)
Set the minimum and maximum entry number to be processed this information helps to optimize the numbe...
virtual Int_t DropBranch(TBranch *b, Bool_t subbranches=kFALSE)
Remove a branch to the list of branches to be stored in the cache this function is called by TBranch:...
Definition: TTreeCache.cxx:533
auto * a
Definition: textangle.C:12
virtual Bool_t IsEnabled() const
Definition: TTreeCache.h:151
TTree * GetTree() const
Definition: TTreeCache.h:149
virtual Int_t ReadBufferPrefetch(char *buf, Long64_t pos, Int_t len)
Used to read a chunk from a block previously fetched.
virtual ~TTreeCache()
Destructor. (in general called by the TFile destructor)
Definition: TTreeCache.cxx:324
Bool_t fReadDirectionSet
! read direction established
Definition: TTreeCache.h:62
Int_t fNReadMiss
Number of blocks read and not found in the cache.
Definition: TTreeCache.h:47
static void SetLearnEntries(Int_t n=10)
Static function to set the number of entries to be used in learning mode The default value for n is 1...
std::unique_ptr< MissCache > fMissCache
! Cache contents for misses
Definition: TTreeCache.h:105
Long64_t fCurrentClusterStart
! Start of the cluster(s) where the current content was picked out
Definition: TTreeCache.h:42
virtual Bool_t FillBuffer()
Fill the cache buffer with the branches in the cache.
Int_t fNReadPref
Number of blocks that were prefetched.
Definition: TTreeCache.h:49
const Bool_t kFALSE
Definition: RtypesCore.h:90
void SetOptimizeMisses(Bool_t opt)
Start of methods for the miss cache.
Definition: TTreeCache.cxx:675
friend bool operator<(const Entry &a, const Entry &b)
Definition: TTreeCache.h:91
Bool_t fReverseRead
! reading in reverse mode
Definition: TTreeCache.h:58
Bool_t fFirstTime
! save the fact that we processes the first entry
Definition: TTreeCache.h:60
Long64_t fEntryCurrent
! current lowest entry number in the cache
Definition: TTreeCache.h:40
TTreeCache()
Default Constructor.
Definition: TTreeCache.cxx:305
virtual void Print(Option_t *option="") const
Print cache statistics.
double Double_t
Definition: RtypesCore.h:57
Bool_t fIsLearning
! true if cache is in learning mode
Definition: TTreeCache.h:54
virtual Int_t GetEntryMax() const
Definition: TTreeCache.h:144
int type
Definition: TGX11.cxx:120
unsigned long long ULong64_t
Definition: RtypesCore.h:72
void ResetMissCache()
Reset all the miss cache training.
Definition: TTreeCache.cxx:689
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Overload to make sure that the object specific.
IOPos(Long64_t pos, Int_t len)
Definition: TTreeCache.h:79
Int_t fNMissReadOk
Number of blocks read, not found in the primary cache, and found in the secondary cache...
Definition: TTreeCache.h:46
EPrefillType fPrefillType
Whether a pre-filling is enabled (and if applicable which type)
Definition: TTreeCache.h:64
Long64_t fFirstMiss
! set to the event # of the first miss.
Definition: TTreeCache.h:73
Int_t fFillTimes
! how many times we can fill the current buffer
Definition: TTreeCache.h:59
virtual Int_t SetBufferSize(Int_t buffersize)
Change the underlying buffer size of the cache.
virtual Bool_t IsLearning() const
Definition: TTreeCache.h:152
Int_t fNbranches
! Number of branches in the cache
Definition: TTreeCache.h:44
Bool_t ProcessMiss(Long64_t pos, int len)
! Given a file read not in the miss cache, handle (possibly) loading the data.
Definition: TTreeCache.cxx:855
Long64_t fFirstEntry
! save the value of the first entry
Definition: TTreeCache.h:61
Definition: file.py:1
Int_t fLen
Position in file of cache entry.
Definition: TTreeCache.h:82
TList * fBrNames
! list of branch names in the cache
Definition: TTreeCache.h:52
you should not use this method at all Int_t Int_t Double_t Double_t Double_t Int_t Double_t Double_t Double_t Double_t b
Definition: TRolke.cxx:630
Bool_t fFirstBuffer
! true if first buffer is used for prefetching
Definition: TTreeCache.h:56
static Int_t fgLearnEntries
number of entries used for learning mode
Definition: TTreeCache.h:65
Double_t GetMissEfficiency() const
The total efficiency of the &#39;miss cache&#39; - defined as the ratio of blocks found in the cache versus t...
Definition: tree.py:1
A TTree represents a columnar dataset.
Definition: TTree.h:78
Bool_t fIsManual
! true if cache is StopLearningPhase was used
Definition: TTreeCache.h:55
A TTree is a list of TBranches.
Definition: TBranch.h:88
Long64_t fEntryNext
! next entry number where cache must be filled
Definition: TTreeCache.h:41
std::vector< TBranch * > fBranches
! list of branches that we read on misses.
Definition: TTreeCache.h:94
virtual Int_t GetEntryMin() const
Definition: TTreeCache.h:143
virtual void UpdateBranches(TTree *tree)
Update pointer to current Tree and recompute pointers to the branches in the cache.
const Bool_t kTRUE
Definition: RtypesCore.h:89
TTreeCache & operator=(const TTreeCache &)=delete
const Int_t n
Definition: legend1.C:16
Double_t GetEfficiency() const
Give the total efficiency of the primary cache...
static Int_t GetLearnEntries()
Static function returning the number of entries used to train the cache see SetLearnEntries.
virtual void Disable()
Definition: TTreeCache.h:136
ULong64_t fIndex
! Location in fData corresponding to this entry.
Definition: TTreeCache.h:90
ECacheAction
TTreeCache flushing semantics.
Definition: TFile.h:70