ROOT logo
ROOT » TREE » TREE » TTreeCacheUnzip

class TTreeCacheUnzip: public TTreeCache

Parallel Unzipping

TTreeCache has been modified to create an additional thread to unzip
the buffers that are sitting on the cache, in that way we could
improve the overall ROOT performance by executing two tasks in
parallel.
The order is important and for the moment we will do it following
the sorting algorithm in TTreeCache since they are usually read by
entry.
By default the unzipping cache will need only 10% of the buffer for
TTreCache so be careful with the memory (in a normal case you need
10MB for TTreeCache and 1MB for the unizp. buffer), to change it use
TTreeCache::SetUnzipBufferSize(Long64_t bufferSize)
where bufferSize must be passed in bytes.

This is a implemented in a similar way to a consumer-producer model
where the the producer will be TTreCache by transfering the data
and the consumer will be additional Thread trying to unzip it.

Function Members (Methods)

public:
TTreeCacheUnzip()
TTreeCacheUnzip(TTree* tree, Int_t buffersize = 0)
virtual~TTreeCacheUnzip()
voidTObject::AbstractMethod(const char* method) const
virtual voidAddBranch(TBranch* b, Bool_t subbranches = kFALSE)
virtual voidAddBranch(const char* branch, Bool_t subbranches = kFALSE)
virtual voidTTreeCache::AddBranch(TBranch* b, Bool_t subbranches = kFALSE)
virtual voidTTreeCache::AddBranch(const char* branch, Bool_t subbranches = kFALSE)
virtual voidTObject::AppendPad(Option_t* option = "")
virtual voidTObject::Browse(TBrowser* b)
static TClass*Class()
static TClass*TTreeCache::Class()
static TClass*TFileCacheRead::Class()
static TClass*TObject::Class()
virtual const char*TObject::ClassName() const
virtual voidTObject::Clear(Option_t* = "")
virtual TObject*TObject::Clone(const char* newname = "") const
virtual Int_tTObject::Compare(const TObject* obj) const
virtual voidTObject::Copy(TObject& object) const
virtual voidTObject::Delete(Option_t* option = "")MENU
virtual Int_tTObject::DistancetoPrimitive(Int_t px, Int_t py)
virtual voidTObject::Draw(Option_t* option = "")
virtual voidTObject::DrawClass() constMENU
virtual TObject*TObject::DrawClone(Option_t* option = "") constMENU
virtual voidTObject::Dump() constMENU
virtual voidTObject::Error(const char* method, const char* msgfmt) const
virtual voidTObject::Execute(const char* method, const char* params, Int_t* error = 0)
virtual voidTObject::Execute(TMethod* method, TObjArray* params, Int_t* error = 0)
virtual voidTObject::ExecuteEvent(Int_t event, Int_t px, Int_t py)
virtual voidTObject::Fatal(const char* method, const char* msgfmt) const
virtual Bool_tFillBuffer()
virtual TObject*TObject::FindObject(const char* name) const
virtual TObject*TObject::FindObject(const TObject* obj) const
virtual Int_tTFileCacheRead::GetBufferSize() const
virtual Option_t*TObject::GetDrawOption() const
static Long_tTObject::GetDtorOnly()
Double_tTTreeCache::GetEfficiency()
Double_tTTreeCache::GetEfficiencyRel()
virtual const char*TObject::GetIconName() const
static Int_tTTreeCache::GetLearnEntries()
virtual const char*TObject::GetName() const
Int_tGetNFound()
Int_tGetNMissed()
Int_tGetNUnzip()
virtual char*TObject::GetObjectInfo(Int_t px, Int_t py) const
static Bool_tTObject::GetObjectStat()
virtual Option_t*TObject::GetOption() const
TTree*TTreeCache::GetOwner() const
static TTreeCacheUnzip::EParUnzipModeGetParallelUnzip()
Int_tGetRecordHeader(char* buf, Int_t maxbytes, Int_t& nbytes, Int_t& objlen, Int_t& keylen)
virtual Bool_tGetSkipZip()
virtual Bool_tTTreeCache::GetSkipZip()
virtual const char*TObject::GetTitle() const
TTree*TTreeCache::GetTree() const
virtual UInt_tTObject::GetUniqueID() const
Int_tGetUnzipBuffer(char** buf, Long64_t pos, Int_t len, Bool_t* free)
virtual Bool_tTObject::HandleTimer(TTimer* timer)
virtual ULong_tTObject::Hash() const
virtual voidTObject::Info(const char* method, const char* msgfmt) const
virtual Bool_tTObject::InheritsFrom(const char* classname) const
virtual Bool_tTObject::InheritsFrom(const TClass* cl) const
virtual voidTObject::Inspect() constMENU
voidTObject::InvertBit(UInt_t f)
virtual TClass*IsA() const
virtual TClass*TTreeCache::IsA() const
Bool_tIsActiveThread()
virtual Bool_tTFileCacheRead::IsAsyncReading() const
virtual Bool_tTObject::IsEqual(const TObject* obj) const
virtual Bool_tTObject::IsFolder() const
Bool_tTTreeCache::IsLearning() const
Bool_tTObject::IsOnHeap() const
static Bool_tIsParallelUnzip()
Bool_tIsQueueEmpty()
virtual Bool_tTObject::IsSortable() const
Bool_tTObject::IsZombie() const
virtual voidTObject::ls(Option_t* option = "") const
voidTObject::MayNotUse(const char* method) const
virtual Bool_tTObject::Notify()
static voidTObject::operator delete(void* ptr)
static voidTObject::operator delete(void* ptr, void* vp)
static voidTObject::operator delete[](void* ptr)
static voidTObject::operator delete[](void* ptr, void* vp)
void*TObject::operator new(size_t sz)
void*TObject::operator new(size_t sz, void* vp)
void*TObject::operator new[](size_t sz)
void*TObject::operator new[](size_t sz, void* vp)
TObject&TObject::operator=(const TObject& rhs)
virtual voidTObject::Paint(Option_t* option = "")
virtual voidTObject::Pop()
virtual voidTFileCacheRead::Prefetch(Long64_t pos, Int_t len)
virtual voidTFileCacheRead::Print(Option_t* option = "") const
virtual voidTObject::Print(Option_t* option = "") const
Int_tProcessQueue()
virtual Int_tTObject::Read(const char* name)
virtual Int_tTTreeCache::ReadBuffer(char* buf, Long64_t pos, Int_t len)
virtual Int_tTFileCacheRead::ReadBuffer(char* buf, Long64_t pos, Int_t len)
virtual voidTObject::RecursiveRemove(TObject* obj)
voidTObject::ResetBit(UInt_t f)
virtual voidResetCache()
virtual voidTTreeCache::ResetCache()
virtual voidTObject::SaveAs(const char* filename = "", Option_t* option = "") constMENU
virtual voidTObject::SavePrimitive(basic_ostream<char,char_traits<char> >& out, Option_t* option = "")
voidSendSignal()
voidTObject::SetBit(UInt_t f)
voidTObject::SetBit(UInt_t f, Bool_t set)
virtual voidTObject::SetDrawOption(Option_t* option = "")MENU
static voidTObject::SetDtorOnly(void* obj)
virtual voidSetEntryRange(Long64_t emin, Long64_t emax)
virtual voidTTreeCache::SetEntryRange(Long64_t emin, Long64_t emax)
virtual voidTFileCacheRead::SetFile(TFile* file)
static voidTTreeCache::SetLearnEntries(Int_t n = 100)
static voidTObject::SetObjectStat(Bool_t stat)
static Int_tSetParallelUnzip(TTreeCacheUnzip::EParUnzipMode option = TTreeCacheUnzip::kEnable)
virtual voidSetSkipZip(Bool_t skip = kTRUE)
virtual voidTTreeCache::SetSkipZip(Bool_t skip = kTRUE)
virtual voidTObject::SetUniqueID(UInt_t uid)
voidSetUnzipBufferSize(Long64_t bufferSize)
virtual voidShowMembers(TMemberInspector& insp, char* parent)
virtual voidTTreeCache::ShowMembers(TMemberInspector& insp, char* parent)
virtual voidTFileCacheRead::ShowMembers(TMemberInspector& insp, char* parent)
virtual voidTObject::ShowMembers(TMemberInspector& insp, char* parent)
virtual voidTFileCacheRead::Sort()
voidTTreeCache::StartLearningPhase()
virtual voidStopLearningPhase()
virtual voidTTreeCache::StopLearningPhase()
virtual voidStreamer(TBuffer& b)
virtual voidTTreeCache::Streamer(TBuffer& b)
virtual voidTFileCacheRead::Streamer(TBuffer& b)
virtual voidTObject::Streamer(TBuffer& b)
voidStreamerNVirtual(TBuffer& b)
voidTTreeCache::StreamerNVirtual(TBuffer& b)
voidTFileCacheRead::StreamerNVirtual(TBuffer& b)
voidTObject::StreamerNVirtual(TBuffer& b)
virtual voidTObject::SysError(const char* method, const char* msgfmt) const
Bool_tTObject::TestBit(UInt_t f) const
Int_tTObject::TestBits(UInt_t f) const
Int_tUnzipBuffer(char** dest, char* src)
Int_tUnzipCache()
static void*UnzipLoop(void* arg)
virtual voidUpdateBranches(TTree* tree, Bool_t owner = kFALSE)
virtual voidTTreeCache::UpdateBranches(TTree* tree, Bool_t owner = kFALSE)
virtual voidTObject::UseCurrentStyle()
voidWaitForSignal()
virtual voidTObject::Warning(const char* method, const char* msgfmt) const
virtual Int_tTObject::Write(const char* name = 0, Int_t option = 0, Int_t bufsize = 0)
virtual Int_tTObject::Write(const char* name = 0, Int_t option = 0, Int_t bufsize = 0) const
protected:
virtual voidTObject::DoError(int level, const char* location, const char* fmt, va_list va) const
voidTObject::MakeZombie()

Data Members

private:
enum EParUnzipMode { kEnable
kDisable
kForce
};
enum TObject::EStatusBits { kCanDelete
kMustCleanup
kObjInCanvas
kIsReferenced
kHasUUID
kCannotPick
kNoContextMenu
kInvalidObject
};
enum TObject::[unnamed] { kIsOnHeap
kNotDeleted
kZombie
kBitMask
kSingleKey
kOverwrite
kWriteDelete
};
protected:
Bool_tfActiveThread
Bool_tTFileCacheRead::fAsyncReading
TList*TTreeCache::fBrNames! list of branch names in the cache
TObjArray*TTreeCache::fBranches! List of branches to be stored in the cache
char*TFileCacheRead::fBuffer[fBufferSize] buffer of contiguous prefetched blocks
Int_tTFileCacheRead::fBufferLenCurrent buffer length (<= fBufferSize)
Int_tTFileCacheRead::fBufferSizeAllocated size of fBuffer (at a given time)
Int_tTFileCacheRead::fBufferSizeMinOriginal size of fBuffer
Int_tTFileCacheRead::fBytesToPrefetchHelpers to allow the incremental async prefetch
Long64_tTTreeCache::fEntryMax! last entry in the cache
Long64_tTTreeCache::fEntryMin! first entry in the cache
Long64_tTTreeCache::fEntryNext! next entry number where cache must be filled
TFile*TFileCacheRead::fFilePointer to file
Int_tTFileCacheRead::fFirstIndexToPrefetch
Bool_tTTreeCache::fIsLearning! true if cache is in learning mode
Bool_tTTreeCache::fIsManual! true if cache is StopLearningPhase was used
Bool_tTFileCacheRead::fIsSortedTrue if fSeek array is sorted
Bool_tTFileCacheRead::fIsTransferredTrue when fBuffer contains something valid
Int_t*TFileCacheRead::fLen[fNb] Length of long buffers
TMutex*fMutexBufferMutex to protect the unzipping buffer 'fUnzipBuffer'
TMutex*fMutexCache
TMutex*fMutexListMutex to protect the list of inflated buffer
Int_tfNFound! number of blocks that were found in the cache
Int_tfNMissed! number of blocks that were not found in the cache and were unzipped
Int_tTTreeCache::fNReadMissNumber of blocks read and not found in the chache
Int_tTTreeCache::fNReadOkNumber of blocks read and found in the cache
Int_tTTreeCache::fNReadPrefNumber of blocks that were prefetched
Int_tfNUnzip! number of blocks that were unzipped
Int_tTFileCacheRead::fNbNumber of long buffers
Int_tTTreeCache::fNbranches! Number of branches in the cache
Bool_tfNewTransferUsed to indicate the second thread taht a new transfer is in progress
Int_tTFileCacheRead::fNseekNumber of blocks to be prefetched
Int_tfNseekMax! fNseek can change so we need to know its max size
Int_tTFileCacheRead::fNtotTotal size of prefetched blocks
TTree*TTreeCache::fOwner! pointer to the owner Tree/chain
Bool_tfParallelIndicate if we want to activate the parallelism (for this instance)
Long64_t*TFileCacheRead::fPos[fNb] start of long buffers
Long64_tfPosWrite
Long64_t*TFileCacheRead::fSeek[fNseek] Position on file of buffers to be prefetched
Int_t*TFileCacheRead::fSeekIndex[fNseek] sorted index table of fSeek
Int_t*TFileCacheRead::fSeekLen[fNseek] Length of buffers to be prefetched
Int_t*TFileCacheRead::fSeekPos[fNseek] Position of sorted blocks in fBuffer
Int_tTFileCacheRead::fSeekSizeAllocated size of fSeek
Long64_t*TFileCacheRead::fSeekSort[fNseek] Position on file of buffers to be prefetched (sorted)
Int_t*TFileCacheRead::fSeekSortLen[fNseek] Length of buffers to be prefetched (sorted)
Bool_tfSkipZipsay if we should skip the uncompression of all buffers
char*fTmpBuffer! [fTmpBufferSz] buffer of contiguous unzipped blocks
Int_tfTmpBufferSz! Size for the fTmpBuffer (default is 10KB... used to unzip a buffer)
TTree*TTreeCache::fTree! pointer to the current Tree
char*fUnzipBuffer! [fTotBytes] buffer of contiguous unzipped blocks
Long64_tfUnzipBufferSize! Size for the fUnzipBuffer (default is 2*fBufferSize)
TCondition*fUnzipCondition
Int_tfUnzipEnd! Unzipped buffers go from fUnzipStart to fUnzipEnd
Int_t*fUnzipLen! [fNseek] Length of buffers to be unzipped
Int_tfUnzipNext! From fUnzipEnd to fUnzipNext we have to buffer that will be unzipped soon
Int_t*fUnzipPos! [fNseek] Position of sorted blocks in fUnzipBuffer
Int_tfUnzipStart! This will give uf the start index (fSeekSort)
TThread*fUnzipThread
Long64_tTTreeCache::fZipBytes! Total compressed size of branches in cache
static Int_tTTreeCache::fgLearnEntriesNumber of entries used for learning mode
static TTreeCacheUnzip::EParUnzipModefgParallelIndicate if we want to activate the parallelism
static Double_tfgRelBuffSizeThis is the percentage of the TTreeCacheUnzip that will be used

Class Charts

Inheritance Inherited Members Includes Libraries
Class Charts

Function documentation

TTreeCacheUnzip()
 Default Constructor.
TTreeCacheUnzip(TTree* tree, Int_t buffersize = 0)
 Constructor.
void Init()
 Initialization procedure common to all the constructors
~TTreeCacheUnzip()
 destructor. (in general called by the TFile destructor
 destructor. (in general called by the TFile destructor)
void AddBranch(TBranch *b, Bool_t subbranches /*= kFALSE*/)
add a branch to the list of branches to be stored in the cache
this function is called by TBranch::GetBasket
void AddBranch(const char *branch, Bool_t subbranches /*= kFALSE*/)
add a branch to the list of branches to be stored in the cache
this function is called by TBranch::GetBasket
Bool_t FillBuffer()
 Fill the cache buffer with the branches in the cache.
void SetEntryRange(Long64_t emin, Long64_t emax)
 Set the minimum and maximum entry number to be processed
 this information helps to optimize the number of baskets to read
 when prefetching the branch buffers.
void StopLearningPhase()
 It's the same as TTreeCache::StopLearningPhase but we guarantee that
 we start the unzipping just after getting the buffers
void UpdateBranches(TTree* tree, Bool_t owner = kFALSE)
update pointer to current Tree and recompute pointers to the branches in the cache
Bool_t IsParallelUnzip()
 Static function that tells wether the multithreading unzipping
 is activated
Bool_t IsActiveThread()
 This indicates if the thread is active in this moment...
 this variable is very important because if we change it from true to
 false the thread will stop... ( see StopThreadTreeCacheUnzip() )
Bool_t IsQueueEmpty()
 It says if the queue is empty... useful to see if we have to process
 it.
Int_t ProcessQueue()
 This will traverse the queue and read all the buffers to put them
 in the cache... after reading each buffer it will send a signal and
 if someone is waiting for it, it will wake up and will check the queue
 again to see if the buffer it was waiting for was the buffer that was
 just processed.
 It works that way because I can not have one signal per-buffer...
 instead there is only one signal and all the methods waiting for buffer
 will know that a "new" buffer has been processed (even if that particular
 method is not the one waiting for the buffer)
void SendSignal()
 This will send the signal corresponfing to the queue... normally used
 when we want to start processing the list of buffers.
Int_t SetParallelUnzip(TTreeCacheUnzip::EParUnzipMode option = TTreeCacheUnzip::kEnable)
 Static function that(de)activates multithreading unzipping
 The possible options are:
 kEnable _Enable_ it, which causes an automatic detection and launches the
 additional thread if the number of cores in the machine is greater than one
 kDisable _Disable_ will not activate the additional thread.
 kForce _Force_ will start the additional thread even if there is only one core.
 the default will be taken as kEnable.
 returns 0 if there was an error, 1 otherwise.
Int_t StartThreadUnzip()
 The Thread is only a part of the TTreeCache but it is the part that
 waits for info in the queue and process it... unfortunatly, a Thread is
 not an object an we have to deal with it in the old C-Style way
 Returns 0 if the thread was initialized or 1 if it was already running
Int_t StopThreadUnzip()
 To stop the thread we only need to change the value of the variable
 fActiveThread to false and the loop will stop (of course, we will have)
 to do the cleaning after that.
 Note: The syncronization part is important here or we will try to delete
       teh object while it's still processing the queue
void WaitForSignal()
 This is the counter part of SendSignal() and is used to wait for a buffer
 that is in the queue and will be processed soon (instead of making a new
 call)
void* UnzipLoop(void* arg)
 This is a static function.
 This is the call that will be executed in the Thread generated by
 StartThreadTreeCacheUnzip... what we want to do is to inflate the next
 series of buffers leaving them in the second cache.
 Returns 0 when it finishes
Int_t GetRecordHeader(char* buf, Int_t maxbytes, Int_t& nbytes, Int_t& objlen, Int_t& keylen)
 Read the logical record header from the buffer buf.
 That must be the pointer tho the header part not the object by itself and
 must contain data of at least maxbytes
 Returns nread;
 In output arguments:
    nbytes : number of bytes in record
             if negative, this is a deleted record
             if 0, cannot read record, wrong value of argument first
    objlen : uncompressed object size
    keylen : length of logical record header
 Note that the arguments objlen and keylen are returned only
 if maxbytes >=16
 Note: This was adapted from TFile... so some things dont apply
void ResetCache()
 This will delete the list of buffers that are in the unzipping cache
 and will reset certain values in the cache.
 This name is ambiguos because the method doesn't reset the whole cache,
 only the part related to the unzipping
 Note: This method is completely different from TTreeCache::ResetCache(),
 in that method we were cleaning the prefetching buffer while here we
 delete the information about the unzipping buffers
Int_t GetUnzipBuffer(char** buf, Long64_t pos, Int_t len, Bool_t* free)
 We try to read a buffer that has already been unzipped
 Returns -1 in case of read failure, 0 in case it's not in the
 cache and n>0 in case read from cache (number of bytes copied).
 pos and len are the original values as were passed to ReadBuffer
 but instead we will return the inflated buffer.
 Note!! : If *buf == 0 we will allocate the buffer and it will be the
 responsability of the caller to free it... it is useful for example
 to pass it to the creator of TBuffer
void SetUnzipBufferSize(Long64_t bufferSize)
 Sets the size for the unzipping cache... by default it should be
 two times the size of the prefetching cache
Int_t UnzipBuffer(char** dest, char* src)
 UNzips a ROOT specific buffer... by reading the header at the beginning.
 returns the size of the inflated buffer or -1 if error
 Note!! : If *dest == 0 we will allocate the buffer and it will be the
 responsability of the caller to free it... it is useful for example
 to pass it to the creator of TBuffer
 src is the original buffer with the record (header+compressed data)
 *dest is the inflated buffer (including the header)
Int_t UnzipCache()
 This inflates all the buffers in the cache.. passing the data to a new
 buffer that will only wait there to be read...
 We can not inflate all the buffers in the cache so we will try to do
 it by parts... there is a member calle fUnzipBufferSize which will
 tell us the size we can allocate for this cache so we will divide
 the prefeteched cche in chunks of this size and we will try to unzip then
 note that we will  unzip in the order they were put into the cache not
 the order of the transference so it has to be read in that order or the
 pre-unzipping will be useless.
 pos and len are used to see where we have to start unzipping...
 and for how long..
 returns 0 in normal conditions or -1 if error
TTreeCacheUnzip(const TTreeCacheUnzip& )
TTreeCacheUnzip& operator=(const TTreeCacheUnzip& )
EParUnzipMode GetParallelUnzip()
 Methods related to the thread
Bool_t GetSkipZip()
{ return fSkipZip; }
void SetSkipZip(Bool_t skip = kTRUE)
{ fSkipZip = skip; }
Int_t GetNUnzip()
 Methods to get stats
{ return fNUnzip; }
Int_t GetNFound()
{ return fNFound; }
Int_t GetNMissed()
{ return fNMissed; }