;){kind=link}
;){kind=link}
;){kind=link}
;){kind=link}
| library: libProof #include "TAdaptivePacketizer.h" |
| TAdaptivePacketizer(TDSet* dset, TList* slaves, Long64_t first, Long64_t num, TList* input) | |
| virtual | ~TAdaptivePacketizer() |
| void | TObject::AbstractMethod(const char* method) const |
| virtual void | TObject::AppendPad(Option_t* option = "") |
| virtual void | TObject::Browse(TBrowser* b) |
| Int_t | CalculatePacketSize(TObject* slstat) |
| static TClass* | Class() |
| virtual const char* | TObject::ClassName() const |
| virtual void | TObject::Clear(Option_t* = "") |
| virtual TObject* | TObject::Clone(const char* newname = "") const |
| virtual Int_t | TObject::Compare(const TObject* obj) const |
| virtual void | TObject::Copy(TObject& object) const |
| virtual void | TObject::Delete(Option_t* option = "") |
| virtual Int_t | TObject::DistancetoPrimitive(Int_t px, Int_t py) |
| virtual void | TObject::Draw(Option_t* option = "") |
| virtual void | TObject::DrawClass() const |
| virtual TObject* | TObject::DrawClone(Option_t* option = "") const |
| virtual void | TObject::Dump() const |
| virtual void | TObject::Error(const char* method, const char* msgfmt) const |
| virtual void | TObject::Execute(const char* method, const char* params, Int_t* error = 0) |
| virtual void | TObject::Execute(TMethod* method, TObjArray* params, Int_t* error = 0) |
| virtual void | TObject::ExecuteEvent(Int_t event, Int_t px, Int_t py) |
| virtual void | TObject::Fatal(const char* method, const char* msgfmt) const |
| virtual TObject* | TObject::FindObject(const char* name) const |
| virtual TObject* | TObject::FindObject(const TObject* obj) const |
| virtual Long64_t | GetBytesRead() const |
| virtual Option_t* | TObject::GetDrawOption() const |
| static Long_t | TObject::GetDtorOnly() |
| virtual Long64_t | GetEntriesProcessed() const |
| virtual Long64_t | GetEntriesProcessed(TSlave* sl) const |
| virtual const char* | TObject::GetIconName() const |
| virtual Float_t | GetInitTime() const |
| virtual const char* | TObject::GetName() const |
| virtual TDSetElement* | GetNextPacket(TSlave* sl, TMessage* r) |
| virtual char* | TObject::GetObjectInfo(Int_t px, Int_t py) const |
| static Bool_t | TObject::GetObjectStat() |
| virtual Option_t* | TObject::GetOption() const |
| virtual Float_t | GetProcTime() const |
| virtual const char* | TObject::GetTitle() const |
| virtual UInt_t | TObject::GetUniqueID() const |
| virtual ULong_t | TObject::Hash() const |
| virtual void | TObject::Info(const char* method, const char* msgfmt) const |
| virtual Bool_t | TObject::InheritsFrom(const char* classname) const |
| virtual Bool_t | TObject::InheritsFrom(const TClass* cl) const |
| virtual void | TObject::Inspect() const |
| void | TObject::InvertBit(UInt_t f) |
| virtual TClass* | IsA() const |
| virtual Bool_t | TObject::IsEqual(const TObject* obj) const |
| virtual Bool_t | TObject::IsFolder() const |
| Bool_t | TObject::IsOnHeap() const |
| virtual Bool_t | TObject::IsSortable() const |
| Bool_t | TVirtualPacketizer::IsValid() const |
| Bool_t | TObject::IsZombie() const |
| virtual void | TObject::ls(Option_t* option = "") const |
| void | TObject::MayNotUse(const char* method) const |
| virtual Bool_t | TObject::Notify() |
| static void | TObject::operator delete(void* ptr) |
| static void | TObject::operator delete(void* ptr, void* vp) |
| static void | TObject::operator delete[](void* ptr) |
| static void | TObject::operator delete[](void* ptr, void* vp) |
| void* | TObject::operator new(size_t sz) |
| void* | TObject::operator new(size_t sz, void* vp) |
| void* | TObject::operator new[](size_t sz) |
| void* | TObject::operator new[](size_t sz, void* vp) |
| virtual void | TObject::Paint(Option_t* option = "") |
| virtual void | TObject::Pop() |
| virtual void | TObject::Print(Option_t* option = "") const |
| virtual Int_t | TObject::Read(const char* name) |
| virtual void | TObject::RecursiveRemove(TObject* obj) |
| void | TObject::ResetBit(UInt_t f) |
| virtual void | TObject::SaveAs(const char* filename = "", Option_t* option = "") const |
| virtual void | TObject::SavePrimitive(ostream& out, Option_t* option = "") |
| void | TObject::SetBit(UInt_t f) |
| void | TObject::SetBit(UInt_t f, Bool_t set) |
| virtual void | TObject::SetDrawOption(Option_t* option = "") |
| static void | TObject::SetDtorOnly(void* obj) |
| static void | TObject::SetObjectStat(Bool_t stat) |
| virtual void | TObject::SetUniqueID(UInt_t uid) |
| virtual void | ShowMembers(TMemberInspector& insp, char* parent) |
| virtual void | TVirtualPacketizer::StopProcess(Bool_t abort) |
| virtual void | Streamer(TBuffer& b) |
| void | StreamerNVirtual(TBuffer& b) |
| virtual void | TObject::SysError(const char* method, const char* msgfmt) const |
| Bool_t | TObject::TestBit(UInt_t f) const |
| Int_t | TObject::TestBits(UInt_t f) const |
| virtual void | TObject::UseCurrentStyle() |
| virtual void | TObject::Warning(const char* method, const char* msgfmt) const |
| virtual Int_t | TObject::Write(const char* name = "0", Int_t option = 0, Int_t bufsize = 0) |
| virtual Int_t | TObject::Write(const char* name = "0", Int_t option = 0, Int_t bufsize = 0) const |
| virtual void | TObject::DoError(int level, const char* location, const char* fmt, va_list va) const |
| Long64_t | TVirtualPacketizer::GetEntries(Bool_t tree, TDSetElement* e) |
| void | TObject::MakeZombie() |
| TAdaptivePacketizer() | |
| TAdaptivePacketizer(const TAdaptivePacketizer&) | |
| TDSetElement* | CreateNewPacket(TDSetElement* base, Long64_t first, Long64_t num) |
| TAdaptivePacketizer::TFileStat* | GetNextActive() |
| TAdaptivePacketizer::TFileStat* | GetNextUnAlloc(TAdaptivePacketizer::TFileNode* node = 0) |
| virtual Bool_t | HandleTimer(TTimer* timer) |
| TAdaptivePacketizer::TFileNode* | NextActiveNode() |
| TAdaptivePacketizer::TFileNode* | NextUnAllocNode() |
| void | operator=(const TAdaptivePacketizer&) |
| void | RemoveActive(TAdaptivePacketizer::TFileStat* file) |
| void | RemoveActiveNode(TAdaptivePacketizer::TFileNode*) |
| void | RemoveUnAllocNode(TAdaptivePacketizer::TFileNode*) |
| void | Reset() |
| void | SplitEventList(TDSet* dset) |
| void | ValidateFiles(TDSet* dset, TList* slaves) |
| enum TObject::EStatusBits { | kCanDelete | |
| kMustCleanup | ||
| kObjInCanvas | ||
| kIsReferenced | ||
| kHasUUID | ||
| kCannotPick | ||
| kNoContextMenu | ||
| kInvalidObject | ||
| }; | ||
| enum TObject::[unnamed] { | kIsOnHeap | |
| kNotDeleted | ||
| kZombie | ||
| kBitMask | ||
| kSingleKey | ||
| kOverwrite | ||
| kWriteDelete | ||
| }; |
| static Int_t | fgMaxSlaveCnt | maximum number of slaves per filenode |
| Bool_t | TVirtualPacketizer::fValid | Constructed properly? |
| Bool_t | TVirtualPacketizer::fStop | Termination of Process() requested? |
| Long64_t | fProcessed | number of entries processed |
| Long64_t | fBytesRead | number of bytes processed |
| TList* | fPackets | all processed packets |
| Long64_t | fTotalEntries | total number of entries to be distributed |
| Long_t | fStartTime | time offset |
| Float_t | fInitTime | time before processing |
| Float_t | fTimeUpdt | time between updates |
| TNtupleD* | fCircProg | Keeps circular info for "instantenous" |
| Int_t | fCircN | Circularity |
| TList* | fFileNodes | nodes with files |
| TList* | fUnAllocated | nodes with unallocated files |
| TList* | fActive | nodes with unfinished files |
| TMap* | fSlaveStats | slave status, keyed by correspondig TSlave |
| TTimer* | fProgress | progress updates timer |
| Int_t | fMaxPerfIdx | maximum of our slaves' performance index |
| Float_t | fFractionOfRemoteFiles | fraction of TDSetElements |
| Long64_t | fNEventsOnRemLoc | number of events in currently |
| Float_t | fProcTime | sum of proc time of all packets so far |
TAdaptivePacketizer
This packetizer is based on TPacketizer but uses different
load-balancing algorithms and data structures.
Two main improvements in the load-balancing strategy:
- First one was to change the order in which the files are assigned
to the computing nodes in such a way that network transfers are
evenly distributed in the query time. Transfer of the remote files
was often becoming a bottleneck at the end of a query.
- The other improvement is the use of time-based packet size. We
measure the processing rate of all the nodes and calculate the
packet size, so that it takes certain amount of time. In this way
packetizer prevents the situation where the query can’t finish
because of one slow node.
The data structures: TFileStat, TFileNode and TSlaveStat are
enriched + changed and TFileNode::Compare method is changed.
Constructor
Check existence of file/dir/tree an get number of entries. Assumes the files have been setup.
Splits the eventlist into parts for each file. Each part is assigned to the apropriate TDSetElement.
Creates a new TDSetElement from base packet starting from first entry with num entries. The function returns a new created objects which have to be deleted.
Calculates the packet size based on performance of this slave and est. time left untill the end of the query.
Get next packet; A meaningfull difference to TPacketizer is the fact that this packetizer, for each worker, tries to predict whether the worker will finish processing it's local files before the end of the query. If yes, it allocates, to those workers, files from non-slave filenodes or from slaves that are overloaded. The check is done every time a new file needs to be assigned.