Logo ROOT  
Reference Guide
TVirtualPacketizer.h
Go to the documentation of this file.
1 // @(#)root/proofplayer:$Id$
2 // Author: Maarten Ballintijn 9/7/2002
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2002, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 #ifndef ROOT_TVirtualPacketizer
13 #define ROOT_TVirtualPacketizer
14 
15 //////////////////////////////////////////////////////////////////////////
16 // //
17 // TVirtualPacketizer //
18 // //
19 // Packetizer is a load balancing object created for each query. //
20 // It generates packets to be processed on PROOF worker servers. //
21 // A packet is an event range (begin entry and number of entries) or //
22 // object range (first object and number of objects) in a TTree //
23 // (entries) or a directory (objects) in a file. //
24 // Packets are generated taking into account the performance of the //
25 // remote machine, the time it took to process a previous packet on //
26 // the remote machine, the locality of the database files, etc. //
27 // //
28 // TVirtualPacketizer includes common parts of PROOF packetizers. //
29 // Look in subclasses for details. //
30 // The default packetizer is TPacketizerAdaptive. //
31 // To use an alternative one, for instance - the TPacketizer, call: //
32 // proof->SetParameter("PROOF_Packetizer", "TPacketizer"); //
33 // //
34 //////////////////////////////////////////////////////////////////////////
35 
36 #include "TObject.h"
37 #include "TSlave.h"
38 #include "TProofProgressStatus.h"
39 #include "TTime.h"
40 
41 
42 class TDSet;
43 class TDSetElement;
44 class TList;
45 class TMap;
46 class TMessage;
47 class TNtuple;
48 class TNtupleD;
49 class TProofProgressInfo;
50 class TSlave;
51 
52 
53 class TVirtualPacketizer : public TObject {
54 
55 public: // public because of Sun CC bug
56  class TVirtualSlaveStat;
57 
58 protected:
59  enum EUseEstOpt { // Option for usage of estimated values
60  kEstOff = 0,
63  };
64 
65  // General configuration parameters
66  Double_t fMinPacketTime; // minimum packet time
67  Double_t fMaxPacketTime; // maximum packet time
68  TList *fConfigParams; // List of configuration parameters
69 
70  TMap *fSlaveStats; // slave status, keyed by correspondig TSlave
71 
72  TProofProgressStatus *fProgressStatus; // pointer to status in the player.
73  TTimer *fProgress; // progress updates timer
74 
75  Long64_t fTotalEntries; // total number of entries to be distributed;
76  // not used in the progressive packetizer
77  TList *fFailedPackets;// a list of packets that failed while processing
78 
79  // Members for progress info
80  TTime fStartTime; // time offset
81  Float_t fInitTime; // time before processing
82  Float_t fProcTime; // time since start of processing
83  Float_t fTimeUpdt; // time between updates
84  TNtupleD *fCircProg; // Keeps circular info for "instantenous"
85  // rate calculations
86  Long_t fCircN; // Circularity
87 
88  TNtuple *fProgressPerf; // {Active workers, evt rate, MBs read} as a function of processing time
89  Float_t fProcTimeLast; // Time of the last measurement
90  Int_t fActWrksLast; // Active workers at fProcTimeLast
91  Float_t fEvtRateLast; // Evt rate at fProcTimeLast
92  Float_t fMBsReadLast; // MBs read at fProcTimeLast
93  Float_t fEffSessLast; // Number of effective sessions at fProcTimeLast
94  Bool_t fAWLastFill; // Whether to fill the last measurement
95  Float_t fReportPeriod; // Time between reports if nothing changes (estimated proc time / 100)
96 
97  EUseEstOpt fUseEstOpt; // Control usage of estimated values for the progress info
98 
99  Bool_t fValid; // Constructed properly?
100  Bool_t fStop; // Termination of Process() requested?
101 
102  TString fDataSet; // Name of the dataset being processed (for dataset-driven runs)
103 
104  TList *fInput; // Input list
105 
107  TVirtualPacketizer(const TVirtualPacketizer &); // no implementation, will generate
108  void operator=(const TVirtualPacketizer &); // error on accidental usage
109 
111  Long64_t GetEntries(Bool_t tree, TDSetElement *e); // Num of entries or objects
112  virtual Bool_t HandleTimer(TTimer *timer);
113 
114 public:
115  enum EStatusBits { kIsInitializing = BIT(16), kIsDone = BIT(17), kIsTree = BIT(18) };
116  virtual ~TVirtualPacketizer();
117 
118  virtual Int_t AssignWork(TDSet* /*dset*/, Long64_t /*first*/, Long64_t /*num*/) { return -1; }
119  Bool_t IsValid() const { return fValid; }
122  { ent = GetEntriesProcessed(); bytes = GetBytesRead(); calls = GetReadCalls(); return 0; }
125  virtual TDSetElement *GetNextPacket(TSlave *sl, TMessage *r);
126  virtual void SetInitTime();
127  virtual void StopProcess(Bool_t abort, Bool_t stoptimer = kFALSE);
129  void SetFailedPackets(TList *list) { fFailedPackets = list; }
130  virtual Int_t AddWorkers(TList *workers);
131 
135  Float_t GetInitTime() const { return fInitTime; }
136  Float_t GetProcTime() const { return fProcTime; }
137  TNtuple *GetProgressPerf(Bool_t steal = kFALSE) { if (steal) { TNtuple *n = fProgressPerf; fProgressPerf = 0; return n;
138  } else { return fProgressPerf;} }
139  TList *GetConfigParams(Bool_t steal = kFALSE) { if (steal) { TList *l = fConfigParams; fConfigParams = 0; return l;
140  } else { return fConfigParams;} }
141  virtual void MarkBad(TSlave * /*s*/, TProofProgressStatus * /*status*/, TList ** /*missingFiles*/) { return; }
142  virtual Int_t AddProcessed(TSlave * /*sl*/, TProofProgressStatus * /*st*/,
143  Double_t /*lat*/, TList ** /*missingFiles*/) { return 0; }
147 
148  TMap *GetSlaveStats() const { return fSlaveStats; }
149 
150  virtual Int_t GetActiveWorkers() { return -1; }
151 
152  ClassDef(TVirtualPacketizer,0) //Generate work packets for parallel processing
153 };
154 
155 //------------------------------------------------------------------------------
156 
158 
159 friend class TPacketizerAdaptive;
160 friend class TPacketizer;
161 
162 protected:
163  TString fWrkFQDN; // Worker FQDN
164  TSlave *fSlave; // corresponding TSlave record
165  TProofProgressStatus *fStatus; // status as of the last finished packet
166 
167 public:
168  const char *GetName() const { return fWrkFQDN.Data(); }
169  const char *GetOrdinal() const { return fSlave->GetOrdinal(); }
171  Double_t GetProcTime() const { return fStatus?fStatus->GetProcTime():-1; }
175 };
176 
177 #endif
TVirtualPacketizer::fProgressPerf
TNtuple * fProgressPerf
Definition: TVirtualPacketizer.h:88
l
auto * l
Definition: textangle.C:4
TPacketizer
Definition: TPacketizer.h:39
n
const Int_t n
Definition: legend1.C:16
TVirtualPacketizer::GetActiveWorkers
virtual Int_t GetActiveWorkers()
Definition: TVirtualPacketizer.h:150
TVirtualPacketizer::fEvtRateLast
Float_t fEvtRateLast
Definition: TVirtualPacketizer.h:91
TVirtualPacketizer::fMBsReadLast
Float_t fMBsReadLast
Definition: TVirtualPacketizer.h:92
TSlave
Definition: TSlave.h:46
first
Definition: first.py:1
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:91
e
#define e(i)
Definition: RSha256.hxx:121
TVirtualPacketizer::fProgress
TTimer * fProgress
Definition: TVirtualPacketizer.h:73
TVirtualPacketizer::fStartTime
TTime fStartTime
Definition: TVirtualPacketizer.h:80
TProofProgressStatus::GetEntries
Long64_t GetEntries() const
Definition: TProofProgressStatus.h:53
TVirtualPacketizer::fMinPacketTime
Double_t fMinPacketTime
Definition: TVirtualPacketizer.h:66
TVirtualPacketizer::TVirtualPacketizer
TVirtualPacketizer(TList *input, TProofProgressStatus *st=0)
Constructor.
Definition: TVirtualPacketizer.cxx:64
TNtuple
Definition: TNtuple.h:28
TVirtualPacketizer::SetInitTime
virtual void SetInitTime()
Set the initialization time.
Definition: TVirtualPacketizer.cxx:429
TVirtualPacketizer::TVirtualSlaveStat::fSlave
TSlave * fSlave
Definition: TVirtualPacketizer.h:164
TVirtualPacketizer::fTimeUpdt
Float_t fTimeUpdt
Definition: TVirtualPacketizer.h:83
TVirtualPacketizer::fEffSessLast
Float_t fEffSessLast
Definition: TVirtualPacketizer.h:93
TString::Data
const char * Data() const
Definition: TString.h:369
TVirtualPacketizer::GetStatus
TProofProgressStatus * GetStatus()
Definition: TVirtualPacketizer.h:144
TSlave.h
tree
Definition: tree.py:1
TProofProgressStatus::GetReadCalls
Long64_t GetReadCalls() const
Definition: TProofProgressStatus.h:55
TVirtualPacketizer::GetInitTime
Float_t GetInitTime() const
Definition: TVirtualPacketizer.h:135
TVirtualPacketizer::GetSlaveStats
TMap * GetSlaveStats() const
Definition: TVirtualPacketizer.h:148
r
ROOT::R::TRInterface & r
Definition: Object.C:4
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TVirtualPacketizer::HandleTimer
virtual Bool_t HandleTimer(TTimer *timer)
Send progress message to client.
Definition: TVirtualPacketizer.cxx:283
TTime.h
Float_t
float Float_t
Definition: RtypesCore.h:57
TVirtualPacketizer::TVirtualSlaveStat
Definition: TVirtualPacketizer.h:157
TVirtualPacketizer::GetProgressPerf
TNtuple * GetProgressPerf(Bool_t steal=kFALSE)
Definition: TVirtualPacketizer.h:137
Int_t
int Int_t
Definition: RtypesCore.h:45
TProofProgressStatus::GetProcTime
Double_t GetProcTime() const
Definition: TProofProgressStatus.h:57
TVirtualPacketizer::GetReadCalls
Long64_t GetReadCalls() const
Definition: TVirtualPacketizer.h:133
TVirtualPacketizer::AddWorkers
virtual Int_t AddWorkers(TList *workers)
Adds new workers.
Definition: TVirtualPacketizer.cxx:443
TVirtualPacketizer::TVirtualSlaveStat::GetOrdinal
const char * GetOrdinal() const
Definition: TVirtualPacketizer.h:169
TVirtualPacketizer::MarkBad
virtual void MarkBad(TSlave *, TProofProgressStatus *, TList **)
Definition: TVirtualPacketizer.h:141
TVirtualPacketizer::kEstAverage
@ kEstAverage
Definition: TVirtualPacketizer.h:62
TVirtualPacketizer::GetProcTime
Float_t GetProcTime() const
Definition: TVirtualPacketizer.h:136
TString
Definition: TString.h:136
Bool_t
bool Bool_t
Definition: RtypesCore.h:63
TDSetElement
Definition: TDSet.h:66
TProofProgressStatus.h
TVirtualPacketizer::fCircN
Long_t fCircN
Definition: TVirtualPacketizer.h:86
bool
TVirtualPacketizer::fCircProg
TNtupleD * fCircProg
Definition: TVirtualPacketizer.h:84
TVirtualPacketizer::GetTotalEntries
Long64_t GetTotalEntries() const
Definition: TVirtualPacketizer.h:124
TVirtualPacketizer::fValid
Bool_t fValid
Definition: TVirtualPacketizer.h:99
TVirtualPacketizer::kEstOff
@ kEstOff
Definition: TVirtualPacketizer.h:60
TPacketizerAdaptive
Definition: TPacketizerAdaptive.h:48
TVirtualPacketizer::kIsInitializing
@ kIsInitializing
Definition: TVirtualPacketizer.h:115
TVirtualPacketizer::StopProcess
virtual void StopProcess(Bool_t abort, Bool_t stoptimer=kFALSE)
Stop process.
Definition: TVirtualPacketizer.cxx:243
TVirtualPacketizer::fMaxPacketTime
Double_t fMaxPacketTime
Definition: TVirtualPacketizer.h:67
TVirtualPacketizer::TVirtualSlaveStat::GetName
const char * GetName() const
Returns name of object.
Definition: TVirtualPacketizer.h:168
TVirtualPacketizer::CreateNewPacket
TDSetElement * CreateNewPacket(TDSetElement *base, Long64_t first, Long64_t num)
Creates a new TDSetElement from from base packet starting from the first entry with num entries.
Definition: TVirtualPacketizer.cxx:254
TTimer
Definition: TTimer.h:51
TVirtualPacketizer::SetProgressStatus
void SetProgressStatus(TProofProgressStatus *st)
Definition: TVirtualPacketizer.h:145
TProofProgressStatus
Definition: TProofProgressStatus.h:25
TVirtualPacketizer::fProcTime
Float_t fProcTime
Definition: TVirtualPacketizer.h:82
TVirtualPacketizer::TVirtualSlaveStat::GetProcTime
Double_t GetProcTime() const
Definition: TVirtualPacketizer.h:171
TVirtualPacketizer::GetEstEntriesProcessed
virtual Int_t GetEstEntriesProcessed(Float_t, Long64_t &ent, Long64_t &bytes, Long64_t &calls)
Definition: TVirtualPacketizer.h:121
TVirtualPacketizer::fStop
Bool_t fStop
Definition: TVirtualPacketizer.h:100
TVirtualPacketizer::GetCumProcTime
Double_t GetCumProcTime() const
Definition: TVirtualPacketizer.h:134
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TVirtualPacketizer::GetEntriesProcessed
Long64_t GetEntriesProcessed() const
Definition: TVirtualPacketizer.h:120
TProofProgressStatus::GetRate
Double_t GetRate() const
Definition: TProofProgressStatus.h:60
Long_t
long Long_t
Definition: RtypesCore.h:54
TTime
Definition: TTime.h:27
TVirtualPacketizer::fProgressStatus
TProofProgressStatus * fProgressStatus
Definition: TVirtualPacketizer.h:72
TVirtualPacketizer::fAWLastFill
Bool_t fAWLastFill
Definition: TVirtualPacketizer.h:94
TVirtualPacketizer::fInitTime
Float_t fInitTime
Definition: TVirtualPacketizer.h:81
TVirtualPacketizer::fSlaveStats
TMap * fSlaveStats
Definition: TVirtualPacketizer.h:70
BIT
#define BIT(n)
Definition: Rtypes.h:85
TVirtualPacketizer::TVirtualSlaveStat::AddProcessed
virtual TProofProgressStatus * AddProcessed(TProofProgressStatus *st)=0
TVirtualPacketizer::EUseEstOpt
EUseEstOpt
Definition: TVirtualPacketizer.h:59
TVirtualPacketizer::TVirtualSlaveStat::GetEntriesProcessed
Long64_t GetEntriesProcessed() const
Definition: TVirtualPacketizer.h:170
TDSet
Definition: TDSet.h:153
TVirtualPacketizer::fTotalEntries
Long64_t fTotalEntries
Definition: TVirtualPacketizer.h:75
TVirtualPacketizer::fUseEstOpt
EUseEstOpt fUseEstOpt
Definition: TVirtualPacketizer.h:97
TVirtualPacketizer::AssignWork
virtual Int_t AssignWork(TDSet *, Long64_t, Long64_t)
Definition: TVirtualPacketizer.h:118
TVirtualPacketizer::fDataSet
TString fDataSet
Definition: TVirtualPacketizer.h:102
TVirtualPacketizer::kIsDone
@ kIsDone
Definition: TVirtualPacketizer.h:115
TVirtualPacketizer::SetTotalEntries
void SetTotalEntries(Long64_t ent)
Definition: TVirtualPacketizer.h:146
Double_t
double Double_t
Definition: RtypesCore.h:59
TNtupleD
Definition: TNtupleD.h:28
TVirtualPacketizer::GetConfigParams
TList * GetConfigParams(Bool_t steal=kFALSE)
Definition: TVirtualPacketizer.h:139
TVirtualPacketizer::operator=
void operator=(const TVirtualPacketizer &)
TObject.h
TVirtualPacketizer::TVirtualSlaveStat::GetAvgRate
Float_t GetAvgRate()
Definition: TVirtualPacketizer.h:172
TVirtualPacketizer::AddProcessed
virtual Int_t AddProcessed(TSlave *, TProofProgressStatus *, Double_t, TList **)
Definition: TVirtualPacketizer.h:142
TVirtualPacketizer
Definition: TVirtualPacketizer.h:53
TVirtualPacketizer::TVirtualSlaveStat::GetProgressStatus
TProofProgressStatus * GetProgressStatus()
Definition: TVirtualPacketizer.h:173
TVirtualPacketizer::TVirtualSlaveStat::fStatus
TProofProgressStatus * fStatus
Definition: TVirtualPacketizer.h:165
TVirtualPacketizer::GetCurrentRate
virtual Float_t GetCurrentRate(Bool_t &all)
Definition: TVirtualPacketizer.h:123
TObject
Definition: TObject.h:37
TVirtualPacketizer::GetFailedPackets
TList * GetFailedPackets()
Definition: TVirtualPacketizer.h:128
ClassDef
#define ClassDef(name, id)
Definition: Rtypes.h:325
TVirtualPacketizer::kEstCurrent
@ kEstCurrent
Definition: TVirtualPacketizer.h:61
TVirtualPacketizer::fInput
TList * fInput
Definition: TVirtualPacketizer.h:104
TProofProgressInfo
Definition: TProof.h:161
TVirtualPacketizer::fProcTimeLast
Float_t fProcTimeLast
Definition: TVirtualPacketizer.h:89
TVirtualPacketizer::fConfigParams
TList * fConfigParams
Definition: TVirtualPacketizer.h:68
TProofProgressStatus::GetBytesRead
Long64_t GetBytesRead() const
Definition: TProofProgressStatus.h:54
TVirtualPacketizer::kIsTree
@ kIsTree
Definition: TVirtualPacketizer.h:115
TProofProgressStatus::GetCurrentRate
Double_t GetCurrentRate() const
Get current rate. Rteunr the average rate if the current is not defined.
Definition: TProofProgressStatus.cxx:117
TVirtualPacketizer::IsValid
Bool_t IsValid() const
Definition: TVirtualPacketizer.h:119
TMessage
Definition: TMessage.h:33
TVirtualPacketizer::GetBytesRead
Long64_t GetBytesRead() const
Definition: TVirtualPacketizer.h:132
TVirtualPacketizer::fFailedPackets
TList * fFailedPackets
Definition: TVirtualPacketizer.h:77
TVirtualPacketizer::~TVirtualPacketizer
virtual ~TVirtualPacketizer()
Destructor.
Definition: TVirtualPacketizer.cxx:171
TVirtualPacketizer::TVirtualSlaveStat::fWrkFQDN
TString fWrkFQDN
Definition: TVirtualPacketizer.h:163
TVirtualPacketizer::GetNextPacket
virtual TDSetElement * GetNextPacket(TSlave *sl, TMessage *r)
Get next packet.
Definition: TVirtualPacketizer.cxx:234
TVirtualPacketizer::GetEntries
Long64_t GetEntries(Bool_t tree, TDSetElement *e)
Get entries.
Definition: TVirtualPacketizer.cxx:184
TVirtualPacketizer::SetFailedPackets
void SetFailedPackets(TList *list)
Definition: TVirtualPacketizer.h:129
TVirtualPacketizer::fReportPeriod
Float_t fReportPeriod
Definition: TVirtualPacketizer.h:95
TObject::EStatusBits
EStatusBits
Definition: TObject.h:57
TList
Definition: TList.h:44
TVirtualPacketizer::fActWrksLast
Int_t fActWrksLast
Definition: TVirtualPacketizer.h:90
int
TMap
Definition: TMap.h:40
TSlave::GetOrdinal
const char * GetOrdinal() const
Definition: TSlave.h:131