Logo ROOT  
Reference Guide
TSelVerifyDataSet.cxx
Go to the documentation of this file.
1// @(#)root/proof:$Id$
2// Author: Sangsu Ryu 28/06/2011
3
4/*************************************************************************
5 * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12/** \class TSelVerifyDataSet
13\ingroup proofkernel
14
15Selector to verify dataset in parallel on workers
16
17*/
18
19#define TSelVerifyDataSet_cxx
20
21#include "TSelVerifyDataSet.h"
22#include "TDataSetManager.h"
23#include "TDSet.h"
24#include "TParameter.h"
25#include "TTree.h"
26#include "TFile.h"
27#include "TNamed.h"
28#include "TSystem.h"
29#include "TROOT.h"
30#include "TFileStager.h"
31#include "TProofDebug.h"
32#include "TProofServ.h"
33#include "TFileCollection.h"
34#include "TFileInfo.h"
35
37
38////////////////////////////////////////////////////////////////////////////////
39/// Constructor
40
42{
44}
45
46////////////////////////////////////////////////////////////////////////////////
47/// Constructor
48
50{
52}
53
54////////////////////////////////////////////////////////////////////////////////
55/// Initialize members
56
58{
59 fFopt = -1;
60 fSopt = 0;
61 fRopt = 0;
62
63 fAllf = 0;
64 fCheckstg = 0;
65 fNonStgf = 0;
66 fReopen = 0;
67 fTouch = 0;
68 fStgf = 0;
69 fNoaction = 0;
70 fFullproc = 0;
71 fLocateonly = 0;
72 fStageonly = 0;
73 fDoall = 0;
74 fGetlistonly = 0;
75 fScanlist = 0;
76 fDbg = 0;
77
79 fTouched = 0;
80 fOpened = 0;
81 fDisappeared = 0;
82 fSubDataSet = 0;
83}
84
85////////////////////////////////////////////////////////////////////////////////
86/// Worker Begin
87
89{
90 TString dsname, opts;
91
92 TNamed* par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_VerifyDataSet"));
93 if (par) {
94 dsname = par->GetTitle();
95 } else {
96 Abort("cannot find dataset name: cannot continue", kAbortProcess);
97 return;
98 }
99
100 par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_VerifyDataSetOption"));
101 if (par) {
102 opts = par->GetTitle();
103 } else {
104 Abort("cannot find verify options: cannot continue", kAbortProcess);
105 return;
106 }
107
108 par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_MSS"));
109 if (par) {
110 fMss = par->GetTitle();
111 PDB(kSelector, 2) Info("SlaveBegin", "dataset MSS: '%s'", fMss.Data());
112 }
113
114 par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_StageOption"));
115 if (par) {
116 fStageopts = par->GetTitle();
117 PDB(kSelector, 2) Info("SlaveBegin", "dataset stage options: '%s'", fStageopts.Data());
118 }
119
120 // Extract the directives
121 UInt_t o = 0;
122 if (!opts.IsNull()) {
123 // Selection options
124 if (strstr(opts, "allfiles:") || strchr(opts, 'A'))
126 else if (strstr(opts, "staged:") || strchr(opts, 'D'))
128 // Pre-action options
129 if (strstr(opts, "open:") || strchr(opts, 'O'))
131 if (strstr(opts, "touch:") || strchr(opts, 'T'))
133 if (strstr(opts, "nostagedcheck:") || strchr(opts, 'I'))
135 // Process options
136 if (strstr(opts, "noaction:") || strchr(opts, 'N'))
138 if (strstr(opts, "locateonly:") || strchr(opts, 'L'))
140 if (strstr(opts, "stageonly:") || strchr(opts, 'S'))
142 // Auxilliary options
143 if (strstr(opts, "verbose:") || strchr(opts, 'V'))
145 } else {
146 // Default
148 }
149
150 PDB(kSelector, 1) Info("SlaveBegin", "o=%d", o);
151 // File selection
152 fFopt = ((o & TDataSetManager::kAllFiles)) ? -1 : 0;
153 if (fFopt >= 0) {
155 fFopt = 10;
156 } else {
157 if ((o & TDataSetManager::kReopen)) fFopt++;
158 if ((o & TDataSetManager::kTouch)) fFopt++;
159 }
160 if ((o & TDataSetManager::kNoStagedCheck)) fFopt += 100;
161 } else {
163 Warning("SlaveBegin", "kAllFiles mode: ignoring kStagedFiles or kReopen"
164 " or kTouch requests");
165 }
166 if ((o & TDataSetManager::kNoStagedCheck)) fFopt -= 100;
167 }
168 PDB(kSelector, 1) Info("SlaveBegin", "fFopt=%d", fFopt);
169
170 // Type of action
171 fSopt = ((o & TDataSetManager::kNoAction)) ? -1 : 0;
172 if (fSopt >= 0) {
174 Error("SlaveBegin", "kLocateOnly and kStageOnly cannot be processed concurrently");
175 return;
176 }
177 if ((o & TDataSetManager::kLocateOnly)) fSopt = 1;
178 if ((o & TDataSetManager::kStageOnly)) fSopt = 2;
180 Warning("SlaveBegin", "kNoAction mode: ignoring kLocateOnly or kStageOnly requests");
181 }
182 PDB(kSelector, 1) Info("SlaveBegin", "fSopt=%d", fSopt);
183
185
186 // File selection, Reopen and Touch options
187 fAllf = (fFopt == -1) ? kTRUE : kFALSE;
188 fCheckstg = (fFopt >= 100 || fFopt < -1) ? kFALSE : kTRUE;
189 if (fFopt >= 0) fFopt %= 100;
190 fNonStgf = (fFopt >= 0 && fFopt < 10) ? kTRUE : kFALSE;
191 fReopen = (fFopt >= 1 && fFopt < 10) ? kTRUE : kFALSE;
192 fTouch = (fFopt >= 2 && fFopt < 10) ? kTRUE : kFALSE;
193 fStgf = (fFopt == 10) ? kTRUE : kFALSE;
194
195 PDB(kSelector, 1) Info("SlaveBegin",
196 "fAllf=%d fCheckstg=%d fNonStgf=%d fReopen=%d fTouch=%d fStgf=%d",
198
199 // File processing options
200 fNoaction = (fSopt == -1) ? kTRUE : kFALSE;
201 fFullproc = (fSopt == 0) ? kTRUE : kFALSE;
202 fLocateonly = (fSopt == 1) ? kTRUE : kFALSE;
203 fStageonly = (fSopt == 2) ? kTRUE : kFALSE;
204
205 PDB(kSelector, 1) Info("SlaveBegin",
206 "fNoaction=%d fFullproc=%d fLocateonly=%d fStageonly=%d",
208
209 // Run options
210 fDoall = (fRopt == 0) ? kTRUE : kFALSE;
211 fGetlistonly = (fRopt == 1) ? kTRUE : kFALSE;
212 fScanlist = (fRopt == 2) ? kTRUE : kFALSE;
213
214 PDB(kSelector, 1) Info("SlaveBegin",
215 "fDoall=%d fGetlistonly=%d fScanlist=%d",
217
218 TString hostname(TUrl(gSystem->HostName()).GetHostFQDN());
219 TString thisordinal = gProofServ ? gProofServ->GetOrdinal() : "n.d";
220 TString title =
221 TString::Format("TSelVerifyDataSet_%s_%s", hostname.Data(), thisordinal.Data());
222 fSubDataSet= new TFileCollection(dsname, title);
223}
224
225////////////////////////////////////////////////////////////////////////////////
226/// Process a single entry
227
229{
230 TDSetElement *fCurrent = 0;
231 TPair *elemPair = 0;
232 if (fInput && (elemPair = dynamic_cast<TPair *>
233 (fInput->FindObject("PROOF_CurrentElement")))) {
234 if ((fCurrent = dynamic_cast<TDSetElement *>(elemPair->Value())))
235 Info("Process", "entry %lld: file: '%s'", entry, fCurrent->GetName());
236 }
237 if (!fCurrent) {
238 Error("Process", "entry %lld: current element not found!", entry);
239 return kFALSE;
240 }
241
242 TFileInfo *fileInfo = dynamic_cast<TFileInfo*>(fCurrent->GetAssocObj(0));
243 if (!fileInfo) {
244 Error("Process", "can not get TFileInfo; returning");
245 return kFALSE;
246 }
247
248 PDB(kSelector, 1) {
249 Info("Process", "input fileinfo: ");
250 fileInfo->Print("L");
251 }
252
253 TFileStager *stager = 0;
254 Bool_t createStager = kFALSE;
255
256 TFileInfo* newfileinfo = new TFileInfo(*fileInfo);
257 newfileinfo->SetIndex(fileInfo->GetIndex());
258
259 if (fDoall || fGetlistonly) {
260
261 stager = (fMss && strlen(fMss) > 0) ? TFileStager::Open(fMss) : 0;
262 createStager = (stager) ? kFALSE : kTRUE;
263
264 // Check which files have been staged, this can be replaced by a bulk command,
265 // once it exists in the xrdclient
266
267 // For real time monitoring
269
270 Bool_t changed = kFALSE;
271 Bool_t touched = kFALSE;
272 Bool_t disappeared = kFALSE;
273
274 TDataSetManager::CheckStagedStatus(newfileinfo, fFopt, -1, 0, stager, createStager,
275 fDbg, changed, touched, disappeared);
276
277 if (changed) fChangedDs = kTRUE;
278 if (touched) fTouched++;
279 if (disappeared) fDisappeared++;
280
281 SafeDelete(stager);
282
283 PDB(kSelector, 1) Info("Process",
284 "fChangedDs = %d, fTouched = %d disappeared = %d",
286
287 // If required to only get the list we are done
288 if (fGetlistonly) {
289 Info("Process", "updated fileinfo: ");
290 newfileinfo->Print("F");
291 fSubDataSet->Add(newfileinfo);
292 return kTRUE;
293 }
294 }
295
296 if (!fNoaction && (fDoall || fScanlist)) {
297
298 // Point to the fileinfo
299 //newStagedFiles = (!fDoall && fScanlist && flist) ? flist : newStagedFiles;
300 if (!fDoall && fScanlist) {
301 SafeDelete(newfileinfo);
302 newfileinfo = new TFileInfo(*fileInfo);
303 newfileinfo->SetIndex(fileInfo->GetIndex());
304 }
305
306 // Loop over now staged files
307 PDB(kSelector, 1) Info("Process",
308 "file appear to be newly staged; %s",
309 newfileinfo->GetFirstUrl()->GetUrl());
310
311 // If staging files, prepare the stager
312 if (fLocateonly || fStageonly) {
313 stager = (fMss && strlen(fMss) > 0) ? TFileStager::Open(fMss) : 0;
314 createStager = (stager) ? kFALSE : kTRUE;
315 }
316
317 // Process the file
318 Bool_t changed = kFALSE;
319 Bool_t opened = kFALSE;
320 TDataSetManager::ProcessFile(newfileinfo, fSopt, fCheckstg, fDoall, stager, createStager, fStageopts,
321 fDbg, changed, opened);
322
323 if (changed) fChangedDs = kTRUE;
324 if (opened) fOpened++;
325 }
326
327 PDB(kSelector, 1) {
328 Info("Process", "updated fileinfo: ");
329 newfileinfo->Print("L");
330 }
331 fSubDataSet->Add(newfileinfo);
332
333 return kTRUE;
334}
335
336////////////////////////////////////////////////////////////////////////////////
337/// Worker Terminate
338
340{
341 if (fSubDataSet) {
343 if (fSubDataSet->GetNFiles() > 0) {
345 Info("SlaveTerminate",
346 "sub-dataset '%s' added to the output list (%lld files)",
348 }
349 // Add information for registration
350 fOutput->Add(new TNamed(TString::Format("DATASET_%s", fSubDataSet->GetName()).Data(),"OT:sortidx:"));
351 fOutput->Add(new TNamed("PROOFSERV_RegisterDataSet", ""));
352 }
353
354 // Send the number of files disppeared, opened and mark 'changed'if any fileinfo in the dataset has changed
355 TString hostname(TUrl(gSystem->HostName()).GetHostFQDN());
356 TString thisordinal = gProofServ ? gProofServ->GetOrdinal() : "n.d";
357 TString sfdisppeared= TString::Format("PROOF_NoFilesDisppeared_%s_%s", hostname.Data(), thisordinal.Data());
358 fOutput->Add(new TParameter<Int_t>(sfdisppeared.Data(), fDisappeared));
359 TString sfOpened= TString::Format("PROOF_NoFilesOpened_%s_%s", hostname.Data(), thisordinal.Data());
360 fOutput->Add(new TParameter<Int_t>(sfOpened.Data(), fOpened));
361 TString sfTouched = TString::Format("PROOF_NoFilesTouched_%s_%s", hostname.Data(), thisordinal.Data());
362 fOutput->Add(new TParameter<Int_t>(sfTouched.Data(), fTouched));
363 TString schanged= TString::Format("PROOF_DataSetChanged_%s_%s", hostname.Data(), thisordinal.Data());
364 fOutput->Add(new TParameter<Bool_t>(schanged.Data(), fChangedDs));
365}
#define SafeDelete(p)
Definition: RConfig.hxx:543
unsigned int UInt_t
Definition: RtypesCore.h:44
const Bool_t kFALSE
Definition: RtypesCore.h:90
bool Bool_t
Definition: RtypesCore.h:61
long long Long64_t
Definition: RtypesCore.h:71
const Bool_t kTRUE
Definition: RtypesCore.h:89
#define ClassImp(name)
Definition: Rtypes.h:361
#define PDB(mask, level)
Definition: TProofDebug.h:56
R__EXTERN TProofServ * gProofServ
Definition: TProofServ.h:347
R__EXTERN TSystem * gSystem
Definition: TSystem.h:556
Manages an element of a TDSet.
Definition: TDSet.h:66
TObject * GetAssocObj(Long64_t i, Bool_t isentry=kFALSE)
Get i-th associated object.
Definition: TDSet.cxx:652
static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles, Int_t newstagedfiles, TFileStager *stager, Bool_t createStager, Bool_t dbg, Bool_t &changed, Bool_t &touched, Bool_t &disappeared)
Check stage status of the file described by "fileInfo".
static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall, TFileStager *stager, Bool_t createStager, const char *stageopts, Bool_t dbg, Bool_t &changed, Bool_t &opened)
Locate, stage, or fully validate file "fileInfo".
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
Int_t Update(Long64_t avgsize=-1)
Update accumulated information about the elements of the collection (e.g.
Long64_t GetNFiles() const
Int_t Add(TFileInfo *info)
Add TFileInfo to the collection.
Class describing a generic file including meta information.
Definition: TFileInfo.h:36
void Print(Option_t *options="") const
Print information about this object.
Definition: TFileInfo.cxx:477
Int_t GetIndex() const
Definition: TFileInfo.h:96
TUrl * GetFirstUrl() const
Definition: TFileInfo.h:69
void SetIndex(Int_t idx)
Definition: TFileInfo.h:97
static TFileStager * Open(const char *stager)
Open a stager, after having loaded the relevant plug-in.
virtual void Add(TObject *obj)
Definition: TList.h:87
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:577
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:877
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:891
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:865
Class used by TMap to store (key,value) pairs.
Definition: TMap.h:102
TObject * Value() const
Definition: TMap.h:121
Named parameter, streamable and storable.
Definition: TParameter.h:37
const char * GetOrdinal() const
Definition: TProofServ.h:253
Selector to verify dataset in parallel on workers.
void InitMembers()
Initialize members.
virtual Bool_t Process(Long64_t entry)
Process a single entry.
TSelVerifyDataSet()
Constructor.
virtual void SlaveBegin(TTree *tree)
Worker Begin.
virtual void SlaveTerminate()
Worker Terminate.
TFileCollection * fSubDataSet
TList * fInput
List of objects available during processing.
Definition: TSelector.h:43
@ kAbortProcess
Definition: TSelector.h:36
TSelectorList * fOutput
! List of objects created during processing
Definition: TSelector.h:44
virtual void Abort(const char *why, EAbort what=kAbortProcess)
Abort processing.
Definition: TSelector.cxx:116
Basic string class.
Definition: TString.h:131
const char * Data() const
Definition: TString.h:364
Bool_t IsNull() const
Definition: TString.h:402
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition: TString.cxx:2311
virtual void DispatchOneEvent(Bool_t pendingOnly=kFALSE)
Dispatch a single event.
Definition: TSystem.cxx:427
virtual const char * HostName()
Return the system's host name.
Definition: TSystem.cxx:301
A TTree represents a columnar dataset.
Definition: TTree.h:78
This class represents a WWW compatible URL.
Definition: TUrl.h:35
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:387
const char * GetHostFQDN() const
Return fully qualified domain name of url host.
Definition: TUrl.cxx:469