Logo ROOT   6.08/07
Reference Guide
TSelVerifyDataSet.cxx
Go to the documentation of this file.
1 // @(#)root/proof:$Id$
2 // Author: Sangsu Ryu 28/06/2011
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /** \class TSelVerifyDataSet
13 \ingroup proofkernel
14 
15 Selector to verify dataset in parallel on workers
16 
17 */
18 
19 #define TSelVerifyDataSet_cxx
20 
21 #include "TSelVerifyDataSet.h"
22 #include "TDataSetManager.h"
23 #include "TDSet.h"
24 #include "TParameter.h"
25 #include "TTree.h"
26 #include "TFile.h"
27 #include "TNamed.h"
28 #include "TSystem.h"
29 #include "TROOT.h"
30 #include "TEnv.h"
31 #include "TFileStager.h"
32 #include "TProofDebug.h"
33 #include "TProofServ.h"
34 #include "TFileCollection.h"
35 #include "TFileInfo.h"
36 
38 
39 ////////////////////////////////////////////////////////////////////////////////
40 /// Constructor
41 
43 {
44  InitMembers();
45 }
46 
47 ////////////////////////////////////////////////////////////////////////////////
48 /// Constructor
49 
51 {
52  InitMembers();
53 }
54 
55 ////////////////////////////////////////////////////////////////////////////////
56 /// Initialize members
57 
59 {
60  fFopt = -1;
61  fSopt = 0;
62  fRopt = 0;
63 
64  fAllf = 0;
65  fCheckstg = 0;
66  fNonStgf = 0;
67  fReopen = 0;
68  fTouch = 0;
69  fStgf = 0;
70  fNoaction = 0;
71  fFullproc = 0;
72  fLocateonly = 0;
73  fStageonly = 0;
74  fDoall = 0;
75  fGetlistonly = 0;
76  fScanlist = 0;
77  fDbg = 0;
78 
80  fTouched = 0;
81  fOpened = 0;
82  fDisappeared = 0;
83  fSubDataSet = 0;
84 }
85 
86 ////////////////////////////////////////////////////////////////////////////////
87 /// Worker Begin
88 
90 {
91  TString dsname, opts;
92 
93  TNamed* par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_VerifyDataSet"));
94  if (par) {
95  dsname = par->GetTitle();
96  } else {
97  Abort("cannot find dataset name: cannot continue", kAbortProcess);
98  return;
99  }
100 
101  par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_VerifyDataSetOption"));
102  if (par) {
103  opts = par->GetTitle();
104  } else {
105  Abort("cannot find verify options: cannot continue", kAbortProcess);
106  return;
107  }
108 
109  par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_MSS"));
110  if (par) {
111  fMss = par->GetTitle();
112  PDB(kSelector, 2) Info("SlaveBegin", "dataset MSS: '%s'", fMss.Data());
113  }
114 
115  par = dynamic_cast<TNamed*>(fInput->FindObject("PROOF_StageOption"));
116  if (par) {
117  fStageopts = par->GetTitle();
118  PDB(kSelector, 2) Info("SlaveBegin", "dataset stage options: '%s'", fStageopts.Data());
119  }
120 
121  // Extract the directives
122  UInt_t o = 0;
123  if (!opts.IsNull()) {
124  // Selection options
125  if (strstr(opts, "allfiles:") || strchr(opts, 'A'))
127  else if (strstr(opts, "staged:") || strchr(opts, 'D'))
129  // Pre-action options
130  if (strstr(opts, "open:") || strchr(opts, 'O'))
132  if (strstr(opts, "touch:") || strchr(opts, 'T'))
134  if (strstr(opts, "nostagedcheck:") || strchr(opts, 'I'))
136  // Process options
137  if (strstr(opts, "noaction:") || strchr(opts, 'N'))
139  if (strstr(opts, "locateonly:") || strchr(opts, 'L'))
141  if (strstr(opts, "stageonly:") || strchr(opts, 'S'))
143  // Auxilliary options
144  if (strstr(opts, "verbose:") || strchr(opts, 'V'))
146  } else {
147  // Default
149  }
150 
151  PDB(kSelector, 1) Info("SlaveBegin", "o=%d", o);
152  // File selection
153  fFopt = ((o & TDataSetManager::kAllFiles)) ? -1 : 0;
154  if (fFopt >= 0) {
155  if ((o & TDataSetManager::kStagedFiles)) {
156  fFopt = 10;
157  } else {
158  if ((o & TDataSetManager::kReopen)) fFopt++;
159  if ((o & TDataSetManager::kTouch)) fFopt++;
160  }
161  if ((o & TDataSetManager::kNoStagedCheck)) fFopt += 100;
162  } else {
164  Warning("SlaveBegin", "kAllFiles mode: ignoring kStagedFiles or kReopen"
165  " or kTouch requests");
166  }
167  if ((o & TDataSetManager::kNoStagedCheck)) fFopt -= 100;
168  }
169  PDB(kSelector, 1) Info("SlaveBegin", "fFopt=%d", fFopt);
170 
171  // Type of action
172  fSopt = ((o & TDataSetManager::kNoAction)) ? -1 : 0;
173  if (fSopt >= 0) {
175  Error("SlaveBegin", "kLocateOnly and kStageOnly cannot be processed concurrently");
176  return;
177  }
178  if ((o & TDataSetManager::kLocateOnly)) fSopt = 1;
179  if ((o & TDataSetManager::kStageOnly)) fSopt = 2;
180  } else if ((o & TDataSetManager::kLocateOnly) || (o & TDataSetManager::kStageOnly)) {
181  Warning("SlaveBegin", "kNoAction mode: ignoring kLocateOnly or kStageOnly requests");
182  }
183  PDB(kSelector, 1) Info("SlaveBegin", "fSopt=%d", fSopt);
184 
186 
187  // File selection, Reopen and Touch options
188  fAllf = (fFopt == -1) ? kTRUE : kFALSE;
189  fCheckstg = (fFopt >= 100 || fFopt < -1) ? kFALSE : kTRUE;
190  if (fFopt >= 0) fFopt %= 100;
191  fNonStgf = (fFopt >= 0 && fFopt < 10) ? kTRUE : kFALSE;
192  fReopen = (fFopt >= 1 && fFopt < 10) ? kTRUE : kFALSE;
193  fTouch = (fFopt >= 2 && fFopt < 10) ? kTRUE : kFALSE;
194  fStgf = (fFopt == 10) ? kTRUE : kFALSE;
195 
196  PDB(kSelector, 1) Info("SlaveBegin",
197  "fAllf=%d fCheckstg=%d fNonStgf=%d fReopen=%d fTouch=%d fStgf=%d",
199 
200  // File processing options
201  fNoaction = (fSopt == -1) ? kTRUE : kFALSE;
202  fFullproc = (fSopt == 0) ? kTRUE : kFALSE;
203  fLocateonly = (fSopt == 1) ? kTRUE : kFALSE;
204  fStageonly = (fSopt == 2) ? kTRUE : kFALSE;
205 
206  PDB(kSelector, 1) Info("SlaveBegin",
207  "fNoaction=%d fFullproc=%d fLocateonly=%d fStageonly=%d",
209 
210  // Run options
211  fDoall = (fRopt == 0) ? kTRUE : kFALSE;
212  fGetlistonly = (fRopt == 1) ? kTRUE : kFALSE;
213  fScanlist = (fRopt == 2) ? kTRUE : kFALSE;
214 
215  PDB(kSelector, 1) Info("SlaveBegin",
216  "fDoall=%d fGetlistonly=%d fScanlist=%d",
218 
219  TString hostname(TUrl(gSystem->HostName()).GetHostFQDN());
220  TString thisordinal = gProofServ ? gProofServ->GetOrdinal() : "n.d";
221  TString title =
222  TString::Format("TSelVerifyDataSet_%s_%s", hostname.Data(), thisordinal.Data());
223  fSubDataSet= new TFileCollection(dsname, title);
224 }
225 
226 ////////////////////////////////////////////////////////////////////////////////
227 /// Process a single entry
228 
230 {
231  TDSetElement *fCurrent = 0;
232  TPair *elemPair = 0;
233  if (fInput && (elemPair = dynamic_cast<TPair *>
234  (fInput->FindObject("PROOF_CurrentElement")))) {
235  if ((fCurrent = dynamic_cast<TDSetElement *>(elemPair->Value())))
236  Info("Process", "entry %lld: file: '%s'", entry, fCurrent->GetName());
237  }
238  if (!fCurrent) {
239  Error("Process", "entry %lld: current element not found!", entry);
240  return kFALSE;
241  }
242 
243  TFileInfo *fileInfo = dynamic_cast<TFileInfo*>(fCurrent->GetAssocObj(0));
244  if (!fileInfo) {
245  Error("Process", "can not get TFileInfo; returning");
246  return kFALSE;
247  }
248 
249  PDB(kSelector, 1) {
250  Info("Process", "input fileinfo: ");
251  fileInfo->Print("L");
252  }
253 
254  TFileStager *stager = 0;
255  Bool_t createStager = kFALSE;
256 
257  TFileInfo* newfileinfo = new TFileInfo(*fileInfo);
258  newfileinfo->SetIndex(fileInfo->GetIndex());
259 
260  if (fDoall || fGetlistonly) {
261 
262  stager = (fMss && strlen(fMss) > 0) ? TFileStager::Open(fMss) : 0;
263  createStager = (stager) ? kFALSE : kTRUE;
264 
265  // Check which files have been staged, this can be replaced by a bulk command,
266  // once it exists in the xrdclient
267 
268  // For real time monitoring
270 
271  Bool_t changed = kFALSE;
272  Bool_t touched = kFALSE;
273  Bool_t disappeared = kFALSE;
274 
275  TDataSetManager::CheckStagedStatus(newfileinfo, fFopt, -1, 0, stager, createStager,
276  fDbg, changed, touched, disappeared);
277 
278  if (changed) fChangedDs = kTRUE;
279  if (touched) fTouched++;
280  if (disappeared) fDisappeared++;
281 
282  SafeDelete(stager);
283 
284  PDB(kSelector, 1) Info("Process",
285  "fChangedDs = %d, fTouched = %d disappeared = %d",
287 
288  // If required to only get the list we are done
289  if (fGetlistonly) {
290  Info("Process", "updated fileinfo: ");
291  newfileinfo->Print("F");
292  fSubDataSet->Add(newfileinfo);
293  return kTRUE;
294  }
295  }
296 
297  if (!fNoaction && (fDoall || fScanlist)) {
298 
299  // Point to the fileinfo
300  //newStagedFiles = (!fDoall && fScanlist && flist) ? flist : newStagedFiles;
301  if (!fDoall && fScanlist) {
302  SafeDelete(newfileinfo);
303  newfileinfo = new TFileInfo(*fileInfo);
304  newfileinfo->SetIndex(fileInfo->GetIndex());
305  }
306 
307  // Loop over now staged files
308  PDB(kSelector, 1) Info("Process",
309  "file appear to be newly staged; %s",
310  newfileinfo->GetFirstUrl()->GetUrl());
311 
312  // If staging files, prepare the stager
313  if (fLocateonly || fStageonly) {
314  stager = (fMss && strlen(fMss) > 0) ? TFileStager::Open(fMss) : 0;
315  createStager = (stager) ? kFALSE : kTRUE;
316  }
317 
318  // Process the file
319  Bool_t changed = kFALSE;
320  Bool_t opened = kFALSE;
321  TDataSetManager::ProcessFile(newfileinfo, fSopt, fCheckstg, fDoall, stager, createStager, fStageopts,
322  fDbg, changed, opened);
323 
324  if (changed) fChangedDs = kTRUE;
325  if (opened) fOpened++;
326  }
327 
328  PDB(kSelector, 1) {
329  Info("Process", "updated fileinfo: ");
330  newfileinfo->Print("L");
331  }
332  fSubDataSet->Add(newfileinfo);
333 
334  return kTRUE;
335 }
336 
337 ////////////////////////////////////////////////////////////////////////////////
338 /// Worker Terminate
339 
341 {
342  if (fSubDataSet) {
343  fSubDataSet->Update();
344  if (fSubDataSet->GetNFiles() > 0) {
346  Info("SlaveTerminate",
347  "sub-dataset '%s' added to the output list (%lld files)",
349  }
350  // Add information for registration
351  fOutput->Add(new TNamed(TString::Format("DATASET_%s", fSubDataSet->GetName()).Data(),"OT:sortidx:"));
352  fOutput->Add(new TNamed("PROOFSERV_RegisterDataSet", ""));
353  }
354 
355  // Send the number of files disppeared, opened and mark 'changed'if any fileinfo in the dataset has changed
356  TString hostname(TUrl(gSystem->HostName()).GetHostFQDN());
357  TString thisordinal = gProofServ ? gProofServ->GetOrdinal() : "n.d";
358  TString sfdisppeared= TString::Format("PROOF_NoFilesDisppeared_%s_%s", hostname.Data(), thisordinal.Data());
359  fOutput->Add(new TParameter<Int_t>(sfdisppeared.Data(), fDisappeared));
360  TString sfOpened= TString::Format("PROOF_NoFilesOpened_%s_%s", hostname.Data(), thisordinal.Data());
361  fOutput->Add(new TParameter<Int_t>(sfOpened.Data(), fOpened));
362  TString sfTouched = TString::Format("PROOF_NoFilesTouched_%s_%s", hostname.Data(), thisordinal.Data());
363  fOutput->Add(new TParameter<Int_t>(sfTouched.Data(), fTouched));
364  TString schanged= TString::Format("PROOF_DataSetChanged_%s_%s", hostname.Data(), thisordinal.Data());
365  fOutput->Add(new TParameter<Bool_t>(schanged.Data(), fChangedDs));
366 }
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:51
double par[1]
Definition: unuranDistr.cxx:38
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:899
TSelectorList * fOutput
! List of objects created during processing
Definition: TSelector.h:50
long long Long64_t
Definition: RtypesCore.h:69
void Print(Option_t *options="") const
Print information about this object.
Definition: TFileInfo.cxx:477
This class represents a WWW compatible URL.
Definition: TUrl.h:41
virtual void SlaveTerminate()
Worker Terminate.
TFileCollection * fSubDataSet
Basic string class.
Definition: TString.h:137
bool Bool_t
Definition: RtypesCore.h:59
const Bool_t kFALSE
Definition: Rtypes.h:92
void SetIndex(Int_t idx)
Definition: TFileInfo.h:111
static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall, TFileStager *stager, Bool_t createStager, const char *stageopts, Bool_t dbg, Bool_t &changed, Bool_t &opened)
Locate, stage, or fully validate file "fileInfo".
Int_t GetIndex() const
Definition: TFileInfo.h:110
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:387
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:497
Manages an element of a TDSet.
Definition: TDSet.h:68
Int_t Update(Long64_t avgsize=-1)
Update accumulated information about the elements of the collection (e.g.
#define SafeDelete(p)
Definition: RConfig.h:507
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString...
Definition: TString.cxx:2335
#define PDB(mask, level)
Definition: TProofDebug.h:58
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:33
TSelVerifyDataSet()
Constructor.
TUrl * GetFirstUrl() const
Definition: TFileInfo.h:83
std::vector< std::vector< double > > Data
static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles, Int_t newstagedfiles, TFileStager *stager, Bool_t createStager, Bool_t dbg, Bool_t &changed, Bool_t &touched, Bool_t &disappeared)
Check stage status of the file described by "fileInfo".
TObject * Value() const
Definition: TMap.h:125
virtual void SlaveBegin(TTree *tree)
Worker Begin.
Named parameter, streamable and storable.
Definition: TParameter.h:49
R__EXTERN TSystem * gSystem
Definition: TSystem.h:549
virtual void Abort(const char *why, EAbort what=kAbortProcess)
Abort processing.
Definition: TSelector.cxx:116
Int_t Add(TFileInfo *info)
Add TFileInfo to the collection.
Long64_t GetNFiles() const
unsigned int UInt_t
Definition: RtypesCore.h:42
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:925
void InitMembers()
Initialize members.
Class used by TMap to store (key,value) pairs.
Definition: TMap.h:106
#define ClassImp(name)
Definition: Rtypes.h:279
virtual const char * HostName()
Return the system&#39;s host name.
Definition: TSystem.cxx:308
static TFileStager * Open(const char *stager)
Open a stager, after having loaded the relevant plug-in.
virtual void DispatchOneEvent(Bool_t pendingOnly=kFALSE)
Dispatch a single event.
Definition: TSystem.cxx:434
virtual Bool_t Process(Long64_t entry)
Process a single entry.
Bool_t IsNull() const
Definition: TString.h:387
TObject * GetAssocObj(Long64_t i, Bool_t isentry=kFALSE)
Get i-th associated object.
Definition: TDSet.cxx:633
TList * fInput
List of objects available during processing.
Definition: TSelector.h:49
R__EXTERN TProofServ * gProofServ
Definition: TProofServ.h:361
virtual void Add(TObject *obj)
Definition: TList.h:81
Class that contains a list of TFileInfo&#39;s and accumulated meta data information about its entries...
A TTree object has a header with a name and a title.
Definition: TTree.h:98
Class describing a generic file including meta information.
Definition: TFileInfo.h:50
Selector to verify dataset in parallel on workers.
const Bool_t kTRUE
Definition: Rtypes.h:91
const char * GetOrdinal() const
Definition: TProofServ.h:267
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:911
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:52
const char * Data() const
Definition: TString.h:349