Logo ROOT  
Reference Guide
TProofMonSenderSQL.cxx
Go to the documentation of this file.
1// @(#)root/proofplayer:$Id$
2// Author: G.Ganis July 2011
3
4/*************************************************************************
5 * Copyright (C) 1995-2004, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12
13/** \class TProofMonSenderSQL
14\ingroup proofkernel
15
16TProofMonSender implementation for the SQL writers
17
18*/
19
20#include "TProofMonSenderSQL.h"
21
22#include "TDSet.h"
23#include "TFileInfo.h"
24#include "THashList.h"
25#include "TList.h"
26#include "TPluginManager.h"
27#include "TProofDebug.h"
28#include "TROOT.h"
29#include "TSystem.h"
30#include "TObjString.h"
31#include "TVirtualMonitoring.h"
32
33////////////////////////////////////////////////////////////////////////////////
34/// Main constructor
35
36TProofMonSenderSQL::TProofMonSenderSQL(const char *serv, const char *user,
37 const char *pass, const char *table,
38 const char *dstab, const char *filestab)
39 : TProofMonSender(serv,"ProofMonSenderSQL"),
40 fDSetSendOpts("bulk,table=proofquerydsets"),
41 fFilesSendOpts("bulk,table=proofqueryfiles")
42{
43 fWriter = 0;
44 // Init the sender instance using the plugin manager
45 TPluginHandler *h = 0;
46 if ((h = gROOT->GetPluginManager()->FindHandler("TVirtualMonitoringWriter", "SQL"))) {
47 if (h->LoadPlugin() != -1) {
48 fWriter = (TVirtualMonitoringWriter *) h->ExecPlugin(4, serv, user, pass, table);
50 }
51 }
52 // Flag this instance as valid if the writer initialization succeeded
54
55 // Set default send control options
59 fSummaryVrs = 2;
61 fFileInfoVrs = 1;
62
63 // Transfer verbosity requirements
64 PDB(kMonitoring,1) if (fWriter) fWriter->Verbose(kTRUE);
65
66 // Reformat the send options strings, if needed
67 if (dstab && strlen(dstab) > 0) fDSetSendOpts.Form("bulk,table=%s", dstab);
68 if (filestab && strlen(filestab) > 0) fFilesSendOpts.Form("bulk,table=%s", filestab);
69}
70
71////////////////////////////////////////////////////////////////////////////////
72/// Destructor
73
75{
77}
78
79////////////////////////////////////////////////////////////////////////////////
80/// Send 'summary' record for the table 'proofquerylog'.
81///
82/// There are three versions of this record, corresponding the evolution
83/// in time of the monitoring requirements.
84///
85/// The default version 2 corresponds to the table created with the following command:
86///
87/// CREATE TABLE proofquerylog (
88/// id int(11) NOT NULL auto_increment,
89/// proofuser varchar(32) NOT NULL,
90/// proofgroup varchar(32) default NULL,
91/// querybegin datetime default NULL,
92/// queryend datetime default NULL,
93/// walltime int(11) default NULL,
94/// cputime float default NULL,
95/// bytesread bigint(20) default NULL,
96/// events bigint(20) default NULL,
97/// totevents bigint(20) default NULL,
98/// workers int(11) default NULL,
99/// querytag varchar(64) NOT NULL,
100/// vmemmxw bigint(20) default NULL,
101/// rmemmxw bigint(20) default NULL,
102/// vmemmxm bigint(20) default NULL,
103/// rmemmxm bigint(20) default NULL,
104/// numfiles int(11) default NULL,
105/// missfiles int(11) default NULL,
106/// status int(11) default NULL,
107/// rootver varchar(32) NOT NULL,
108/// PRIMARY KEY (id) );
109///
110/// Version 1 corresponds to the table created with the following command:
111/// ('user','begin','end' instead of 'proofuser', 'querybegin', 'queryend';
112/// no 'status', 'missfiles', 'rootver'; 'dataset' field with name(s) of
113/// processed dataset(s))
114///
115/// CREATE TABLE proofquerylog (
116/// id int(11) NOT NULL auto_increment,
117/// user varchar(32) NOT NULL,
118/// proofgroup varchar(32) default NULL,
119/// begin datetime default NULL,
120/// end datetime default NULL,
121/// walltime int(11) default NULL,
122/// cputime float default NULL,
123/// bytesread bigint(20) default NULL,
124/// events bigint(20) default NULL,
125/// totevents bigint(20) default NULL,
126/// workers int(11) default NULL,
127/// querytag varchar(64) NOT NULL,
128/// vmemmxw bigint(20) default NULL,
129/// rmemmxw bigint(20) default NULL,
130/// vmemmxm bigint(20) default NULL,
131/// rmemmxm bigint(20) default NULL,
132/// numfiles int(11) default NULL,
133/// dataset varchar(512) NOT NULL,
134/// PRIMARY KEY (id) );
135
137{
138 //
139 // Version 0 corresponds to the table created with the following command:
140 // ('group' instead of 'proofgroup'; no 'querytag', 'vmemmxw',
141 // 'rmemmxw', 'vmemmxm', 'rmemmxm', 'numfiles', 'dataset')
142 //
143 // CREATE TABLE proofquerylog (
144 // id int(11) NOT NULL auto_increment,
145 // user varchar(32) NOT NULL,
146 // group varchar(32) default NULL,
147 // begin datetime default NULL,
148 // end datetime default NULL,
149 // walltime int(11) default NULL,
150 // cputime float default NULL,
151 // bytesread bigint(20) default NULL,
152 // events bigint(20) default NULL,
153 // totevents bigint(20) default NULL,
154 // workers int(11) default NULL,
155 // PRIMARY KEY (id) );
156 //
157 // Return 0 on success, -1 on any failure.
158
159 if (!IsValid()) {
160 Error("SendSummary", "invalid instance: do nothing!");
161 return -1;
162 }
163
164 // Are we requested to send this info?
166
167 PDB(kMonitoring,1) Info("SendSummary", "preparing (qid: '%s')", dumid);
168
169 // Make sure we have something to send
170 if (!recs || (recs && recs->GetSize() <= 0)) {
171 Error("SendSummary", "records list undefined or empty!");
172 return -1;
173 }
174 TList *xrecs = recs;
175
176 TObject *dsn = 0;
177 TNamed *nm = 0;
178 // We may need to correct some variable names first
179 if (fSummaryVrs > 1) {
180 if ((nm = (TNamed *) recs->FindObject("user"))) nm->SetName("proofuser");
181 if ((nm = (TNamed *) recs->FindObject("begin"))) nm->SetName("querybegin");
182 if ((nm = (TNamed *) recs->FindObject("end"))) nm->SetName("queryend");
183 if ((dsn = recs->FindObject("dataset"))) recs->Remove(dsn);
184 } else if (fSummaryVrs == 0) {
185 // Only the first records
186 xrecs = new TList;
187 xrecs->SetOwner(kFALSE);
188 TIter nxr(recs);
189 TObject *o = 0;
190 while ((o = nxr())) {
191 if (!strcmp(o->GetName(), "vmemmxw")) break;
192 xrecs->Add(o);
193 }
194 }
195
196 PDB(kMonitoring,1) Info("SendSummary", "sending (%d entries)", xrecs->GetSize());
197
198 // Now we are ready to send
199 Bool_t rc = fWriter->SendParameters(xrecs, dumid);
200
201 // Restore the "dataset" entry in the list
202 if (fSummaryVrs > 1 && dsn && xrecs == recs) {
203 TObject *num = recs->FindObject("numfiles");
204 if (num)
205 recs->AddBefore(num, dsn);
206 else
207 recs->Add(dsn);
208 }
209 if (xrecs != recs) SafeDelete(xrecs);
210
211 // Done
212 return (rc ? 0 : -1);
213}
214
215////////////////////////////////////////////////////////////////////////////////
216/// Post information about the processed dataset(s). The information is taken
217/// from the TDSet object 'dset' and integrated with the missing files
218/// information in the list 'missing'. The string 'qid' is the uninque
219/// ID of the query; 'begin' the starting time.
220///
221/// The record is formatted for the table 'proofquerydsets'.
222///
223/// There are two versions of this record, with or without the starting time.
224/// The starting time could be looked up from the summary record, if available.
225///
226/// The default version 1 corresponds to the table created with the following command:
227///
228/// CREATE TABLE proofquerydsets (
229/// id int(11) NOT NULL auto_increment,
230/// dsn varchar(512) NOT NULL,
231/// querytag varchar(64) NOT NULL,
232/// querybegin datetime default NULL,
233/// numfiles int(11) default NULL,
234/// missfiles int(11) default NULL,
235/// PRIMARY KEY (id),
236/// KEY ix_querytag (querytag) );
237///
238/// Version 0 corresponds to the table created with the following command:
239/// (no 'querybegin')
240///
241/// CREATE TABLE proofquerydsets (
242/// id int(11) NOT NULL auto_increment,
243/// dsn varchar(512) NOT NULL,
244/// querytag varchar(64) NOT NULL,
245/// numfiles int(11) default NULL,
246/// missfiles int(11) default NULL,
247/// PRIMARY KEY (id),
248/// KEY ix_querytag (querytag) );
249///
250/// The information is posted with a bulk insert.
251///
252/// Returns 0 on success, -1 on failure.
253
255 const char *begin, const char *qid)
256{
257 if (!IsValid()) {
258 Error("SendDataSetInfo", "invalid instance: do nothing!");
259 return -1;
260 }
261
262 // Are we requested to send this info?
264
265 // The query id (tag) must be given
266 if (!qid || (qid && strlen(qid) <= 0)) {
267 Error("SendDataSetInfo", "query id (tag) undefined!");
268 return -1;
269 }
270 // The dataset must be given
271 if (!dset) {
272 Error("SendDataSetInfo", "TDSet object undefined! (qid: '%s')", qid);
273 return -1;
274 }
275
276 PDB(kMonitoring,1) Info("SendDataSetInfo", "preparing (qid: '%s')", qid);
277
278 TList plets;
279 // Extract the information and save it into the relevant multiplets
280 TString dss(dset->GetName()), ds;
281 Ssiz_t from = 0;
282 while ((dss.Tokenize(ds, from , "[,| ]"))) {
283 // Create a new TDSetPlet and add it to the list
284 plets.Add(new TDSetPlet(ds.Data(), dset));
285 }
286
287 // Now try to count the files
288 TDSetPlet *plet = 0;
289 TIter nxpl(&plets);
290 TObject *o = 0;
291 TDSetElement *e = 0, *ee = 0;
292 TDSet *dsete = 0;
293 TIter nxe(dset->GetListOfElements());
294 TString dse;
295 while ((o = nxe())) {
296 if ((e = dynamic_cast<TDSetElement *>(o))) {
297 dse = e->GetDataSet();
298 if (!dse.IsNull()) {
299 nxpl.Reset();
300 while ((plet = (TDSetPlet *) nxpl())) {
301 if (dse == plet->GetName()) {
302 plet->fFiles += 1;
303 break;
304 }
305 }
306 }
307 } else if ((dsete = dynamic_cast<TDSet *>(o))) {
308 PDB(kMonitoring,1)
309 Info("SendDataSetInfo", "dset '%s' (%d files)",
310 o->GetName(), dsete->GetListOfElements()->GetSize());
311 TIter nxee(dsete->GetListOfElements());
312 while ((ee = (TDSetElement *) nxee())) {
313 dse = ee->GetDataSet();
314 if (!dse.IsNull()) {
315 nxpl.Reset();
316 while ((plet = (TDSetPlet *) nxpl())) {
317 if (dse == plet->GetName()) {
318 plet->fFiles += 1;
319 plet->fDSet = dsete;
320 break;
321 }
322 }
323 }
324 }
325 } else {
326 Warning("SendDataSetInfo", "ignoring unknown element type: '%s'", o->ClassName());
327 }
328 }
329
330 // Now try to include the missing files info
331 if (missing) {
332 TFileInfo *fi = 0;
333 TIter nxm(missing);
334 TString dsfi, fn;
335 while ((fi = (TFileInfo *) nxm())) {
336 dsfi = fi->GetTitle();
337 if (!dsfi.IsNull() && dsfi != "TFileInfo") {
338 nxpl.Reset();
339 while ((plet = (TDSetPlet *) nxpl())) {
340 if (dsfi == plet->GetName()) {
341 fn = fi->GetCurrentUrl()->GetUrl();
342 if (plet->fDSet && plet->fDSet->GetListOfElements() &&
343 !(plet->fDSet->GetListOfElements()->FindObject(fn))) plet->fFiles += 1;
344 plet->fMissing += 1;
345 break;
346 }
347 }
348 }
349 }
350 }
351
352 // Now we can prepare the input for SendParameters
353 TList values;
354 TString ent("dsn,querytag,querybegin,numfiles,missfiles");
355 if (fDataSetInfoVrs == 0) ent = "dsn,querytag,numfiles,missfiles";
356 values.Add(new TObjString(ent.Data()));
357 nxpl.Reset();
358 while ((plet = (TDSetPlet *) nxpl())) {
359 if (fDataSetInfoVrs == 0)
360 ent.Form("'%s','%s',%d,%d", plet->GetName(), qid, plet->fFiles, plet->fMissing);
361 else
362 ent.Form("'%s','%s','%s',%d,%d", plet->GetName(), qid, begin, plet->fFiles, plet->fMissing);
363 values.Add(new TObjString(ent.Data()));
364 }
365
366 PDB(kMonitoring,1)
367 Info("SendDataSetInfo", "sending (%d entries)", values.GetSize());
368
369 // Now we are ready to send
371
372 // Done
373 return (rc ? 0 : -1);
374}
375
376////////////////////////////////////////////////////////////////////////////////
377/// Post information about the requested files. The information is taken
378/// from the TDSet object 'dset' and integrated with the missing files
379/// information in the list 'missing'. The string 'qid' is the unique
380/// ID of the query; 'begin' the starting time.
381///
382/// The record is formatted for the table 'proofqueryfiles'.
383///
384/// There are two versions of this record, with or without the starting time.
385/// The starting time could be looked up from the summary record, if available.
386///
387/// The default version 1 corresponds to the table created with the following command:
388///
389/// CREATE TABLE proofqueryfiles (
390/// id int(11) NOT NULL auto_increment,
391/// lfn varchar(255) NOT NULL,
392/// path varchar(2048) NOT NULL,
393/// querytag varchar(64) NOT NULL,
394/// querybegin datetime default NULL,
395/// status enum('Ok','Failed') NOT NULL default 'Ok',
396/// PRIMARY KEY (id),
397/// KEY ix_querytag (querytag) );
398///
399/// Version 0 corresponds to the table created with the following command:
400/// (no 'querybegin')
401///
402/// CREATE TABLE proofqueryfiles (
403/// id int(11) NOT NULL auto_increment,
404/// lfn varchar(255) NOT NULL,
405/// path varchar(2048) NOT NULL,
406/// querytag varchar(64) NOT NULL,
407/// status enum('Ok','Failed') NOT NULL default 'Ok',
408/// PRIMARY KEY (id),
409/// KEY ix_querytag (querytag) );
410///
411/// The information is posted with a bulk insert.
412///
413/// Returns 0 on success, -1 on failure.
414
416 const char *begin, const char *qid)
417{
418 if (!IsValid()) {
419 Error("SendFileInfo", "invalid instance: do nothing!");
420 return -1;
421 }
422
423 // Are we requested to send this info?
425
426 // The query id (tag) must be given
427 if (!qid || (qid && strlen(qid) <= 0)) {
428 Error("SendFileInfo", "query id (tag) undefined!");
429 return -1;
430 }
431 // The dataset must be given
432 if (!dset) {
433 Error("SendFileInfo", "TDSet object undefined! (qid: '%s')", qid);
434 return -1;
435 }
436
437 PDB(kMonitoring,1) Info("SendFileInfo", "preparing (qid: '%s')", qid);
438 THashList hmiss;
439 if (missing) {
440 TIter nxfm(missing);
441 TFileInfo *fi = 0;
442 while ((fi = (TFileInfo *)nxfm())) {
443 hmiss.Add(new TObjString(fi->GetCurrentUrl()->GetUrl()));
444 }
445 PDB(kMonitoring,2) hmiss.Print();
446 }
447
448 TList values;
449 TString ent("lfn,path,querytag,querybegin,status");
450 if (fFileInfoVrs == 0) ent = "lfn,path,querytag,status";
451 values.Add(new TObjString(ent.Data()));
452
453 // Create the file-plets
454 TObject *o = 0;
455 TDSetElement *e = 0, *ee = 0;
456 TDSet *dsete = 0;
457 TIter nxe(dset->GetListOfElements());
458 TString fne, status;
459 while ((o = nxe())) {
460 if ((e = dynamic_cast<TDSetElement *>(o))) {
461 fne = e->GetName();
462 // Try to determine the status
463 status = "Ok";
464 if (hmiss.FindObject(fne)) status = "Failed";
465 if (fFileInfoVrs == 0)
466 ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
467 qid, status.Data());
468 else
469 ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
470 qid, begin, status.Data());
471 values.Add(new TObjString(ent.Data()));
472 } else if ((dsete = dynamic_cast<TDSet *>(o))) {
473 PDB(kMonitoring,1)
474 Info("SendFileInfo", "dset '%s' (%d files)",
475 o->GetName(), dsete->GetListOfElements()->GetSize());
476 TIter nxee(dsete->GetListOfElements());
477 while ((ee = (TDSetElement *) nxee())) {
478 fne = ee->GetName();
479 // Try to determine the status
480 status = "Ok";
481 if (hmiss.FindObject(fne)) status = "Failed";
482 if (fFileInfoVrs == 0)
483 ent.Form("'%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
484 qid, status.Data());
485 else
486 ent.Form("'%s','%s','%s','%s','%s'", gSystem->BaseName(fne), gSystem->GetDirName(fne).Data(),
487 qid, begin, status.Data());
488 values.Add(new TObjString(ent.Data()));
489 }
490 } else {
491 Warning("SendFileInfo", "ignoring unknown element type: '%s'", o->ClassName());
492 }
493 }
494
495 PDB(kMonitoring,1) Info("SendFileInfo", "sending (%d entries)", values.GetSize());
496
497 // Now we are ready to send
499
500 // Done
501 return (rc ? 0 : -1);
502}
#define SafeDelete(p)
Definition: RConfig.hxx:543
#define h(i)
Definition: RSha256.hxx:106
#define e(i)
Definition: RSha256.hxx:103
const Bool_t kFALSE
Definition: RtypesCore.h:90
const Bool_t kTRUE
Definition: RtypesCore.h:89
#define PDB(mask, level)
Definition: TProofDebug.h:56
#define gROOT
Definition: TROOT.h:406
R__EXTERN TSystem * gSystem
Definition: TSystem.h:556
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Definition: TCollection.h:182
Manages an element of a TDSet.
Definition: TDSet.h:66
This class implements a data set to be used for PROOF processing.
Definition: TDSet.h:153
TList * GetListOfElements() const
Definition: TDSet.h:231
Class describing a generic file including meta information.
Definition: TFileInfo.h:36
TUrl * GetCurrentUrl() const
Return the current url.
Definition: TFileInfo.cxx:248
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:34
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:262
void Reset()
Definition: TCollection.h:252
A doubly linked list.
Definition: TList.h:44
virtual void Add(TObject *obj)
Definition: TList.h:87
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:821
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:577
virtual void AddBefore(const TObject *before, TObject *obj)
Insert object before object before in the list.
Definition: TList.cxx:195
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
Collectable string class.
Definition: TObjString.h:28
Mother of all ROOT objects.
Definition: TObject.h:37
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:357
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:187
virtual const char * ClassName() const
Returns name of class to which the object belongs.
Definition: TObject.cxx:128
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:877
R__ALWAYS_INLINE Bool_t IsZombie() const
Definition: TObject.h:149
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:694
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:891
void ResetBit(UInt_t f)
Definition: TObject.h:186
@ kInvalidObject
if object ctor succeeded but object should not be used
Definition: TObject.h:68
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:865
virtual ~TProofMonSenderSQL()
Destructor.
Int_t SendFileInfo(TDSet *, TList *, const char *, const char *)
Post information about the requested files.
Int_t SendDataSetInfo(TDSet *, TList *, const char *, const char *)
Post information about the processed dataset(s).
Int_t SendSummary(TList *, const char *)
Send 'summary' record for the table 'proofquerylog'.
TProofMonSenderSQL(const char *serv, const char *user, const char *pass, const char *table="proof.proofquerylog", const char *dstab=0, const char *filestab=0)
Main constructor.
TVirtualMonitoringWriter * fWriter
Provides the interface for PROOF monitoring to different writers.
Bool_t IsValid() const
Basic string class.
Definition: TString.h:131
const char * Data() const
Definition: TString.h:364
Bool_t IsNull() const
Definition: TString.h:402
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2289
virtual const char * BaseName(const char *pathname)
Base name of a file name. Base name of /user/root is root.
Definition: TSystem.cxx:930
virtual TString GetDirName(const char *pathname)
Return the directory name in pathname.
Definition: TSystem.cxx:1027
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
Definition: TUrl.cxx:387
virtual Bool_t SendParameters(TList *, const char *=0)
virtual void Verbose(Bool_t)
static constexpr double nm