Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TDataSetManagerFile.cxx
Go to the documentation of this file.
1// @(#)root/base:$Id$
2// Author: Jan Fiete Grosse-Oetringhaus, 04.06.07
3
4/*************************************************************************
5 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12//////////////////////////////////////////////////////////////////////////
13// //
14// TDataSetManagerFile //
15// //
16// Implementation of TDataSetManager handling datasets from root //
17// files under a specific directory path //
18// //
19//////////////////////////////////////////////////////////////////////////
20#include <sys/stat.h>
21
22#include "TDataSetManagerFile.h"
23
24#include "Riostream.h"
25#include "TDatime.h"
26#include "TEnv.h"
27#include "TFileCollection.h"
28#include "TFileInfo.h"
29#include "TFile.h"
30#include "TFileStager.h"
31#include "TLockFile.h"
32#include "TMap.h"
33#include "TRegexp.h"
34#include "TMD5.h"
35#include "TMacro.h"
36#include "TMessage.h"
37#include "TObjString.h"
38#include "TSystem.h"
39#include "TError.h"
40#include "TPRegexp.h"
41#include "TVirtualMonitoring.h"
42#include "THashList.h"
43#include "TKey.h"
44#include "TTree.h"
45#include "TParameter.h"
46
48 TObjString *fGrp; // Group
49 TObjString *fUsr; // User
50 TObjString *fMd5; // Checksum
51 TObjString *fLss; // 'ls' string
52 Long64_t fMtime; // modification time
53 LsTreeEntry_t(const char *g, const char *u, const char *cs, const char *ls, Long64_t m) :
54 fMtime(m) { fGrp = new TObjString(g); fUsr = new TObjString(u);
55 fMd5 = new TObjString(cs); fLss = new TObjString(ls); }
57};
58
60
61////////////////////////////////////////////////////////////////////////////////
62///
63/// Main constructor
64
66 const char *user, const char *ins)
67 : TDataSetManager(group, user, ins)
68{
69 // Parse options
70 ParseInitOpts(ins);
71
72 // Init the instance
73 Init();
74}
75
76////////////////////////////////////////////////////////////////////////////////
77///
78/// Main constructor
79
81 : TDataSetManager("", "", ins)
82{
83 // Parse options
84 ParseInitOpts(ins);
85
86 // Init the instance
87 Init();
88}
89
90////////////////////////////////////////////////////////////////////////////////
91/// Do the real inititialization
92
94{
96 if (!fUser.IsNull() && !fGroup.IsNull() && !fDataSetDir.IsNull()) {
97
98 // Make sure that the dataset dir exists
99 TString dir;
100 dir.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
101 if (gSystem->AccessPathName(dir)) {
102 if (gSystem->mkdir(dir, kTRUE) != 0) {
103 TString emsg = dir;
104 // Read only dataset info system: switch to COMMON
111 dir.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
112 if (gSystem->AccessPathName(dir)) {
113 Error("Init",
114 "could not attach to a valid the dataset dir; paths tried:");
115 Error("Init", " %s", emsg.Data());
116 Error("Init", " %s", dir.Data());
118 return;
119 }
120 }
121 else if (fOpenPerms) {
122
123 // Directory creation was OK: let's open permissions if requested
124 TString t;
125 Int_t rr = 0;
126
127 t.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(), fUser.Data());
128 rr += gSystem->Chmod(t.Data(), 0777);
129
130 t.Form("%s/%s", fDataSetDir.Data(), fGroup.Data());
131 rr += gSystem->Chmod(t.Data(), 0777);
132
133 rr += gSystem->Chmod(fDataSetDir.Data(), 0777);
134
135 if (rr < 0) {
136 t.Form("%s/%s/%s", fDataSetDir.Data(), fGroup.Data(),
137 fUser.Data());
138 Warning("Init",
139 "problems setting perms of dataset directory %s (#%d)",
140 t.Data(), TSystem::GetErrno());
141 }
142
143 }
144 }
145
146 // If not in sandbox, construct the base URI using session defaults
147 // (group, user) (syntax: /group/user/dsname[#[subdir/]objname])
149 fBase.SetUri(TString(Form("/%s/%s/", fGroup.Data(), fUser.Data())));
150
151 // Local or remote?
152 TString locPath;
153 TFile::EFileType pathType = TFile::GetType(fDataSetDir, "READ", &locPath);
154 if (pathType == TFile::kLocal) {
155 // Remote Url pointing to this machine
156 fDataSetDir = locPath;
157 if (gDebug > 0)
158 Info("Init", "repository '%s' is local", fDataSetDir.Data());
159 } else if (pathType != TFile::kDefault && pathType != TFile::kFile) {
161 if (gDebug > 0)
162 Info("Init", "repository '%s' is remote", fDataSetDir.Data());
163 }
164
165 // Read locking path from kDataSet_LockLocation
167 if (!gSystem->AccessPathName(lockloc, kReadPermission)) {
168 // Open the file in RAW mode
169 lockloc += "?filetype=raw";
170 TFile *f = TFile::Open(lockloc);
171 if (f && !(f->IsZombie())) {
172 const Int_t blen = 8192;
173 char buf[blen];
174 Long64_t rest = f->GetSize();
175 while (rest > 0) {
176 Long64_t len = (rest > blen - 1) ? blen - 1 : rest;
177 if (f->ReadBuffer(buf, len)) {
178 fDataSetLockFile = "";
179 break;
180 }
181 buf[len] = '\0';
182 fDataSetLockFile += buf;
183 rest -= len;
184 }
185 f->Close();
186 SafeDelete(f);
188 } else {
189 lockloc.ReplaceAll("?filetype=raw", "");
190 Warning("Init", "could not open remore file '%s' with the lock location", lockloc.Data());
191 }
192 }
193 if (fDataSetLockFile.IsNull()) {
194 fDataSetLockFile.Form("%s-dataset-lock", fDataSetDir.Data());
198 }
201 if (!strcmp(lu.GetProtocol(), "file")) {
202 // Add host and port
203 TUrl u(fDataSetDir);
204 TString srv(fDataSetDir);
205 srv.Remove(srv.Index(u.GetFile()));
206 fDataSetLockFile.Insert(0, srv);
207 }
208 }
209 }
210
211 // Limit in seconds after a lock automatically expires
212 fLockFileTimeLimit = 120;
213
214 // Default validity of the cache
215 fCacheUpdatePeriod = gEnv->GetValue("ProofDataSet.CacheUpdatePeriod", 0);
216
217 // If the MSS url was not given, check if one is defined via env
218 if (fMSSUrl.IsNull())
219 fMSSUrl = gEnv->GetValue("ProofDataSet.MSSUrl", "");
220 // Default highest priority for xrd-backends
221 fStageOpts = gEnv->GetValue("DataSet.StageOpts", "p=3");
222
223 // File to check updates and its locking path
225
226 // Init the local cache directory if the repository is remote
228 fLocalCacheDir = "";
230}
231////////////////////////////////////////////////////////////////////////////////
232/// Init the local cache if required
233
235{
237
238 // Check if the caller has given specific instructions
239 TString useCache;
240 if (TestBit(TDataSetManager::kUseCache)) useCache = "yes";
241 if (TestBit(TDataSetManager::kDoNotUseCache)) useCache = "no";
242 if (useCache.IsNull()) useCache = gEnv->GetValue("DataSet.UseCache", "");
243 if (useCache.IsNull() && gSystem->Getenv("DATASETCACHE"))
244 useCache = gSystem->Getenv("DATASETCACHE");
245 useCache.ToLower();
246 if (!useCache.IsNull())
247 fUseCache = (useCache == "no" || useCache == "0") ? kFALSE : kTRUE;
248
249 if (fUseCache) {
250 fLocalCacheDir = gSystem->Getenv("DATASETLOCALCACHEDIR");
252 fLocalCacheDir = gEnv->GetValue("DataSet.LocalCacheDir", "");
253 if (!fLocalCacheDir.IsNull()) {
254 // Make sure that the non-default local cache directory exists and is writable
256 if (gSystem->mkdir(fLocalCacheDir, kTRUE) != 0) {
257 // Switch to default
258 Warning("InitLocalCache",
259 "non-default local cache directory '%s' could not be created"
260 " - switching to default", fLocalCacheDir.Data());
261 fLocalCacheDir = "";
262 }
263 }
264 if (!fLocalCacheDir.IsNull() &&
266 Warning("InitLocalCache",
267 "non-default local cache directory '%s' is not writable"
268 " - switching to default",
269 fDataSetDir.Data());
270 fLocalCacheDir = "";
271 }
272 }
273 // If not defined yet try the (unique) default
274 if (fLocalCacheDir.IsNull()) {
275 // Add something related to fDataSetDir
276 TString uds(fDataSetDir.Data());
277 uds.ReplaceAll("/","%");
278 uds.ReplaceAll(":","%");
282 } else {
283 fLocalCacheDir.Form("%s/%s/%s/%s", gSystem->TempDirectory(),
285 }
286 // Make sure that the local cache dir exists and is writable
288 // Disable
289 Warning("InitLocalCache",
290 "local cache directory '%s' could not be created"
291 " - disabling cache", fLocalCacheDir.Data());
293 }
294 if (!fLocalCacheDir.IsNull() &&
296 Warning("InitLocalCache",
297 "local cache directory '%s' is not writable - disabling cache",
298 fDataSetDir.Data());
300 }
301 if (!fUseCache) fLocalCacheDir = "";
302 }
303 }
304 // Done
305 return;
306}
307
308////////////////////////////////////////////////////////////////////////////////
309/// Parse the input string and set the init bits accordingly
310/// Format is
311/// dir:`<datasetdir>` [mss:`<mss-url>`] [opt:`<base-options>`]
312/// The `<datasetdir>` is mandatory.
313/// See TDataSetManager::ParseInitOpts for the available
314/// base options.
315/// The base options are already initialized by the base constructor
316
318{
321
322 // Needs something in
323 if (!ins || strlen(ins) <= 0) return;
324
325 // Extract elements
326 Int_t from = 0;
327 TString s(ins), tok;
328 while (s.Tokenize(tok, from, " ")) {
329 if (tok.BeginsWith("dir:"))
330 fDataSetDir = tok(4, tok.Length());
331 if (tok.BeginsWith("mss:"))
332 fMSSUrl = tok(4, tok.Length());
333 if (tok == "perms:open")
335 }
336
337 // The directory is mandatory
338 if (fDataSetDir.IsNull()) return;
339
340 // Object is valid
342}
343
344////////////////////////////////////////////////////////////////////////////////
345/// Returns path of the indicated dataset. The extension is '.root' for all files
346/// except for 'dsName==ls' which have extension '.txt'.
347/// If 'local' is kTRUE the local cache path is returned instead in the form
348/// `<cachedir>`/`<group>`.`<user>`.`<dsName>`.`<ext>`.
349/// NB: contains a static TString for result, so copy result before using twice.
350
352 const char *user,
353 const char *dsName,
354 TString &md5path, Bool_t local)
355{
358
359 if (fgCommonDataSetTag == user)
360 user = fCommonUser;
361
362 const char *ext = (!strcmp(dsName, "ls")) ? ".txt" : ".root";
363 static TString result;
364 if (local) {
365 result.Form("%s/%s.%s.%s%s", fLocalCacheDir.Data(), group, user, dsName, ext);
366 md5path.Form("%s/%s.%s.%s.md5sum", fLocalCacheDir.Data(), group, user, dsName);
367 } else {
368 result.Form("%s/%s/%s/%s%s", fDataSetDir.Data(), group, user, dsName, ext);
369 md5path.Form("%s/%s/%s/%s.md5sum", fDataSetDir.Data(), group, user, dsName);
370 }
371 if (gDebug > 0)
372 Info("GetDataSetPath","paths: %s, %s ", result.Data(), md5path.Data());
373 return result;
374}
375
376////////////////////////////////////////////////////////////////////////////////
377/// Save into the `<datasetdir>/kDataSet_DataSetList` file the name of the updated
378/// or created or modified dataset. For still existing datasets, fill the
379/// modification date in seconds anf the checksum.
380/// Returns 0 on success, -1 on error
381
382Int_t TDataSetManagerFile::NotifyUpdate(const char *group, const char *user,
383 const char *dsName, Long_t mtime, const char *checksum)
384{
385 // Update / create list for the owner
386 Long_t lsmtime = 0;
387 TString lschecksum;
388 Int_t lsrc = -1;
389 if ((lsrc = CreateLsFile(group, user, lsmtime, lschecksum)) < 0) {
390 Warning("NotifyUpdate", "problems (re-)creating the dataset lists for '/%s/%s'",
391 group, user);
392 }
393
395 TString dspath = TString::Format("/%s/%s/%s", group, user, dsName);
396 // Check if the global file exists
398 // Load the info in form of TMacro
399 TMD5 *oldMd5 = 0, *newMd5 = 0;
400 if (hasListFile && !(oldMd5 = TMD5::FileChecksum(fListFile.Data()))) {
401 Error("NotifyUpdate", "problems calculating old checksum of %s", fListFile.Data());
402 return -1;
403 }
404 // Create the TMacro object, filling it with the existing file, if any
405 TMacro mac;
406 if (hasListFile) mac.ReadFile(fListFile.Data());
407 // Locate the line to change or delete
408 TObjString *os = mac.GetLineWith(dspath);
409 if (os) {
410 // Delete the line if required so
411 if (!strcmp(checksum, "removed")) {
412 mac.GetListOfLines()->Remove(os);
413 SafeDelete(os);
414 } else {
415 // Update the information
416 os->SetString(TString::Format("%ld %s %s", mtime, dspath.Data(), checksum));
417 }
418 } else {
419 if (!strcmp(checksum, "removed")) {
420 Warning("NotifyUpdate", "entry for removed dataset '%s' not found!", dspath.Data());
421 } else {
422 // Add new line
423 mac.AddLine(TString::Format("%ld %s %s", mtime, dspath.Data(), checksum));
424 }
425 }
426 // Locate the ls line now, is needed
427 TString lspath = TString::Format("/%s/%s/ls", group, user);
428 os = mac.GetLineWith(lspath);
429 if (os) {
430 // Delete the line if required so
431 if (lsrc == 1) {
432 mac.GetListOfLines()->Remove(os);
433 SafeDelete(os);
434 } else {
435 // Update the information
436 os->SetString(TString::Format("%ld %s %s", lsmtime, lspath.Data(), lschecksum.Data()));
437 }
438 } else {
439 if (lsrc == 0) {
440 // Add new line
441 mac.AddLine(TString::Format("%ld %s %s", lsmtime, lspath.Data(), lschecksum.Data()));
442 }
443 }
444 // Write off the new content
446 if (fOpenPerms) {
447 if (gSystem->Chmod(fListFile.Data(), 0666) < 0) {
448 Warning("NotifyUpdate",
449 "can't set permissions of dataset list file %s (#%d)",
451 }
452 }
453 if (!(newMd5 = TMD5::FileChecksum(fListFile.Data()))) {
454 Error("NotifyUpdate", "problems calculating new checksum of %s", fListFile.Data());
455 SafeDelete(oldMd5);
456 return -1;
457 }
458 if (oldMd5 && (*newMd5 == *oldMd5))
459 Warning("NotifyUpdate", "checksum for %s did not change!", fListFile.Data());
460 // Cleanup
461 SafeDelete(oldMd5);
462 SafeDelete(newMd5);
463 }
464 // Done
465 return 0;
466}
467
468////////////////////////////////////////////////////////////////////////////////
469/// Create or recreate the dataset lists for 'uri'.
470/// The list are saved in text form in 'uri'/ls.txt for fast browsing and in
471/// 'uri'/ls.root in form of TMacro for optimized and portable transfer.
472/// Return 0 on success, 1 if the file was empty, -1 on error
473
474Int_t TDataSetManagerFile::CreateLsFile(const char *group, const char *user,
475 Long_t &mtime, TString &checksum)
476{
477 mtime = 0;
478 checksum = "";
479 // Create temporary file; we cannot lock now because we would (dead-)lock
480 // during ShowDataSets
481 TString tmpfile;
482 tmpfile.Form("%s/%s/%s/ls.tmp.txt", fDataSetDir.Data(), group, user);
483
484 // Redirect output to 'tmpfile'
486 if (gSystem->RedirectOutput(tmpfile.Data(), "w", &rh) != 0) {
487 Error("CreateLsFile", "problems redirecting output to %s (errno: %d)",
488 tmpfile.Data(), TSystem::GetErrno());
489 return -1;
490 }
491 // Create the list
492 TString uri;
493 uri.Form("/%s/%s", group, user);
494 ShowDataSets(uri, "forcescan:noheader:");
495 // Restore output to standard streams
496 if (gSystem->RedirectOutput(0, 0, &rh) != 0) {
497 Error("CreateLsFile", "problems restoring output to standard streams (errno: %d)",
499 return -1;
500 }
501 // We can lock now
503 // Rename the temp file
504 TString lsfile;
505 lsfile.Form("%s/%s/%s/ls.txt", fDataSetDir.Data(), group, user);
506 // Remove the file, if existing
507 if (!gSystem->AccessPathName(lsfile) && gSystem->Unlink(lsfile) != 0) {
508 Error("CreateLsFile", "problems unlinking old file '%s' (errno: %d)",
509 lsfile.Data(), TSystem::GetErrno());
510 return -1;
511 }
512 // Save the new file only if non empty
513 FileStat_t st;
514 if (gSystem->GetPathInfo(tmpfile, st) == 0 && st.fSize > 0) {
515 if (gSystem->Rename(tmpfile, lsfile) != 0) {
516 Error("CreateLsFile", "problems renaming '%s' to '%s' (errno: %d)",
517 tmpfile.Data(), lsfile.Data(), TSystem::GetErrno());
518 return -1;
519 }
520#ifndef WIN32
521 // Make sure that the ownership and permissions are those expected
522 FileStat_t udirst;
523 if (!fIsRemote && gSystem->GetPathInfo(gSystem->GetDirName(tmpfile), udirst) == 0) {
524 if (chown(lsfile.Data(), udirst.fUid, udirst.fGid) != 0) {
525 Warning("CreateLsFile", "problems setting ownership on file '%s' (errno: %d)",
526 lsfile.Data(), TSystem::GetErrno());
527 }
528 if (fOpenPerms) {
529 if (gSystem->Chmod(lsfile.Data(), 0666) < 0) {
530 Warning("NotifyUpdate",
531 "can't set permissions of list file %s (#%d)",
532 lsfile.Data(), TSystem::GetErrno());
533 }
534 }
535 else if (chmod(lsfile.Data(), 0644) != 0) {
536 Warning("CreateLsFile", "problems setting permissions on file '%s' (errno: %d)",
537 lsfile.Data(), TSystem::GetErrno());
538 }
539 }
540#endif
541 mtime = st.fMtime;
542 TMD5 *md5 = TMD5::FileChecksum(lsfile);
543 if (!md5) {
544 Error("CreateLsFile", "problems calculating checksum for '%s'", lsfile.Data());
545 } else {
546 checksum = md5->AsString();
547 SafeDelete(md5);
548 }
549 } else {
550 if (!gSystem->AccessPathName(tmpfile) && gSystem->Unlink(tmpfile) != 0) {
551 Error("CreateLsFile", "problems unlinking temp file '%s' (errno: %d)",
552 tmpfile.Data(), TSystem::GetErrno());
553 return -1;
554 }
555 // No datasets anymore
556 return 1;
557 }
558 // Done
559 return 0;
560}
561
562////////////////////////////////////////////////////////////////////////////////
563/// Adds the dataset in the folder of group, user to the list in target.
564/// If dsName is defined, only the information about the specified dataset
565/// is processed.
566///
567/// The unsigned int 'option' is forwarded to GetDataSet and BrowseDataSet.
568/// Available options (to be .or.ed):
569/// kPrint print the dataset content
570/// kQuotaUpdate update quotas
571/// kExport use export naming
572/// kList get a list of dataset names
573///
574/// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
575/// NB2: for options "kPrint" and "kQuotaUpdate" return is null.
576
578 const char *dsName,
580{
581 TString userDirPath;
582 userDirPath.Form("%s/%s/%s", fDataSetDir.Data(), group, user);
583 void *userDir = gSystem->OpenDirectory(userDirPath);
584 if (!userDir)
585 return kFALSE;
586
587 // Options
588 Bool_t printing = (option & kPrint) ? kTRUE : kFALSE;
589 Bool_t exporting = (option & kExport) ? kTRUE : kFALSE;
590 Bool_t updating = (option & kQuotaUpdate) ? kTRUE : kFALSE;
591 Bool_t printout = (printing && (option & kDebug)) ? kTRUE : kFALSE;
592 Bool_t listing = (option & kList) ? kTRUE : kFALSE;
593
594 // If printing is required add kReadShort to the options
595 if (printing || updating)
597
598 // The last three options are mutually exclusive
599 if (((Int_t)printing + (Int_t)exporting + (Int_t)updating + (Int_t)listing) > 1) {
600 Error("BrowseDataSets",
601 "only one of kPrint, kQuotaUpdate, kExport or kList can be specified at once");
602 return kFALSE;
603 }
604 Bool_t fillmap = (!exporting && !printing && !updating) ? kTRUE : kFALSE;
605
606 // Output object
607 TMap *outmap = (fillmap || exporting || listing) ? (TMap *)target : (TMap *)0;
608 TList *outlist = (printing) ? (TList *)target : (TList *)0;
609
610 TRegexp rg("^[^./][^/]*.root$"); //check that it is a root file, not starting with "."
611
612 TRegexp *reds = 0;
613 if (dsName && strlen(dsName) > 0) reds = new TRegexp(dsName, kTRUE);
614
615 TMap *userMap = 0, *datasetMap = 0;
616 // loop over datasets
617 const char *dsEnt = 0;
618 while ((dsEnt = gSystem->GetDirEntry(userDir))) {
619 TString datasetFile(dsEnt);
620 if (datasetFile.Index(rg) != kNPOS) {
621 TString datasetName(datasetFile(0, datasetFile.Length()-5));
622
623 // Check dataset name, if required
624 if (reds && datasetName.Index(*reds) == kNPOS) continue;
625
626 if (gDebug > 0)
627 Info("GetDataSets", "found dataset %s of user %s in group %s",
628 datasetName.Data(), user, group);
629
630 TFileCollection *fileList = GetDataSet(group, user, datasetName, option);
631 if (!fileList) {
632 Error("GetDataSets", "dataset %s (user %s, group %s) could not be opened",
633 datasetName.Data(), user, group);
634 continue;
635 }
636 if (gDebug > 0)
637 fileList->Print();
638
639 // We found a dataset, now add it to the map
640
641 // COMMON dataset transition
642 const char *mapGroup = group;
643 if (fCommonGroup == mapGroup)
644 mapGroup = fgCommonDataSetTag.Data();
645 const char *mapUser = user;
646 if (fCommonUser == mapUser)
647 mapUser = fgCommonDataSetTag.Data();
648
649 if (fillmap && !listing && outmap) {
650 if (!(userMap = dynamic_cast<TMap*> (outmap->GetValue(mapGroup)))) {
651 userMap = new TMap;
652 userMap->SetOwner();
653 outmap->Add(new TObjString(mapGroup), userMap);
654 }
655
656 if (!(datasetMap = dynamic_cast<TMap*> (userMap->GetValue(mapUser)))) {
657 datasetMap = new TMap;
658 datasetMap->SetOwner();
659 userMap->Add(new TObjString(mapUser), datasetMap);
660 }
661 }
662
663 // Action depends on option
664 if (exporting) {
665
666 // Just format the dataset name with group and user
667 TString dsNameFormatted(Form("/%s/%s/%s", mapGroup,
668 mapUser, datasetName.Data()));
669 if (outmap)
670 outmap->Add(new TObjString(dsNameFormatted), fileList);
671
672 } else if (updating) {
673
674 // Update quotas
675 GetQuota(mapGroup, mapUser, datasetName.Data(), fileList);
676
677 } else if (printing) {
678
679 // Prepare the output list
680 if (outlist) {
681 TString dsNameFormatted(Form("/%s/%s/%s", mapGroup,
682 mapUser, datasetName.Data()));
683 // Magic number?
684 if (dsNameFormatted.Length() < 42)
685 dsNameFormatted.Resize(42);
686
687 // Done
688 outlist->Add(fileList->ExportInfo(dsNameFormatted));
689 if (printout) {
690 // Recreating the LS file
691 TObjString *os = (TObjString *) outlist->Last();
692 if (os) Printf("%s", os->GetName());
693 }
694 }
695 } else if (listing) {
696
697 // Just a list of available datasets
698 if (outmap) {
699 outmap->Add(new TObjString(TString::Format("/%s/%s/%s", mapGroup, mapUser, datasetName.Data())),
700 new TObjString(""));
701 }
702 } else {
703 if (fillmap && datasetMap)
704 datasetMap->Add(new TObjString(datasetName), fileList);
705 }
706 }
707 }
708 gSystem->FreeDirectory(userDir);
709 SafeDelete(reds);
710
711 return kTRUE;
712}
713
714////////////////////////////////////////////////////////////////////////////////
715/// General purpose call to go through the existing datasets.
716/// If `<user>` is 0 or "*", act on all datasets for the given `<group>`.
717/// If `<group>` is 0 or "*", act on all datasets.
718/// If `<dsName>` is defined, only the information about the specified dataset
719/// is processed.
720/// Action depends on option; available options:
721///
722/// kExport Return a TMap object containing all the information about
723/// datasets in the form:
724/// { `<group>`, `<map of users>` }
725/// |
726/// { `<map of datasets>`, `<dataset>`}
727/// (`<dataset>` are TFileCollection objects)
728/// kShowDefault as kExport with in addition a default selection including
729/// the datasets from the current user, the ones from the group
730/// and the common ones
731///
732/// kPrint print the dataset content; no output is returned
733/// kList get a list of available dataset names
734/// kForceScan Re-open files while processing kPrint (do not use the
735/// pre-processed information)
736/// kNoHeaderPrint Labelling header is not printed
737/// kQuotaUpdate update {group, user} quotas; no output is returned
738///
739/// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
740/// NB2: for options "kPrint" and "kQuoatUpdate" return is null.
741
742TMap *TDataSetManagerFile::GetDataSets(const char *group, const char *user,
743 const char *dsName, UInt_t option)
744{
747
748 if (user && fgCommonDataSetTag == user)
749 user = fCommonUser;
750
751 // Special treatment for the COMMON user
752 Bool_t notCommonUser = kTRUE;
753 if ((user && fCommonUser == user) &&
754 (group && fCommonGroup == group)) notCommonUser = kFALSE;
755
756 // convert * to "nothing"
757 if (group && (strcmp(group, "*") == 0 || !group[0]))
758 group = 0;
759 if (user && (strcmp(user, "*") == 0 || !user[0]))
760 user = 0;
761
762 Bool_t printing = (option & kPrint) ? kTRUE : kFALSE;
763 Bool_t forcescan = (option & kForceScan) ? kTRUE : kFALSE;
764 Bool_t printheader = (option & kNoHeaderPrint) ? kFALSE : kTRUE;
765 Bool_t exporting = (option & kExport) ? kTRUE : kFALSE;
766 Bool_t updating = (option & kQuotaUpdate) ? kTRUE : kFALSE;
767 Bool_t refreshingls = (option & kRefreshLs) ? kTRUE : kFALSE;
768 Bool_t listing = (option & kList) ? kTRUE : kFALSE;
769
770 // The last three options are mutually exclusive
771 if (((Int_t)printing + (Int_t)exporting + (Int_t)updating + (Int_t)listing) > 1) {
772 Error("GetDataSets", "only one of '?P', '?Q', '?E' or '?L' can be specified at once");
773 return 0;
774 }
775
776 TObject *result = 0;
777 if (printing) {
778 // The output is a list of strings
779 TList *ol = new TList();
780 ol->SetOwner();
781 result = ol;
782 } else if (exporting || !updating || listing) {
783 TMap *om = new TMap;
784 om->SetOwner();
785 result = om;
786 }
787
788 if (gDebug > 0)
789 Info("GetDataSets", "opening dir %s", fDataSetDir.Data());
790
791 Long_t m;
792 TString s;
793 if (option & kShowDefault) {
794 // Add the common ones
795 if (refreshingls) {
796 if (CreateLsFile(fCommonGroup, fCommonUser, m, s) != 0)
797 Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
799 } else if (!printing || forcescan || (printing &&
802 }
803 user = 0;
804 } else {
805 // Fill the information at least once
806 if (!notCommonUser) notCommonUser = kTRUE;
807 }
808
809 // Fill the information only once
810 if (notCommonUser) {
811 // group, user defined, no looping needed
812 if (user && group && strchr(user, '*') && strchr(group, '*')) {
813 if (refreshingls) {
814 if (CreateLsFile(group, user, m, s) != 0)
815 Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
816 group, user);
817 } else if (!printing || forcescan || (printing &&
818 FillLsDataSet(group, user, dsName, (TList *)result, option) != 0)) {
819 BrowseDataSets(group, user, dsName, option, result);
820 }
821 if (!printing) return (TMap *)result;
822 } else {
823 TRegexp *reg = (group && strlen(group) > 0) ? new TRegexp(group, kTRUE) : 0;
824 TRegexp *reu = (user && strlen(user) > 0) ? new TRegexp(user, kTRUE) : 0;
825 // Loop needed, either on the local cache or on the real thing
826 if (printing && !forcescan &&
827 fUseCache && CheckLocalCache(group, user, 0, option) == 0) {
828 // Loop on the local cache
829 Int_t from = 0;
830 TString locupdate, dsn, grp, usr;
831 locupdate.Form("%s/%s", fLocalCacheDir.Data(), kDataSet_DataSetList);
832 TMacro uptmac(locupdate);
833 TIter nxl(uptmac.GetListOfLines());
834 TObjString *os = 0;
835 while ((os = (TObjString *) nxl())) {
836 if (!(os->GetString().Contains("/ls"))) continue;
837 from = 0;
838 if (!(os->GetString().Tokenize(dsn, from, " "))) continue;
839 if (!(os->GetString().Tokenize(dsn, from, " "))) continue;
840 from = 0;
841 // Get group and apply filter on group
842 if (!(dsn.Tokenize(grp, from, "/")) || (reg && (grp.Index(*reg) == kNPOS))) continue;
843 // Get user and apply filter on user
844 if (!(dsn.Tokenize(usr, from, "/")) || (reu && (usr.Index(*reu) == kNPOS))) continue;
845 // Now get the info
846 if (FillLsDataSet(grp, usr, dsName, (TList *)result, option) != 0) {
847 // Use the standard way opening all the files
848 BrowseDataSets(grp, usr, dsName, option, result);
849 }
850 }
851 } else {
852 // Loop needed on the real thing
853 void *dataSetDir = 0;
854 if ((dataSetDir = gSystem->OpenDirectory(fDataSetDir))) {
855 // loop over groups
856 const char *eg = 0;
857 while ((eg = gSystem->GetDirEntry(dataSetDir))) {
858
859 if (strcmp(eg, ".") == 0 || strcmp(eg, "..") == 0)
860 continue;
861
862 if (reg && (TString(eg).Index(*reg) == kNPOS))
863 continue;
864
865 TString groupDirPath;
866 groupDirPath.Form("%s/%s", fDataSetDir.Data(), eg);
867
868 // Make sure it is a directory
869 FileStat_t dirSt;
870 if (gSystem->GetPathInfo(groupDirPath, dirSt) != 0 || !R_ISDIR(dirSt.fMode))
871 continue;
872
873 void *groupDir = gSystem->OpenDirectory(groupDirPath);
874 if (!groupDir)
875 continue;
876
877 // loop over users
878 const char *eu = 0;
879 while ((eu = gSystem->GetDirEntry(groupDir))) {
880
881 if (strcmp(eu, ".") == 0 || strcmp(eu, "..") == 0)
882 continue;
883
884 if (reu && (TString(eu).Index(*reu) == kNPOS))
885 continue;
886
887 // If we have the ls.macro use that
888 if (refreshingls) {
889 if (CreateLsFile(eg, eu, m, s) != 0)
890 Warning("GetDataSets", "problems recreating 'ls' info for {%s,%s}",
891 eg, eu);
892 } else if (!printing || forcescan || (printing &&
893 FillLsDataSet(eg, eu, dsName, (TList *)result, option) != 0)) {
894 // Use the standard way opening all the files
895 BrowseDataSets(eg, eu, dsName, option, result);
896 }
897 }
898 gSystem->FreeDirectory(groupDir);
899 }
900 gSystem->FreeDirectory(dataSetDir);
901 }
902 }
904 SafeDelete(reu);
905 }
906 }
907 // Print the result, if required
908 if (printing) {
909 TList *output = (TList *)result;
910 output->Sort();
911 if (printheader) {
912 Printf("Dataset repository: %s", fDataSetDir.Data());
913 Printf("Dataset URI | # Files | Default tree | # Events | Disk | Staged");
914 }
915 TIter iter4(output);
916 TObjString *os = 0;
917 while ((os = dynamic_cast<TObjString*> (iter4()))) {
918 if (os->GetString().BeginsWith("file:")) {
919 // Path of the file to be browsed
920 TString path(os->GetString()(5, os->GetString().Length()));
921 RedirectHandle_t rh(path.Data());
922 gSystem->ShowOutput(&rh);
923 fflush(stderr);
924 } else {
925 // Simple line
926 Printf("%s", os->String().Data());
927 }
928 }
929 // Cleanup
931 result = 0;
932 }
933
934 return (TMap *)result;
935}
936
937////////////////////////////////////////////////////////////////////////////////
938/// Check for the 'ls.txt' for 'group' and 'user' and fill the path for the
939/// ls file in 'out'.
940/// If 'dsname' is defined, open the file and extract the relevant line.
941/// Return 0 on success, -1 on failure
942
943Int_t TDataSetManagerFile::FillLsDataSet(const char *group, const char *user,
944 const char *dsname, TList *out, UInt_t option)
945{
946 // Check inputs
947 if (!group || strlen(group) <= 0 || !user || strlen(user) <= 0 || !out) {
948 Error("FillLsDataSet", "at least one of the inputs is invalid (%s,%s,%p)", group, user, out);
949 return -1;
950 }
951
952 // File with the TMacro
953 Int_t crc = -1;
954 TString lsfile, lsmd5file;
955 if (!fUseCache || (fUseCache && (crc = CheckLocalCache(group, user, "ls", option)) <= 0)) {
956 Bool_t local = (crc == 0) ? kTRUE : kFALSE;
957 lsfile = GetDataSetPath(group, user, "ls", lsmd5file, local);
958 } else {
959 // The dataset does not exist anymore
960 return 0;
961 }
962
963 if (gSystem->AccessPathName(lsfile, kFileExists)) {
964 if (gDebug > 0)
965 Info("FillLsDataSet", "file '%s' does not exists", lsfile.Data());
966 return -1;
967 }
968 if (gSystem->AccessPathName(lsfile, kReadPermission)) {
969 Warning("FillLsDataSet", "file '%s' exists cannot be read (permission denied)", lsfile.Data());
970 return -1;
971 }
972
973 if (dsname && strlen(dsname) > 0) {
974 // Read the macro
975 TMacro *mac = new TMacro(lsfile.Data());
976 if (!mac) {
977 Error("FillLsDataSet", "could not initialize TMacro from '%s'", lsfile.Data());
978 return -1;
979 }
980 // Prepare the string to search for
981 TString fullname = TString::Format("/%s/%s/%s", group, user, dsname);
982 Bool_t wc = (fullname.Contains("*")) ? kTRUE : kFALSE;
983 if (wc) fullname.ReplaceAll("*", ".*");
984 TRegexp reds(fullname);
985 TIter nxl(mac->GetListOfLines());
986 TObjString *o;
987 Int_t nf = 0;
988 while ((o = (TObjString *) nxl())) {
989 if (o->GetString().Index(reds) != kNPOS) {
990 out->Add(o->Clone());
991 nf++;
992 if (!wc) break;
993 }
994 }
995 if (nf > 0 && gDebug > 0)
996 Info("FillLsDataSet", "no match for dataset uri '/%s/%s/%s'", group, user, dsname);
997 // Delete the macro
998 SafeDelete(mac);
999 } else {
1000 // Fill in the file information
1001 out->Add(new TObjString(TString::Format("file:%s", lsfile.Data())));
1002 }
1003 // Done
1004 return 0;
1005}
1006
1007////////////////////////////////////////////////////////////////////////////////
1008///
1009/// Returns the dataset `<dsName>` of user `<user>` in group `<group>`.
1010/// If checksum is non-zero, it will contain the pointer to a TMD5 sum object
1011/// with the checksum of the file, has to be deleted by the user.
1012/// If option has the bi kReadShort set, the shortobject is read, that does not
1013/// contain the list of files. This is much faster.
1014
1016 const char *user,
1017 const char *dsName,
1018 UInt_t option,
1019 TMD5 **checksum)
1020{
1021 TFileCollection *fileList = 0;
1022 Bool_t readshort = (option & kReadShort) ? kTRUE : kFALSE;
1023 // Check is the file is in the cache
1024 Int_t crc = -1;
1025 TString path, md5path;
1026 if (readshort || !fUseCache ||
1027 (!readshort && fUseCache && (crc = CheckLocalCache(group, user, dsName, option)) <= 0)) {
1028 Bool_t local = (crc == 0) ? kTRUE : kFALSE;
1029 path = GetDataSetPath(group, user, dsName, md5path, local);
1030 } else {
1031 // The dataset does not exist (anymore?)
1032 if (gDebug > 0)
1033 Info("GetDataSet", "dataset %s does not exist", path.Data());
1034 return fileList;
1035 }
1036
1037 // Now we lock because we are going to use the file, if there
1039
1040 // Check if the file can be opened for reading
1041 if (gSystem->AccessPathName(path, kFileExists)) {
1042 if (gDebug > 0)
1043 Info("GetDataSet", "file '%s' does not exists", path.Data());
1044 return fileList;
1045 }
1047 Warning("GetDataSet", "file '%s' exists cannot be read (permission denied)", path.Data());
1048 return fileList;
1049 }
1050
1051 // Get checksum
1052 if (checksum) {
1053 // save md5 sum
1054 *checksum = TMD5::ReadChecksum(md5path);
1055 if (!(*checksum)) {
1056 Error("GetDataSet", "could not get checksum of %s from %s", path.Data(), md5path.Data());
1057 return fileList;
1058 }
1059 }
1060
1061 TFile *f = TFile::Open(path.Data());
1062 if (!f) {
1063 Error("GetDataSet", "could not open file %s", path.Data());
1064 if (checksum) SafeDelete(*checksum);
1065 return fileList;
1066 }
1067
1068 if (option & kReadShort)
1069 fileList = dynamic_cast<TFileCollection*> (f->Get("dataset_short"));
1070
1071 if (!fileList)
1072 fileList = dynamic_cast<TFileCollection*> (f->Get("dataset"));
1073
1074 f->Close();
1075 SafeDelete(f);
1076
1077 return fileList;
1078}
1079
1080////////////////////////////////////////////////////////////////////////////////
1081/// Check if the local cache information for group, user, dsName is up-to-date
1082/// If not, make the relevant updates
1083/// Return 0 if OK, 1 if the dataset does not exists anymore, -1 on failure
1084
1086 const char *dsName, UInt_t option)
1087{
1088 // Check first if the global update info is uptodate
1089 static TMacro *uptmac = 0;
1090 Bool_t need_last_update = (option & kNoCacheUpdate) ? kFALSE : kTRUE;
1091 TString locupdtim, locupdate, remupdate;
1092 locupdtim.Form("%s/%s.update", fLocalCacheDir.Data(), kDataSet_DataSetList);
1093 locupdate.Form("%s/%s", fLocalCacheDir.Data(), kDataSet_DataSetList);
1094 remupdate.Form("%s/%s", fDataSetDir.Data(), kDataSet_DataSetList);
1095 need_last_update = (gSystem->AccessPathName(locupdate)) ? kTRUE : need_last_update;
1096 TDatime now;
1097 UInt_t tnow = now.Convert();
1098 FileStat_t timst, locst, remst;
1099 if (need_last_update && !gSystem->AccessPathName(locupdtim)) {
1100 if (gSystem->GetPathInfo(locupdtim, timst) == 0) {
1101 need_last_update = kFALSE;
1102 if ((Int_t)tnow > timst.fMtime + fCacheUpdatePeriod) need_last_update = kTRUE;
1103 }
1104 }
1105 if (need_last_update) {
1106 if (gSystem->GetPathInfo(remupdate, remst) != 0) {
1107 Error("CheckLocalCache", "cannot get info for remote file '%s' - ignoring", remupdate.Data());
1108 return -1;
1109 }
1110 if (gSystem->GetPathInfo(locupdate, locst) == 0) {
1111 need_last_update = kFALSE;
1112 if (remst.fMtime > locst.fMtime) {
1113 need_last_update = kTRUE;
1114 } else {
1115 if (!gSystem->AccessPathName(locupdtim))
1116 if (gSystem->Utime(locupdtim, tnow, 0) != 0)
1117 Warning("CheckLocalCache",
1118 "cannot set modification time on file '%s' (errno: %d)",
1119 locupdtim.Data(), TSystem::GetErrno());
1120 }
1121 }
1122 }
1123 // Get the file, if needed
1124 if (need_last_update) {
1125 if (!TFile::Cp(remupdate, locupdate, kFALSE)) {
1126 Error("CheckLocalCache", "cannot get remote file '%s' - ignoring", remupdate.Data());
1127 return -1;
1128 }
1129 // Set the modification time
1130 if (gSystem->Utime(locupdate, remst.fMtime, 0) != 0) {
1131 Warning("CheckLocalCache", "cannot set modification time on file '%s' (errno: %d)",
1132 locupdate.Data(), TSystem::GetErrno());
1133 }
1134 // Touch or create the file to control updates
1135 if (gSystem->AccessPathName(locupdtim)) {
1136 FILE *ftim = fopen(locupdtim.Data(), "w");
1137 if (!ftim) {
1138 Warning("CheckLocalCache", "problems create file '%s' (errno: %d)",
1139 locupdtim.Data(), TSystem::GetErrno());
1140 } else {
1141 if (fclose(ftim) != 0)
1142 Warning("CheckLocalCache", "problems close file '%s' (errno: %d)",
1143 locupdtim.Data(), TSystem::GetErrno());
1144 if (gSystem->Utime(locupdtim, now.Convert(), 0) != 0)
1145 Warning("CheckLocalCache",
1146 "cannot set modification time on file '%s' (errno: %d)",
1147 locupdtim.Data(), TSystem::GetErrno());
1148 }
1149 }
1150 // Update macro info
1151 SafeDelete(uptmac);
1152 uptmac = new TMacro(locupdate);
1153 } else {
1154 // Touch or create the file to control updates
1155 if (gSystem->AccessPathName(locupdtim)) {
1156 FILE *ftim = fopen(locupdtim.Data(), "w");
1157 if (!ftim) {
1158 Warning("CheckLocalCache", "problems create file '%s' (errno: %d)",
1159 locupdtim.Data(), TSystem::GetErrno());
1160 } else {
1161 if (fclose(ftim) != 0)
1162 Warning("CheckLocalCache", "problems close file '%s' (errno: %d)",
1163 locupdtim.Data(), TSystem::GetErrno());
1164 if (gSystem->GetPathInfo(locupdate, locst) == 0) {
1165 if (gSystem->Utime(locupdtim, locst.fMtime, 0) != 0)
1166 Warning("CheckLocalCache",
1167 "cannot set modification time on file '%s' (errno: %d)",
1168 locupdtim.Data(), TSystem::GetErrno());
1169 } else {
1170 Warning("CheckLocalCache", "cannot get info for file '%s'"
1171 " - will not touch '%s'", locupdate.Data(), locupdtim.Data());
1172 }
1173 }
1174 }
1175 if (!uptmac) uptmac = new TMacro(locupdate);
1176 }
1177
1178 // If we are just interested in the global dataset list we are done
1179 if (!dsName || strlen(dsName) <= 0)
1180 return 0;
1181
1182 // Read the information
1183 TString ds, locpath, path, locmd5path, md5path, remmd5s;
1184 TMD5 *locmd5 = 0;
1185 // The paths ...
1186 path = GetDataSetPath(group, user, dsName, md5path);
1187 locpath = GetDataSetPath(group, user, dsName, locmd5path, kTRUE);
1188 ds.Form("/%s/%s/%s", group, user, dsName);
1189 TObjString *os = uptmac->GetLineWith(ds);
1190 if (!os) {
1191 // DataSet does not exist anymore
1192 if (strcmp(dsName, "ls"))
1193 Warning("CheckLocalCache", "dataset '%s' does not exists anymore", ds.Data());
1194 return 1;
1195 }
1196 // Extract the relevant information
1197 TString s;
1198 Int_t from = 0;
1199 while (os->GetString().Tokenize(s, from, " ")) {
1200 if (!s.IsDigit() && s != ds) {
1201 remmd5s = s;
1202 }
1203 }
1204 if (remmd5s == "---") {
1205 // DataSet does not exist anymore
1206 if (strcmp(dsName, "ls"))
1207 Warning("CheckLocalCache", "dataset '%s' does not exists anymore", ds.Data());
1208 return 1;
1209 }
1210 Bool_t need_update = (option & kNoCacheUpdate) ? kFALSE : kTRUE;
1211 if (!gSystem->AccessPathName(locpath)) {
1212 if (need_update) {
1213 need_update = kFALSE;
1214 locmd5 = TMD5::ReadChecksum(locmd5path);
1215 if (!locmd5 && !(locmd5 = TMD5::FileChecksum(locpath))) {
1216 Warning("CheckLocalCache", "cannot get checksum of '%s' - assuming match failed", ds.Data());
1217 need_update = kTRUE;
1218 } else {
1219 if (remmd5s != locmd5->AsString()) need_update = kTRUE;
1220 }
1221 }
1222 } else {
1223 need_update = kTRUE;
1224 }
1225 // Get the file, if needed
1226 if (need_update) {
1227 SafeDelete(locmd5);
1228 if (!TFile::Cp(path, locpath, kFALSE)) {
1229 Error("CheckLocalCache", "cannot get remote file '%s' - ignoring", path.Data());
1230 return -1;
1231 }
1232 // Calculate and save the new checksum
1233 locmd5 = TMD5::FileChecksum(locpath);
1234 if (locmd5) {
1235 if (remmd5s != locmd5->AsString())
1236 Warning("CheckLocalCache", "checksum for freshly downloaded file '%s' does not match the"
1237 " one posted in '%s'", locpath.Data(), kDataSet_DataSetList);
1238 if (TMD5::WriteChecksum(locmd5path, locmd5) != 0)
1239 Warning("CheckLocalCache", "problems saving checksum to '%s' (errno: %d)",
1240 locmd5path.Data(), TSystem::GetErrno());
1241 } else {
1242 Warning("CheckLocalCache", "problems calculating checksum for '%s'", locpath.Data());
1243 }
1244 }
1245 SafeDelete(locmd5);
1246 // Done
1247 return 0;
1248}
1249
1250////////////////////////////////////////////////////////////////////////////////
1251/// Clear cached information matching uri
1252
1254{
1255 // Open the top directory
1256 void *dirp = gSystem->OpenDirectory(fLocalCacheDir.Data());
1257 if (!dirp) {
1258 Error("ClearCache", "cannot open directory '%s' (errno: %d)",
1260 return -1;
1261 }
1262 TRegexp *re = 0;
1263 if (uri && strlen(uri) > 0) {
1264 if (strcmp(uri, "*") && strcmp(uri, "/*") && strcmp(uri, "/*/") &&
1265 strcmp(uri, "/*/*") && strcmp(uri, "/*/*/") && strcmp(uri, "/*/*/*")) {
1266 TString u(uri);
1267 // Remove leading '/'
1268 if (u(0) == '/') u.Remove(0,1);
1269 // Change '/' to '%'
1270 u.ReplaceAll("/", ".");
1271 // Init the regular expression
1272 u.ReplaceAll("*", ".*");
1273 re = new TRegexp(u.Data());
1274 }
1275 }
1276
1277 Printf(" Dataset repository: %s", fDataSetDir.Data());
1278 Printf(" Local cache directory: %s", fLocalCacheDir.Data());
1279
1280 Long64_t totsz = 0, nf = 0;
1281 FileStat_t st;
1282 TString path;
1283 const char *e = 0;
1284 while ((e = gSystem->GetDirEntry(dirp))) {
1285 // Skip basic entries
1286 if (!strcmp(e,".") || !strcmp(e,"..")) continue;
1287 // Apply regular expression, if requested
1288 if (re && TString(e).Index(*re) == kNPOS) continue;
1289 // Group directory
1290 path.Form("%s/%s", fLocalCacheDir.Data(), e);
1291 // Get file information
1292 if (gSystem->GetPathInfo(path, st) != 0) {
1293 Warning("ShowCache", "problems 'stat'-ing '%s' (errno: %d)",
1294 path.Data(), TSystem::GetErrno());
1295 continue;
1296 }
1297 // Count
1298 totsz += st.fSize;
1299 nf++;
1300 // Remove the file
1301 if (gSystem->Unlink(path) != 0) {
1302 Warning("ClearCache", "problems unlinking '%s' (errno: %d)",
1303 path.Data(), TSystem::GetErrno());
1304 }
1305 }
1306 gSystem->FreeDirectory(dirp);
1307 SafeDelete(re);
1308
1309 // Notify totals
1310 Printf(" %lld bytes (%lld files) have been freed", totsz, nf);
1311
1312 // Done
1313 return 0;
1314}
1315
1316////////////////////////////////////////////////////////////////////////////////
1317/// Show cached information matching uri
1318
1320{
1321 // Open the top directory
1322 void *dirp = gSystem->OpenDirectory(fLocalCacheDir.Data());
1323 if (!dirp) {
1324 Error("ShowCache", "cannot open directory '%s' (errno: %d)",
1326 return -1;
1327 }
1328 TRegexp *re = 0;
1329 if (uri && strlen(uri) > 0) {
1330 if (strcmp(uri, "*") && strcmp(uri, "/*") && strcmp(uri, "/*/") &&
1331 strcmp(uri, "/*/*") && strcmp(uri, "/*/*/") && strcmp(uri, "/*/*/*")) {
1332 TString u(uri);
1333 // Remove leading '/'
1334 if (u(0) == '/') u.Remove(0,1);
1335 // Change '/' to '%'
1336 u.ReplaceAll("/", ".");
1337 // Init the regular expression
1338 u.ReplaceAll("*", ".*");
1339 re = new TRegexp(u.Data());
1340 }
1341 }
1342
1343 Printf(" Dataset repository: %s", fDataSetDir.Data());
1344 Printf(" Local cache directory: %s", fLocalCacheDir.Data());
1345 Printf(" Last modified Size(bytes) File");
1346
1347 Long64_t totsz = 0, nf = 0;
1348 FileStat_t st;
1349 TString path, sz;
1350 const char *e = 0;
1351 while ((e = gSystem->GetDirEntry(dirp))) {
1352 // Skip basic entries
1353 if (!strcmp(e,".") || !strcmp(e,"..")) continue;
1354 // Apply regular expression, if requested
1355 if (re && TString(e).Index(*re) == kNPOS) continue;
1356 // Group directory
1357 path.Form("%s/%s", fLocalCacheDir.Data(), e);
1358 // Get file information
1359 if (gSystem->GetPathInfo(path, st) != 0) {
1360 Warning("ShowCache", "problems 'stat'-ing '%s' (errno: %d)",
1361 path.Data(), TSystem::GetErrno());
1362 continue;
1363 }
1364 // Count
1365 totsz += st.fSize;
1366 nf++;
1367 // Get modification time in human readable form
1368 TDatime tmod(st.fMtime);
1369 sz.Form("%lld", st.fSize);
1370 sz.Resize(12);
1371 Printf(" %s %s %s", tmod.AsSQLString(), sz.Data(), e);
1372 }
1373 gSystem->FreeDirectory(dirp);
1374 SafeDelete(re);
1375
1376 // Notify totals
1377 Printf(" %lld files, %lld bytes", nf, totsz);
1378
1379 // Done
1380 return 0;
1381}
1382
1383////////////////////////////////////////////////////////////////////////////////
1384///
1385/// Writes indicated dataset.
1386/// If option has the bit kFileMustExist set, the file must still exist,
1387/// otherwise the new dataset is not written (returns 3 in this case).
1388/// If checksum is non-zero the files current checksum is checked against it,
1389/// if it does not match the file is not written (the function returns 2 in this
1390/// case, if the file has disappeared it is also not written (i.e. checksum
1391/// implies the bit kFileMustExist set in option).
1392/// Returns != 0 for success, 0 for error
1393
1394Int_t TDataSetManagerFile::WriteDataSet(const char *group, const char *user,
1395 const char *dsName, TFileCollection *dataset,
1396 UInt_t option, TMD5 *checksum)
1397{
1398 TString md5path, path, md5sum;
1399 Long_t mtime = 0;
1401
1402 Bool_t checkIfExists = ((option & kFileMustExist) || checksum) ? kTRUE : kFALSE;
1403
1404 path = GetDataSetPath(group, user, dsName, md5path);
1405
1406 if (checkIfExists) {
1407 // check if file still exists, otherwise it was deleted in the meanwhile and is not written here
1408 Long_t tmp;
1409 if (gSystem->GetPathInfo(path, 0, (Long_t*) 0, 0, &tmp) != 0) {
1410 if (gDebug > 0)
1411 Info("WriteDataSet", "Dataset disappeared. Discarding update.");
1412 return 3;
1413 }
1414 }
1415
1416 if (checksum) {
1417 // verify md5 sum, otherwise the file was changed in the meanwhile and is not overwritten here
1418 TMD5 *checksum2 = TMD5::FileChecksum(path);
1419 if (!checksum2) {
1420 Error("WriteDataSet", "Could not get checksum of %s", path.Data());
1421 return 0;
1422 }
1423
1424 Bool_t checksumAgrees = (*checksum == *checksum2);
1425 delete checksum2;
1426
1427 if (!checksumAgrees) {
1428 if (gDebug > 0)
1429 Info("WriteDataSet", "Dataset changed. Discarding update.");
1430 return 2;
1431 }
1432 }
1433
1434 // write first in ".<file>" then rename to recover from crash during writing
1435 TString tempFile(path);
1436 Int_t index = -1;
1437 while (tempFile.Index("/", index+1) >= 0)
1438 index = tempFile.Index("/", index+1);
1439
1440 tempFile.Insert(index+1, ".");
1441
1442 TFile *f = TFile::Open(tempFile, "RECREATE");
1443 if (!f) {
1444 Error("WriteDataSet", "Could not open dataset for writing %s", tempFile.Data());
1445 return 0;
1446 }
1447
1448 // write full TFileCollection
1449 dataset->Write("dataset", TObject::kSingleKey | TObject::kOverwrite);
1450
1451 // write only metadata
1452 THashList *list = dataset->GetList();
1453 dataset->SetList(0);
1454 dataset->Write("dataset_short", TObject::kSingleKey | TObject::kOverwrite);
1455
1456 f->Close();
1457 delete f;
1458
1459 // Restore full list
1460 dataset->SetList(list);
1461
1462 // file is written, rename to real filename
1463 if (gSystem->Rename(tempFile, path) != 0) {
1464 Error("WriteDataSet", "renaming %s to %s failed; dataset might be corrupted",
1465 tempFile.Data(), path.Data());
1466 // Cleanup any MD5 sum information
1467 if (!gSystem->AccessPathName(md5path, kWritePermission) && gSystem->Unlink(md5path) != 0)
1468 Error("WriteDataSet", "unlink of %s failed", md5path.Data());
1469 return 0;
1470 }
1471 else if (fOpenPerms) {
1472 if (gSystem->Chmod(path.Data(), 0666) < 0) {
1473 Warning("NotifyUpdate",
1474 "can't set permissions of dataset file %s (#%d)",
1475 path.Data(), TSystem::GetErrno());
1476 }
1477 }
1478
1479 // Save md5 sum, otherwise the file was changed in the meanwhile and is not overwritten here
1480 if (ChecksumDataSet(path, md5path, md5sum) != 0) {
1481 Error("WriteDataSet", "problems calculating checksum of %s", path.Data());
1482 return 0;
1483 }
1484 else if (fOpenPerms) {
1485 if (gSystem->Chmod(md5path.Data(), 0666) < 0) {
1486 Warning("NotifyUpdate",
1487 "can't set permissions of dataset MD5 checksum file %s (#%d)",
1488 md5path.Data(), TSystem::GetErrno());
1489 }
1490 }
1491
1492 FileStat_t st;
1493 if (gSystem->GetPathInfo(path, st) != 0) {
1494 Error("WriteDataSet", "could not 'stat' the version of '%s'!", path.Data());
1495 return 0;
1496 }
1497 mtime= st.fMtime;
1498 }
1499
1500 // The repository was updated
1501 if (NotifyUpdate(group, user, dsName, mtime, md5sum) != 0)
1502 Warning("WriteDataSet", "problems notifying update with 'NotifyUpdate'");
1503
1504 return 1;
1505}
1506
1507////////////////////////////////////////////////////////////////////////////////
1508/// Calculate the checksum of the indicated dataset at 'path' and save it to the
1509/// appropriate file 'md5path'. The MD5 string is returned in 'md5sum'.
1510/// Return 0 on success, -1 on error.
1511
1513 const char *md5path, TString &checksum)
1514{
1515 checksum = "";
1516 // Check inputs
1517 if (!path || strlen(path) <= 0 || !md5path || strlen(md5path) <= 0) {
1518 Error("ChecksumDataSet", "one or more inputs are invalid ('%s','%s')",
1519 path, md5path);
1520 return -1;
1521 }
1522 // Calculate md5 sum
1523 TMD5 *md5sum = TMD5::FileChecksum(path);
1524 if (!md5sum) {
1525 Error("ChecksumDataSet", "problems calculating checksum of '%s'", path);
1526 return -1;
1527 }
1528 // Save it to a file
1529 if (TMD5::WriteChecksum(md5path, md5sum) != 0) {
1530 Error("ChecksumDataSet", "problems saving checksum to '%s'", md5path);
1531 SafeDelete(md5sum);
1532 return -1;
1533 }
1534 // Fill output
1535 checksum = md5sum->AsString();
1536 // Done
1537 SafeDelete(md5sum);
1538 return 0;
1539}
1540
1541////////////////////////////////////////////////////////////////////////////////
1542/// Removes the indicated dataset
1543
1545 const char *dsName)
1546{
1547 TString md5path, path;
1549
1550 path = GetDataSetPath(group, user, dsName, md5path);
1551 Int_t rc = 0;
1552 // Remove the main file
1553 if ((rc = gSystem->Unlink(path)) != 0)
1554 Warning("RemoveDataSet", "problems removing main file '%s' (errno: %d)",
1555 path.Data(), TSystem::GetErrno());
1556 // Remove the checksum file
1557 if (gSystem->Unlink(md5path) != 0)
1558 Warning("RemoveDataSet", "problems removing chcksum file '%s' (errno: %d)",
1559 md5path.Data(), TSystem::GetErrno());
1560 }
1561
1562 // The repository was updated
1563 if (gSystem->AccessPathName(path, kFileExists)) {
1564 if (NotifyUpdate(group, user, dsName, 0, "removed") != 0)
1565 Warning("RemoveDataSet", "problems notifying update with 'NotifyUpdate'");
1566 // Success
1567 return kTRUE;
1568 }
1569 // Failure
1570 return kFALSE;
1571}
1572
1573////////////////////////////////////////////////////////////////////////////////
1574/// Checks if the indicated dataset exits
1575
1577 const char *dsName)
1578{
1580
1581 TString md5path, path(GetDataSetPath(group, user, dsName, md5path));
1582
1583 return (gSystem->AccessPathName(path) == kFALSE);
1584}
1585
1586////////////////////////////////////////////////////////////////////////////////
1587/// Register a dataset, perfoming quota checkings and verification, if required.
1588/// If a dataset with the same name already exists the action fails unless 'opts'
1589/// contains 'O', in which case the old dataset is overwritten, or contains 'U',
1590/// in which case 'newDataSet' is added to the existing dataset (duplications are
1591/// ignored, if any).
1592/// If 'opts' contains 'V' the dataset files are also verified (if the dataset manager
1593/// is configured to allow so). By default the dataset is not verified.
1594/// If 'opts' contains 'T' the in the dataset object (status bits, meta,...)
1595/// is trusted, i.e. not reset (if the dataset manager is configured to allow so).
1596/// Returns 0 on success, -1 on failure
1597
1599 TFileCollection *newDataSet,
1600 const char *opts)
1601{
1603 return -1;
1604
1605 // Get the dataset name
1606 TString dsName;
1607 if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE) == kFALSE) {
1608 Error("RegisterDataSet", "problem parsing uri: %s", uri);
1609 return -1;
1610 }
1611
1612 // The dataset
1613 TFileCollection *dataSet = newDataSet;
1614 // Check option
1615 TString opt(opts);
1616 // If in update mode, retrieve the existing dataset, if any
1617 if (opt.Contains("U", TString::kIgnoreCase)) {
1618 // Fail if it exists already
1619 if (ExistsDataSet(fGroup, fUser, dsName)) {
1620 // Retrieve the dataset
1621 if (!(dataSet = GetDataSet(fGroup, fUser, dsName))) {
1622 // Dataset name does exist
1623 Warning("RegisterDataSet",
1624 "dataset '%s' claimed to exists but retrieval failed - ignoring", uri);
1625 dataSet = newDataSet;
1626 } else {
1627 // Add new dataset to existing one
1628 dataSet->Add(newDataSet);
1629 }
1630 }
1631 } else if (!opt.Contains("O", TString::kIgnoreCase)) {
1632 // Fail if it exists already
1633 if (ExistsDataSet(fGroup, fUser, dsName)) {
1634 //Dataset name does exist
1635 Error("RegisterDataSet", "dataset '%s' exists already", uri);
1636 return -1;
1637 }
1638 }
1639
1640 // A temporary list to hold the unique members (i.e. the very set)
1641 TList *uniqueFileList = new TList();
1642 TIter nextFile(dataSet->GetList());
1643 TFileInfo *prevFile = (TFileInfo*)nextFile();
1644 uniqueFileList->Add(prevFile);
1645 while (TFileInfo *obj = (TFileInfo*)nextFile()) {
1646 // Add entities only once to the temporary list
1647 if (!uniqueFileList->FindObject(obj->GetFirstUrl()->GetUrl()))
1648 uniqueFileList->Add(obj);
1649 }
1650
1651 // Clear dataSet and add contents of uniqueFileList needed otherwise
1652 // THashList deletes the objects even when nodelete is set
1653 dataSet->GetList()->SetOwner(0);
1654 dataSet->GetList()->Clear("nodelete");
1655 dataSet->GetList()->SetOwner(1);
1656 dataSet->GetList()->AddAll(uniqueFileList);
1657 uniqueFileList->SetOwner(kFALSE);
1658 delete uniqueFileList;
1659
1660 // Enforce certain settings
1661 Bool_t reset = kTRUE;
1662 if (opt.Contains("T", TString::kIgnoreCase)) {
1664 Warning("RegisterDataSet", "configured to not trust the information"
1665 " provided by users: ignoring request");
1666 } else {
1667 reset = kFALSE;
1668 }
1669 }
1670 if (reset) {
1671 dataSet->SetName(dsName);
1672 dataSet->ResetBitAll(TFileInfo::kStaged);
1673 dataSet->ResetBitAll(TFileInfo::kCorrupted);
1674 dataSet->RemoveMetaData();
1675 }
1676
1677 // Verify the dataset if required
1678 if (opt.Contains("V", TString::kIgnoreCase)) {
1680 // Reopen files and notify
1681 if (TDataSetManager::ScanDataSet(dataSet, 1, 0, 0, kTRUE ) < 0) {
1682 Error("RegisterDataSet", "problems verifying the dataset");
1683 return -1;
1684 }
1685 } else {
1686 Warning("RegisterDataSet", "user-driven verification not allowed: ignoring request");
1687 }
1688 }
1689
1690 // Update accumulated information
1691 dataSet->Update(fAvgFileSize);
1692
1694 if (dataSet->GetTotalSize() <= 0) {
1695 Error("RegisterDataSet", "datasets without size information are not accepted:");
1696 if (fAvgFileSize < 0) {
1697 Error("RegisterDataSet", "you may want to define an average"
1698 " file size to get an estimated dataset size");
1699 }
1700 return -1;
1701 }
1702 // now check the quota
1704 Long64_t used = GetGroupUsed(fGroup) + dataSet->GetTotalSize();
1705
1706 Info("RegisterDataSet", "your group %s uses %.1f GB + %.1f GB for the new dataset; "
1707 "the available quota is %.1f GB", fGroup.Data(),
1708 (Float_t) GetGroupUsed(fGroup) / 1073741824,
1709 (Float_t) dataSet->GetTotalSize() / 1073741824,
1710 (Float_t) GetGroupQuota(fGroup) / 1073741824);
1711 if (used > GetGroupQuota(fGroup)) {
1712 Error("RegisterDataSet", "quota exceeded");
1713 return -1;
1714 }
1715 }
1716
1717 Bool_t success = WriteDataSet(fGroup, fUser, dsName, dataSet);
1718 if (!success)
1719 Error("RegisterDataSet", "could not write dataset: %s", dsName.Data());
1720
1721 // Done
1722 return ((success) ? 0 : -1);
1723}
1724////////////////////////////////////////////////////////////////////////////////
1725/// Scans the dataset indicated by `<uri>` and returns the number of missing files.
1726/// Returns -1 if any failure occurs, >= 0 on success.
1727/// For more details, see documentation of
1728/// ScanDataSet(TFileCollection *dataset, const char *option)
1729
1731{
1732 TString dsName, dsTree;
1733 if ((opt & kSetDefaultTree)) {
1735 if (ParseUri(uri, 0, 0, &dsName, &dsTree, kTRUE)) {
1736 TFileCollection *dataset = GetDataSet(fGroup, fUser, dsName);
1737 if (!dataset) return -1;
1738 dataset->SetDefaultTreeName(dsTree.Data());
1739 Int_t rc = WriteDataSet(fGroup, fUser, dsName, dataset);
1740 delete dataset;
1741 return (rc == 0) ? -1 : 0;
1742 }
1743 }
1744 } else {
1746 if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE, kTRUE)) {
1747 if (!(dsName.Contains("*"))) {
1748 if (ScanDataSet(fGroup, fUser, dsName, opt) > 0)
1749 return GetNDisapparedFiles();
1750 } else {
1751 TString luri = TString::Format("/%s/%s/%s", fGroup.Data(), fUser.Data(), dsName.Data());
1752 TMap *fcs = GetDataSets(luri, kList);
1753 if (!fcs) return -1;
1754 fcs->Print();
1755 Int_t ndisappeared = 0;
1756 TIter nxd(fcs);
1757 TObjString *d = 0;
1758 while ((d = (TObjString *) nxd())) {
1759 if (!(d->GetString().IsNull())) {
1760 TString dsn(d->GetName());
1761 if (dsn.Contains("/")) dsn.Remove(0, dsn.Last('/') + 1);
1762 if (ScanDataSet(fGroup, fUser, dsn, opt) > 0) {
1763 ndisappeared += GetNDisapparedFiles();
1764 } else {
1765 Warning("ScanDataSet", "problems processing dataset: %s", d->GetName());
1766 }
1767 } else {
1768 Warning("ScanDataSet", "empty string found in map while processing: %s", uri);
1769 }
1770 }
1771 SafeDelete(fcs);
1772 return ndisappeared;
1773 }
1774 }
1775 }
1776 }
1777 return -1;
1778}
1779
1780////////////////////////////////////////////////////////////////////////////////
1781/// See documentation of ScanDataSet(TFileCollection *dataset, UInt_t option)
1782
1783Int_t TDataSetManagerFile::ScanDataSet(const char *group, const char *user,
1784 const char *dsName, UInt_t option)
1785{
1787 return -1;
1788
1789 TFileCollection *dataset = GetDataSet(group, user, dsName);
1790 if (!dataset)
1791 return -1;
1792
1793 // File selection
1794 Int_t fopt = ((option & kAllFiles)) ? -1 : 0;
1795 if (fopt >= 0) {
1796 if ((option & kStagedFiles)) {
1797 fopt = 10;
1798 } else {
1799 if ((option & kReopen)) fopt++;
1800 if ((option & kTouch)) fopt++;
1801 }
1802 if ((option & kNoStagedCheck)) fopt += 100;
1803 } else {
1804 if ((option & kStagedFiles) || (option & kReopen) || (option & kTouch)) {
1805 Warning("ScanDataSet", "kAllFiles mode: ignoring kStagedFiles or kReopen"
1806 " or kTouch requests");
1807 }
1808 if ((option & kNoStagedCheck)) fopt -= 100;
1809 }
1810
1811 // Type of action
1812 Int_t sopt = ((option & kNoAction)) ? -1 : 0;
1813 if (sopt >= 0) {
1814 if ((option & kLocateOnly) && (option & kStageOnly)) {
1815 Error("ScanDataSet", "kLocateOnly and kStageOnly cannot be processed concurrently");
1816 return -1;
1817 }
1818 if ((option & kLocateOnly)) sopt = 1;
1819 if ((option & kStageOnly)) sopt = 2;
1820 } else if ((option & kLocateOnly) || (option & kStageOnly)) {
1821 Warning("ScanDataSet", "kNoAction mode: ignoring kLocateOnly or kStageOnly requests");
1822 }
1823
1824 Bool_t dbg = ((option & kDebug)) ? kTRUE : kFALSE;
1825 // Do the scan
1826 Int_t result = TDataSetManager::ScanDataSet(dataset, fopt, sopt, 0, dbg,
1828 (TList *)0, fAvgFileSize, fMSSUrl.Data(), -1, fStageOpts.Data());
1829 if (result == 2) {
1830 if (WriteDataSet(group, user, dsName, dataset) == 0) {
1831 delete dataset;
1832 return -2;
1833 }
1834 }
1835 delete dataset;
1836
1837 return result;
1838}
1839
1840////////////////////////////////////////////////////////////////////////////////
1841///
1842/// Returns all datasets for the `<group>` and `<user>` specified by `<uri>`.
1843/// If `<user>` is 0, it returns all datasets for the given `<group>`.
1844/// If `<group>` is 0, it returns all datasets.
1845/// The returned TMap contains:
1846/// `<group>` --> `<map of users>` --> `<map of datasets>` --> `<dataset>` (TFileCollection)
1847///
1848/// The unsigned int 'option' is forwarded to GetDataSet and BrowseDataSet.
1849/// Available options (to be .or.ed):
1850/// kShowDefault a default selection is shown that include the ones from
1851/// the current user, the ones from the group and the common ones
1852/// kPrint print the dataset content
1853/// kQuotaUpdate update quotas
1854/// kExport use export naming
1855///
1856/// NB1: options "kPrint", "kQuoatUpdate" and "kExport" are mutually exclusive
1857/// NB2: for options "kPrint" and "kQuoatUpdate" return is null.
1858
1860{
1861 TString dsUser, dsGroup, dsName;
1862
1863 if (((option & kPrint) || (option & kExport)) && strlen(uri) <= 0)
1865
1866 if (ParseUri(uri, &dsGroup, &dsUser, &dsName, 0, kFALSE, kTRUE))
1867 return GetDataSets(dsGroup, dsUser, dsName, option);
1868 return (TMap *)0;
1869}
1870
1871////////////////////////////////////////////////////////////////////////////////
1872/// Utility function used in various methods for user dataset upload.
1873
1874TFileCollection *TDataSetManagerFile::GetDataSet(const char *uri, const char *opts)
1875{
1876 TString dsUser, dsGroup, dsName, ss(opts);
1877
1878 TFileCollection *fc = 0;
1879 if (!strchr(uri, '*')) {
1880 if (!ParseUri(uri, &dsGroup, &dsUser, &dsName)) return fc;
1881 UInt_t opt = (ss.Contains("S:") || ss.Contains("short:")) ? kReadShort : 0;
1882 ss.ReplaceAll("S:","");
1883 ss.ReplaceAll("short:","");
1884 fc = GetDataSet(dsGroup, dsUser, dsName, opt);
1885 } else {
1886 TMap *fcs = GetDataSets(uri);
1887 if (!fcs) return fc;
1888 TIter nxd(fcs);
1889 TObject *k = 0;
1890 TFileCollection *xfc = 0;
1891 while ((k = nxd()) && (xfc = (TFileCollection *) fcs->GetValue(k))) {
1892 if (!fc) {
1893 // The first one
1894 fc = xfc;
1895 fcs->Remove(k);
1896 } else {
1897 // Add
1898 fc->Add(xfc);
1899 }
1900 }
1901 }
1902
1903 if (fc && !ss.IsNull()) {
1904 // Build up the subset
1905 TFileCollection *sfc = 0;
1906 TString s;
1907 Int_t from = 0;
1908 while (ss.Tokenize(s, from, ",")) {
1909 TFileCollection *xfc = fc->GetFilesOnServer(s.Data());
1910 if (xfc) {
1911 if (sfc) {
1912 sfc->Add(xfc);
1913 delete xfc;
1914 } else {
1915 sfc = xfc;
1916 }
1917 }
1918 }
1919 // Cleanup
1920 delete fc;
1921 fc = sfc;
1922 }
1923 // Done
1924 return fc;
1925}
1926
1927////////////////////////////////////////////////////////////////////////////////
1928/// Removes the indicated dataset
1929
1931{
1932 TString dsName;
1933
1935 if (ParseUri(uri, 0, 0, &dsName, 0, kTRUE)) {
1936 Bool_t rc = RemoveDataSet(fGroup, fUser, dsName);
1937 if (rc) return kTRUE;
1938 Error("RemoveDataSet", "error removing dataset %s", dsName.Data());
1939 }
1940 }
1941 return kFALSE;
1942}
1943
1944////////////////////////////////////////////////////////////////////////////////
1945/// Checks if the indicated dataset exits
1946
1948{
1949 TString dsUser, dsGroup, dsName;
1950
1951 if (ParseUri(uri, &dsGroup, &dsUser, &dsName))
1952 return ExistsDataSet(dsGroup, dsUser, dsName);
1953 return kFALSE;
1954}
1955
1956////////////////////////////////////////////////////////////////////////////////
1957/// updates the used space maps
1958
1960{
1961 // Clear used space entries
1964
1965 // Scan the existing datasets
1966 GetDataSets(0, 0, 0, (UInt_t)kQuotaUpdate);
1967}
1968
1969////////////////////////////////////////////////////////////////////////////////
1970/// Gets last dataset modification time. Returns -1 on error, or number of
1971/// seconds since epoch on success
1972
1974{
1975 TString group, user, name, md5path;
1976 if (!ParseUri(uri, &group, &user, &name)) {
1977 return -1;
1978 }
1979
1980 TString path( GetDataSetPath(group, user, name, md5path) );
1981
1982 Long_t modTime;
1983 if (gSystem->GetPathInfo(path.Data(),
1984 (Long_t *)0, (Long_t *)0, (Long_t *)0, &modTime)) {
1985 return -1;
1986 }
1987
1988 return modTime;
1989}
#define SafeDelete(p)
Definition RConfig.hxx:525
#define d(i)
Definition RSha256.hxx:102
#define f(i)
Definition RSha256.hxx:104
#define g(i)
Definition RSha256.hxx:105
#define e(i)
Definition RSha256.hxx:103
long Long_t
Definition RtypesCore.h:54
float Float_t
Definition RtypesCore.h:57
constexpr Bool_t kFALSE
Definition RtypesCore.h:94
constexpr Ssiz_t kNPOS
Definition RtypesCore.h:117
long long Long64_t
Definition RtypesCore.h:69
constexpr Bool_t kTRUE
Definition RtypesCore.h:93
#define ClassImp(name)
Definition Rtypes.h:382
const char *const kDataSet_LocalCache
const char *const kDataSet_DataSetList
const char *const kDataSet_LockLocation
R__EXTERN TEnv * gEnv
Definition TEnv.h:170
Option_t Option_t option
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t target
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t UChar_t len
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void reg
char name[80]
Definition TGX11.cxx:110
Int_t gDebug
Definition TROOT.cxx:597
char * Form(const char *fmt,...)
Formats a string in a circular formatting buffer.
Definition TString.cxx:2489
void Printf(const char *fmt,...)
Formats a string in a circular formatting buffer and prints the string.
Definition TString.cxx:2503
@ kFileExists
Definition TSystem.h:42
@ kReadPermission
Definition TSystem.h:45
@ kWritePermission
Definition TSystem.h:44
Bool_t R_ISDIR(Int_t mode)
Definition TSystem.h:113
R__EXTERN TSystem * gSystem
Definition TSystem.h:561
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
void Print(Option_t *option="") const override
Default print for collections, calls Print(option, 1).
Int_t NotifyUpdate(const char *group, const char *user, const char *dspath, Long_t mtime, const char *checksum=0) override
Save into the <datasetdir>/kDataSet_DataSetList file the name of the updated or created or modified d...
Int_t ChecksumDataSet(const char *path, const char *md5path, TString &checksum)
Calculate the checksum of the indicated dataset at 'path' and save it to the appropriate file 'md5pat...
TMap * GetDataSets(const char *uri, UInt_t option=TDataSetManager::kExport) override
Returns all datasets for the <group> and <user> specified by <uri>.
Int_t WriteDataSet(const char *group, const char *user, const char *dsName, TFileCollection *dataset, UInt_t option=0, TMD5 *checksum=0)
Writes indicated dataset.
Int_t ShowCache(const char *uri=0) override
Show cached information matching uri.
Int_t CreateLsFile(const char *group, const char *user, Long_t &mtime, TString &checksum)
Create or recreate the dataset lists for 'uri'.
Bool_t ExistsDataSet(const char *group, const char *user, const char *dsName)
Checks if the indicated dataset exits.
const char * GetDataSetPath(const char *group, const char *user, const char *dsName)
void ParseInitOpts(const char *opts) override
Parse the input string and set the init bits accordingly Format is dir:<datasetdir> [mss:<mss-url>] [...
Bool_t BrowseDataSets(const char *group, const char *user, const char *dsName, UInt_t option, TObject *target)
Adds the dataset in the folder of group, user to the list in target.
Int_t RegisterDataSet(const char *uri, TFileCollection *dataSet, const char *opt) override
Register a dataset, perfoming quota checkings and verification, if required.
void InitLocalCache()
Init the local cache if required.
Bool_t RemoveDataSet(const char *group, const char *user, const char *dsName)
Removes the indicated dataset.
Int_t CheckLocalCache(const char *group, const char *user, const char *dsName="ls", UInt_t option=0)
Check if the local cache information for group, user, dsName is up-to-date If not,...
Int_t ScanDataSet(const char *group, const char *user, const char *dsName, UInt_t option=kReopen|kDebug)
See documentation of ScanDataSet(TFileCollection *dataset, UInt_t option)
Int_t FillLsDataSet(const char *group, const char *user, const char *dsName, TList *out, UInt_t option)
Check for the 'ls.txt' for 'group' and 'user' and fill the path for the ls file in 'out'.
void UpdateUsedSpace() override
updates the used space maps
Long_t GetModTime(const char *uri)
Gets last dataset modification time.
void Init()
Do the real inititialization.
TFileCollection * GetDataSet(const char *uri, const char *srv=0) override
Utility function used in various methods for user dataset upload.
Int_t ClearCache(const char *uri=0) override
Clear cached information matching uri.
static TString fgCommonDataSetTag
virtual Long64_t GetGroupUsed(const char *group)
Returns the used space of that group.
Int_t ScanDataSet(const char *uri, const char *opt)
Scans the dataset indicated by 'uri' following the 'opts' directives.
void GetQuota(const char *group, const char *user, const char *dsName, TFileCollection *dataset)
Gets quota information from this dataset.
virtual void ShowDataSets(const char *uri="*", const char *opt="")
Prints formatted information about the dataset 'uri'.
Int_t GetNDisapparedFiles() const
Bool_t ParseUri(const char *uri, TString *dsGroup=0, TString *dsUser=0, TString *dsName=0, TString *dsTree=0, Bool_t onlyCurrent=kFALSE, Bool_t wildcards=kFALSE)
Parses a (relative) URI that describes a DataSet on the cluster.
virtual Long64_t GetGroupQuota(const char *group)
returns the quota a group is allowed to have
This class stores the date and time with a precision of one second in an unsigned 32 bit word (950130...
Definition TDatime.h:37
const char * AsSQLString() const
Return the date & time in SQL compatible string format, like: 1997-01-15 20:16:28.
Definition TDatime.cxx:152
UInt_t Convert(Bool_t toGMT=kFALSE) const
Convert fDatime from TDatime format to the standard time_t format.
Definition TDatime.cxx:182
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
Definition TEnv.cxx:491
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
TObjString * ExportInfo(const char *name=nullptr, Int_t popt=0)
Export the relevant info as a string; use 'name' as collection name, if defined, else use GetName().
void Print(Option_t *option="") const override
Prints the contents of the TFileCollection.
Int_t Add(TFileInfo *info)
Add TFileInfo to the collection.
TFileCollection * GetFilesOnServer(const char *server)
Return the subset of files served by 'server'.
Class describing a generic file including meta information.
Definition TFileInfo.h:39
A ROOT file is an on-disk file, usually with extension .root, that stores objects in a file-system-li...
Definition TFile.h:53
static EFileType GetType(const char *name, Option_t *option="", TString *prefix=nullptr)
Resolve the file type as a function of the protocol field in 'name'.
Definition TFile.cxx:4821
EFileType
File type.
Definition TFile.h:202
@ kDefault
Definition TFile.h:202
@ kLocal
Definition TFile.h:202
@ kFile
Definition TFile.h:202
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
Definition TFile.cxx:5003
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault, Int_t netopt=0)
Create / open a file.
Definition TFile.cxx:4089
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition THashList.h:34
A doubly linked list.
Definition TList.h:38
TObject * FindObject(const char *name) const override
Find an object in this list using its name.
Definition TList.cxx:576
void Add(TObject *obj) override
Definition TList.h:83
TObject * Remove(TObject *obj) override
Remove object from the list.
Definition TList.cxx:820
TObject * Last() const override
Return the last object in the list. Returns 0 when list is empty.
Definition TList.cxx:691
A scoped lock based on files.
Definition TLockFile.h:19
This code implements the MD5 message-digest algorithm.
Definition TMD5.h:44
const char * AsString() const
Return message digest as string.
Definition TMD5.cxx:220
static TMD5 * FileChecksum(const char *file)
Returns checksum of specified file.
Definition TMD5.cxx:474
static TMD5 * ReadChecksum(const char *file)
Returns checksum stored in ASCII in specified file.
Definition TMD5.cxx:422
static Int_t WriteChecksum(const char *file, const TMD5 *md5)
Writes checksum in ASCII format to specified file.
Definition TMD5.cxx:452
Class supporting a collection of lines with C++ code.
Definition TMacro.h:31
virtual TObjString * AddLine(const char *text)
Add line with text in the list of lines of this macro.
Definition TMacro.cxx:138
virtual Int_t ReadFile(const char *filename)
Read lines in filename in this macro.
Definition TMacro.cxx:329
void SaveSource(FILE *fp)
Save macro source in file pointer fp.
Definition TMacro.cxx:371
TList * GetListOfLines() const
Definition TMacro.h:51
virtual TObjString * GetLineWith(const char *text) const
Search the first line containing text.
Definition TMacro.cxx:297
TMap implements an associative array of (key,value) pairs using a THashTable for efficient retrieval ...
Definition TMap.h:40
void DeleteAll()
Remove all (key,value) pairs from the map AND delete the keys AND values when they are allocated on t...
Definition TMap.cxx:168
void Add(TObject *obj) override
This function may not be used (but we need to provide it since it is a pure virtual in TCollection).
Definition TMap.cxx:54
TObject * Remove(TObject *key) override
Remove the (key,value) pair with key from the map.
Definition TMap.cxx:296
TObject * GetValue(const char *keyname) const
Returns a pointer to the value associated with keyname as name of the key.
Definition TMap.cxx:236
Collectable string class.
Definition TObjString.h:28
const TString & GetString() const
Definition TObjString.h:46
TString & String()
Definition TObjString.h:48
void SetString(const char *s)
Definition TObjString.h:45
const char * GetName() const override
Returns name of object.
Definition TObjString.h:38
Mother of all ROOT objects.
Definition TObject.h:41
@ kOverwrite
overwrite existing object with same name
Definition TObject.h:92
@ kSingleKey
write collection with single key
Definition TObject.h:91
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition TObject.h:199
virtual TObject * Clone(const char *newname="") const
Make a clone of an object using the Streamer facility.
Definition TObject.cxx:229
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition TObject.cxx:979
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition TObject.cxx:786
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition TObject.cxx:993
void ResetBit(UInt_t f)
Definition TObject.h:198
@ kInvalidObject
if object ctor succeeded but object should not be used
Definition TObject.h:72
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition TObject.cxx:967
Regular expression class.
Definition TRegexp.h:31
Basic string class.
Definition TString.h:139
Ssiz_t Length() const
Definition TString.h:417
void ToLower()
Change string to lower-case.
Definition TString.cxx:1182
TString & Insert(Ssiz_t pos, const char *s)
Definition TString.h:661
Bool_t EndsWith(const char *pat, ECaseCompare cmp=kExact) const
Return true if string ends with the specified string.
Definition TString.cxx:2244
const char * Data() const
Definition TString.h:376
Bool_t IsDigit() const
Returns true if all characters in string are digits (0-9) or white spaces, i.e.
Definition TString.cxx:1830
TString & ReplaceAll(const TString &s1, const TString &s2)
Definition TString.h:704
void Resize(Ssiz_t n)
Resize the string. Truncate or add blanks as necessary.
Definition TString.cxx:1152
@ kIgnoreCase
Definition TString.h:277
Ssiz_t Last(char c) const
Find last occurrence of a character c.
Definition TString.cxx:931
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition TString.cxx:2264
Bool_t BeginsWith(const char *s, ECaseCompare cmp=kExact) const
Definition TString.h:623
Bool_t IsNull() const
Definition TString.h:414
TString & Remove(Ssiz_t pos)
Definition TString.h:685
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition TString.cxx:2378
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition TString.cxx:2356
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition TString.h:632
Ssiz_t Index(const char *pat, Ssiz_t i=0, ECaseCompare cmp=kExact) const
Definition TString.h:651
virtual Int_t RedirectOutput(const char *name, const char *mode="a", RedirectHandle_t *h=nullptr)
Redirect standard output (stdout, stderr) to the specified file.
Definition TSystem.cxx:1715
static Int_t GetErrno()
Static function returning system error number.
Definition TSystem.cxx:276
virtual int Chmod(const char *file, UInt_t mode)
Set the file permission bits. Returns -1 in case or error, 0 otherwise.
Definition TSystem.cxx:1508
virtual void FreeDirectory(void *dirp)
Free a directory.
Definition TSystem.cxx:845
virtual void * OpenDirectory(const char *name)
Open a directory. Returns 0 if directory does not exist.
Definition TSystem.cxx:836
virtual const char * Getenv(const char *env)
Get environment variable.
Definition TSystem.cxx:1665
virtual void ShowOutput(RedirectHandle_t *h)
Display the content associated with the redirection described by the opaque handle 'h'.
Definition TSystem.cxx:1725
virtual int mkdir(const char *name, Bool_t recursive=kFALSE)
Make a file system directory.
Definition TSystem.cxx:906
int GetPathInfo(const char *path, Long_t *id, Long_t *size, Long_t *flags, Long_t *modtime)
Get info about a file: id, size, flags, modification time.
Definition TSystem.cxx:1398
virtual Bool_t AccessPathName(const char *path, EAccessMode mode=kFileExists)
Returns FALSE if one can access a file using the specified access mode.
Definition TSystem.cxx:1296
virtual const char * GetDirEntry(void *dirp)
Get a directory entry. Returns 0 if no more entries.
Definition TSystem.cxx:853
virtual int Rename(const char *from, const char *to)
Rename a file.
Definition TSystem.cxx:1350
virtual int Utime(const char *file, Long_t modtime, Long_t actime)
Set the a files modification and access times.
Definition TSystem.cxx:1527
virtual TString GetDirName(const char *pathname)
Return the directory name in pathname.
Definition TSystem.cxx:1032
virtual int Unlink(const char *name)
Unlink, i.e.
Definition TSystem.cxx:1381
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition TSystem.cxx:1482
Bool_t SetUri(const TString &uri)
Parse URI and set the member variables accordingly, returns kTRUE if URI validates,...
Definition TUri.cxx:601
This class represents a WWW compatible URL.
Definition TUrl.h:33
const char * GetFile() const
Definition TUrl.h:69
const char * GetProtocol() const
Definition TUrl.h:64
Int_t fMode
Definition TSystem.h:125
Long64_t fSize
Definition TSystem.h:128
Int_t fGid
Definition TSystem.h:127
Long_t fMtime
Definition TSystem.h:129
Int_t fUid
Definition TSystem.h:126
LsTreeEntry_t(const char *g, const char *u, const char *cs, const char *ls, Long64_t m)
TMarker m
Definition textangle.C:8
static void output()