Logo ROOT  
Reference Guide
TDataSetManager.h
Go to the documentation of this file.
1 // @(#)root/proof:$Id$
2 // Author: Jan Fiete Grosse-Oetringhaus, 08.08.07
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 #ifndef ROOT_TDataSetManager
13 #define ROOT_TDataSetManager
14 
15 //////////////////////////////////////////////////////////////////////////
16 // //
17 // TDataSetManager //
18 // //
19 // This class contains functions to handle datasets in PROOF //
20 // It is the layer between TProofServ and the file system that stores //
21 // the datasets. //
22 // //
23 //////////////////////////////////////////////////////////////////////////
24 
25 #include "TObject.h"
26 #include "TString.h"
27 #include "TMap.h"
28 #include "TUri.h"
29 
30 
31 class TFileCollection;
32 class TFileInfo;
33 class TMD5;
34 class TUrl;
36 class TFileStager;
37 
38 class TDataSetManager : public TObject {
39 
40 private:
41  TDataSetManager(const TDataSetManager&); // not implemented
42  TDataSetManager& operator=(const TDataSetManager&); // not implemented
43 
44 protected:
45  TString fGroup; // Group to which the owner of this session belongs
46  TString fUser; // Owner of the session
47  TString fCommonUser; // User that stores the COMMON datasets
48  TString fCommonGroup; // Group that stores the COMMON datasets
49 
50  TUri fBase; // Base URI used to parse dataset names
51 
52  TMap fGroupQuota; // Group quotas (read from config file)
53  TMap fGroupUsed; // <group> --> <used bytes> (TParameter)
54  TMap fUserUsed; // <group> --> <map of users> --> <value>
55 
56  Long64_t fAvgFileSize; // Average file size to be used to estimate the dataset size (in MB)
57 
58  Int_t fNTouchedFiles; // Number of files touched in the last ScanDataSet operation
59  Int_t fNOpenedFiles; // Number of files opened in the last ScanDataSet operation
60  Int_t fNDisappearedFiles; // Number of files disappared in the last ScanDataSet operation
61 
62  TString fGroupConfigFile; // Path to the group config file
63  Long_t fMTimeGroupConfig; // Last modification of the group config file
64 
65  static TString fgCommonDataSetTag; // Name for common datasets, default: COMMON
66 
67  static TList *fgDataSetSrvMaps; // List of TPair(TRegexp, TObjString) for mapping server coordinates
68  // for dataset files (init from DataSet.SrvMap)
69 
70  virtual TMap *GetGroupUsedMap() { return &fGroupUsed; }
71  virtual TMap *GetUserUsedMap() { return &fUserUsed; }
73  Int_t GetNOpenedFiles() const { return fNOpenedFiles; }
75  void GetQuota(const char *group, const char *user, const char *dsName, TFileCollection *dataset);
76  void PrintDataSet(TFileCollection *fc, Int_t popt = 0);
77  void PrintUsedSpace();
78  Bool_t ReadGroupConfig(const char *cf = 0);
79  virtual void UpdateUsedSpace();
80 
81  static Long64_t ToBytes(const char *size = 0);
82 
83 public:
85  kCheckQuota = BIT(15), // quota checking enabled
86  kAllowRegister = BIT(16), // allow registration of a new dataset
87  kAllowVerify = BIT(17), // allow verification of a dataset (requires registration permit)
88  kTrustInfo = BIT(18), // during registration, trust the available information provided by the user
89  kIsSandbox = BIT(19), // dataset dir is in the user sandbox (simplified naming)
90  kUseCache = BIT(20), // force the usage of cache
91  kDoNotUseCache = BIT(21) // disable the cache
92  };
93 
94  enum EDataSetWorkOpts { // General (bits 1-8)
95  kDebug = 0x1, kShowDefault = 0x2, kPrint = 0x4, kExport = 0x8,
96  kQuotaUpdate = 0x10, kSetDefaultTree = 0x20, kForceScan = 0x40,
98  // File-based specific (bits 9-16)
99  kReopen = 0x100, kTouch = 0x200, kMaxFiles = 0x400, kReadShort = 0x800,
100  kFileMustExist = 0x1000,
101  kNoAction = 0x2000, kLocateOnly = 0x4000, kStageOnly = 0x8000,
102  // Auxilliary bits (bits 17-)
103  kNoCacheUpdate = 0x10000, kRefreshLs = 0x20000, kList = 0x40000,
104  kAllFiles = 0x80000, kStagedFiles = 0x100000, kNoStagedCheck = 0x200000
105  };
106 
107  TDataSetManager(const char *group = 0, const char *user = 0, const char *options = 0);
108  virtual ~TDataSetManager();
109 
110  virtual Int_t ClearCache(const char *uri);
111  virtual Long64_t GetAvgFileSize() const { return fAvgFileSize; }
112  virtual TFileCollection *GetDataSet(const char *uri, const char *server = 0);
113  virtual TMap *GetDataSets(const char *uri, UInt_t /*option*/ = TDataSetManager::kExport);
114  virtual TMap *GetSubDataSets(const char *uri, const char *excludeservers);
115 
116  virtual Long64_t GetGroupQuota(const char *group);
117  virtual TMap *GetGroupQuotaMap() { return &fGroupQuota; }
118  virtual Long64_t GetGroupUsed(const char *group);
119  virtual Bool_t ExistsDataSet(const char *uri);
120  virtual void MonitorUsedSpace(TVirtualMonitoringWriter *monitoring);
121  virtual Int_t NotifyUpdate(const char *group = 0, const char *user = 0,
122  const char *dspath = 0, Long_t mtime = 0, const char *checksum = 0);
123  Bool_t ParseUri(const char *uri, TString *dsGroup = 0, TString *dsUser = 0,
124  TString *dsName = 0, TString *dsTree = 0,
125  Bool_t onlyCurrent = kFALSE, Bool_t wildcards = kFALSE);
126  virtual void ParseInitOpts(const char *opts);
127  virtual Bool_t RemoveDataSet(const char *uri);
128  virtual Int_t RegisterDataSet(const char *uri, TFileCollection *dataSet, const char *opt);
129  Int_t ScanDataSet(const char *uri, const char *opt);
130  virtual Int_t ScanDataSet(const char *uri, UInt_t option = kReopen | kDebug);
131  void SetScanCounters(Int_t t = -1, Int_t o = -1, Int_t d = -1);
132  virtual Int_t ShowCache(const char *uri);
133  virtual void ShowQuota(const char *opt);
134 
135  virtual void ShowDataSets(const char *uri = "*", const char *opt = "");
136 
137  static TString CreateUri(const char *dsGroup = 0, const char *dsUser = 0,
138  const char *dsName = 0, const char *dsTree = 0);
139  static Bool_t CheckDataSetSrvMaps(TUrl *furl, TString &fn, TList *srvmaplist = 0);
140  static TList *GetDataSetSrvMaps();
141  static TList *ParseDataSetSrvMaps(const TString &srvmaps);
142  static Int_t ScanDataSet(TFileCollection *dataset, Int_t fopt, Int_t sopt = 0, Int_t ropt = 0,
143  Bool_t dbg = kFALSE,
144  Int_t *touched = 0, Int_t *opened = 0, Int_t *disappeared = 0,
145  TList *flist = 0, Long64_t avgsz = -1, const char *mss = 0,
146  Int_t maxfiles = -1, const char *stageopts = 0);
147  static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles,
148  Int_t newstagedfiles, TFileStager* stager, Bool_t createStager,
149  Bool_t dbg, Bool_t &changed, Bool_t& touched,
150  Bool_t& disappeared);
151  static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall,
152  TFileStager* stager, Bool_t createStager, const char* stageopts,
153  Bool_t dbg, Bool_t &changed, Bool_t& opened);
154 
155  static Int_t ScanFile(TFileInfo *fileinfo, Bool_t notify);
156  static Int_t FillMetaData(TFileInfo *fi, TDirectory *d, const char *rdir = "/");
157 
158  ClassDef(TDataSetManager, 0) // Abstract data set manager class
159 };
160 
161 #endif
TDataSetManager::kAllFiles
@ kAllFiles
Definition: TDataSetManager.h:104
TDataSetManager::GetSubDataSets
virtual TMap * GetSubDataSets(const char *uri, const char *excludeservers)
Partition dataset 'ds' accordingly to the servers.
Definition: TDataSetManager.cxx:1006
TDataSetManager::ExistsDataSet
virtual Bool_t ExistsDataSet(const char *uri)
Checks if the indicated dataset exits.
Definition: TDataSetManager.cxx:439
TDataSetManager::fGroupUsed
TMap fGroupUsed
Definition: TDataSetManager.h:53
TDataSetManager::kReopen
@ kReopen
Definition: TDataSetManager.h:99
TFileInfo
Class describing a generic file including meta information.
Definition: TFileInfo.h:39
TDataSetManager::ReadGroupConfig
Bool_t ReadGroupConfig(const char *cf=0)
Read group config file 'cf'.
Definition: TDataSetManager.cxx:206
TDataSetManager::GetUserUsedMap
virtual TMap * GetUserUsedMap()
Definition: TDataSetManager.h:71
TDataSetManager::kNoStagedCheck
@ kNoStagedCheck
Definition: TDataSetManager.h:104
TDataSetManager::GetDataSets
virtual TMap * GetDataSets(const char *uri, UInt_t=TDataSetManager::kExport)
Returns all datasets for the <group> and <user> specified by <uri>.
Definition: TDataSetManager.cxx:464
TDataSetManager::EDataSetWorkOpts
EDataSetWorkOpts
Definition: TDataSetManager.h:94
TDataSetManager::RegisterDataSet
virtual Int_t RegisterDataSet(const char *uri, TFileCollection *dataSet, const char *opt)
Register a dataset, perfoming quota checkings, if needed.
Definition: TDataSetManager.cxx:761
TDataSetManager::ShowDataSets
virtual void ShowDataSets(const char *uri="*", const char *opt="")
Prints formatted information about the dataset 'uri'.
Definition: TDataSetManager.cxx:1098
fc
static struct mg_connection * fc(struct mg_context *ctx)
Definition: civetweb.c:3728
TDataSetManager::CheckDataSetSrvMaps
static Bool_t CheckDataSetSrvMaps(TUrl *furl, TString &fn, TList *srvmaplist=0)
Check if the dataset server mappings apply to the url defined by 'furl'.
Definition: TDataSetManager.cxx:1930
TDataSetManager::ClearCache
virtual Int_t ClearCache(const char *uri)
Clear cached information matching uri.
Definition: TDataSetManager.cxx:786
TDataSetManager::kNoAction
@ kNoAction
Definition: TDataSetManager.h:101
TDataSetManager::kSetDefaultTree
@ kSetDefaultTree
Definition: TDataSetManager.h:96
TDataSetManager::fgCommonDataSetTag
static TString fgCommonDataSetTag
Definition: TDataSetManager.h:65
TDataSetManager::kDoNotUseCache
@ kDoNotUseCache
Definition: TDataSetManager.h:91
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TDataSetManager::GetGroupUsed
virtual Long64_t GetGroupUsed(const char *group)
Returns the used space of that group.
Definition: TDataSetManager.cxx:714
TDataSetManager::kList
@ kList
Definition: TDataSetManager.h:103
TVirtualMonitoringWriter
Definition: TVirtualMonitoring.h:35
Int_t
int Int_t
Definition: RtypesCore.h:45
TDataSetManager::fBase
TUri fBase
Definition: TDataSetManager.h:50
TDataSetManager::GetGroupQuota
virtual Long64_t GetGroupQuota(const char *group)
returns the quota a group is allowed to have
Definition: TDataSetManager.cxx:734
TDataSetManager::fGroup
TString fGroup
Definition: TDataSetManager.h:45
TUri.h
TDataSetManager::GetAvgFileSize
virtual Long64_t GetAvgFileSize() const
Definition: TDataSetManager.h:111
TDataSetManager
Definition: TDataSetManager.h:38
TString
Basic string class.
Definition: TString.h:136
TDataSetManager::GetGroupUsedMap
virtual TMap * GetGroupUsedMap()
Definition: TDataSetManager.h:70
TDataSetManager::kAllowVerify
@ kAllowVerify
Definition: TDataSetManager.h:87
TDataSetManager::fGroupQuota
TMap fGroupQuota
Definition: TDataSetManager.h:52
Bool_t
bool Bool_t
Definition: RtypesCore.h:63
TDataSetManager::kForceScan
@ kForceScan
Definition: TDataSetManager.h:96
TDataSetManager::ScanFile
static Int_t ScanFile(TFileInfo *fileinfo, Bool_t notify)
Open the file described by 'fileinfo' to extract the relevant meta-information.
Definition: TDataSetManager.cxx:1654
TString.h
TDataSetManager::kNoCacheUpdate
@ kNoCacheUpdate
Definition: TDataSetManager.h:103
TDataSetManager::kStageOnly
@ kStageOnly
Definition: TDataSetManager.h:101
TDataSetManager::fAvgFileSize
Long64_t fAvgFileSize
Definition: TDataSetManager.h:56
TDataSetManager::kPrint
@ kPrint
Definition: TDataSetManager.h:95
TDataSetManager::ParseInitOpts
virtual void ParseInitOpts(const char *opts)
Parse the opts string and set the init bits accordingly Available options: Cq: set kCheckQuota Ar: se...
Definition: TDataSetManager.cxx:149
TDataSetManager::FillMetaData
static Int_t FillMetaData(TFileInfo *fi, TDirectory *d, const char *rdir="/")
Navigate the directory 'd' (and its subdirectories) looking for TTree objects.
Definition: TDataSetManager.cxx:1801
TDataSetManager::GetQuota
void GetQuota(const char *group, const char *user, const char *dsName, TFileCollection *dataset)
Gets quota information from this dataset.
Definition: TDataSetManager.cxx:550
TDataSetManager::kExport
@ kExport
Definition: TDataSetManager.h:95
TDataSetManager::PrintUsedSpace
void PrintUsedSpace()
Prints the quota.
Definition: TDataSetManager.cxx:638
group
Definition: TWinNTSystem.h:59
TDataSetManager::UpdateUsedSpace
virtual void UpdateUsedSpace()
updates the used space maps
Definition: TDataSetManager.cxx:752
TDataSetManager::GetNOpenedFiles
Int_t GetNOpenedFiles() const
Definition: TDataSetManager.h:73
TDataSetManager::fgDataSetSrvMaps
static TList * fgDataSetSrvMaps
Definition: TDataSetManager.h:67
TDataSetManager::fMTimeGroupConfig
Long_t fMTimeGroupConfig
Definition: TDataSetManager.h:63
TDataSetManager::GetNTouchedFiles
Int_t GetNTouchedFiles() const
Definition: TDataSetManager.h:72
TDataSetManager::fNOpenedFiles
Int_t fNOpenedFiles
Definition: TDataSetManager.h:59
TDataSetManager::EDataSetStatusBits
EDataSetStatusBits
Definition: TDataSetManager.h:84
TDataSetManager::kMaxFiles
@ kMaxFiles
Definition: TDataSetManager.h:99
TDataSetManager::PrintDataSet
void PrintDataSet(TFileCollection *fc, Int_t popt=0)
Formatted printout of the content of TFileCollection 'fc'.
Definition: TDataSetManager.cxx:1043
TDataSetManager::kNoHeaderPrint
@ kNoHeaderPrint
Definition: TDataSetManager.h:97
TDataSetManager::kShowDefault
@ kShowDefault
Definition: TDataSetManager.h:95
TDataSetManager::CheckStagedStatus
static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles, Int_t newstagedfiles, TFileStager *stager, Bool_t createStager, Bool_t dbg, Bool_t &changed, Bool_t &touched, Bool_t &disappeared)
Check stage status of the file described by "fileInfo".
Definition: TDataSetManager.cxx:1414
TFileStager
Definition: TFileStager.h:36
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TDataSetManager::GetGroupQuotaMap
virtual TMap * GetGroupQuotaMap()
Definition: TDataSetManager.h:117
TDataSetManager::~TDataSetManager
virtual ~TDataSetManager()
Destructor.
Definition: TDataSetManager.cxx:127
Long_t
long Long_t
Definition: RtypesCore.h:54
TDataSetManager::MonitorUsedSpace
virtual void MonitorUsedSpace(TVirtualMonitoringWriter *monitoring)
Log info to the monitoring server.
Definition: TDataSetManager.cxx:673
TDataSetManager::kTouch
@ kTouch
Definition: TDataSetManager.h:99
BIT
#define BIT(n)
Definition: Rtypes.h:85
TDataSetManager::fGroupConfigFile
TString fGroupConfigFile
Definition: TDataSetManager.h:62
UInt_t
unsigned int UInt_t
Definition: RtypesCore.h:46
TDataSetManager::operator=
TDataSetManager & operator=(const TDataSetManager &)
TDataSetManager::kAllowRegister
@ kAllowRegister
Definition: TDataSetManager.h:86
TDataSetManager::RemoveDataSet
virtual Bool_t RemoveDataSet(const char *uri)
Removes the indicated dataset.
Definition: TDataSetManager.cxx:430
TDataSetManager::ParseDataSetSrvMaps
static TList * ParseDataSetSrvMaps(const TString &srvmaps)
Create a server mapping list from the content of 'srvmaps' Return the list (owned by the caller) or 0...
Definition: TDataSetManager.cxx:1867
TDataSetManager::kRefreshLs
@ kRefreshLs
Definition: TDataSetManager.h:103
TDataSetManager::ShowQuota
virtual void ShowQuota(const char *opt)
Display quota information.
Definition: TDataSetManager.cxx:585
TMD5
This code implements the MD5 message-digest algorithm.
Definition: TMD5.h:44
TDataSetManager::kStagedFiles
@ kStagedFiles
Definition: TDataSetManager.h:104
TDataSetManager::fNDisappearedFiles
Int_t fNDisappearedFiles
Definition: TDataSetManager.h:60
TDataSetManager::kQuotaUpdate
@ kQuotaUpdate
Definition: TDataSetManager.h:96
TDataSetManager::fNTouchedFiles
Int_t fNTouchedFiles
Definition: TDataSetManager.h:58
TUrl
This class represents a WWW compatible URL.
Definition: TUrl.h:33
TDataSetManager::GetDataSet
virtual TFileCollection * GetDataSet(const char *uri, const char *server=0)
Utility function used in various methods for user dataset upload.
Definition: TDataSetManager.cxx:421
TDataSetManager::fCommonUser
TString fCommonUser
Definition: TDataSetManager.h:47
TDataSetManager::NotifyUpdate
virtual Int_t NotifyUpdate(const char *group=0, const char *user=0, const char *dspath=0, Long_t mtime=0, const char *checksum=0)
Save into the <datasetdir>/dataset.list file the name of the last updated or created or modified data...
Definition: TDataSetManager.cxx:773
TDataSetManager::SetScanCounters
void SetScanCounters(Int_t t=-1, Int_t o=-1, Int_t d=-1)
Update scan counters.
Definition: TDataSetManager.cxx:1981
TDataSetManager::TDataSetManager
TDataSetManager(const TDataSetManager &)
TDataSetManager::kDebug
@ kDebug
Definition: TDataSetManager.h:95
TDataSetManager::kReadShort
@ kReadShort
Definition: TDataSetManager.h:99
TDataSetManager::GetNDisapparedFiles
Int_t GetNDisapparedFiles() const
Definition: TDataSetManager.h:74
TObject.h
TUri
This class represents a RFC 3986 compatible URI.
Definition: TUri.h:35
TDataSetManager::GetDataSetSrvMaps
static TList * GetDataSetSrvMaps()
Static getter for server mapping list.
Definition: TDataSetManager.cxx:1919
TDataSetManager::kLocateOnly
@ kLocateOnly
Definition: TDataSetManager.h:101
TObject
Mother of all ROOT objects.
Definition: TObject.h:37
ClassDef
#define ClassDef(name, id)
Definition: Rtypes.h:325
TDataSetManager::ProcessFile
static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall, TFileStager *stager, Bool_t createStager, const char *stageopts, Bool_t dbg, Bool_t &changed, Bool_t &opened)
Locate, stage, or fully validate file "fileInfo".
Definition: TDataSetManager.cxx:1560
TMap.h
d
#define d(i)
Definition: RSha256.hxx:102
TDirectory
Describe directory structure in memory.
Definition: TDirectory.h:45
TDataSetManager::fUserUsed
TMap fUserUsed
Definition: TDataSetManager.h:54
TDataSetManager::kCheckQuota
@ kCheckQuota
Definition: TDataSetManager.h:85
TDataSetManager::ToBytes
static Long64_t ToBytes(const char *size=0)
Static utility function to gt the number of bytes from a string representation in the form "<digit><s...
Definition: TDataSetManager.cxx:387
TDataSetManager::kTrustInfo
@ kTrustInfo
Definition: TDataSetManager.h:88
TDataSetManager::CreateUri
static TString CreateUri(const char *dsGroup=0, const char *dsUser=0, const char *dsName=0, const char *dsTree=0)
Creates URI for the dataset manger in the form '[[/dsGroup/]dsUser/]dsName[#dsObjPath]',...
Definition: TDataSetManager.cxx:805
TDataSetManager::kFileMustExist
@ kFileMustExist
Definition: TDataSetManager.h:100
TDataSetManager::ShowCache
virtual Int_t ShowCache(const char *uri)
Show cached information matching uri.
Definition: TDataSetManager.cxx:795
TDataSetManager::kIsSandbox
@ kIsSandbox
Definition: TDataSetManager.h:89
TDataSetManager::ParseUri
Bool_t ParseUri(const char *uri, TString *dsGroup=0, TString *dsUser=0, TString *dsName=0, TString *dsTree=0, Bool_t onlyCurrent=kFALSE, Bool_t wildcards=kFALSE)
Parses a (relative) URI that describes a DataSet on the cluster.
Definition: TDataSetManager.cxx:842
TDataSetManager::kUseCache
@ kUseCache
Definition: TDataSetManager.h:90
TDataSetManager::fUser
TString fUser
Definition: TDataSetManager.h:46
TDataSetManager::ScanDataSet
Int_t ScanDataSet(const char *uri, const char *opt)
Scans the dataset indicated by 'uri' following the 'opts' directives.
Definition: TDataSetManager.cxx:497
TDataSetManager::fCommonGroup
TString fCommonGroup
Definition: TDataSetManager.h:48
TList
A doubly linked list.
Definition: TList.h:44
int
TFileCollection
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
Definition: TFileCollection.h:39
TMap
TMap implements an associative array of (key,value) pairs using a THashTable for efficient retrieval ...
Definition: TMap.h:40