Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
TDataSetManager.h
Go to the documentation of this file.
1// @(#)root/proof:$Id$
2// Author: Jan Fiete Grosse-Oetringhaus, 08.08.07
3
4/*************************************************************************
5 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6 * All rights reserved. *
7 * *
8 * For the licensing terms see $ROOTSYS/LICENSE. *
9 * For the list of contributors see $ROOTSYS/README/CREDITS. *
10 *************************************************************************/
11
12#ifndef ROOT_TDataSetManager
13#define ROOT_TDataSetManager
14
15//////////////////////////////////////////////////////////////////////////
16// //
17// TDataSetManager //
18// //
19// This class contains functions to handle datasets in PROOF //
20// It is the layer between TProofServ and the file system that stores //
21// the datasets. //
22// //
23//////////////////////////////////////////////////////////////////////////
24
25#include "TObject.h"
26#include "TString.h"
27#include "TMap.h"
28#include "TUri.h"
29
30
31class TFileCollection;
32class TFileInfo;
33class TMD5;
34class TUrl;
36class TFileStager;
37
38class TDataSetManager : public TObject {
39
40private:
41 TDataSetManager(const TDataSetManager&); // not implemented
42 TDataSetManager& operator=(const TDataSetManager&); // not implemented
43
44protected:
45 TString fGroup; // Group to which the owner of this session belongs
46 TString fUser; // Owner of the session
47 TString fCommonUser; // User that stores the COMMON datasets
48 TString fCommonGroup; // Group that stores the COMMON datasets
49
50 TUri fBase; // Base URI used to parse dataset names
51
52 TMap fGroupQuota; // Group quotas (read from config file)
53 TMap fGroupUsed; // <group> --> <used bytes> (TParameter)
54 TMap fUserUsed; // <group> --> <map of users> --> <value>
55
56 Long64_t fAvgFileSize; // Average file size to be used to estimate the dataset size (in MB)
57
58 Int_t fNTouchedFiles; // Number of files touched in the last ScanDataSet operation
59 Int_t fNOpenedFiles; // Number of files opened in the last ScanDataSet operation
60 Int_t fNDisappearedFiles; // Number of files disappared in the last ScanDataSet operation
61
62 TString fGroupConfigFile; // Path to the group config file
63 Long_t fMTimeGroupConfig; // Last modification of the group config file
64
65 static TString fgCommonDataSetTag; // Name for common datasets, default: COMMON
66
67 static TList *fgDataSetSrvMaps; // List of TPair(TRegexp, TObjString) for mapping server coordinates
68 // for dataset files (init from DataSet.SrvMap)
69
70 virtual TMap *GetGroupUsedMap() { return &fGroupUsed; }
71 virtual TMap *GetUserUsedMap() { return &fUserUsed; }
75 void GetQuota(const char *group, const char *user, const char *dsName, TFileCollection *dataset);
76 void PrintDataSet(TFileCollection *fc, Int_t popt = 0);
77 void PrintUsedSpace();
78 Bool_t ReadGroupConfig(const char *cf = 0);
79 virtual void UpdateUsedSpace();
80
81 static Long64_t ToBytes(const char *size = 0);
82
83public:
85 kCheckQuota = BIT(15), // quota checking enabled
86 kAllowRegister = BIT(16), // allow registration of a new dataset
87 kAllowVerify = BIT(17), // allow verification of a dataset (requires registration permit)
88 kTrustInfo = BIT(18), // during registration, trust the available information provided by the user
89 kIsSandbox = BIT(19), // dataset dir is in the user sandbox (simplified naming)
90 kUseCache = BIT(20), // force the usage of cache
91 kDoNotUseCache = BIT(21) // disable the cache
92 };
93
94 enum EDataSetWorkOpts { // General (bits 1-8)
95 kDebug = 0x1, kShowDefault = 0x2, kPrint = 0x4, kExport = 0x8,
96 kQuotaUpdate = 0x10, kSetDefaultTree = 0x20, kForceScan = 0x40,
98 // File-based specific (bits 9-16)
99 kReopen = 0x100, kTouch = 0x200, kMaxFiles = 0x400, kReadShort = 0x800,
101 kNoAction = 0x2000, kLocateOnly = 0x4000, kStageOnly = 0x8000,
102 // Auxilliary bits (bits 17-)
103 kNoCacheUpdate = 0x10000, kRefreshLs = 0x20000, kList = 0x40000,
104 kAllFiles = 0x80000, kStagedFiles = 0x100000, kNoStagedCheck = 0x200000
105 };
106
107 TDataSetManager(const char *group = 0, const char *user = 0, const char *options = 0);
108 ~TDataSetManager() override;
109
110 virtual Int_t ClearCache(const char *uri);
111 virtual Long64_t GetAvgFileSize() const { return fAvgFileSize; }
112 virtual TFileCollection *GetDataSet(const char *uri, const char *server = 0);
113 virtual TMap *GetDataSets(const char *uri, UInt_t /*option*/ = TDataSetManager::kExport);
114 virtual TMap *GetSubDataSets(const char *uri, const char *excludeservers);
115
116 virtual Long64_t GetGroupQuota(const char *group);
117 virtual TMap *GetGroupQuotaMap() { return &fGroupQuota; }
118 virtual Long64_t GetGroupUsed(const char *group);
119 virtual Bool_t ExistsDataSet(const char *uri);
120 virtual void MonitorUsedSpace(TVirtualMonitoringWriter *monitoring);
121 virtual Int_t NotifyUpdate(const char *group = 0, const char *user = 0,
122 const char *dspath = 0, Long_t mtime = 0, const char *checksum = 0);
123 Bool_t ParseUri(const char *uri, TString *dsGroup = 0, TString *dsUser = 0,
124 TString *dsName = 0, TString *dsTree = 0,
125 Bool_t onlyCurrent = kFALSE, Bool_t wildcards = kFALSE);
126 virtual void ParseInitOpts(const char *opts);
127 virtual Bool_t RemoveDataSet(const char *uri);
128 virtual Int_t RegisterDataSet(const char *uri, TFileCollection *dataSet, const char *opt);
129 Int_t ScanDataSet(const char *uri, const char *opt);
130 virtual Int_t ScanDataSet(const char *uri, UInt_t option = kReopen | kDebug);
131 void SetScanCounters(Int_t t = -1, Int_t o = -1, Int_t d = -1);
132 virtual Int_t ShowCache(const char *uri);
133 virtual void ShowQuota(const char *opt);
134
135 virtual void ShowDataSets(const char *uri = "*", const char *opt = "");
136
137 static TString CreateUri(const char *dsGroup = 0, const char *dsUser = 0,
138 const char *dsName = 0, const char *dsTree = 0);
139 static Bool_t CheckDataSetSrvMaps(TUrl *furl, TString &fn, TList *srvmaplist = 0);
140 static TList *GetDataSetSrvMaps();
141 static TList *ParseDataSetSrvMaps(const TString &srvmaps);
142 static Int_t ScanDataSet(TFileCollection *dataset, Int_t fopt, Int_t sopt = 0, Int_t ropt = 0,
143 Bool_t dbg = kFALSE,
144 Int_t *touched = 0, Int_t *opened = 0, Int_t *disappeared = 0,
145 TList *flist = 0, Long64_t avgsz = -1, const char *mss = 0,
146 Int_t maxfiles = -1, const char *stageopts = 0);
147 static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles,
148 Int_t newstagedfiles, TFileStager* stager, Bool_t createStager,
149 Bool_t dbg, Bool_t &changed, Bool_t& touched,
150 Bool_t& disappeared);
151 static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall,
152 TFileStager* stager, Bool_t createStager, const char* stageopts,
153 Bool_t dbg, Bool_t &changed, Bool_t& opened);
154
155 static Int_t ScanFile(TFileInfo *fileinfo, Bool_t notify);
156 static Int_t FillMetaData(TFileInfo *fi, TDirectory *d, const char *rdir = "/");
157
158 ClassDefOverride(TDataSetManager, 0) // Abstract data set manager class
159};
160
161#endif
#define d(i)
Definition RSha256.hxx:102
size_t size(const MatrixT &matrix)
retrieve the size of a square matrix
bool Bool_t
Definition RtypesCore.h:63
int Int_t
Definition RtypesCore.h:45
long Long_t
Definition RtypesCore.h:54
unsigned int UInt_t
Definition RtypesCore.h:46
constexpr Bool_t kFALSE
Definition RtypesCore.h:94
long long Long64_t
Definition RtypesCore.h:69
#define BIT(n)
Definition Rtypes.h:90
#define ClassDefOverride(name, id)
Definition Rtypes.h:346
Option_t Option_t option
static TString fgCommonDataSetTag
static Bool_t CheckStagedStatus(TFileInfo *fileInfo, Int_t fopt, Int_t maxfiles, Int_t newstagedfiles, TFileStager *stager, Bool_t createStager, Bool_t dbg, Bool_t &changed, Bool_t &touched, Bool_t &disappeared)
Check stage status of the file described by "fileInfo".
virtual TFileCollection * GetDataSet(const char *uri, const char *server=0)
Utility function used in various methods for user dataset upload.
virtual Long64_t GetGroupUsed(const char *group)
Returns the used space of that group.
Int_t ScanDataSet(const char *uri, const char *opt)
Scans the dataset indicated by 'uri' following the 'opts' directives.
Bool_t ReadGroupConfig(const char *cf=0)
Read group config file 'cf'.
~TDataSetManager() override
Destructor.
static TList * GetDataSetSrvMaps()
Static getter for server mapping list.
virtual void UpdateUsedSpace()
updates the used space maps
static void ProcessFile(TFileInfo *fileInfo, Int_t sopt, Bool_t checkstg, Bool_t doall, TFileStager *stager, Bool_t createStager, const char *stageopts, Bool_t dbg, Bool_t &changed, Bool_t &opened)
Locate, stage, or fully validate file "fileInfo".
static TList * ParseDataSetSrvMaps(const TString &srvmaps)
Create a server mapping list from the content of 'srvmaps' Return the list (owned by the caller) or 0...
static Int_t ScanFile(TFileInfo *fileinfo, Bool_t notify)
Open the file described by 'fileinfo' to extract the relevant meta-information.
static TString CreateUri(const char *dsGroup=0, const char *dsUser=0, const char *dsName=0, const char *dsTree=0)
Creates URI for the dataset manger in the form '[[/dsGroup/]dsUser/]dsName[#dsObjPath]',...
static Int_t FillMetaData(TFileInfo *fi, TDirectory *d, const char *rdir="/")
Navigate the directory 'd' (and its subdirectories) looking for TTree objects.
void PrintUsedSpace()
Prints the quota.
static Bool_t CheckDataSetSrvMaps(TUrl *furl, TString &fn, TList *srvmaplist=0)
Check if the dataset server mappings apply to the url defined by 'furl'.
void GetQuota(const char *group, const char *user, const char *dsName, TFileCollection *dataset)
Gets quota information from this dataset.
void PrintDataSet(TFileCollection *fc, Int_t popt=0)
Formatted printout of the content of TFileCollection 'fc'.
virtual TMap * GetDataSets(const char *uri, UInt_t=TDataSetManager::kExport)
Returns all datasets for the <group> and <user> specified by <uri>.
void SetScanCounters(Int_t t=-1, Int_t o=-1, Int_t d=-1)
Update scan counters.
virtual Bool_t ExistsDataSet(const char *uri)
Checks if the indicated dataset exits.
TDataSetManager & operator=(const TDataSetManager &)
virtual TMap * GetGroupQuotaMap()
virtual void ShowDataSets(const char *uri="*", const char *opt="")
Prints formatted information about the dataset 'uri'.
virtual TMap * GetUserUsedMap()
virtual void ShowQuota(const char *opt)
Display quota information.
virtual Int_t NotifyUpdate(const char *group=0, const char *user=0, const char *dspath=0, Long_t mtime=0, const char *checksum=0)
Save into the <datasetdir>/dataset.list file the name of the last updated or created or modified data...
virtual Bool_t RemoveDataSet(const char *uri)
Removes the indicated dataset.
Int_t GetNDisapparedFiles() const
Bool_t ParseUri(const char *uri, TString *dsGroup=0, TString *dsUser=0, TString *dsName=0, TString *dsTree=0, Bool_t onlyCurrent=kFALSE, Bool_t wildcards=kFALSE)
Parses a (relative) URI that describes a DataSet on the cluster.
static TList * fgDataSetSrvMaps
Int_t GetNOpenedFiles() const
virtual Long64_t GetGroupQuota(const char *group)
returns the quota a group is allowed to have
virtual Int_t ClearCache(const char *uri)
Clear cached information matching uri.
virtual void ParseInitOpts(const char *opts)
Parse the opts string and set the init bits accordingly Available options: Cq: set kCheckQuota Ar: se...
virtual TMap * GetSubDataSets(const char *uri, const char *excludeservers)
Partition dataset 'ds' accordingly to the servers.
TDataSetManager(const TDataSetManager &)
virtual Long64_t GetAvgFileSize() const
virtual void MonitorUsedSpace(TVirtualMonitoringWriter *monitoring)
Log info to the monitoring server.
Int_t GetNTouchedFiles() const
virtual Int_t RegisterDataSet(const char *uri, TFileCollection *dataSet, const char *opt)
Register a dataset, perfoming quota checkings, if needed.
static Long64_t ToBytes(const char *size=0)
Static utility function to gt the number of bytes from a string representation in the form "`<digit>`...
virtual TMap * GetGroupUsedMap()
virtual Int_t ShowCache(const char *uri)
Show cached information matching uri.
Describe directory structure in memory.
Definition TDirectory.h:45
Class that contains a list of TFileInfo's and accumulated meta data information about its entries.
Class describing a generic file including meta information.
Definition TFileInfo.h:39
A doubly linked list.
Definition TList.h:38
This code implements the MD5 message-digest algorithm.
Definition TMD5.h:44
TMap implements an associative array of (key,value) pairs using a THashTable for efficient retrieval ...
Definition TMap.h:40
Mother of all ROOT objects.
Definition TObject.h:41
Basic string class.
Definition TString.h:139
This class represents a RFC 3986 compatible URI.
Definition TUri.h:35
This class represents a WWW compatible URL.
Definition TUrl.h:33