54static const std::string
VERSION =
"0.2.0";
57" TDavixFile/" +
VERSION +
" davix/" + Davix::version();
60#define ENVPFX "Davix."
86 if (!str)
return false;
88 if (!strcmp(str,
"n") || !strcmp(str,
"no") || !strcmp(str,
"0") || !strcmp(str,
"false"))
return true;
95 if(!str)
return defvalue;
97 if(strcmp(str,
"n") == 0 || strcmp(str,
"no") == 0 || strcmp(str,
"0") == 0 || strcmp(str,
"false") == 0)
return false;
98 if(strcmp(str,
"y") == 0 || strcmp(str,
"yes") == 0 || strcmp(str,
"1") == 0 || strcmp(str,
"true") == 0)
return true;
108 old_flag |= O_RDONLY;
111 old_flag |= (O_CREAT | O_WRONLY | O_TRUNC);
114 old_flag |= (O_RDWR);
127 davix_set_log_level(0);
130 davix_set_log_level(DAVIX_LOG_WARNING);
133 davix_set_log_level(DAVIX_LOG_VERBOSE);
136 davix_set_log_level(DAVIX_LOG_DEBUG);
139 davix_set_log_level(DAVIX_LOG_ALL);
151 char default_proxy[64];
152 const char *genvvar = 0, *genvvar1 = 0;
154 genvvar =
gEnv->
GetValue(
"Davix.GSI.UserProxy", (
const char *) NULL);
156 ucert = ukey = genvvar;
158 Info(
"TDavixFile_http_get_ucert",
"Found proxy in gEnv");
163 if (getenv(
"X509_USER_PROXY")) {
165 Info(
"TDavixFile_http_get_ucert",
"Found proxy in X509_USER_PROXY");
166 ucert = ukey = getenv(
"X509_USER_PROXY");
171 snprintf(default_proxy,
sizeof(default_proxy),
"/tmp/x509up_u%d",
174 if (access(default_proxy, R_OK) == 0) {
176 Info(
"TDavixFile_http_get_ucert",
"Found proxy in /tmp");
177 ucert = ukey = default_proxy;
182 genvvar =
gEnv->
GetValue(
"Davix.GSI.UserCert", (
const char *) NULL);
183 genvvar1 =
gEnv->
GetValue(
"Davix.GSI.UserKey", (
const char *) NULL);
184 if (genvvar || genvvar1) {
186 Info(
"TDavixFile_http_get_ucert",
"Found cert and key in gEnv");
194 if (getenv(
"X509_USER_CERT"))
195 ucert = getenv(
"X509_USER_CERT");
196 if (getenv(
"X509_USER_KEY"))
197 ukey = getenv(
"X509_USER_KEY");
199 if ((ucert.size() > 0) || (ukey.size() > 0)) {
201 Info(
"TDavixFile_http_get_ucert",
"Found cert and key in gEnv");
210 Davix::X509Credential *cert, Davix::DavixError **err)
214 std::string ucert, ukey;
217 if (ucert.empty() || ukey.empty()) {
218 Davix::DavixError::setupError(err,
"TDavixFile",
219 Davix::StatusCode::AuthentificationError,
220 "Could not set the user's proxy or certificate");
223 return cert->loadFromFilePEM(ukey, ucert,
"", err);
252 DavixError *davixErr = NULL;
260 DavixError *davixErr2 = NULL;
263 std::vector<DavFile> replicasLocal =
file.getReplicas(NULL, &davixErr2);
264 for(
size_t i = 0; i < replicasLocal.size(); i++) {
265 replicas.push_back(replicasLocal[i].getUri().getString());
269 DavixError::clearError(&davixErr2);
274 Error(
"DavixOpen",
"can not open file \"%s\" with davix: %s (%d)",
276 davixErr->getErrMsg().c_str(), davixErr->getStatus());
278 DavixError::clearError(&davixErr);
281 davixPosix->fadvise(fd, 0, 300, Davix::AdviseRandom);
291 DavixError *davixErr = NULL;
293 Error(
"DavixClose",
"can not to close file with davix: %s (%d)",
294 davixErr->getErrMsg().c_str(), davixErr->getStatus());
295 DavixError::clearError(&davixErr);
303 const char *env_var = NULL;
306 Info(
"enableGridMode",
" grid mode enabled !");
308 if( ( env_var = getenv(
"X509_CERT_DIR")) == NULL){
309 env_var=
"/etc/grid-security/certificates/";
311 davixParam->addCertificateAuthorityPath(env_var);
313 Info(
"enableGridMode",
"Adding CAdir %s", env_var);
326template<
typename TRequestParams = Davix::RequestParams>
327static auto awsRegion(TRequestParams *parameters,
const char *region)
328 ->
decltype(parameters->setAwsRegion(region),
void())
330 if (
gDebug > 1)
Info(
"awsRegion",
"Setting S3 Region to '%s' - v4 signature will be used", region);
331 parameters->setAwsRegion(region);
334template<
typename TRequestParams = Davix::RequestParams>
336 Warning(
"setAwsRegion",
"Unable to set AWS region, not supported by this version of davix");
340template<
typename TRequestParams = Davix::RequestParams>
341static auto awsToken(TRequestParams *parameters,
const char *token)
342 ->
decltype(parameters->setAwsToken(token),
void())
344 if (
gDebug > 1)
Info(
"awsToken",
"Setting S3 STS temporary credentials");
345 parameters->setAwsToken(token);
348template<
typename TRequestParams = Davix::RequestParams>
350 Warning(
"awsToken",
"Unable to set AWS token, not supported by this version of davix");
354template<
typename TRequestParams = Davix::RequestParams>
356 ->
decltype(parameters->setAwsAlternate(option),
void())
358 if (
gDebug > 1)
Info(
"awsAlternate",
"Setting S3 path-based bucket option (s3alternate)");
359 parameters->setAwsAlternate(option);
362template<
typename TRequestParams = Davix::RequestParams>
364 Warning(
"awsAlternate",
"Unable to set AWS path-based bucket option (s3alternate), not supported by this version of davix");
368 if(!region.empty()) {
385 const std::string ®ion,
const std::string &token)
388 Info(
"setS3Auth",
" Aws S3 tokens configured");
390 davixParam->setAwsAuthorizationKeys(secret, access);
391 davixParam->setProtocol(RequestProtocol::AwsS3);
401 const char *env_var = NULL, *env_var2 = NULL;
403 davixParam->setTransparentRedirectionSupport(
true);
407 env_var =
gEnv->
GetValue(
"Davix.GSI.CAdir", (
const char *) NULL);
409 davixParam->addCertificateAuthorityPath(env_var);
411 Info(
"parseConfig",
"Add CAdir: %s", env_var);
414 bool ca_check_local = !
isno(
gEnv->
GetValue(
"Davix.GSI.CACheck", (
const char *)
"y"));
417 Info(
"parseConfig",
"Setting CAcheck to %s", ((ca_check_local) ? (
"true") : (
"false")));
420 if (((env_var =
gEnv->
GetValue(
"Davix.S3.SecretKey", getenv(
"S3_SECRET_KEY"))) != NULL)
421 && ((env_var2 =
gEnv->
GetValue(
"Davix.S3.AccessKey", getenv(
"S3_ACCESS_KEY"))) != NULL)) {
422 Info(
"parseConfig",
"Setting S3 SecretKey and AccessKey. Access Key : %s ", env_var2);
423 davixParam->setAwsAuthorizationKeys(env_var, env_var2);
426 if ( (env_var =
gEnv->
GetValue(
"Davix.S3.Region", getenv(
"S3_REGION"))) != NULL) {
430 if( (env_var =
gEnv->
GetValue(
"Davix.S3.Token", getenv(
"S3_TOKEN"))) != NULL) {
434 if( (env_var =
gEnv->
GetValue(
"Davix.S3.Alternate", getenv(
"S3_ALTERNATE"))) != NULL) {
439 env_var =
gEnv->
GetValue(
"Davix.GSI.GridMode", (
const char *)
"y");
449 std::stringstream ss(option);
451 std::vector<std::string> parsed_options;
453 std::string s3seckey, s3acckey, s3region, s3token;
455 while (std::getline(ss, item,
' ')) {
456 parsed_options.push_back(item);
459 for (std::vector<std::string>::iterator it = parsed_options.begin(); it < parsed_options.end(); ++it) {
482 s3token = std::string(it->c_str() + strlen(
s3_token_opt));
492 if (s3seckey.size() > 0) {
493 setS3Auth(s3seckey, s3acckey, s3region, s3token);
507 davixParam->setMetalinkMode(Davix::MetalinkMode::Disable);
517 DavixError *davixErr = NULL;
521 Error(
"DavixStat",
"can not stat the file with davix: %s (%d)",
522 davixErr->getErrMsg().c_str(), davixErr->getStatus());
523 DavixError::clearError(&davixErr);
570 if(!replicas.empty()) {
571 std::stringstream ss;
572 for(
size_t i = 0; i < replicas.size(); i++) {
574 if(i != replicas.size()-1) ss <<
"|";
597 Error(
"Seek",
"seeking from end in archive is not (yet) supported");
603 Info(
"Seek",
" move cursor to %lld"
622 Info(
"ReadBuffer",
"%lld bytes of data read sequentially"
623 " (%d requested)", ret, len);
641 Info(
"ReadBuffer",
"%lld bytes of data read from offset"
642 " %lld (%d requested)", ret, pos, len);
654 d_ptr->
davixPosix->fadvise(fd,
static_cast<dav_off_t
>(offs),
static_cast<dav_size_t
>(len), Davix::AdviseRandom);
657 Info(
"ReadBufferAsync",
"%d bytes of data prefected from offset"
658 " %lld ", len, offs);
675 Info(
"ReadBuffers",
"%lld bytes of data read from a list of %d buffers",
694 Info(
"WriteBuffer",
"%lld bytes of data write"
695 " %d requested", ret, len);
718 std::vector<void *>::iterator
f = std::find(
dirdVec.begin(),
dirdVec.end(), fd);
735 std::vector<void *>::iterator
f = std::find(
dirdVec.begin(),
dirdVec.end(), fd);
748 Info(
"GetSize",
"file size requested: %lld", (
Long64_t)st.st_size);
787 DavixError *davixErr = NULL;
792 Error(
"DavixReadBuffer",
"can not read data with davix: %s (%d)",
793 davixErr->getErrMsg().c_str(), davixErr->getStatus());
794 DavixError::clearError(&davixErr);
807 DavixError *davixErr = NULL;
812 Error(
"DavixWriteBuffer",
"can not write data with davix: %s (%d)",
813 davixErr->getErrMsg().c_str(), davixErr->getStatus());
814 DavixError::clearError(&davixErr);
827 DavixError *davixErr = NULL;
832 Error(
"DavixPReadBuffer",
"can not read data with davix: %s (%d)",
833 davixErr->getErrMsg().c_str(), davixErr->getStatus());
834 DavixError::clearError(&davixErr);
847 DavixError *davixErr = NULL;
849 DavIOVecInput in[nbuf];
850 DavIOVecOuput out[nbuf];
853 for (
Int_t i = 0; i < nbuf; ++i) {
854 in[i].diov_buffer = &buf[lastPos];
855 in[i].diov_offset = pos[i];
856 in[i].diov_size = len[i];
862 Error(
"DavixReadBuffers",
"can not read data with davix: %s (%d)",
863 davixErr->getErrMsg().c_str(), davixErr->getStatus());
864 DavixError::clearError(&davixErr);
const char * s3_seckey_opt
static const std::string VERSION
static void ConfigureDavixLogLevel()
static auto awsRegion(TRequestParams *parameters, const char *region) -> decltype(parameters->setAwsRegion(region), void())
static auto awsAlternate(TRequestParams *parameters, bool option) -> decltype(parameters->setAwsAlternate(option), void())
static int TDavixFile_http_authn_cert_X509(void *userdata, const Davix::SessionInfo &info, Davix::X509Credential *cert, Davix::DavixError **err)
const char * s3_alternate_opt
static auto awsToken(TRequestParams *parameters, const char *token) -> decltype(parameters->setAwsToken(token), void())
const char * s3_token_opt
bool isno(const char *str)
const char * ca_check_opt
const char * open_mode_new
const char * open_mode_update
const char * open_mode_create
const char * s3_acckey_opt
static Context * davix_context_s
const char * grid_mode_opt
int configure_open_flag(const std::string &str, int old_flag)
static void TDavixFile_http_get_ucert(std::string &ucert, std::string &ukey)
static const std::string gUserAgent
bool strToBool(const char *str, bool defvalue)
const char * s3_region_opt
const char * open_mode_read
void Info(const char *location, const char *msgfmt,...)
void Error(const char *location, const char *msgfmt,...)
void Warning(const char *location, const char *msgfmt,...)
typedef void((*Func_t)())
Davix::RequestParams * davixParam
std::vector< void * > dirdVec
void setAwsToken(const std::string &token)
void setS3Auth(const std::string &secret, const std::string &access, const std::string ®ion, const std::string &token)
void setAwsAlternate(const bool &option)
static Davix::Context * getDavixInstance()
Davix::DavPosix * davixPosix
void parseParams(Option_t *option)
intput params
std::vector< std::string > getReplicas()
Int_t DavixStat(const char *url, struct stat *st)
Davix_fd * getDavixFileInstance()
Davix::Context * davixContext
void setAwsRegion(const std::string ®ion)
std::vector< std::string > replicas
void removeDird(void *fd)
virtual Long64_t GetSize() const
Returns the current file size.
void Init(Bool_t init)
Initialize a TFile object.
Long64_t DavixReadBuffers(Davix_fd *fd, char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Long64_t DavixReadBuffer(Davix_fd *fd, char *buf, Int_t len)
Long64_t DavixPReadBuffer(Davix_fd *fd, char *buf, Long64_t pos, Int_t len)
virtual Bool_t WriteBuffer(const char *buffer, Int_t bufferLength)
Write a buffer to the file.
virtual TString GetNewUrl()
Long64_t DavixWriteBuffer(Davix_fd *fd, const char *buf, Int_t len)
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read specified byte range from remote file via HTTP.
TDavixFile(const char *url, Option_t *option="", const char *ftitle="", Int_t compress=ROOT::RCompressionSetting::EDefaults::kUseCompiledDefault)
Open function for TDavixFile.
void enableGridMode()
Enable the grid mode The grid Mode configure automatically all grid-CA path, VOMS authentication and ...
virtual void Seek(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
TDavixFileInternal * d_ptr
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
void eventStop(Double_t t, Long64_t len, bool read=true)
set TFile state info
void setCACheck(Bool_t check)
Enable or disable certificate authority check.
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format.
Int_t fReadCalls
Number of read calls ( not counting the cache calls )
static void SetFileBytesWritten(Long64_t bytes=0)
Long64_t fBytesRead
Number of bytes read from this file.
static Long64_t GetFileBytesWritten()
Static function returning the total number of bytes written to all files.
static void SetFileBytesRead(Long64_t bytes=0)
static void SetFileReadCalls(Int_t readcalls=0)
static Long64_t GetFileBytesRead()
Static function returning the total number of bytes read from all files.
Long64_t fArchiveOffset
!Offset at which file starts in archive
virtual void Init(Bool_t create)
Initialize a TFile object.
Long64_t fBytesWrite
Number of bytes written to this file.
Long64_t fOffset
!Seek offset cache
Long64_t fEND
Last used byte in file.
static Int_t GetFileReadCalls()
Static function returning the total number of read calls from all files.
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
The TTimeStamp encapsulates seconds and ns since EPOCH.
const char * GetUrl(Bool_t withDeflt=kFALSE) const
Return full URL.
EvaluateInfo init(std::vector< RooRealProxy > parameters, std::vector< ArrayWrapper * > wrappers, std::vector< double * > arrays, size_t begin, size_t batchSize)