49 #include <sys/resource.h>
67 maxfiles = _getmaxstdio();
70 if (getrlimit(RLIMIT_NOFILE,&filelimit)==0) {
71 maxfiles = filelimit.rlim_cur;
77 if (maxfiles > kCintFileNumber) {
79 }
else if (maxfiles > 5) {
92 fLocal(isLocal), fHistoOneGo(histoOneGo), fObjectNames()
102 gROOT->GetListOfCleanups()->Add(
this);
110 gROOT->GetListOfCleanups()->Remove(
this);
157 if (!
TFile::Cp(url, localcopy, cpProgress)) {
158 Error(
"AddFile",
"cannot get a local copy of file %s", url);
168 Error(
"AddFile",
"cannot open local copy %s of URL %s",
169 localcopy.
Data(), url);
171 Error(
"AddFile",
"cannot open file %s", url);
232 if (!source->
Cp(localcopy, cpProgress)) {
233 Error(
"AddFile",
"cannot get a local copy of file %s", source->
GetName());
243 Error(
"AddFile",
"cannot open local copy %s of URL %s",
246 Error(
"AddFile",
"cannot open file %s", source->
GetName());
251 if (own || newfile != source) {
264 if (newfile != source && own) {
276 return OutputFile(outputfile,(force?
"RECREATE":
"CREATE"),compressionLevel);
364 path.
Remove(0, path.Last(
':') + 2);
374 allNames.
Add(arr->
At(iname));
390 current_sourcedir = target;
395 while (current_file || current_sourcedir) {
398 if (current_sourcedir && (current_file == 0 || current_sourcedir != target)) {
405 while ( (key = (
TKey*)nextkey())) {
413 if (strcmp(key->GetClassName(),
"TProcessID") == 0) { key->ReadObj();
continue;}
419 oldkeyname = key->GetName();
425 Info(
"MergeRecursive",
"cannot indentify object type (%s), name: %s title: %s",
426 key->GetClassName(), key->GetName(), key->GetTitle());
439 oldkeyname = key->GetName();
445 oldkeyname = key->GetName();
448 oldkeyname = key->GetName();
457 oldkeyname = key->GetName();
461 if (!(type&kNonResetable)) {
464 oldkeyname = key->GetName();
471 if (type & kIncremental) {
474 obj = key->ReadObj();
477 obj = key->ReadObj();
480 Info(
"MergeRecursive",
"could not read object for key {%s, %s}",
481 key->GetName(), key->GetTitle());
484 if (cl->
IsTObject() && cl != obj->IsA()) {
485 Error(
"MergeRecursive",
"TKey and object retrieve disagree on type (%s vs %s). Continuing with %s.",
486 key->GetClassName(), obj->IsA()->
GetName(), obj->IsA()->
GetName());
498 if (type & kIncremental || alreadyseen) {
512 if (onlyListed) type &= ~kOnlyListed;
515 if (!status)
return status;
519 if (alreadyseen)
continue;
526 if (nextsource == 0) {
529 func(obj, &inputs, &info);
541 Info(
"MergeRecursive",
"could not read object for key {%s, %s}; skipping file %s",
542 key->GetName(), key->GetTitle(), nextsource->
GetName());
543 nextsource = (
TFile*)sourcelist->
After(nextsource);
557 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
564 nextsource = (
TFile*)sourcelist->
After( nextsource );
565 }
while (nextsource);
569 func(obj, &inputs, &info);
579 if (alreadyseen)
continue;
583 listHargs.
Form(
"(TCollection*)0x%lx,(TFileMergeInfo*)0x%lx", (
ULong_t)&listH,(
ULong_t)&info);
587 if (nextsource == 0) {
593 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
594 obj->
GetName(), key->GetName());
606 Info(
"MergeRecursive",
"could not read object for key {%s, %s}; skipping file %s",
607 key->GetName(), key->GetTitle(), nextsource->
GetName());
608 nextsource = (
TFile*)sourcelist->
After(nextsource);
621 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
627 nextsource = (
TFile*)sourcelist->
After( nextsource );
642 if (alreadyseen)
continue;
646 listHargs.
Form(
"((TCollection*)0x%lx)", (
ULong_t)&listH);
650 if (nextsource == 0) {
655 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
656 obj->
GetName(), key->GetName());
668 Info(
"MergeRecursive",
"could not read object for key {%s, %s}; skipping file %s",
669 key->GetName(), key->GetTitle(), nextsource->
GetName());
670 nextsource = (
TFile*)sourcelist->
After(nextsource);
683 Error(
"MergeRecursive",
"calling Merge() on '%s' with the corresponding object in '%s'",
689 nextsource = (
TFile*)sourcelist->
After( nextsource );
710 oldkeyname = key->GetName();
718 if (!(type&kIncremental) ||
dynamic_cast<TDirectory*
>(
obj)->GetFile() != target) {
736 if ( target->
WriteObjectAny( (
void*)obj, cl, oldkeyname, canBeMerged ?
"OverWrite" :
"" ) <= 0) {
745 current_file = current_file ? (
TFile*)sourcelist->
After(current_file) : (
TFile*)sourcelist->
First();
749 current_sourcedir = 0;
753 if (!(type&kIncremental)) {
781 Info(
"PartialMerge",
"will merge the results to the file %s\n"
782 "since you didn't specify a merge filename",
783 TUrl(outf).GetFile());
797 if (!file || (file && file->
IsZombie())) {
798 Error(
"PartialMerge",
"one-file case: problem attaching to file");
803 Error(
"PartialMerge",
"one-file case: could not copy '%s' to '%s'",
813 Warning(
"PartialMerge",
"problems removing temporary local file '%s'", u.GetFile());
838 p = p(0, p.Index(
':',0));
853 Error(
"Merge",
"error during merge of your ROOT files");
893 Error(
"OpenExcessFiles",
"cannot get a local copy of file %s", url->
GetName());
903 Error(
"OpenExcessFiles",
"cannot open local copy %s of URL %s",
906 Error(
"OpenExcessFiles",
"cannot open file %s", url->
GetName());
926 Fatal(
"RecursiveRemove",
"Output file of the TFile Merger (targeting %s) has been deleted (likely due to a TTree larger than 100Gb)",
fOutputFilename.
Data());
940 if (newmax < sysmax) {
const char * GetName() const
Returns name of object.
virtual void Clear(Option_t *="")
virtual Int_t GetEntries() const
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Int_t fPrintLevel
How much information to print out at run time.
ROOT::MergeFunc_t GetMerge() const
Return the wrapper around Merge.
virtual void Delete(Option_t *option="")
Remove all objects from the list AND delete all heap based objects.
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
virtual TList * GetListOfKeys() const
ClassImp(TSeqCollection) Int_t TSeqCollection TIter next(this)
Return index of object in collection.
Collectable string class.
TObject * FindObject(const char *name) const
Find object using its name.
This class represents a WWW compatible URL.
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
virtual TList * GetList() const
Bool_t IsTObject() const
Return kTRUE is the class inherits from TObject.
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
virtual Bool_t Merge(Bool_t=kTRUE)
Merge the files.
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Normal merge, overwritting the output file.
Bool_t OpenExcessFiles()
Open up to fMaxOpenedFiles of the excess files.
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Short_t Min(Short_t a, Short_t b)
virtual TDirectory * mkdir(const char *name, const char *title="")
Create a sub-directory and return a pointer to the created directory.
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Int_t GetEntriesFast() const
virtual const char * GetPath() const
Returns the full path of the directory.
Bool_t fCompressionChange
True if the output and input have different compression level (default kFALSE)
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Only the objects specified in fObjectNames list.
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=1, Int_t netopt=0)
Create / open a file.
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
virtual void RecursiveRemove(TObject *obj)
Intercept the case where the output TFile is deleted!
const char * Data() const
virtual void Fatal(const char *method, const char *msgfmt,...) const
Issue fatal error message.
virtual int Unlink(const char *name)
Unlink, i.e. remove, a file.
Merge all type of objects (default)
virtual TObject * After(const TObject *obj) const
Returns the object after object obj.
Int_t GetCompressionLevel() const
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
TList * fExcessFiles
! List of TObjString containing the name of the files not yet added to fFileList due to user or syste...
Long64_t(* MergeFunc_t)(void *, TCollection *, TFileMergeInfo *)
void Clear()
Clear string without changing its capacity.
Skip objects specified in fObjectNames list.
TString & Append(const char *cs)
virtual Int_t WriteObjectAny(const void *, const char *, const char *, Option_t *="", Int_t=0)
Merge the input file with the content of the output file (if already exising).
Book space in a file, create I/O buffers, to fill them, (un)compress them.
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Bool_t fHistoOneGo
Merger histos in one go (default is kTRUE)
void SetMsgPrefix(const char *prefix)
Set the prefix to be used when printing informational message.
virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress=kTRUE)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
TFileMerger(const TFileMerger &)
virtual Bool_t OutputFile(const char *url, Bool_t force)
Open merger output file.
TFile * fOutputFile
The outputfile for merging.
TClassRef R__TTree_Class("TTree")
TList * fMergeList
list of TObjString containing the name of the files need to be merged
virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
static const Int_t kCpProgress
TString fOutputFilename
The name of the outputfile for merging.
R__EXTERN TSystem * gSystem
static Int_t R__GetSystemMaxOpenedFiles()
Return the maximum number of allowed opened files minus some wiggle room for CINT or at least of the ...
virtual Int_t Write(const char *name=0, Int_t opt=0, Int_t bufsiz=0)
Write memory objects to this file.
virtual void Execute(const char *method, const char *params, Int_t *error=0)
Execute method on this object with the given parameter string, e.g.
This class provides file copy and merging services.
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Collection abstract base class.
Keep compression level unchanged for each input files.
void Destructor(void *obj, Bool_t dtorOnly=kFALSE)
Explicitly call destructor for object.
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Bool_t TestBit(UInt_t f) const
TString fMsgPrefix
Prefix to be used when printing informational message (default TFileMerger)
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
virtual const char * GetName() const
Returns name of object.
The ROOT global object gROOT contains a list of all defined classes.
virtual TDirectory * GetDirectory(const char *apath, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory named "apath".
TString fObjectNames
List of object names to be either merged exclusively or skipped.
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
ClassImp(TFileMerger) TClassRef R__TH1_Class("TH1")
void SetMaxOpenedFiles(Int_t newmax)
Set a limit to the number file that TFileMerger will opened at one time.
virtual void SaveSelf(Bool_t=kFALSE)
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
TString & Remove(Ssiz_t pos)
virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type=kRegular|kAll)
Merge all objects in a directory.
virtual Int_t GetSize() const
virtual const char * GetName() const
Returns name of object.
Describe directory structure in memory.
double func(double *x, double *p)
static const Int_t kCintFileNumber
Bool_t fLocal
Makes local copies of merging files if True (default is kTRUE)
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
virtual void Clear(Option_t *option="")
Remove all objects from the list.
Mother of all ROOT objects.
Bool_t fFastMethod
True if using Fast merging algorithm (default)
Only the objects without a MergeAfterReset member function.
virtual TObject * First() const
Return the first object in the list. Returns 0 when list is empty.
TClassRef is used to implement a permanent reference to a TClass object.
virtual TObject * ReadObj()
To read a TObject* from the file.
Bool_t fNoTrees
True if Trees should not be merged (default is kFALSE)
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
virtual Bool_t PartialMerge(Int_t type=kAll|kIncremental)
Merge the files.
virtual void Add(TObject *obj)
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
virtual TDirectory * GetDirectory(const char *namecycle, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory using apath.
virtual ~TFileMerger()
Cleanup.
A TTree object has a header with a name and a title.
Bool_t InheritsFrom(const char *cl) const
Return kTRUE if this class inherits from a class with name "classname".
virtual void PrintFiles(Option_t *options)
Print list of files being merged.
ROOT::ResetAfterMergeFunc_t GetResetAfterMerge() const
Return the wrapper around Merge.
TObject * At(Int_t idx) const
Only the objects with a MergeAfterReset member function.
TMethod * GetMethodWithPrototype(const char *method, const char *proto, Bool_t objectIsConst=kFALSE, ROOT::EFunctionMatchMode mode=ROOT::kConversionMatch)
Find the method with a given prototype.
virtual void Reset()
Reset merger file list.
virtual const char * GetTitle() const
Returns title of object.
Int_t fMaxOpenedFiles
Maximum number of files opened at the same time by the TFileMerger.
TList * fFileList
A list the file (TFile*) which shall be merged.
virtual void Close(Option_t *option="")
Close a file.
Bool_t fExplicitCompLevel
True if the user explicitly requested a compressio level change (default kFALSE)
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.