Logo ROOT   6.14/05
Reference Guide
TFileMerger.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Andreas Peters + Fons Rademakers + Rene Brun 26/5/2005
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13 \class TFileMerger TFileMerger.cxx
14 \ingroup IO
15 
16 This class provides file copy and merging services.
17 
18 It can be used to copy files (not only ROOT files), using TFile or
19 any of its remote file access plugins. It is therefore useful in
20 a Grid environment where the files might be accessible via Castor,
21 rfio, dcap, etc.
22 The merging interface allows files containing histograms and trees
23 to be merged, like the standalone hadd program.
24 */
25 
26 #include "TFileMerger.h"
27 #include "TDirectory.h"
28 #include "TUrl.h"
29 #include "TFile.h"
30 #include "TUUID.h"
31 #include "TSystem.h"
32 #include "TKey.h"
33 #include "THashList.h"
34 #include "TObjString.h"
35 #include "TClass.h"
36 #include "TMethodCall.h"
37 #include "Riostream.h"
38 #include "TFileMergeInfo.h"
39 #include "TClassRef.h"
40 #include "TROOT.h"
41 #include "TMemFile.h"
42 #include "TVirtualMutex.h"
43 
44 #ifdef WIN32
45 // For _getmaxstdio
46 #include <stdio.h>
47 #else
48 // For getrlimit
49 #include <sys/time.h>
50 #include <sys/resource.h>
51 #endif
52 
54 
55 TClassRef R__TH1_Class("TH1");
56 TClassRef R__TTree_Class("TTree");
57 
58 static const Int_t kCpProgress = BIT(14);
59 static const Int_t kCintFileNumber = 100;
60 ////////////////////////////////////////////////////////////////////////////////
61 /// Return the maximum number of allowed opened files minus some wiggle room
62 /// for CINT or at least of the standard library (stdio).
63 
65 {
66  int maxfiles;
67 #ifdef WIN32
68  maxfiles = _getmaxstdio();
69 #else
70  rlimit filelimit;
71  if (getrlimit(RLIMIT_NOFILE,&filelimit)==0) {
72  maxfiles = filelimit.rlim_cur;
73  } else {
74  // We could not get the value from getrlimit, let's return a reasonable default.
75  maxfiles = 512;
76  }
77 #endif
78  if (maxfiles > kCintFileNumber) {
79  return maxfiles - kCintFileNumber;
80  } else if (maxfiles > 5) {
81  return maxfiles - 5;
82  } else {
83  return maxfiles;
84  }
85 }
86 
87 ////////////////////////////////////////////////////////////////////////////////
88 /// Create file merger object.
89 
91  : fMaxOpenedFiles( R__GetSystemMaxOpenedFiles() ),
92  fLocal(isLocal), fHistoOneGo(histoOneGo)
93 {
96 
98  gROOT->GetListOfCleanups()->Add(this);
99 }
100 
101 ////////////////////////////////////////////////////////////////////////////////
102 /// Cleanup.
103 
105 {
106  {
108  gROOT->GetListOfCleanups()->Remove(this);
109  }
111 }
112 
113 ////////////////////////////////////////////////////////////////////////////////
114 /// Reset merger file list.
115 
117 {
118  fFileList.Clear();
119  fMergeList.Clear();
122 }
123 
124 ////////////////////////////////////////////////////////////////////////////////
125 /// Add file to file merger.
126 
127 Bool_t TFileMerger::AddFile(const char *url, Bool_t cpProgress)
128 {
129  if (fPrintLevel > 0) {
130  Printf("%s Source file %d: %s", fMsgPrefix.Data(), fFileList.GetEntries() + fExcessFiles.GetEntries() + 1, url);
131  }
132 
133  TFile *newfile = 0;
134  TString localcopy;
135 
136  if (fFileList.GetEntries() >= (fMaxOpenedFiles-1)) {
137 
138  TObjString *urlObj = new TObjString(url);
139  fMergeList.Add(urlObj);
140 
141  urlObj = new TObjString(url);
142  urlObj->SetBit(kCpProgress);
143  fExcessFiles.Add(urlObj);
144  return kTRUE;
145  }
146 
147  // We want gDirectory untouched by anything going on here
149 
150  if (fLocal) {
151  TUUID uuid;
152  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
153  if (!TFile::Cp(url, localcopy, cpProgress)) {
154  Error("AddFile", "cannot get a local copy of file %s", url);
155  return kFALSE;
156  }
157  newfile = TFile::Open(localcopy, "READ");
158  } else {
159  newfile = TFile::Open(url, "READ");
160  }
161 
162  // Zombie files should also be skipped
163  if (newfile && newfile->IsZombie()) {
164  delete newfile;
165  newfile = 0;
166  }
167 
168  if (!newfile) {
169  if (fLocal)
170  Error("AddFile", "cannot open local copy %s of URL %s",
171  localcopy.Data(), url);
172  else
173  Error("AddFile", "cannot open file %s", url);
174  return kFALSE;
175  } else {
176  if (fOutputFile && fOutputFile->GetCompressionLevel() != newfile->GetCompressionLevel()) fCompressionChange = kTRUE;
177 
178  newfile->SetBit(kCanDelete);
179  fFileList.Add(newfile);
180 
181  TObjString *urlObj = new TObjString(url);
182  fMergeList.Add(urlObj);
183 
184  return kTRUE;
185  }
186 }
187 
188 ////////////////////////////////////////////////////////////////////////////////
189 /// Add the TFile to this file merger and *do not* give ownership of the TFile to this
190 /// object.
191 ///
192 /// Return kTRUE if the addition was successful.
193 
195 {
196  return AddFile(source,kFALSE,cpProgress);
197 }
198 
199 ////////////////////////////////////////////////////////////////////////////////
200 /// Add the TFile to this file merger and give ownership of the TFile to this
201 /// object (unless kFALSE is returned).
202 ///
203 /// Return kTRUE if the addition was successful.
204 
206 {
207  return AddFile(source,kTRUE,cpProgress);
208 }
209 
210 ////////////////////////////////////////////////////////////////////////////////
211 /// Add the TFile to this file merger and give ownership of the TFile to this
212 /// object (unless kFALSE is returned).
213 ///
214 /// Return kTRUE if the addition was successful.
215 
216 Bool_t TFileMerger::AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
217 {
218  if (source == 0 || source->IsZombie()) {
219  return kFALSE;
220  }
221 
222  if (fPrintLevel > 0) {
223  Printf("%s Source file %d: %s",fMsgPrefix.Data(),fFileList.GetEntries()+1,source->GetName());
224  }
225 
226  TFile *newfile = 0;
227  TString localcopy;
228 
229  // We want gDirectory untouched by anything going on here
231  if (fLocal && !source->InheritsFrom(TMemFile::Class())) {
232  TUUID uuid;
233  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
234  if (!source->Cp(localcopy, cpProgress)) {
235  Error("AddFile", "cannot get a local copy of file %s", source->GetName());
236  return kFALSE;
237  }
238  newfile = TFile::Open(localcopy, "READ");
239  // Zombie files should also be skipped
240  if (newfile && newfile->IsZombie()) {
241  delete newfile;
242  newfile = 0;
243  }
244  } else {
245  newfile = source;
246  }
247 
248  if (!newfile) {
249  if (fLocal)
250  Error("AddFile", "cannot open local copy %s of URL %s",
251  localcopy.Data(), source->GetName());
252  else
253  Error("AddFile", "cannot open file %s", source->GetName());
254  return kFALSE;
255  } else {
256  if (fOutputFile && fOutputFile->GetCompressionSettings() != newfile->GetCompressionSettings()) fCompressionChange = kTRUE;
257 
258  if (own || newfile != source) {
259  newfile->SetBit(kCanDelete);
260  } else {
261  newfile->ResetBit(kCanDelete);
262  }
263  fFileList.Add(newfile);
264 
265  TObjString *urlObj = new TObjString(source->GetName());
266  fMergeList.Add(urlObj);
267 
268  if (newfile != source && own) {
269  delete source;
270  }
271  return kTRUE;
272  }
273 }
274 
275 ////////////////////////////////////////////////////////////////////////////////
276 /// Open merger output file.
277 
278 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force, Int_t compressionLevel)
279 {
280  return OutputFile(outputfile,(force?"RECREATE":"CREATE"),compressionLevel);
281 }
282 
283 ////////////////////////////////////////////////////////////////////////////////
284 /// Open merger output file.
285 
286 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force)
287 {
288  Bool_t res = OutputFile(outputfile,(force?"RECREATE":"CREATE"),1); // 1 is the same as the default from the TFile constructor.
290  return res;
291 }
292 
293 ////////////////////////////////////////////////////////////////////////////////
294 /// Open merger output file.
295 ///
296 /// The 'mode' parameter is passed to the TFile constructor as the option, it
297 /// should be one of 'NEW','CREATE','RECREATE','UPDATE'
298 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
299 
300 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode, Int_t compressionLevel)
301 {
302  // We want gDirectory untouched by anything going on here
304  if (TFile *outputFile = TFile::Open(outputfile, mode, "", compressionLevel))
305  return OutputFile(std::unique_ptr<TFile>(outputFile));
306 
307  Error("OutputFile", "cannot open the MERGER output file %s", fOutputFilename.Data());
308  return kFALSE;
309 }
310 
311 ////////////////////////////////////////////////////////////////////////////////
312 /// Set an output file opened externally by the users
313 
314 Bool_t TFileMerger::OutputFile(std::unique_ptr<TFile> outputfile)
315 {
316  if (!outputfile || outputfile->IsZombie()) {
317  Error("OutputFile", "cannot open the MERGER output file %s", (outputfile) ? outputfile->GetName() : "");
318  return kFALSE;
319  }
320 
321  if (!outputfile->IsWritable()) {
322  Error("OutputFile", "output file %s is not writable", outputfile->GetName());
323  return kFALSE;
324  }
325 
327 
328  TFile *oldfile = fOutputFile;
329  fOutputFile = 0; // This avoids the complaint from RecursiveRemove about the file being deleted which is here
330  // spurrious. (see RecursiveRemove).
331  SafeDelete(oldfile);
332 
333  fOutputFilename = outputfile->GetName();
334  // We want gDirectory untouched by anything going on here
336  fOutputFile = outputfile.release(); // Transfer the ownership of the file.
337 
338  return kTRUE;
339 }
340 
341 ////////////////////////////////////////////////////////////////////////////////
342 /// Open merger output file. 'mode' is passed to the TFile constructor as the option, it should
343 /// be one of 'NEW','CREATE','RECREATE','UPDATE'
344 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
345 
346 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode /* = "RECREATE" */)
347 {
348  Bool_t res = OutputFile(outputfile,mode,1); // 1 is the same as the default from the TFile constructor.
350  return res;
351 }
352 
353 ////////////////////////////////////////////////////////////////////////////////
354 /// Print list of files being merged.
355 
357 {
358  fFileList.Print(options);
359  fExcessFiles.Print(options);
360 }
361 
362 ////////////////////////////////////////////////////////////////////////////////
363 /// Merge the files.
364 ///
365 /// If no output file was specified it will write into
366 /// the file "FileMerger.root" in the working directory. Returns true
367 /// on success, false in case of error.
368 
370 {
371  return PartialMerge(kAll | kRegular);
372 }
373 
374 ////////////////////////////////////////////////////////////////////////////////
375 /// Merge all objects in a directory
376 ///
377 /// The type is defined by the bit values in TFileMerger::EPartialMergeType.
378 
379 Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type /* = kRegular | kAll */)
380 {
381  Bool_t status = kTRUE;
382  Bool_t onlyListed = kFALSE;
383  if (fPrintLevel > 0) {
384  Printf("%s Target path: %s",fMsgPrefix.Data(),target->GetPath());
385  }
386 
387  // Get the dir name
388  TString path(target->GetPath());
389  // coverity[unchecked_value] 'target' is from a file so GetPath always returns path starting with filename:
390  path.Remove(0, path.Last(':') + 2);
391 
392  Int_t nguess = sourcelist->GetSize()+1000;
393  THashList allNames(nguess);
394  allNames.SetOwner(kTRUE);
395  // If the mode is set to skipping list objects, add names to the allNames list
396  if (type & kSkipListed) {
397  TObjArray *arr = fObjectNames.Tokenize(" ");
398  arr->SetOwner(kFALSE);
399  for (Int_t iname=0; iname<arr->GetEntriesFast(); iname++)
400  allNames.Add(arr->At(iname));
401  delete arr;
402  }
403  ((THashList*)target->GetList())->Rehash(nguess);
404  ((THashList*)target->GetListOfKeys())->Rehash(nguess);
405 
406  TFileMergeInfo info(target);
407  info.fIOFeatures = fIOFeatures;
408  info.fOptions = fMergeOptions;
409  if (fFastMethod && ((type&kKeepCompression) || !fCompressionChange) ) {
410  info.fOptions.Append(" fast");
411  }
412 
413  TFile *current_file;
414  TDirectory *current_sourcedir;
415  if (type & kIncremental) {
416  current_file = 0;
417  current_sourcedir = target;
418  } else {
419  current_file = (TFile*)sourcelist->First();
420  current_sourcedir = current_file->GetDirectory(path);
421  }
422  while (current_file || current_sourcedir) {
423  // When current_sourcedir != 0 and current_file == 0 we are going over the target
424  // for an incremental merge.
425  if (current_sourcedir && (current_file == 0 || current_sourcedir != target)) {
426 
427  // loop over all keys in this directory
428  TIter nextkey( current_sourcedir->GetListOfKeys() );
429  TKey *key;
430  TString oldkeyname;
431 
432  while ( (key = (TKey*)nextkey())) {
433 
434  // Keep only the highest cycle number for each key for mergeable objects. They are stored
435  // in the (hash) list consecutively and in decreasing order of cycles, so we can continue
436  // until the name changes. We flag the case here and we act consequently later.
437  Bool_t alreadyseen = (oldkeyname == key->GetName()) ? kTRUE : kFALSE;
438 
439  // Read in but do not copy directly the processIds.
440  if (strcmp(key->GetClassName(),"TProcessID") == 0) { key->ReadObj(); continue;}
441 
442  // If we have already seen this object [name], we already processed
443  // the whole list of files for this objects and we can just skip it
444  // and any related cycles.
445  if (allNames.FindObject(key->GetName())) {
446  oldkeyname = key->GetName();
447  continue;
448  }
449 
450  TClass *cl = TClass::GetClass(key->GetClassName());
451  if (!cl) {
452  Info("MergeRecursive", "cannot indentify object type (%s), name: %s title: %s",
453  key->GetClassName(), key->GetName(), key->GetTitle());
454  continue;
455  }
456  // For mergeable objects we add the names in a local hashlist handling them
457  // again (see above)
458  if (cl->GetMerge() || cl->InheritsFrom(TDirectory::Class()) ||
459  (cl->IsTObject() &&
460  (cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ||
461  cl->GetMethodWithPrototype("Merge", "TCollection*"))))
462  allNames.Add(new TObjString(key->GetName()));
463 
464  if (fNoTrees && cl->InheritsFrom(R__TTree_Class)) {
465  // Skip the TTree objects and any related cycles.
466  oldkeyname = key->GetName();
467  continue;
468  }
469  // Check if only the listed objects are to be merged
470  if (type & kOnlyListed) {
471  onlyListed = kFALSE;
472  oldkeyname = key->GetName();
473  oldkeyname += " ";
474  onlyListed = fObjectNames.Contains(oldkeyname);
475  oldkeyname = key->GetName();
476  if ((!onlyListed) && (!cl->InheritsFrom(TDirectory::Class()))) continue;
477  }
478 
479  if (!(type&kResetable && type&kNonResetable)) {
480  // If neither or both are requested at the same time, we merger both types.
481  if (!(type&kResetable)) {
482  if (cl->GetResetAfterMerge()) {
483  // Skip the object with a reset after merge routine (TTree and other incrementally mergeable objects)
484  oldkeyname = key->GetName();
485  continue;
486  }
487  }
488  if (!(type&kNonResetable)) {
489  if (!cl->GetResetAfterMerge()) {
490  // Skip the object without a reset after merge routine (Histograms and other non incrementally mergeable objects)
491  oldkeyname = key->GetName();
492  continue;
493  }
494  }
495  }
496  // read object from first source file
497  TObject *obj;
498  if (type & kIncremental) {
499  obj = current_sourcedir->GetList()->FindObject(key->GetName());
500  if (!obj) {
501  obj = key->ReadObj();
502  }
503  } else {
504  obj = key->ReadObj();
505  }
506  if (!obj) {
507  Info("MergeRecursive", "could not read object for key {%s, %s}",
508  key->GetName(), key->GetTitle());
509  continue;
510  }
511  if (cl->IsTObject() && cl != obj->IsA()) {
512  Error("MergeRecursive", "TKey and object retrieve disagree on type (%s vs %s). Continuing with %s.",
513  key->GetClassName(), obj->IsA()->GetName(), obj->IsA()->GetName());
514  cl = obj->IsA();
515  }
516  Bool_t canBeMerged = kTRUE;
517 
518  if ( cl->InheritsFrom( TDirectory::Class() ) ) {
519  // it's a subdirectory
520 
521  target->cd();
522  TDirectory *newdir;
523 
524  // For incremental or already seen we may have already a directory created
525  if (type & kIncremental || alreadyseen) {
526  newdir = target->GetDirectory(obj->GetName());
527  if (!newdir) {
528  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
529  }
530  } else {
531  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
532  }
533 
534  // newdir is now the starting point of another round of merging
535  // newdir still knows its depth within the target file via
536  // GetPath(), so we can still figure out where we are in the recursion
537 
538  // If this folder is a onlyListed object, merge everything inside.
539  if (onlyListed) type &= ~kOnlyListed;
540  status = MergeRecursive(newdir, sourcelist, type);
541  if (onlyListed) type |= kOnlyListed;
542  if (!status) return status;
543  } else if (cl->GetMerge()) {
544 
545  // Check if already treated
546  if (alreadyseen) continue;
547 
548  TList inputs;
550 
551  // Loop over all source files and merge same-name object
552  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
553  if (nextsource == 0) {
554  // There is only one file in the list
555  ROOT::MergeFunc_t func = cl->GetMerge();
556  func(obj, &inputs, &info);
557  info.fIsFirst = kFALSE;
558  } else {
559  do {
560  // make sure we are at the correct directory level by cd'ing to path
561  TDirectory *ndir = nextsource->GetDirectory(path);
562  if (ndir) {
563  ndir->cd();
564  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
565  if (key2) {
566  TObject *hobj = key2->ReadObj();
567  if (!hobj) {
568  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
569  key->GetName(), key->GetTitle(), nextsource->GetName());
570  nextsource = (TFile*)sourcelist->After(nextsource);
571  continue;
572  }
573  // Set ownership for collections
574  if (hobj->InheritsFrom(TCollection::Class())) {
575  ((TCollection*)hobj)->SetOwner();
576  }
577  hobj->ResetBit(kMustCleanup);
578  inputs.Add(hobj);
579  if (!oneGo) {
580  ROOT::MergeFunc_t func = cl->GetMerge();
581  Long64_t result = func(obj, &inputs, &info);
582  info.fIsFirst = kFALSE;
583  if (result < 0) {
584  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
585  obj->GetName(), nextsource->GetName());
586  }
587  inputs.Delete();
588  }
589  }
590  }
591  nextsource = (TFile*)sourcelist->After( nextsource );
592  } while (nextsource);
593  // Merge the list, if still to be done
594  if (oneGo || info.fIsFirst) {
595  ROOT::MergeFunc_t func = cl->GetMerge();
596  func(obj, &inputs, &info);
597  info.fIsFirst = kFALSE;
598  inputs.Delete();
599  }
600  }
601  } else if (cl->IsTObject() &&
602  cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ) {
603  // Object implements Merge(TCollection*,TFileMergeInfo*) and has a reflex dictionary ...
604 
605  // Check if already treated
606  if (alreadyseen) continue;
607 
608  TList listH;
609  TString listHargs;
610  listHargs.Form("(TCollection*)0x%lx,(TFileMergeInfo*)0x%lx", (ULong_t)&listH,(ULong_t)&info);
611 
612  // Loop over all source files and merge same-name object
613  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
614  if (nextsource == 0) {
615  // There is only one file in the list
616  Int_t error = 0;
617  obj->Execute("Merge", listHargs.Data(), &error);
618  info.fIsFirst = kFALSE;
619  if (error) {
620  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
621  obj->GetName(), key->GetName());
622  }
623  } else {
624  while (nextsource) {
625  // make sure we are at the correct directory level by cd'ing to path
626  TDirectory *ndir = nextsource->GetDirectory(path);
627  if (ndir) {
628  ndir->cd();
629  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
630  if (key2) {
631  TObject *hobj = key2->ReadObj();
632  if (!hobj) {
633  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
634  key->GetName(), key->GetTitle(), nextsource->GetName());
635  nextsource = (TFile*)sourcelist->After(nextsource);
636  continue;
637  }
638  // Set ownership for collections
639  if (hobj->InheritsFrom(TCollection::Class())) {
640  ((TCollection*)hobj)->SetOwner();
641  }
642  hobj->ResetBit(kMustCleanup);
643  listH.Add(hobj);
644  Int_t error = 0;
645  obj->Execute("Merge", listHargs.Data(), &error);
646  info.fIsFirst = kFALSE;
647  if (error) {
648  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
649  obj->GetName(), nextsource->GetName());
650  }
651  listH.Delete();
652  }
653  }
654  nextsource = (TFile*)sourcelist->After( nextsource );
655  }
656  // Merge the list, if still to be done
657  if (info.fIsFirst) {
658  Int_t error = 0;
659  obj->Execute("Merge", listHargs.Data(), &error);
660  info.fIsFirst = kFALSE;
661  listH.Delete();
662  }
663  }
664  } else if (cl->IsTObject() &&
665  cl->GetMethodWithPrototype("Merge", "TCollection*") ) {
666  // Object implements Merge(TCollection*) and has a reflex dictionary ...
667 
668  // Check if already treated
669  if (alreadyseen) continue;
670 
671  TList listH;
672  TString listHargs;
673  listHargs.Form("((TCollection*)0x%lx)", (ULong_t)&listH);
674 
675  // Loop over all source files and merge same-name object
676  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
677  if (nextsource == 0) {
678  // There is only one file in the list
679  Int_t error = 0;
680  obj->Execute("Merge", listHargs.Data(), &error);
681  if (error) {
682  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
683  obj->GetName(), key->GetName());
684  }
685  } else {
686  while (nextsource) {
687  // make sure we are at the correct directory level by cd'ing to path
688  TDirectory *ndir = nextsource->GetDirectory(path);
689  if (ndir) {
690  ndir->cd();
691  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
692  if (key2) {
693  TObject *hobj = key2->ReadObj();
694  if (!hobj) {
695  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
696  key->GetName(), key->GetTitle(), nextsource->GetName());
697  nextsource = (TFile*)sourcelist->After(nextsource);
698  continue;
699  }
700  // Set ownership for collections
701  if (hobj->InheritsFrom(TCollection::Class())) {
702  ((TCollection*)hobj)->SetOwner();
703  }
704  hobj->ResetBit(kMustCleanup);
705  listH.Add(hobj);
706  Int_t error = 0;
707  obj->Execute("Merge", listHargs.Data(), &error);
708  info.fIsFirst = kFALSE;
709  if (error) {
710  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
711  obj->GetName(), nextsource->GetName());
712  }
713  listH.Delete();
714  }
715  }
716  nextsource = (TFile*)sourcelist->After( nextsource );
717  }
718  // Merge the list, if still to be done
719  if (info.fIsFirst) {
720  Int_t error = 0;
721  obj->Execute("Merge", listHargs.Data(), &error);
722  info.fIsFirst = kFALSE;
723  listH.Delete();
724  }
725  }
726  } else {
727  // Object is of no type that we can merge
728  canBeMerged = kFALSE;
729  }
730 
731  // now write the merged histogram (which is "in" obj) to the target file
732  // note that this will just store obj in the current directory level,
733  // which is not persistent until the complete directory itself is stored
734  // by "target->SaveSelf()" below
735  target->cd();
736 
737  oldkeyname = key->GetName();
738  //!!if the object is a tree, it is stored in globChain...
739  if(cl->InheritsFrom( TDirectory::Class() )) {
740  //printf("cas d'une directory\n");
741  // Do not delete the directory if it is part of the output
742  // and we are in incremental mode (because it will be reuse
743  // and has not been written to disk (for performance reason).
744  // coverity[var_deref_model] the IsA()->InheritsFrom guarantees that the dynamic_cast will succeed.
745  if (!(type&kIncremental) || dynamic_cast<TDirectory*>(obj)->GetFile() != target) {
746  delete obj;
747  }
748  } else if (cl->InheritsFrom( TCollection::Class() )) {
749  // Don't overwrite, if the object were not merged.
750  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kSingleKey | TObject::kOverwrite : TObject::kSingleKey) <= 0 ) {
751  status = kFALSE;
752  }
753  ((TCollection*)obj)->SetOwner();
754  delete obj;
755  } else {
756  // Don't overwrite, if the object were not merged.
757  // NOTE: this is probably wrong for emulated objects.
758  if (cl->IsTObject()) {
759  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kOverwrite : 0) <= 0) {
760  status = kFALSE;
761  }
762  } else {
763  if ( target->WriteObjectAny( (void*)obj, cl, oldkeyname, canBeMerged ? "OverWrite" : "" ) <= 0) {
764  status = kFALSE;
765  }
766  }
767  cl->Destructor(obj); // just in case the class is not loaded.
768  }
769  info.Reset();
770  } // while ( ( TKey *key = (TKey*)nextkey() ) )
771  }
772  current_file = current_file ? (TFile*)sourcelist->After(current_file) : (TFile*)sourcelist->First();
773  if (current_file) {
774  current_sourcedir = current_file->GetDirectory(path);
775  } else {
776  current_sourcedir = 0;
777  }
778  }
779  // save modifications to the target directory.
780  if (!(type&kIncremental)) {
781  // In case of incremental build, we will call Write on the top directory/file, so we do not need
782  // to call SaveSelf explicilty.
783  target->SaveSelf(kTRUE);
784  }
785  return status;
786 }
787 
788 ////////////////////////////////////////////////////////////////////////////////
789 /// Merge the files. If no output file was specified it will write into
790 /// the file "FileMerger.root" in the working directory. Returns true
791 /// on success, false in case of error.
792 /// The type is defined by the bit values in EPartialMergeType:
793 /// kRegular : normal merge, overwritting the output file
794 /// kIncremental : merge the input file with the content of the output file (if already exising) (default)
795 /// kAll : merge all type of objects (default)
796 /// kResetable : merge only the objects with a MergeAfterReset member function.
797 /// kNonResetable : merge only the objects without a MergeAfterReset member function.
798 ///
799 /// If the type is set to kIncremental the output file is done deleted at the end of
800 /// this operation. If the type is not set to kIncremental, the output file is closed.
801 
803 {
804  if (!fOutputFile) {
805  TString outf(fOutputFilename);
806  if (outf.IsNull()) {
807  outf.Form("file:%s/FileMerger.root", gSystem->TempDirectory());
808  Info("PartialMerge", "will merge the results to the file %s\n"
809  "since you didn't specify a merge filename",
810  TUrl(outf).GetFile());
811  }
812  if (!OutputFile(outf.Data())) {
813  return kFALSE;
814  }
815  }
816 
817  // Special treament for the single file case ...
818  if ((fFileList.GetEntries() == 1) && !fExcessFiles.GetEntries() &&
819  !(in_type & kIncremental) && !fCompressionChange && !fExplicitCompLevel) {
820  fOutputFile->Close();
822 
823  TFile *file = (TFile *) fFileList.First();
824  if (!file || (file && file->IsZombie())) {
825  Error("PartialMerge", "one-file case: problem attaching to file");
826  return kFALSE;
827  }
828  Bool_t result = kTRUE;
829  if (!(result = file->Cp(fOutputFilename))) {
830  Error("PartialMerge", "one-file case: could not copy '%s' to '%s'",
831  file->GetPath(), fOutputFilename.Data());
832  return kFALSE;
833  }
834  if (file->TestBit(kCanDelete)) file->Close();
835 
836  // Remove the temporary file
837  if (fLocal && !file->InheritsFrom(TMemFile::Class())) {
838  TUrl u(file->GetPath(), kTRUE);
839  if (gSystem->Unlink(u.GetFile()) != 0)
840  Warning("PartialMerge", "problems removing temporary local file '%s'", u.GetFile());
841  }
842  fFileList.Clear();
843  return result;
844  }
845 
847 
849 
850  Bool_t result = kTRUE;
851  Int_t type = in_type;
852  while (result && fFileList.GetEntries()>0) {
853  result = MergeRecursive(fOutputFile, &fFileList, type);
854 
855  // Remove local copies if there are any
856  TIter next(&fFileList);
857  TFile *file;
858  while ((file = (TFile*) next())) {
859  // close the files
860  if (file->TestBit(kCanDelete)) file->Close();
861  // remove the temporary files
862  if(fLocal && !file->InheritsFrom(TMemFile::Class())) {
863  TString p(file->GetPath());
864  // coverity[unchecked_value] Index is return a value with range or NPos to select the whole name.
865  p = p(0, p.Index(':',0));
866  gSystem->Unlink(p);
867  }
868  }
869  fFileList.Clear();
870  if (result && fExcessFiles.GetEntries() > 0) {
871  // We merge the first set of files in the output,
872  // we now need to open the next set and make
873  // sure we accumulate into the output, so we
874  // switch to incremental merging (if not already set)
875  type = type | kIncremental;
876  result = OpenExcessFiles();
877  }
878  }
879  if (!result) {
880  Error("Merge", "error during merge of your ROOT files");
881  } else {
882  // Close or write is required so the file is complete.
883  if (in_type & kIncremental) {
885  } else {
886  fOutputFile->Close();
887  }
888  }
889 
890  // Cleanup
891  if (in_type & kIncremental) {
892  Clear();
893  } else {
896  }
897  return result;
898 }
899 
900 ////////////////////////////////////////////////////////////////////////////////
901 /// Open up to fMaxOpenedFiles of the excess files.
902 
904 {
905  if (fPrintLevel > 0) {
906  Printf("%s Opening the next %d files", fMsgPrefix.Data(), TMath::Min(fExcessFiles.GetEntries(), fMaxOpenedFiles - 1));
907  }
908  Int_t nfiles = 0;
909  TIter next(&fExcessFiles);
910  TObjString *url = 0;
911  TString localcopy;
912  // We want gDirectory untouched by anything going on here
914  while( nfiles < (fMaxOpenedFiles-1) && ( url = (TObjString*)next() ) ) {
915  TFile *newfile = 0;
916  if (fLocal) {
917  TUUID uuid;
918  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
919  if (!TFile::Cp(url->GetName(), localcopy, url->TestBit(kCpProgress))) {
920  Error("OpenExcessFiles", "cannot get a local copy of file %s", url->GetName());
921  return kFALSE;
922  }
923  newfile = TFile::Open(localcopy, "READ");
924  } else {
925  newfile = TFile::Open(url->GetName(), "READ");
926  }
927 
928  if (!newfile) {
929  if (fLocal)
930  Error("OpenExcessFiles", "cannot open local copy %s of URL %s",
931  localcopy.Data(), url->GetName());
932  else
933  Error("OpenExcessFiles", "cannot open file %s", url->GetName());
934  return kFALSE;
935  } else {
937 
938  newfile->SetBit(kCanDelete);
939  fFileList.Add(newfile);
940  ++nfiles;
941  fExcessFiles.Remove(url);
942  }
943  }
944  return kTRUE;
945 }
946 
947 ////////////////////////////////////////////////////////////////////////////////
948 /// Intercept the case where the output TFile is deleted!
949 
951 {
952  if (obj == fOutputFile) {
953  Fatal("RecursiveRemove","Output file of the TFile Merger (targeting %s) has been deleted (likely due to a TTree larger than 100Gb)", fOutputFilename.Data());
954  }
955 
956 }
957 
958 ////////////////////////////////////////////////////////////////////////////////
959 /// Set a limit to the number of files that TFileMerger will open simultaneously.
960 ///
961 /// If the request is higher than the system limit, we reset it to the system limit.
962 /// If the request is less than two, we reset it to 2 (one for the output file and one for the input file).
963 
965 {
967  if (newmax < sysmax) {
968  fMaxOpenedFiles = newmax;
969  } else {
970  fMaxOpenedFiles = sysmax;
971  }
972  if (fMaxOpenedFiles < 2) {
973  fMaxOpenedFiles = 2;
974  }
975 }
976 
977 ////////////////////////////////////////////////////////////////////////////////
978 /// Set the prefix to be used when printing informational message.
979 
980 void TFileMerger::SetMsgPrefix(const char *prefix)
981 {
982  fMsgPrefix = prefix;
983 }
984 
virtual void Clear(Option_t *="")
Definition: TObject.h:100
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:785
Int_t fPrintLevel
How much information to print out at run time.
Definition: TFileMerger.h:46
TList fExcessFiles
! List of TObjString containing the name of the files not yet added to fFileList due to user or syste...
Definition: TFileMerger.h:56
An array of TObjects.
Definition: TObjArray.h:37
virtual TList * GetListOfKeys() const
Definition: TDirectory.h:150
virtual void Delete(Option_t *option="")
Remove all objects from the list AND delete all heap based objects.
Definition: TList.cxx:467
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:854
long long Long64_t
Definition: RtypesCore.h:69
Collectable string class.
Definition: TObjString.h:28
Long64_t(* MergeFunc_t)(void *, TCollection *, TFileMergeInfo *)
Definition: Rtypes.h:111
const char Option_t
Definition: RtypesCore.h:62
This class represents a WWW compatible URL.
Definition: TUrl.h:35
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
#define BIT(n)
Definition: Rtypes.h:78
virtual Bool_t Merge(Bool_t=kTRUE)
Merge the files.
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:47
virtual Int_t GetEntries() const
Definition: TCollection.h:177
Normal merge, overwritting the output file.
Definition: TFileMerger.h:65
Bool_t OpenExcessFiles()
Open up to fMaxOpenedFiles of the excess files.
overwrite existing object with same name
Definition: TObject.h:88
#define gROOT
Definition: TROOT.h:410
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:172
Basic string class.
Definition: TString.h:131
Short_t Min(Short_t a, Short_t b)
Definition: TMathBase.h:168
int Int_t
Definition: RtypesCore.h:41
virtual TDirectory * mkdir(const char *name, const char *title="")
Create a sub-directory "a" or a hierarchy of sub-directories "a/b/c/...".
bool Bool_t
Definition: RtypesCore.h:59
R__EXTERN TVirtualMutex * gROOTMutex
Definition: TROOT.h:57
TFileMerger(const TFileMerger &)=delete
TString fMergeOptions
Options (in string format) to be passed down to the Merge functions.
Definition: TFileMerger.h:47
TObject * At(Int_t idx) const
Definition: TObjArray.h:165
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:262
Bool_t fCompressionChange
True if the output and input have different compression level (default kFALSE)
Definition: TFileMerger.h:45
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:694
virtual TObject * FindObject(const char *name) const
Delete a TObjLink object.
Definition: TList.cxx:574
if object in a list can be deleted
Definition: TObject.h:58
Only the objects specified in fObjectNames list.
Definition: TFileMerger.h:73
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=1, Int_t netopt=0)
Create / open a file.
Definition: TFile.cxx:3976
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
Definition: TFile.cxx:4874
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition: TUUID.h:42
virtual void RecursiveRemove(TObject *obj)
Intercept the case where the output TFile is deleted!
TList fMergeList
list of TObjString containing the name of the files need to be merged
Definition: TFileMerger.h:55
virtual int Unlink(const char *name)
Unlink, i.e. remove, a file.
Definition: TSystem.cxx:1357
Merge all type of objects (default)
Definition: TFileMerger.h:70
void Class()
Definition: Class.C:29
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:34
void Clear()
Clear string without changing its capacity.
Definition: TString.cxx:1151
Skip objects specified in fObjectNames list.
Definition: TFileMerger.h:74
TIOFeatures * fIOFeatures
TString & Append(const char *cs)
Definition: TString.h:559
virtual Int_t WriteObjectAny(const void *, const char *, const char *, Option_t *="", Int_t=0)
Definition: TDirectory.h:200
virtual TList * GetList() const
Definition: TDirectory.h:149
Merge the input file with the content of the output file (if already exising).
Definition: TFileMerger.h:66
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition: TSystem.cxx:1458
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:24
Bool_t fHistoOneGo
Merger histos in one go (default is kTRUE)
Definition: TFileMerger.h:53
ROOT::MergeFunc_t GetMerge() const
void SetMsgPrefix(const char *prefix)
Set the prefix to be used when printing informational message.
virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress=kTRUE)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
virtual Bool_t OutputFile(const char *url, Bool_t force)
Open merger output file.
TFile * fOutputFile
The outputfile for merging.
Definition: TFileMerger.h:40
R__ALWAYS_INLINE Bool_t IsZombie() const
Definition: TObject.h:134
TClassRef R__TTree_Class("TTree")
A doubly linked list.
Definition: TList.h:44
const char * GetName() const
Returns name of object.
Definition: TObjString.h:39
virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
TClassRef R__TH1_Class("TH1")
static const Int_t kCpProgress
Definition: TFileMerger.cxx:58
TString fOutputFilename
The name of the outputfile for merging.
Definition: TFileMerger.h:41
virtual TObject * First() const
Return the first object in the list. Returns 0 when list is empty.
Definition: TList.cxx:655
R__EXTERN TSystem * gSystem
Definition: TSystem.h:540
static Int_t R__GetSystemMaxOpenedFiles()
Return the maximum number of allowed opened files minus some wiggle room for CINT or at least of the ...
Definition: TFileMerger.cxx:64
write collection with single key
Definition: TObject.h:87
virtual Int_t Write(const char *name=0, Int_t opt=0, Int_t bufsiz=0)
Write memory objects to this file.
Definition: TFile.cxx:2336
TList fFileList
A list the file (TFile*) which shall be merged.
Definition: TFileMerger.h:39
virtual void Execute(const char *method, const char *params, Int_t *error=0)
Execute method on this object with the given parameter string, e.g.
Definition: TObject.cxx:277
This class provides file copy and merging services.
Definition: TFileMerger.h:30
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:818
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
Definition: TObject.cxx:443
Collection abstract base class.
Definition: TCollection.h:63
Keep compression level unchanged for each input files.
Definition: TFileMerger.h:75
void Destructor(void *obj, Bool_t dtorOnly=kFALSE)
Explicitly call destructor for object.
Definition: TClass.cxx:5149
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2264
Int_t GetEntriesFast() const
Definition: TObjArray.h:64
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:880
TString fMsgPrefix
Prefix to be used when printing informational message (default TFileMerger)
Definition: TFileMerger.h:49
The ROOT global object gROOT contains a list of all defined classes.
Definition: TClass.h:75
virtual TObject * After(const TObject *obj) const
Returns the object after object obj.
Definition: TList.cxx:327
Bool_t InheritsFrom(const char *cl) const
Return kTRUE if this class inherits from a class with name "classname".
Definition: TClass.cxx:4688
virtual TDirectory * GetDirectory(const char *apath, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory named "apath".
virtual const char * GetPath() const
Returns the full path of the directory.
Definition: TDirectory.cxx:987
TString fObjectNames
List of object names to be either merged exclusively or skipped.
Definition: TFileMerger.h:54
if object destructor must call RecursiveRemove()
Definition: TObject.h:60
#define Printf
Definition: TGeoToOCC.h:18
void SetMaxOpenedFiles(Int_t newmax)
Set a limit to the number of files that TFileMerger will open simultaneously.
const Bool_t kFALSE
Definition: RtypesCore.h:88
virtual void SaveSelf(Bool_t=kFALSE)
Definition: TDirectory.h:181
#define SafeDelete(p)
Definition: RConfig.h:529
TString & Remove(Ssiz_t pos)
Definition: TString.h:668
virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type=kRegular|kAll)
Merge all objects in a directory.
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2172
#define ClassImp(name)
Definition: Rtypes.h:359
ROOT::ResetAfterMergeFunc_t GetResetAfterMerge() const
Describe directory structure in memory.
Definition: TDirectory.h:34
int type
Definition: TGX11.cxx:120
unsigned long ULong_t
Definition: RtypesCore.h:51
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:619
static const Int_t kCintFileNumber
Definition: TFileMerger.cxx:59
Bool_t fLocal
Makes local copies of merging files if True (default is kTRUE)
Definition: TFileMerger.h:52
#define R__LOCKGUARD(mutex)
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2887
virtual void Clear(Option_t *option="")
Remove all objects from the list.
Definition: TList.cxx:399
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
Definition: TUUID.cxx:533
Bool_t IsNull() const
Definition: TString.h:402
Mother of all ROOT objects.
Definition: TObject.h:37
Bool_t fFastMethod
True if using Fast merging algorithm (default)
Definition: TFileMerger.h:42
Only the objects without a MergeAfterReset member function.
Definition: TFileMerger.h:68
TClassRef is used to implement a permanent reference to a TClass object.
Definition: TClassRef.h:29
virtual const char * GetTitle() const
Returns title of object.
Definition: TObject.cxx:401
Bool_t IsTObject() const
Return kTRUE is the class inherits from TObject.
Definition: TClass.cxx:5668
virtual TObject * ReadObj()
To read a TObject* from the file.
Definition: TKey.cxx:722
Bool_t fNoTrees
True if Trees should not be merged (default is kFALSE)
Definition: TFileMerger.h:43
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
Definition: TDirectory.cxx:497
virtual Bool_t PartialMerge(Int_t type=kAll|kIncremental)
Merge the files.
virtual void Add(TObject *obj)
Definition: TList.h:87
Definition: file.py:1
TIOFeatures * fIOFeatures
IO features to use in the output file.
Definition: TFileMerger.h:48
virtual TDirectory * GetDirectory(const char *namecycle, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory using apath.
Definition: TDirectory.cxx:400
virtual ~TFileMerger()
Cleanup.
void ResetBit(UInt_t f)
Definition: TObject.h:171
virtual void PrintFiles(Option_t *options)
Print list of files being merged.
virtual void Fatal(const char *method, const char *msgfmt,...) const
Issue fatal error message.
Definition: TObject.cxx:908
Bool_t IsWritable() const
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
Only the objects with a MergeAfterReset member function.
Definition: TFileMerger.h:67
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:357
virtual Int_t GetSize() const
Return the capacity of the collection, i.e.
Definition: TCollection.h:182
TMethod * GetMethodWithPrototype(const char *method, const char *proto, Bool_t objectIsConst=kFALSE, ROOT::EFunctionMatchMode mode=ROOT::kConversionMatch)
Find the method with a given prototype.
Definition: TClass.cxx:4277
virtual void Reset()
Reset merger file list.
const Bool_t kTRUE
Definition: RtypesCore.h:87
Int_t GetCompressionLevel() const
Definition: TFile.h:384
Int_t fMaxOpenedFiles
Maximum number of files opened at the same time by the TFileMerger.
Definition: TFileMerger.h:51
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:866
Int_t GetCompressionSettings() const
Definition: TFile.h:390
virtual void Close(Option_t *option="")
Close a file.
Definition: TFile.cxx:917
Bool_t fExplicitCompLevel
True if the user explicitly requested a compressio level change (default kFALSE)
Definition: TFileMerger.h:44
const char * Data() const
Definition: TString.h:364