ROOT  6.06/09
Reference Guide
TFileMerger.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Andreas Peters + Fons Rademakers + Rene Brun 26/5/2005
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13 \class TFileMerger TFileMerger.cxx
14 \ingroup IO
15 
16 This class provides file copy and merging services.
17 
18 It can be used to copy files (not only ROOT files), using TFile or
19 any of its remote file access plugins. It is therefore usefull in
20 a Grid environment where the files might be accessable via Castor,
21 rfio, dcap, etc.
22 The merging interface allows files containing histograms and trees
23 to be merged, like the standalone hadd program.
24 */
25 
26 #include "TFileMerger.h"
27 #include "TDirectory.h"
28 #include "TUrl.h"
29 #include "TFile.h"
30 #include "TUUID.h"
31 #include "TSystem.h"
32 #include "TKey.h"
33 #include "THashList.h"
34 #include "TObjString.h"
35 #include "TClass.h"
36 #include "TMethodCall.h"
37 #include "Riostream.h"
38 #include "TFileMergeInfo.h"
39 #include "TClassRef.h"
40 #include "TROOT.h"
41 #include "TMemFile.h"
42 
43 #ifdef WIN32
44 // For _getmaxstdio
45 #include <stdio.h>
46 #else
47 // For getrlimit
48 #include <sys/time.h>
49 #include <sys/resource.h>
50 #endif
51 
53 
54 TClassRef R__TH1_Class("TH1");
56 
57 static const Int_t kCpProgress = BIT(14);
58 static const Int_t kCintFileNumber = 100;
59 ////////////////////////////////////////////////////////////////////////////////
60 /// Return the maximum number of allowed opened files minus some wiggle room
61 /// for CINT or at least of the standard library (stdio).
62 
64 {
65  int maxfiles;
66 #ifdef WIN32
67  maxfiles = _getmaxstdio();
68 #else
69  rlimit filelimit;
70  if (getrlimit(RLIMIT_NOFILE,&filelimit)==0) {
71  maxfiles = filelimit.rlim_cur;
72  } else {
73  // We could not get the value from getrlimit, let's return a reasonable default.
74  maxfiles = 512;
75  }
76 #endif
77  if (maxfiles > kCintFileNumber) {
78  return maxfiles - kCintFileNumber;
79  } else if (maxfiles > 5) {
80  return maxfiles - 5;
81  } else {
82  return maxfiles;
83  }
84 }
85 
86 ////////////////////////////////////////////////////////////////////////////////
87 /// Create file merger object.
88 
90  : fOutputFile(0), fFastMethod(kTRUE), fNoTrees(kFALSE), fExplicitCompLevel(kFALSE), fCompressionChange(kFALSE),
91  fPrintLevel(0), fMsgPrefix("TFileMerger"), fMaxOpenedFiles( R__GetSystemMaxOpenedFiles() ),
92  fLocal(isLocal), fHistoOneGo(histoOneGo), fObjectNames()
93 {
94  fFileList = new TList;
95 
96  fMergeList = new TList;
98 
99  fExcessFiles = new TList;
101 
102  gROOT->GetListOfCleanups()->Add(this);
103 }
104 
105 ////////////////////////////////////////////////////////////////////////////////
106 /// Cleanup.
107 
109 {
110  gROOT->GetListOfCleanups()->Remove(this);
115 }
116 
117 ////////////////////////////////////////////////////////////////////////////////
118 /// Reset merger file list.
119 
121 {
122  fFileList->Clear();
123  fMergeList->Clear();
124  fExcessFiles->Clear();
126 }
127 
128 ////////////////////////////////////////////////////////////////////////////////
129 /// Add file to file merger.
130 
131 Bool_t TFileMerger::AddFile(const char *url, Bool_t cpProgress)
132 {
133  if (fPrintLevel > 0) {
134  Printf("%s Source file %d: %s",fMsgPrefix.Data(),fFileList->GetEntries()+fExcessFiles->GetEntries()+1,url);
135  }
136 
137  TFile *newfile = 0;
138  TString localcopy;
139 
140  if (fFileList->GetEntries() >= (fMaxOpenedFiles-1)) {
141 
142  TObjString *urlObj = new TObjString(url);
143  fMergeList->Add(urlObj);
144 
145  urlObj = new TObjString(url);
146  urlObj->SetBit(kCpProgress);
147  fExcessFiles->Add(urlObj);
148  return kTRUE;
149  }
150 
151  // We want gDirectory untouched by anything going on here
153 
154  if (fLocal) {
155  TUUID uuid;
156  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
157  if (!TFile::Cp(url, localcopy, cpProgress)) {
158  Error("AddFile", "cannot get a local copy of file %s", url);
159  return kFALSE;
160  }
161  newfile = TFile::Open(localcopy, "READ");
162  } else {
163  newfile = TFile::Open(url, "READ");
164  }
165 
166  if (!newfile) {
167  if (fLocal)
168  Error("AddFile", "cannot open local copy %s of URL %s",
169  localcopy.Data(), url);
170  else
171  Error("AddFile", "cannot open file %s", url);
172  return kFALSE;
173  } else {
175 
176  newfile->SetBit(kCanDelete);
177  fFileList->Add(newfile);
178 
179  TObjString *urlObj = new TObjString(url);
180  fMergeList->Add(urlObj);
181 
182  return kTRUE;
183  }
184 }
185 
186 ////////////////////////////////////////////////////////////////////////////////
187 /// Add the TFile to this file merger and *do not* give ownership of the TFile to this
188 /// object.
189 ///
190 /// Return kTRUE if the addition was successful.
191 
193 {
194  return AddFile(source,kFALSE,cpProgress);
195 }
196 
197 ////////////////////////////////////////////////////////////////////////////////
198 /// Add the TFile to this file merger and give ownership of the TFile to this
199 /// object (unless kFALSE is returned).
200 ///
201 /// Return kTRUE if the addition was successful.
202 
204 {
205  return AddFile(source,kTRUE,cpProgress);
206 }
207 
208 ////////////////////////////////////////////////////////////////////////////////
209 /// Add the TFile to this file merger and give ownership of the TFile to this
210 /// object (unless kFALSE is returned).
211 ///
212 /// Return kTRUE if the addition was successful.
213 
214 Bool_t TFileMerger::AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
215 {
216  if (source == 0) {
217  return kFALSE;
218  }
219 
220  if (fPrintLevel > 0) {
221  Printf("%s Source file %d: %s",fMsgPrefix.Data(),fFileList->GetEntries()+1,source->GetName());
222  }
223 
224  TFile *newfile = 0;
225  TString localcopy;
226 
227  // We want gDirectory untouched by anything going on here
229  if (fLocal && !source->InheritsFrom(TMemFile::Class())) {
230  TUUID uuid;
231  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
232  if (!source->Cp(localcopy, cpProgress)) {
233  Error("AddFile", "cannot get a local copy of file %s", source->GetName());
234  return kFALSE;
235  }
236  newfile = TFile::Open(localcopy, "READ");
237  } else {
238  newfile = source;
239  }
240 
241  if (!newfile) {
242  if (fLocal)
243  Error("AddFile", "cannot open local copy %s of URL %s",
244  localcopy.Data(), source->GetName());
245  else
246  Error("AddFile", "cannot open file %s", source->GetName());
247  return kFALSE;
248  } else {
250 
251  if (own || newfile != source) {
252  newfile->SetBit(kCanDelete);
253  } else {
254  newfile->ResetBit(kCanDelete);
255  }
256  fFileList->Add(newfile);
257 
258  if (!fMergeList) {
259  fMergeList = new TList;
260  }
261  TObjString *urlObj = new TObjString(source->GetName());
262  fMergeList->Add(urlObj);
263 
264  if (newfile != source && own) {
265  delete source;
266  }
267  return kTRUE;
268  }
269 }
270 
271 ////////////////////////////////////////////////////////////////////////////////
272 /// Open merger output file.
273 
274 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force, Int_t compressionLevel)
275 {
276  return OutputFile(outputfile,(force?"RECREATE":"CREATE"),compressionLevel);
277 }
278 
279 ////////////////////////////////////////////////////////////////////////////////
280 /// Open merger output file.
281 
282 Bool_t TFileMerger::OutputFile(const char *outputfile, Bool_t force)
283 {
284  Bool_t res = OutputFile(outputfile,(force?"RECREATE":"CREATE"),1); // 1 is the same as the default from the TFile constructor.
286  return res;
287 }
288 
289 ////////////////////////////////////////////////////////////////////////////////
290 /// Open merger output file.
291 ///
292 /// The 'mode' parameter is passed to the TFile constructor as the option, it
293 /// should be one of 'NEW','CREATE','RECREATE','UPDATE'
294 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
295 
296 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode, Int_t compressionLevel)
297 {
299 
300  TFile *oldfile = fOutputFile;
301  fOutputFile = 0; // This avoids the complaint from RecursiveRemove about the file being deleted which is here spurrious. (see RecursiveRemove).
302  SafeDelete(oldfile);
303 
304  fOutputFilename = outputfile;
305 
306  // We want gDirectory untouched by anything going on here
308  if (!(fOutputFile = TFile::Open(outputfile, mode, "", compressionLevel)) || fOutputFile->IsZombie()) {
309  Error("OutputFile", "cannot open the MERGER output file %s", fOutputFilename.Data());
310  return kFALSE;
311  }
312  return kTRUE;
313 }
314 
315 ////////////////////////////////////////////////////////////////////////////////
316 /// Open merger output file. 'mode' is passed to the TFile constructor as the option, it should
317 /// be one of 'NEW','CREATE','RECREATE','UPDATE'
318 /// 'UPDATE' is usually used in conjunction with IncrementalMerge.
319 
320 Bool_t TFileMerger::OutputFile(const char *outputfile, const char *mode /* = "RECREATE" */)
321 {
322  Bool_t res = OutputFile(outputfile,mode,1); // 1 is the same as the default from the TFile constructor.
324  return res;
325 }
326 
327 ////////////////////////////////////////////////////////////////////////////////
328 /// Print list of files being merged.
329 
331 {
332  fFileList->Print(options);
333  fExcessFiles->Print(options);
334 }
335 
336 ////////////////////////////////////////////////////////////////////////////////
337 /// Merge the files.
338 ///
339 /// If no output file was specified it will write into
340 /// the file "FileMerger.root" in the working directory. Returns true
341 /// on success, false in case of error.
342 
344 {
345  return PartialMerge(kAll | kRegular);
346 }
347 
348 ////////////////////////////////////////////////////////////////////////////////
349 /// Merge all objects in a directory
350 ///
351 /// The type is defined by the bit values in TFileMerger::EPartialMergeType.
352 
353 Bool_t TFileMerger::MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type /* = kRegular | kAll */)
354 {
355  Bool_t status = kTRUE;
356  Bool_t onlyListed = kFALSE;
357  if (fPrintLevel > 0) {
358  Printf("%s Target path: %s",fMsgPrefix.Data(),target->GetPath());
359  }
360 
361  // Get the dir name
362  TString path(target->GetPath());
363  // coverity[unchecked_value] 'target' is from a file so GetPath always returns path starting with filename:
364  path.Remove(0, path.Last(':') + 2);
365 
366  Int_t nguess = sourcelist->GetSize()+1000;
367  THashList allNames(nguess);
368  allNames.SetOwner(kTRUE);
369  // If the mode is set to skipping list objects, add names to the allNames list
370  if (type & kSkipListed) {
371  TObjArray *arr = fObjectNames.Tokenize(" ");
372  arr->SetOwner(kFALSE);
373  for (Int_t iname=0; iname<arr->GetEntriesFast(); iname++)
374  allNames.Add(arr->At(iname));
375  delete arr;
376  }
377  ((THashList*)target->GetList())->Rehash(nguess);
378  ((THashList*)target->GetListOfKeys())->Rehash(nguess);
379 
380  TFileMergeInfo info(target);
381  info.fOptions = fMergeOptions;
382  if (fFastMethod && ((type&kKeepCompression) || !fCompressionChange) ) {
383  info.fOptions.Append(" fast");
384  }
385 
386  TFile *current_file;
387  TDirectory *current_sourcedir;
388  if (type & kIncremental) {
389  current_file = 0;
390  current_sourcedir = target;
391  } else {
392  current_file = (TFile*)sourcelist->First();
393  current_sourcedir = current_file->GetDirectory(path);
394  }
395  while (current_file || current_sourcedir) {
396  // When current_sourcedir != 0 and current_file == 0 we are going over the target
397  // for an incremental merge.
398  if (current_sourcedir && (current_file == 0 || current_sourcedir != target)) {
399 
400  // loop over all keys in this directory
401  TIter nextkey( current_sourcedir->GetListOfKeys() );
402  TKey *key;
403  TString oldkeyname;
404 
405  while ( (key = (TKey*)nextkey())) {
406 
407  // Keep only the highest cycle number for each key for mergeable objects. They are stored
408  // in the (hash) list onsecutively and in decreasing order of cycles, so we can continue
409  // until the name changes. We flag the case here and we act consequently later.
410  Bool_t alreadyseen = (oldkeyname == key->GetName()) ? kTRUE : kFALSE;
411 
412  // Read in but do not copy directly the processIds.
413  if (strcmp(key->GetClassName(),"TProcessID") == 0) { key->ReadObj(); continue;}
414 
415  // If we have already seen this object [name], we already processed
416  // the whole list of files for this objects and we can just skip it
417  // and any related cycles.
418  if (allNames.FindObject(key->GetName())) {
419  oldkeyname = key->GetName();
420  continue;
421  }
422 
423  TClass *cl = TClass::GetClass(key->GetClassName());
424  if (!cl) {
425  Info("MergeRecursive", "cannot indentify object type (%s), name: %s title: %s",
426  key->GetClassName(), key->GetName(), key->GetTitle());
427  continue;
428  }
429  // For mergeable objects we add the names in a local hashlist handling them
430  // again (see above)
431  if (cl->GetMerge() || cl->InheritsFrom(TDirectory::Class()) ||
432  (cl->IsTObject() &&
433  (cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ||
434  cl->GetMethodWithPrototype("Merge", "TCollection*"))))
435  allNames.Add(new TObjString(key->GetName()));
436 
437  if (fNoTrees && cl->InheritsFrom(R__TTree_Class)) {
438  // Skip the TTree objects and any related cycles.
439  oldkeyname = key->GetName();
440  continue;
441  }
442  // Check if only the listed objects are to be merged
443  if (type & kOnlyListed) {
444  onlyListed = kFALSE;
445  oldkeyname = key->GetName();
446  oldkeyname += " ";
447  onlyListed = fObjectNames.Contains(oldkeyname);
448  oldkeyname = key->GetName();
449  if ((!onlyListed) && (!cl->InheritsFrom(TDirectory::Class()))) continue;
450  }
451 
452  if (!(type&kResetable && type&kNonResetable)) {
453  // If neither or both are requested at the same time, we merger both types.
454  if (!(type&kResetable)) {
455  if (cl->GetResetAfterMerge()) {
456  // Skip the object with a reset after merge routine (TTree and other incrementally mergeable objects)
457  oldkeyname = key->GetName();
458  continue;
459  }
460  }
461  if (!(type&kNonResetable)) {
462  if (!cl->GetResetAfterMerge()) {
463  // Skip the object without a reset after merge routine (Histograms and other non incrementally mergeable objects)
464  oldkeyname = key->GetName();
465  continue;
466  }
467  }
468  }
469  // read object from first source file
470  TObject *obj;
471  if (type & kIncremental) {
472  obj = current_sourcedir->GetList()->FindObject(key->GetName());
473  if (!obj) {
474  obj = key->ReadObj();
475  }
476  } else {
477  obj = key->ReadObj();
478  }
479  if (!obj) {
480  Info("MergeRecursive", "could not read object for key {%s, %s}",
481  key->GetName(), key->GetTitle());
482  continue;
483  }
484  if (cl->IsTObject() && cl != obj->IsA()) {
485  Error("MergeRecursive", "TKey and object retrieve disagree on type (%s vs %s). Continuing with %s.",
486  key->GetClassName(), obj->IsA()->GetName(), obj->IsA()->GetName());
487  cl = obj->IsA();
488  }
489  Bool_t canBeMerged = kTRUE;
490 
491  if ( cl->InheritsFrom( TDirectory::Class() ) ) {
492  // it's a subdirectory
493 
494  target->cd();
495  TDirectory *newdir;
496 
497  // For incremental or already seen we may have already a directory created
498  if (type & kIncremental || alreadyseen) {
499  newdir = target->GetDirectory(obj->GetName());
500  if (!newdir) {
501  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
502  }
503  } else {
504  newdir = target->mkdir( obj->GetName(), obj->GetTitle() );
505  }
506 
507  // newdir is now the starting point of another round of merging
508  // newdir still knows its depth within the target file via
509  // GetPath(), so we can still figure out where we are in the recursion
510 
511  // If this folder is a onlyListed object, merge everything inside.
512  if (onlyListed) type &= ~kOnlyListed;
513  status = MergeRecursive(newdir, sourcelist, type);
514  if (onlyListed) type |= kOnlyListed;
515  if (!status) return status;
516  } else if (cl->GetMerge()) {
517 
518  // Check if already treated
519  if (alreadyseen) continue;
520 
521  TList inputs;
522  Bool_t oneGo = fHistoOneGo && cl->InheritsFrom(R__TH1_Class);
523 
524  // Loop over all source files and merge same-name object
525  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
526  if (nextsource == 0) {
527  // There is only one file in the list
529  func(obj, &inputs, &info);
530  info.fIsFirst = kFALSE;
531  } else {
532  do {
533  // make sure we are at the correct directory level by cd'ing to path
534  TDirectory *ndir = nextsource->GetDirectory(path);
535  if (ndir) {
536  ndir->cd();
537  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
538  if (key2) {
539  TObject *hobj = key2->ReadObj();
540  if (!hobj) {
541  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
542  key->GetName(), key->GetTitle(), nextsource->GetName());
543  nextsource = (TFile*)sourcelist->After(nextsource);
544  continue;
545  }
546  // Set ownership for collections
547  if (hobj->InheritsFrom(TCollection::Class())) {
548  ((TCollection*)hobj)->SetOwner();
549  }
550  hobj->ResetBit(kMustCleanup);
551  inputs.Add(hobj);
552  if (!oneGo) {
554  Long64_t result = func(obj, &inputs, &info);
555  info.fIsFirst = kFALSE;
556  if (result < 0) {
557  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
558  obj->GetName(), nextsource->GetName());
559  }
560  inputs.Delete();
561  }
562  }
563  }
564  nextsource = (TFile*)sourcelist->After( nextsource );
565  } while (nextsource);
566  // Merge the list, if still to be done
567  if (oneGo || info.fIsFirst) {
569  func(obj, &inputs, &info);
570  info.fIsFirst = kFALSE;
571  inputs.Delete();
572  }
573  }
574  } else if (cl->IsTObject() &&
575  cl->GetMethodWithPrototype("Merge", "TCollection*,TFileMergeInfo*") ) {
576  // Object implements Merge(TCollection*,TFileMergeInfo*) and has a reflex dictionary ...
577 
578  // Check if already treated
579  if (alreadyseen) continue;
580 
581  TList listH;
582  TString listHargs;
583  listHargs.Form("(TCollection*)0x%lx,(TFileMergeInfo*)0x%lx", (ULong_t)&listH,(ULong_t)&info);
584 
585  // Loop over all source files and merge same-name object
586  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
587  if (nextsource == 0) {
588  // There is only one file in the list
589  Int_t error = 0;
590  obj->Execute("Merge", listHargs.Data(), &error);
591  info.fIsFirst = kFALSE;
592  if (error) {
593  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
594  obj->GetName(), key->GetName());
595  }
596  } else {
597  while (nextsource) {
598  // make sure we are at the correct directory level by cd'ing to path
599  TDirectory *ndir = nextsource->GetDirectory(path);
600  if (ndir) {
601  ndir->cd();
602  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
603  if (key2) {
604  TObject *hobj = key2->ReadObj();
605  if (!hobj) {
606  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
607  key->GetName(), key->GetTitle(), nextsource->GetName());
608  nextsource = (TFile*)sourcelist->After(nextsource);
609  continue;
610  }
611  // Set ownership for collections
612  if (hobj->InheritsFrom(TCollection::Class())) {
613  ((TCollection*)hobj)->SetOwner();
614  }
615  hobj->ResetBit(kMustCleanup);
616  listH.Add(hobj);
617  Int_t error = 0;
618  obj->Execute("Merge", listHargs.Data(), &error);
619  info.fIsFirst = kFALSE;
620  if (error) {
621  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
622  obj->GetName(), nextsource->GetName());
623  }
624  listH.Delete();
625  }
626  }
627  nextsource = (TFile*)sourcelist->After( nextsource );
628  }
629  // Merge the list, if still to be done
630  if (info.fIsFirst) {
631  Int_t error = 0;
632  obj->Execute("Merge", listHargs.Data(), &error);
633  info.fIsFirst = kFALSE;
634  listH.Delete();
635  }
636  }
637  } else if (cl->IsTObject() &&
638  cl->GetMethodWithPrototype("Merge", "TCollection*") ) {
639  // Object implements Merge(TCollection*) and has a reflex dictionary ...
640 
641  // Check if already treated
642  if (alreadyseen) continue;
643 
644  TList listH;
645  TString listHargs;
646  listHargs.Form("((TCollection*)0x%lx)", (ULong_t)&listH);
647 
648  // Loop over all source files and merge same-name object
649  TFile *nextsource = current_file ? (TFile*)sourcelist->After( current_file ) : (TFile*)sourcelist->First();
650  if (nextsource == 0) {
651  // There is only one file in the list
652  Int_t error = 0;
653  obj->Execute("Merge", listHargs.Data(), &error);
654  if (error) {
655  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
656  obj->GetName(), key->GetName());
657  }
658  } else {
659  while (nextsource) {
660  // make sure we are at the correct directory level by cd'ing to path
661  TDirectory *ndir = nextsource->GetDirectory(path);
662  if (ndir) {
663  ndir->cd();
664  TKey *key2 = (TKey*)ndir->GetListOfKeys()->FindObject(key->GetName());
665  if (key2) {
666  TObject *hobj = key2->ReadObj();
667  if (!hobj) {
668  Info("MergeRecursive", "could not read object for key {%s, %s}; skipping file %s",
669  key->GetName(), key->GetTitle(), nextsource->GetName());
670  nextsource = (TFile*)sourcelist->After(nextsource);
671  continue;
672  }
673  // Set ownership for collections
674  if (hobj->InheritsFrom(TCollection::Class())) {
675  ((TCollection*)hobj)->SetOwner();
676  }
677  hobj->ResetBit(kMustCleanup);
678  listH.Add(hobj);
679  Int_t error = 0;
680  obj->Execute("Merge", listHargs.Data(), &error);
681  info.fIsFirst = kFALSE;
682  if (error) {
683  Error("MergeRecursive", "calling Merge() on '%s' with the corresponding object in '%s'",
684  obj->GetName(), nextsource->GetName());
685  }
686  listH.Delete();
687  }
688  }
689  nextsource = (TFile*)sourcelist->After( nextsource );
690  }
691  // Merge the list, if still to be done
692  if (info.fIsFirst) {
693  Int_t error = 0;
694  obj->Execute("Merge", listHargs.Data(), &error);
695  info.fIsFirst = kFALSE;
696  listH.Delete();
697  }
698  }
699  } else {
700  // Object is of no type that we can merge
701  canBeMerged = kFALSE;
702  }
703 
704  // now write the merged histogram (which is "in" obj) to the target file
705  // note that this will just store obj in the current directory level,
706  // which is not persistent until the complete directory itself is stored
707  // by "target->SaveSelf()" below
708  target->cd();
709 
710  oldkeyname = key->GetName();
711  //!!if the object is a tree, it is stored in globChain...
712  if(cl->InheritsFrom( TDirectory::Class() )) {
713  //printf("cas d'une directory\n");
714  // Do not delete the directory if it is part of the output
715  // and we are in incremental mode (because it will be reuse
716  // and has not been written to disk (for performance reason).
717  // coverity[var_deref_model] the IsA()->InheritsFrom guarantees that the dynamic_cast will succeed.
718  if (!(type&kIncremental) || dynamic_cast<TDirectory*>(obj)->GetFile() != target) {
719  delete obj;
720  }
721  } else if (cl->InheritsFrom( TCollection::Class() )) {
722  // Don't overwrite, if the object were not merged.
723  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kSingleKey | TObject::kOverwrite : TObject::kSingleKey) <= 0 ) {
724  status = kFALSE;
725  }
726  ((TCollection*)obj)->SetOwner();
727  delete obj;
728  } else {
729  // Don't overwrite, if the object were not merged.
730  // NOTE: this is probably wrong for emulated objects.
731  if (cl->IsTObject()) {
732  if ( obj->Write( oldkeyname, canBeMerged ? TObject::kOverwrite : 0) <= 0) {
733  status = kFALSE;
734  }
735  } else {
736  if ( target->WriteObjectAny( (void*)obj, cl, oldkeyname, canBeMerged ? "OverWrite" : "" ) <= 0) {
737  status = kFALSE;
738  }
739  }
740  cl->Destructor(obj); // just in case the class is not loaded.
741  }
742  info.Reset();
743  } // while ( ( TKey *key = (TKey*)nextkey() ) )
744  }
745  current_file = current_file ? (TFile*)sourcelist->After(current_file) : (TFile*)sourcelist->First();
746  if (current_file) {
747  current_sourcedir = current_file->GetDirectory(path);
748  } else {
749  current_sourcedir = 0;
750  }
751  }
752  // save modifications to the target directory.
753  if (!(type&kIncremental)) {
754  // In case of incremental build, we will call Write on the top directory/file, so we do not need
755  // to call SaveSelf explicilty.
756  target->SaveSelf(kTRUE);
757  }
758  return status;
759 }
760 
761 ////////////////////////////////////////////////////////////////////////////////
762 /// Merge the files. If no output file was specified it will write into
763 /// the file "FileMerger.root" in the working directory. Returns true
764 /// on success, false in case of error.
765 /// The type is defined by the bit values in EPartialMergeType:
766 /// kRegular : normal merge, overwritting the output file
767 /// kIncremental : merge the input file with the content of the output file (if already exising) (default)
768 /// kAll : merge all type of objects (default)
769 /// kResetable : merge only the objects with a MergeAfterReset member function.
770 /// kNonResetable : merge only the objects without a MergeAfterReset member function.
771 ///
772 /// If the type is set to kIncremental the output file is done deleted at the end of
773 /// this operation. If the type is not set to kIncremental, the output file is closed.
774 
776 {
777  if (!fOutputFile) {
778  TString outf(fOutputFilename);
779  if (outf.IsNull()) {
780  outf.Form("file:%s/FileMerger.root", gSystem->TempDirectory());
781  Info("PartialMerge", "will merge the results to the file %s\n"
782  "since you didn't specify a merge filename",
783  TUrl(outf).GetFile());
784  }
785  if (!OutputFile(outf.Data())) {
786  return kFALSE;
787  }
788  }
789 
790  // Special treament for the single file case ...
791  if ((fFileList->GetEntries() == 1) && !fExcessFiles->GetEntries() &&
792  !(in_type & kIncremental) && !fCompressionChange && !fExplicitCompLevel) {
793  fOutputFile->Close();
795 
796  TFile *file = (TFile *) fFileList->First();
797  if (!file || (file && file->IsZombie())) {
798  Error("PartialMerge", "one-file case: problem attaching to file");
799  return kFALSE;
800  }
801  Bool_t result = kTRUE;
802  if (!(result = file->Cp(fOutputFilename))) {
803  Error("PartialMerge", "one-file case: could not copy '%s' to '%s'",
804  file->GetPath(), fOutputFilename.Data());
805  return kFALSE;
806  }
807  if (file->TestBit(kCanDelete)) file->Close();
808 
809  // Remove the temporary file
810  if (fLocal) {
811  TUrl u(file->GetPath(), kTRUE);
812  if (gSystem->Unlink(u.GetFile()) != 0)
813  Warning("PartialMerge", "problems removing temporary local file '%s'", u.GetFile());
814  }
815  fFileList->Clear();
816  return result;
817  }
818 
820 
822 
823  Bool_t result = kTRUE;
824  Int_t type = in_type;
825  while (result && fFileList->GetEntries()>0) {
826  result = MergeRecursive(fOutputFile, fFileList, type);
827 
828  // Remove local copies if there are any
830  TFile *file;
831  while ((file = (TFile*) next())) {
832  // close the files
833  if (file->TestBit(kCanDelete)) file->Close();
834  // remove the temporary files
835  if(fLocal) {
836  TString p(file->GetPath());
837  // coverity[unchecked_value] Index is return a value with range or NPos to select the whole name.
838  p = p(0, p.Index(':',0));
839  gSystem->Unlink(p);
840  }
841  }
842  fFileList->Clear();
843  if (fExcessFiles->GetEntries() > 0) {
844  // We merge the first set of files in the output,
845  // we now need to open the next set and make
846  // sure we accumulate into the output, so we
847  // switch to incremental merging (if not already set)
848  type = type | kIncremental;
849  OpenExcessFiles();
850  }
851  }
852  if (!result) {
853  Error("Merge", "error during merge of your ROOT files");
854  } else {
855  // Close or write is required so the file is complete.
856  if (in_type & kIncremental) {
858  } else {
859  fOutputFile->Close();
860  }
861  }
862 
863  // Cleanup
864  if (in_type & kIncremental) {
865  Clear();
866  } else {
869  }
870  return result;
871 }
872 
873 ////////////////////////////////////////////////////////////////////////////////
874 /// Open up to fMaxOpenedFiles of the excess files.
875 
877 {
878  if (fPrintLevel > 0) {
879  Printf("%s Opening the next %d files",fMsgPrefix.Data(),TMath::Min(fExcessFiles->GetEntries(),(fMaxOpenedFiles-1)));
880  }
881  Int_t nfiles = 0;
883  TObjString *url = 0;
884  TString localcopy;
885  // We want gDirectory untouched by anything going on here
887  while( nfiles < (fMaxOpenedFiles-1) && ( url = (TObjString*)next() ) ) {
888  TFile *newfile = 0;
889  if (fLocal) {
890  TUUID uuid;
891  localcopy.Form("file:%s/ROOTMERGE-%s.root", gSystem->TempDirectory(), uuid.AsString());
892  if (!TFile::Cp(url->GetName(), localcopy, url->TestBit(kCpProgress))) {
893  Error("OpenExcessFiles", "cannot get a local copy of file %s", url->GetName());
894  return kFALSE;
895  }
896  newfile = TFile::Open(localcopy, "READ");
897  } else {
898  newfile = TFile::Open(url->GetName(), "READ");
899  }
900 
901  if (!newfile) {
902  if (fLocal)
903  Error("OpenExcessFiles", "cannot open local copy %s of URL %s",
904  localcopy.Data(), url->GetName());
905  else
906  Error("OpenExcessFiles", "cannot open file %s", url->GetName());
907  return kFALSE;
908  } else {
910 
911  newfile->SetBit(kCanDelete);
912  fFileList->Add(newfile);
913  ++nfiles;
914  fExcessFiles->Remove(url);
915  }
916  }
917  return kTRUE;
918 }
919 
920 ////////////////////////////////////////////////////////////////////////////////
921 /// Intercept the case where the output TFile is deleted!
922 
924 {
925  if (obj == fOutputFile) {
926  Fatal("RecursiveRemove","Output file of the TFile Merger (targeting %s) has been deleted (likely due to a TTree larger than 100Gb)", fOutputFilename.Data());
927  }
928 
929 }
930 
931 ////////////////////////////////////////////////////////////////////////////////
932 /// Set a limit to the number file that TFileMerger will opened at one time.
933 ///
934 /// If the request is higher than the system limit, we reset it to the system limit.
935 /// If the request is less than two, we reset it to 2 (one for the output file and one for the input file).
936 
938 {
940  if (newmax < sysmax) {
941  fMaxOpenedFiles = newmax;
942  } else {
943  fMaxOpenedFiles = sysmax;
944  }
945  if (fMaxOpenedFiles < 2) {
946  fMaxOpenedFiles = 2;
947  }
948 }
949 
950 ////////////////////////////////////////////////////////////////////////////////
951 /// Set the prefix to be used when printing informational message.
952 
953 void TFileMerger::SetMsgPrefix(const char *prefix)
954 {
955  fMsgPrefix = prefix;
956 }
957 
const char * GetName() const
Returns name of object.
Definition: TObjString.h:42
virtual void Clear(Option_t *="")
Definition: TObject.h:110
virtual Int_t GetEntries() const
Definition: TCollection.h:92
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:823
Int_t fPrintLevel
How much information to print out at run time.
Definition: TFileMerger.h:44
An array of TObjects.
Definition: TObjArray.h:39
ROOT::MergeFunc_t GetMerge() const
Return the wrapper around Merge.
Definition: TClass.cxx:6869
virtual void Delete(Option_t *option="")
Remove all objects from the list AND delete all heap based objects.
Definition: TList.cxx:404
long long Long64_t
Definition: RtypesCore.h:69
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
Definition: TObject.cxx:487
virtual TList * GetListOfKeys() const
Definition: TDirectory.h:155
ClassImp(TSeqCollection) Int_t TSeqCollection TIter next(this)
Return index of object in collection.
Collectable string class.
Definition: TObjString.h:32
const char Option_t
Definition: RtypesCore.h:62
TObject * FindObject(const char *name) const
Find object using its name.
Definition: THashList.cxx:212
This class represents a WWW compatible URL.
Definition: TUrl.h:41
virtual void SetOwner(Bool_t enable=kTRUE)
Set whether this collection is the owner (enable==true) of its content.
#define BIT(n)
Definition: Rtypes.h:120
virtual TList * GetList() const
Definition: TDirectory.h:154
Bool_t IsTObject() const
Return kTRUE is the class inherits from TObject.
Definition: TClass.cxx:5552
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:892
virtual Bool_t Merge(Bool_t=kTRUE)
Merge the files.
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:45
Normal merge, overwritting the output file.
Definition: TFileMerger.h:62
Bool_t OpenExcessFiles()
Open up to fMaxOpenedFiles of the excess files.
#define gROOT
Definition: TROOT.h:340
virtual const char * TempDirectory() const
Return a user configured or systemwide directory to create temporary files in.
Definition: TSystem.cxx:1447
Bool_t IsZombie() const
Definition: TObject.h:141
Basic string class.
Definition: TString.h:137
Short_t Min(Short_t a, Short_t b)
Definition: TMathBase.h:170
TAlienJobStatus * status
Definition: TAlienJob.cxx:51
int Int_t
Definition: RtypesCore.h:41
virtual TDirectory * mkdir(const char *name, const char *title="")
Create a sub-directory and return a pointer to the created directory.
Definition: TDirectory.cxx:955
bool Bool_t
Definition: RtypesCore.h:59
const Bool_t kFALSE
Definition: Rtypes.h:92
virtual TObject * FindObject(const char *name) const
Find an object in this list using its name.
Definition: TList.cxx:496
TString fMergeOptions
Options (in string format) to be passed down to the Merge functions.
Definition: TFileMerger.h:45
Int_t GetEntriesFast() const
Definition: TObjArray.h:66
virtual const char * GetPath() const
Returns the full path of the directory.
Definition: TDirectory.cxx:909
Bool_t fCompressionChange
True if the output and input have different compression level (default kFALSE)
Definition: TFileMerger.h:43
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:732
Long64_t(* MergeFunc_t)(void *, TCollection *, TFileMergeInfo *)
Definition: Rtypes.h:153
Only the objects specified in fObjectNames list.
Definition: TFileMerger.h:70
static TFile * Open(const char *name, Option_t *option="", const char *ftitle="", Int_t compress=1, Int_t netopt=0)
Create / open a file.
Definition: TFile.cxx:3851
virtual Bool_t Cp(const char *dst, Bool_t progressbar=kTRUE, UInt_t buffersize=1000000)
Allows to copy this file to the dst URL.
Definition: TFile.cxx:4749
This class defines a UUID (Universally Unique IDentifier), also known as GUIDs (Globally Unique IDent...
Definition: TUUID.h:44
virtual void RecursiveRemove(TObject *obj)
Intercept the case where the output TFile is deleted!
const char * Data() const
Definition: TString.h:349
virtual void Fatal(const char *method, const char *msgfmt,...) const
Issue fatal error message.
Definition: TObject.cxx:946
#define SafeDelete(p)
Definition: RConfig.h:436
virtual int Unlink(const char *name)
Unlink, i.e. remove, a file.
Definition: TSystem.cxx:1346
Merge all type of objects (default)
Definition: TFileMerger.h:67
virtual TObject * After(const TObject *obj) const
Returns the object after object obj.
Definition: TList.cxx:288
Int_t GetCompressionLevel() const
Definition: TFile.h:357
void Class()
Definition: Class.C:29
THashList implements a hybrid collection class consisting of a hash table and a list to store TObject...
Definition: THashList.h:36
TList * fExcessFiles
! List of TObjString containing the name of the files not yet added to fFileList due to user or syste...
Definition: TFileMerger.h:53
void Clear()
Clear string without changing its capacity.
Definition: TString.cxx:1139
Skip objects specified in fObjectNames list.
Definition: TFileMerger.h:71
TString & Append(const char *cs)
Definition: TString.h:492
virtual Int_t WriteObjectAny(const void *, const char *, const char *, Option_t *="", Int_t=0)
Definition: TDirectory.h:205
Merge the input file with the content of the output file (if already exising).
Definition: TFileMerger.h:63
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:30
virtual void Error(const char *method, const char *msgfmt,...) const
Issue error message.
Definition: TObject.cxx:918
Bool_t fHistoOneGo
Merger histos in one go (default is kTRUE)
Definition: TFileMerger.h:50
void SetMsgPrefix(const char *prefix)
Set the prefix to be used when printing informational message.
virtual Bool_t AddAdoptFile(TFile *source, Bool_t cpProgress=kTRUE)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
TFileMerger(const TFileMerger &)
virtual Bool_t OutputFile(const char *url, Bool_t force)
Open merger output file.
TFile * fOutputFile
The outputfile for merging.
Definition: TFileMerger.h:38
TClassRef R__TTree_Class("TTree")
TList * fMergeList
list of TObjString containing the name of the files need to be merged
Definition: TFileMerger.h:52
A doubly linked list.
Definition: TList.h:47
virtual Bool_t AddFile(TFile *source, Bool_t own, Bool_t cpProgress)
Add the TFile to this file merger and give ownership of the TFile to this object (unless kFALSE is re...
static const Int_t kCpProgress
Definition: TFileMerger.cxx:57
TString fOutputFilename
The name of the outputfile for merging.
Definition: TFileMerger.h:39
R__EXTERN TSystem * gSystem
Definition: TSystem.h:549
static Int_t R__GetSystemMaxOpenedFiles()
Return the maximum number of allowed opened files minus some wiggle room for CINT or at least of the ...
Definition: TFileMerger.cxx:63
virtual Int_t Write(const char *name=0, Int_t opt=0, Int_t bufsiz=0)
Write memory objects to this file.
Definition: TFile.cxx:2248
virtual void Execute(const char *method, const char *params, Int_t *error=0)
Execute method on this object with the given parameter string, e.g.
Definition: TObject.cxx:335
This class provides file copy and merging services.
Definition: TFileMerger.h:30
virtual TObject * Remove(TObject *obj)
Remove object from the list.
Definition: TList.cxx:674
Collection abstract base class.
Definition: TCollection.h:48
Keep compression level unchanged for each input files.
Definition: TFileMerger.h:72
void Destructor(void *obj, Bool_t dtorOnly=kFALSE)
Explicitly call destructor for object.
Definition: TClass.cxx:5040
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2321
Bool_t TestBit(UInt_t f) const
Definition: TObject.h:173
TString fMsgPrefix
Prefix to be used when printing informational message (default TFileMerger)
Definition: TFileMerger.h:46
const char * AsString() const
Return UUID as string. Copy string immediately since it will be reused.
Definition: TUUID.cxx:531
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:51
The ROOT global object gROOT contains a list of all defined classes.
Definition: TClass.h:81
Bool_t IsNull() const
Definition: TString.h:387
virtual TDirectory * GetDirectory(const char *apath, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory named "apath".
TString fObjectNames
List of object names to be either merged exclusively or skipped.
Definition: TFileMerger.h:51
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2240
ClassImp(TFileMerger) TClassRef R__TH1_Class("TH1")
#define Printf
Definition: TGeoToOCC.h:18
void SetMaxOpenedFiles(Int_t newmax)
Set a limit to the number file that TFileMerger will opened at one time.
virtual void SaveSelf(Bool_t=kFALSE)
Definition: TDirectory.h:186
virtual void Print(Option_t *option="") const
Default print for collections, calls Print(option, 1).
TString & Remove(Ssiz_t pos)
Definition: TString.h:616
virtual Bool_t MergeRecursive(TDirectory *target, TList *sourcelist, Int_t type=kRegular|kAll)
Merge all objects in a directory.
virtual Int_t GetSize() const
Definition: TCollection.h:95
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:415
Describe directory structure in memory.
Definition: TDirectory.h:41
int type
Definition: TGX11.cxx:120
unsigned long ULong_t
Definition: RtypesCore.h:51
double func(double *x, double *p)
Definition: stressTF1.cxx:213
static const Int_t kCintFileNumber
Definition: TFileMerger.cxx:58
The TH1 histogram class.
Definition: TH1.h:80
Bool_t fLocal
Makes local copies of merging files if True (default is kTRUE)
Definition: TFileMerger.h:49
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2881
virtual void Clear(Option_t *option="")
Remove all objects from the list.
Definition: TList.cxx:348
Mother of all ROOT objects.
Definition: TObject.h:58
Bool_t fFastMethod
True if using Fast merging algorithm (default)
Definition: TFileMerger.h:40
Only the objects without a MergeAfterReset member function.
Definition: TFileMerger.h:65
virtual TObject * First() const
Return the first object in the list. Returns 0 when list is empty.
Definition: TList.cxx:556
TClassRef is used to implement a permanent reference to a TClass object.
Definition: TClassRef.h:33
virtual TObject * ReadObj()
To read a TObject* from the file.
Definition: TKey.cxx:727
Bool_t fNoTrees
True if Trees should not be merged (default is kFALSE)
Definition: TFileMerger.h:41
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
Definition: TDirectory.cxx:433
virtual Bool_t PartialMerge(Int_t type=kAll|kIncremental)
Merge the files.
virtual void Add(TObject *obj)
Definition: TList.h:81
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:567
virtual TDirectory * GetDirectory(const char *namecycle, Bool_t printError=false, const char *funcname="GetDirectory")
Find a directory using apath.
Definition: TDirectory.cxx:336
virtual ~TFileMerger()
Cleanup.
A TTree object has a header with a name and a title.
Definition: TTree.h:94
double result[121]
void ResetBit(UInt_t f)
Definition: TObject.h:172
Bool_t InheritsFrom(const char *cl) const
Return kTRUE if this class inherits from a class with name "classname".
Definition: TClass.cxx:4579
virtual void PrintFiles(Option_t *options)
Print list of files being merged.
ROOT::ResetAfterMergeFunc_t GetResetAfterMerge() const
Return the wrapper around Merge.
Definition: TClass.cxx:6877
TObject * At(Int_t idx) const
Definition: TObjArray.h:167
Only the objects with a MergeAfterReset member function.
Definition: TFileMerger.h:64
const Bool_t kTRUE
Definition: Rtypes.h:91
TMethod * GetMethodWithPrototype(const char *method, const char *proto, Bool_t objectIsConst=kFALSE, ROOT::EFunctionMatchMode mode=ROOT::kConversionMatch)
Find the method with a given prototype.
Definition: TClass.cxx:4194
virtual void Reset()
Reset merger file list.
virtual const char * GetTitle() const
Returns title of object.
Definition: TObject.cxx:459
TObject * obj
Int_t fMaxOpenedFiles
Maximum number of files opened at the same time by the TFileMerger.
Definition: TFileMerger.h:48
TList * fFileList
A list the file (TFile*) which shall be merged.
Definition: TFileMerger.h:37
virtual void Close(Option_t *option="")
Close a file.
Definition: TFile.cxx:898
Bool_t fExplicitCompLevel
True if the user explicitly requested a compressio level change (default kFALSE)
Definition: TFileMerger.h:42
virtual void Warning(const char *method, const char *msgfmt,...) const
Issue warning message.
Definition: TObject.cxx:904