Logo ROOT  
Reference Guide
TTreeCloner.cxx
Go to the documentation of this file.
1 // @(#)root/tree:$Id$
2 // Author: Philippe Canal 07/11/2005
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /** \class TTreeCloner
13 \ingroup tree
14 
15 Class implementing or helping the various TTree cloning method
16 */
17 
18 #include "TBasket.h"
19 #include "TBranch.h"
20 #include "TBranchClones.h"
21 #include "TBranchElement.h"
22 #include "TStreamerInfo.h"
23 #include "TBranchRef.h"
24 #include "TError.h"
25 #include "TProcessID.h"
26 #include "TTree.h"
27 #include "TTreeCloner.h"
28 #include "TFile.h"
29 #include "TLeafB.h"
30 #include "TLeafI.h"
31 #include "TLeafL.h"
32 #include "TLeafS.h"
33 #include "TLeafO.h"
34 #include "TLeafC.h"
35 #include "TFileCacheRead.h"
36 #include "TTreeCache.h"
37 #include "snprintf.h"
38 
39 #include <algorithm>
40 
41 ////////////////////////////////////////////////////////////////////////////////
42 
44 {
45  if (fObject->fBasketSeek[i1] == fObject->fBasketSeek[i2]) {
46  if (fObject->fBasketEntry[i1] == fObject->fBasketEntry[i2]) {
47  return i1 < i2;
48  }
49  return fObject->fBasketEntry[i1] < fObject->fBasketEntry[i2];
50  }
51  return fObject->fBasketSeek[i1] < fObject->fBasketSeek[i2];
52 }
53 
54 ////////////////////////////////////////////////////////////////////////////////
55 
57 {
58  if (fObject->fBasketEntry[i1] == fObject->fBasketEntry[i2]) {
59  return i1 < i2;
60  }
61  return fObject->fBasketEntry[i1] < fObject->fBasketEntry[i2];
62 }
63 
64 ////////////////////////////////////////////////////////////////////////////////
65 /// Constructor. This object would transfer the data from
66 /// 'from' to 'to' using the method indicated in method.
67 ///
68 /// The value of the parameter 'method' determines in which
69 /// order the branches' baskets are written to the output file.
70 ///
71 /// When a TTree is filled the data is stored in the individual
72 /// branches' basket. Each basket is written individually to
73 /// the disk as soon as it is full. In consequence the baskets
74 /// of branches that contain 'large' data chunk are written to
75 /// the disk more often.
76 ///
77 /// There is currently 3 supported sorting order:
78 ///
79 /// SortBasketsByOffset (the default)
80 /// SortBasketsByBranch
81 /// SortBasketsByEntry
82 ///
83 /// When using SortBasketsByOffset the baskets are written in
84 /// the output file in the same order as in the original file
85 /// (i.e. the basket are sorted on their offset in the original
86 /// file; Usually this also means that the baskets are sorted
87 /// on the index/number of the _last_ entry they contain)
88 ///
89 /// When using SortBasketsByBranch all the baskets of each
90 /// individual branches are stored contiguously. This tends to
91 /// optimize reading speed when reading a small number (1->5) of
92 /// branches, since all their baskets will be clustered together
93 /// instead of being spread across the file. However it might
94 /// decrease the performance when reading more branches (or the full
95 /// entry).
96 ///
97 /// When using SortBasketsByEntry the baskets with the lowest
98 /// starting entry are written first. (i.e. the baskets are
99 /// sorted on the index/number of the first entry they contain).
100 /// This means that on the file the baskets will be in the order
101 /// in which they will be needed when reading the whole tree
102 /// sequentially.
103 
104 TTreeCloner::TTreeCloner(TTree *from, TTree *to, Option_t *method, UInt_t options) :
105  TTreeCloner(from, to, to ? to->GetDirectory() : nullptr, method, options)
106 {
107 
108 }
109 
110 ////////////////////////////////////////////////////////////////////////////////
111 /// Constructor. In place cloning.
112 //// This object would transfer the data from
113 /// 'from' the original location to 'newdirectory' the new directory
114 /// using the sorting method indicated in method.
115 /// It updates the 'from' TTree with the new information.
116 /// See TTreeCloner::TTreeCloner(TTree *from, TTree *to, Option_t *method, UInt_t options)
117 /// for details on the sorting methods.
118 
119 TTreeCloner::TTreeCloner(TTree *from, TDirectory *newdirectory, Option_t *method, UInt_t options) :
120  TTreeCloner(from, from, newdirectory, method, options)
121 {
122 
123 }
124 
125 ////////////////////////////////////////////////////////////////////////////////
126 /// Constructor implementation.
127 TTreeCloner::TTreeCloner(TTree *from, TTree *to, TDirectory *newdirectory, Option_t *method, UInt_t options) :
128  fWarningMsg(),
129  fIsValid(kTRUE),
130  fNeedConversion(kFALSE),
131  fOptions(options),
132  fFromTree(from),
133  fToTree(to),
134  fToDirectory(newdirectory),
135  fToFile(fToDirectory ? fToDirectory->GetFile() : nullptr),
136  fMethod(method),
137  fFromBranches( from ? from->GetListOfLeaves()->GetEntriesFast()+1 : 0),
138  fToBranches( to ? to->GetListOfLeaves()->GetEntriesFast()+1 : 0),
139  fMaxBaskets(CollectBranches()),
140  fBasketBranchNum(new UInt_t[fMaxBaskets]),
141  fBasketNum(new UInt_t[fMaxBaskets]),
142  fBasketSeek(new Long64_t[fMaxBaskets]),
143  fBasketEntry(new Long64_t[fMaxBaskets]),
144  fBasketIndex(new UInt_t[fMaxBaskets]),
145  fPidOffset(0),
146  fCloneMethod(TTreeCloner::kDefault),
147  fToStartEntries(0),
148  fCacheSize(0LL),
149  fFileCache(nullptr),
150  fPrevCache(nullptr)
151 {
152  TString opt(method);
153  opt.ToLower();
154  if (opt.Contains("sortbasketsbybranch")) {
155  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByBranch");
157  } else if (opt.Contains("sortbasketsbyentry")) {
158  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByEntry");
160  } else {
161  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByOffset");
163  }
165 
166  if (fFromTree == nullptr) {
167  if (to)
168  fWarningMsg.Form("An input TTree is required (cloning to %s).",
169  to->GetName());
170  else
171  fWarningMsg.Form("An input and output TTree are required.");
172  if (!(fOptions & kNoWarnings)) {
173  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
174  }
175  fIsValid = kFALSE;
176  }
177  if (fToTree == nullptr) {
178  fWarningMsg.Form("An output TTree is required (cloning %s).",
179  from ? from->GetName() : "no tree");
180  if (!(fOptions & kNoWarnings)) {
181  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
182  }
183  fIsValid = kFALSE;
184  } else if (fToDirectory == nullptr) {
185  fWarningMsg.Form("The output TTree (%s) must be associated with a directory.",
186  fToTree->GetName());
187  if (!(fOptions & kNoWarnings)) {
188  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
189  }
190  fIsValid = kFALSE;
191  } else if (fToFile == nullptr) {
192  fWarningMsg.Form("The output TTree (%s) must be associated with a directory (%s) that is in a file.",
194  if (!(fOptions & kNoWarnings)) {
195  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
196  }
197  fIsValid = kFALSE;
198  } else if (! fToDirectory->IsWritable()) {
199  if (fToDirectory==fToFile) {
200  fWarningMsg.Form("The output TTree (%s) must be associated with a writable file (%s).",
202  } else {
203  fWarningMsg.Form("The output TTree (%s) must be associated with a writable directory (%s in %s).",
205  }
206  if (!(fOptions & kNoWarnings)) {
207  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
208  }
209  fIsValid = kFALSE;
210  }
211 
212  if (fIsValid && (!(fOptions & kNoFileCache))) {
214  }
215 }
216 
217 
218 
219 ////////////////////////////////////////////////////////////////////////////////
220 /// Execute the cloning.
221 
223 {
224  if (!IsValid()) {
225  return kFALSE;
226  }
227  CreateCache();
230  CopyProcessIds();
232  CollectBaskets();
233  SortBaskets();
234  WriteBaskets();
236  RestoreCache();
237  if (IsInPlace())
239 
240  return kTRUE;
241 }
242 
243 ////////////////////////////////////////////////////////////////////////////////
244 /// TTreeCloner destructor
245 
247 {
248  // The file cache was restored to its previous value at the end of Exec,
249  // we can safely delete our cache.
250  delete fFileCache;
251  delete [] fBasketBranchNum;
252  delete [] fBasketNum;
253  delete [] fBasketSeek;
254  delete [] fBasketEntry;
255  delete [] fBasketIndex;
256 }
257 
258 ////////////////////////////////////////////////////////////////////////////////
259 /// Before we can start adding new basket, we need to flush to
260 /// disk the partially filled baskets (the WriteBasket)
261 
263 {
264  if (IsInPlace())
265  return;
266 
267  for(Int_t i=0; i<fToBranches.GetEntriesFast(); ++i) {
269  to->FlushOneBasket(to->GetWriteBasket());
270  }
271 }
272 
273 ////////////////////////////////////////////////////////////////////////////////
274 /// Fill the array of branches, adding the branch 'from' and 'to',
275 /// and matching the sub-branches of the 'from' and 'to' branches.
276 /// Returns the total number of baskets in all the from branch and
277 /// it sub-branches.
278 
280  // Since this is called from the constructor, this can not be a virtual function
281 
282  UInt_t numBaskets = 0;
283  if (from->InheritsFrom(TBranchClones::Class())) {
284  TBranchClones *fromclones = (TBranchClones*) from;
285  TBranchClones *toclones = (TBranchClones*) to;
286  numBaskets += CollectBranches(fromclones->fBranchCount, toclones->fBranchCount);
287 
288  } else if (from->InheritsFrom(TBranchElement::Class())) {
289  Int_t nb = from->GetListOfLeaves()->GetEntriesFast();
290  Int_t fnb = to->GetListOfLeaves()->GetEntriesFast();
291  if (nb != fnb && (nb == 0 || fnb == 0)) {
292  // We might be in the case where one branch is split
293  // while the other is not split. We must reject this match.
294  fWarningMsg.Form("The export branch and the import branch do not have the same split level. (The branch name is %s.)",
295  from->GetName());
296  if (!(fOptions & kNoWarnings)) {
297  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
298  }
300  fIsValid = kFALSE;
301  return 0;
302  }
303  if (((TBranchElement*) from)->GetStreamerType() != ((TBranchElement*) to)->GetStreamerType()) {
304  fWarningMsg.Form("The export branch and the import branch do not have the same streamer type. (The branch name is %s.)",
305  from->GetName());
306  if (!(fOptions & kNoWarnings)) {
307  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
308  }
309  fIsValid = kFALSE;
310  return 0;
311  }
312  TBranchElement *fromelem = (TBranchElement*) from;
313  TBranchElement *toelem = (TBranchElement*) to;
314  if (fromelem->fMaximum > toelem->fMaximum) toelem->fMaximum = fromelem->fMaximum;
315  } else {
316 
317  Int_t nb = from->GetListOfLeaves()->GetEntriesFast();
318  Int_t fnb = to->GetListOfLeaves()->GetEntriesFast();
319  if (nb != fnb) {
320  fWarningMsg.Form("The export branch and the import branch (%s) do not have the same number of leaves (%d vs %d)",
321  from->GetName(), fnb, nb);
322  if (!(fOptions & kNoWarnings)) {
323  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
324  }
325  fIsValid = kFALSE;
326  return 0;
327  }
328  for (Int_t i=0;i<nb;i++) {
329 
330  TLeaf *fromleaf = (TLeaf*)from->GetListOfLeaves()->At(i);
331  TLeaf *toleaf = (TLeaf*)to->GetListOfLeaves()->At(i);
332  if (toleaf->IsA() != fromleaf->IsA() ) {
333  // The data type do not match, we can not do a fast merge.
334  fWarningMsg.Form("The export leaf and the import leaf (%s.%s) do not have the same data type (%s vs %s)",
335  from->GetName(),fromleaf->GetName(),fromleaf->GetTypeName(),toleaf->GetTypeName());
336  if (! (fOptions & kNoWarnings) ) {
337  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
338  }
339  fIsValid = kFALSE;
341  return 0;
342  }
343  toleaf->IncludeRange( fromleaf );
344  }
345 
346  }
347 
348  fFromBranches.AddLast(from);
349  if (!from->TestBit(TBranch::kDoNotUseBufferMap)) {
350  // Make sure that we reset the Buffer's map if needed.
352  }
353  fToBranches.AddLast(to);
354 
355  numBaskets += from->GetWriteBasket();
356  numBaskets += CollectBranches(from->GetListOfBranches(),to->GetListOfBranches());
357 
358  return numBaskets;
359 }
360 
361 ////////////////////////////////////////////////////////////////////////////////
362 /// Fill the array of branches, matching the branches of the 'from' and 'to' arrays.
363 /// Returns the total number of baskets in all the branches.
364 
366 {
367  // Since this is called from the constructor, this can not be a virtual function
368 
369  Int_t fnb = from->GetEntriesFast();
370  Int_t tnb = to->GetEntriesFast();
371  if (!fnb || !tnb) {
372  return 0;
373  }
374 
375  UInt_t numBasket = 0;
376  Int_t fi = 0;
377  Int_t ti = 0;
378  while (ti < tnb) {
379  TBranch* fb = (TBranch*) from->UncheckedAt(fi);
380  TBranch* tb = (TBranch*) to->UncheckedAt(ti);
381  Int_t firstfi = fi;
382  while (strcmp(fb->GetName(), tb->GetName())) {
383  ++fi;
384  if (fi >= fnb) {
385  // continue at the beginning
386  fi = 0;
387  }
388  if (fi==firstfi) {
389  // We tried all the branches and there is not match.
390  fb = 0;
391  break;
392  }
393  fb = (TBranch*) from->UncheckedAt(fi);
394  }
395  if (fb) {
396  numBasket += CollectBranches(fb, tb);
397  ++fi;
398  if (fi >= fnb) {
399  fi = 0;
400  }
401  } else {
402  if (tb->GetMother()==tb) {
403  // Top level branch.
404  if (!(fOptions & kIgnoreMissingTopLevel)) {
405  fWarningMsg.Form("One of the export top level branches (%s) is not present in the import TTree.",
406  tb->GetName());
407  if (!(fOptions & kNoWarnings)) {
408  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
409  }
410  fIsValid = kFALSE;
411  }
412  } else {
413  fWarningMsg.Form("One of the export sub-branches (%s) is not present in the import TTree.",
414  tb->GetName());
415  if (!(fOptions & kNoWarnings)) {
416  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
417  }
418  fIsValid = kFALSE;
419  }
420  }
421  ++ti;
422  }
423  return numBasket;
424 }
425 
426 ////////////////////////////////////////////////////////////////////////////////
427 /// Fill the array of branches, matching the branches of the 'from' and 'to' TTrees
428 /// Returns the total number of baskets in all the branches.
429 
431 {
432  // Since this is called from the constructor, this can not be a virtual function
433 
434  if (!fFromTree || !fToTree) {
435  return 0;
436  }
439 
440  if (fFromTree->GetBranchRef()) {
441  fToTree->BranchRef();
443  }
444  return numBasket;
445 }
446 
447 ////////////////////////////////////////////////////////////////////////////////
448 /// Collect the information about the on-file basket that need
449 /// to be copied.
450 
452 {
454 
455  for(UInt_t i=0,bi=0; i<len; ++i) {
457  for(Int_t b=0; b<from->GetWriteBasket(); ++b,++bi) {
458  fBasketBranchNum[bi] = i;
459  fBasketNum[bi] = b;
460  fBasketSeek[bi] = from->GetBasketSeek(b);
461  //fprintf(stderr,"For %s %d %lld\n",from->GetName(),bi,fBasketSeek[bi]);
462  fBasketEntry[bi] = from->GetBasketEntry()[b];
463  fBasketIndex[bi] = bi;
464  }
465  }
466 }
467 
468 ////////////////////////////////////////////////////////////////////////////////
469 /// Make sure that all the needed TStreamerInfo are
470 /// present in the output file
471 
473 {
474  TFile *fromFile = fFromTree->GetDirectory()->GetFile();
475  TFile *toFile = fToDirectory->GetFile();
476  TList *l = fromFile->GetStreamerInfoList();
477  TIter next(l);
478  TStreamerInfo *oldInfo;
479  while ( (oldInfo = (TStreamerInfo*)next()) ) {
480  if (oldInfo->IsA() != TStreamerInfo::Class()) {
481  continue;
482  }
483  TStreamerInfo *curInfo = 0;
484  TClass *cl = TClass::GetClass(oldInfo->GetName());
485 
486  if (!cl->IsLoaded() || cl->GetNew()) {
487  // Insure that the TStreamerInfo is loaded
488  curInfo = (TStreamerInfo*)cl->GetStreamerInfo(oldInfo->GetClassVersion());
489  if (oldInfo->GetClassVersion()==1) {
490  // We may have a Foreign class let's look using the
491  // checksum:
492  TStreamerInfo *matchInfo = (TStreamerInfo*)cl->FindStreamerInfo(oldInfo->GetCheckSum());
493  if (matchInfo) {
494  curInfo = matchInfo;
495  }
496  }
497  curInfo->ForceWriteInfo(toFile);
498  } else {
499  // If there is no default constructor the GetStreamerInfo
500  // will not work. It also means (hopefully) that an
501  // inheriting class has a streamerInfo in the list (which
502  // will induces the setting of this streamerInfo)
503 
504  oldInfo->ForceWriteInfo(toFile);
505  }
506  }
507  delete l;
508 }
509 
510 ////////////////////////////////////////////////////////////////////////////////
511 /// Transfer the basket from the input file to the output file
512 
514 {
515  if (IsInPlace())
516  return;
517 
518  TBasket *basket = 0;
519  for(Int_t i=0; i<fToBranches.GetEntriesFast(); ++i) {
520  TBranch *from = (TBranch*)fFromBranches.UncheckedAt( i );
521  TBranch *to = (TBranch*)fToBranches.UncheckedAt( i );
522 
523  basket = (!from->GetListOfBaskets()->IsEmpty()) ? from->GetBasket(from->GetWriteBasket()) : 0;
524  if (basket && basket->GetNevBuf()) {
525  basket = (TBasket*)basket->Clone();
526  basket->SetBranch(to);
527  to->AddBasket(*basket, kFALSE, fToStartEntries+from->GetBasketEntry()[from->GetWriteBasket()]);
528  } else {
530  }
531  // In older files, if the branch is a TBranchElement non-terminal 'object' branch, it's basket will contain 0
532  // events, in newer file in the same case, the write basket will be missing.
533  if (from->GetEntries()!=0 && from->GetWriteBasket()==0 && (basket==0 || basket->GetNevBuf()==0)) {
534  to->SetEntries(to->GetEntries()+from->GetEntries());
535  }
536  }
537 }
538 
539 ////////////////////////////////////////////////////////////////////////////////
540 /// Make sure that all the needed TStreamerInfo are
541 /// present in the output file
542 
544 {
545  // NOTE: We actually need to merge the ProcessId somehow :(
546 
547  TFile *fromfile = fFromTree->GetDirectory()->GetFile();
548  TFile *tofile = fToFile;
549 
550  fPidOffset = tofile->GetNProcessIDs();
551 
552  TIter next(fromfile->GetListOfKeys());
553  TKey *key;
554  TDirectory::TContext cur(fromfile);
555  while ((key = (TKey*)next())) {
556  if (!strcmp(key->GetClassName(),"TProcessID")) {
557  TProcessID *pid = (TProcessID*)key->ReadObjectAny(0);
558  if (!pid) continue;
559 
560  //UShort_t out = TProcessID::WriteProcessID(id,tofile);
561  UShort_t out = 0;
562  TObjArray *pids = tofile->GetListOfProcessIDs();
563  Int_t npids = tofile->GetNProcessIDs();
564  Bool_t wasIn = kFALSE;
565  for (Int_t i=0;i<npids;i++) {
566  if (pids->At(i) == pid) {out = (UShort_t)i; wasIn = kTRUE; break;}
567  }
568 
569  if (!wasIn) {
570  TDirectory *dirsav = gDirectory;
571  tofile->cd();
572  tofile->SetBit(TFile::kHasReferences);
573  pids->AddAtAndExpand(pid,npids);
574  pid->IncrementCount();
575  char name[32];
576  snprintf(name,32,"ProcessID%d",npids);
577  pid->Write(name);
578  tofile->IncrementProcessIDs();
579  if (gDebug > 0) {
580  Info("WriteProcessID", "name=%s, file=%s", name, tofile->GetName());
581  }
582  if (dirsav) dirsav->cd();
583  out = (UShort_t)npids;
584  }
585  if (out<fPidOffset) {
586  Error("CopyProcessIDs","Copied %s from %s might already exist!\n",
587  pid->GetName(),fromfile->GetName());
588  }
589  }
590  }
591 }
592 
593 ////////////////////////////////////////////////////////////////////////////////
594 /// Create a TFileCacheRead if it was requested.
595 
597 {
598  if (fCacheSize && fFromTree->GetCurrentFile()) {
600  auto prev = fFromTree->GetReadCache(f);
601  if (fFileCache && prev == fFileCache) {
602  return;
603  }
604  fPrevCache = prev;
605  // Remove the previous cache if any.
606  if (prev) f->SetCacheRead(nullptr, fFromTree);
607  // The constructor attach the new cache.
609  }
610 }
611 
612 ////////////////////////////////////////////////////////////////////////////////
613 /// Restore the TFileCacheRead to its previous value.
614 
616  if (IsValid() && fFileCache && fFromTree->GetCurrentFile()) {
618  f->SetCacheRead(nullptr,fFromTree); // Remove our file cache.
619  f->SetCacheRead(fPrevCache, fFromTree);
620  }
621 }
622 
623 ////////////////////////////////////////////////////////////////////////////////
624 /// Set the entries and import the cluster range of the
625 
627 {
628  if (IsInPlace())
629  return;
630 
631  // First undo, the external call to SetEntries
632  // We could improve the interface to optional tell the TTreeCloner that the
633  // SetEntries was not done.
635 
637 
638  // This is only updated by TTree::Fill upon seeing a Flush event in TTree::Fill
639  // So we need to propagate (this has also the advantage of turning on the
640  // history recording feature of SetAutoFlush for the next iteration)
642 
644 }
645 
646 ////////////////////////////////////////////////////////////////////////////////
647 /// Set the TFile cache size to be used.
648 /// Note that the default is to use the same size as the default TTreeCache for
649 /// the input tree.
650 /// \param size Size of the cache. Zero disable the use of the cache.
651 
653 {
654  fCacheSize = size;
655  if (IsValid() && fFileCache) {
656  if (fCacheSize == 0 || fCacheSize != fFileCache->GetBufferSize()) {
658  f->SetCacheRead(nullptr,fFromTree);
659  delete fFileCache;
660  fFileCache = nullptr;
661  }
662  }
663  // Note if the TFile cache is needed, it will be created at the
664  // beginning of Exec.
665 }
666 
667 ////////////////////////////////////////////////////////////////////////////////
668 /// Sort the basket according to the user request.
669 
671 {
672  // Currently this sort __has to__ preserve the order
673  // of basket for each individual branch.
674 
675  switch (fCloneMethod) {
677  // nothing to do, it is already sorted.
678  break;
679  case kSortBasketsByEntry: {
680  for(UInt_t i = 0; i < fMaxBaskets; ++i) { fBasketIndex[i] = i; }
681  std::sort(fBasketIndex, fBasketIndex+fMaxBaskets, CompareEntry( this) );
682  break;
683  }
685  default: {
686  for(UInt_t i = 0; i < fMaxBaskets; ++i) { fBasketIndex[i] = i; }
687  std::sort(fBasketIndex, fBasketIndex+fMaxBaskets, CompareSeek( this) );
688  break;
689  }
690  }
691 }
692 
693 ////////////////////////////////////////////////////////////////////////////////
694 /// Fill the file cache with the next set of basket.
695 ///
696 /// \param from index of the first lement of fFromBranches to start caching
697 /// \return The index of first element of fFromBranches that is not in the cache
699 {
700  if (!fFileCache) return 0;
701  // Reset the cache
702  fFileCache->Prefetch(0, 0);
703  Long64_t size = 0;
704  for (UInt_t j = from; j < fMaxBaskets; ++j) {
706 
707 
708  Int_t index = fBasketNum[ fBasketIndex[j] ];
709  Long64_t pos = frombr->GetBasketSeek(index);
710  Int_t len = frombr->GetBasketBytes()[index];
711  if (pos && len) {
712  size += len;
713  if (size > fFileCache->GetBufferSize()) {
714  return j;
715  }
716  fFileCache->Prefetch(pos,len);
717  }
718  }
719  return fMaxBaskets;
720 }
721 
722 ////////////////////////////////////////////////////////////////////////////////
723 /// Transfer the basket from the input file to the output file
724 
726 {
727  TBasket *basket = new TBasket();
728  for(UInt_t j = 0, notCached = 0; j<fMaxBaskets; ++j) {
731 
732  TFile *tofile = fToFile;
733  TFile *fromfile = from->GetFile(0);
734 
735  Int_t index = fBasketNum[ fBasketIndex[j] ];
736 
737  Long64_t pos = from->GetBasketSeek(index);
738  if (IsInPlace()) {
739  if (pos != 0) {
740  if (fFileCache && j >= notCached) {
741  notCached = FillCache(notCached);
742  }
743  Int_t len = from->GetBasketBytes()[index];
744  if (len == 0) {
745  len = basket->ReadBasketBytes(pos, fromfile);
746  from->GetBasketBytes()[index] = len;
747  }
748 
749  basket->LoadBasketBuffers(pos,len,fromfile,fFromTree);
751  basket->CopyTo(tofile);
752  to->fBasketSeek[index] = basket->GetSeekKey();
753  }
754  } else if (pos!=0) {
755  if (fFileCache && j >= notCached) {
756  notCached = FillCache(notCached);
757  }
758  if (from->GetBasketBytes()[index] == 0) {
759  from->GetBasketBytes()[index] = basket->ReadBasketBytes(pos, fromfile);
760  }
761  Int_t len = from->GetBasketBytes()[index];
762 
763  basket->LoadBasketBuffers(pos,len,fromfile,fFromTree);
765  basket->CopyTo(tofile);
766  to->AddBasket(*basket,kTRUE,fToStartEntries + from->GetBasketEntry()[index]);
767  } else {
768  TBasket *frombasket = from->GetBasket( index );
769  if (frombasket && frombasket->GetNevBuf()>0) {
770  TBasket *tobasket = (TBasket*)frombasket->Clone();
771  tobasket->SetBranch(to);
772  to->AddBasket(*tobasket, kFALSE, fToStartEntries+from->GetBasketEntry()[index]);
773  to->FlushOneBasket(to->GetWriteBasket());
774  }
775  }
776  }
777  delete basket;
778 }
TFileCacheRead::Prefetch
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
Definition: TFileCacheRead.cxx:201
l
auto * l
Definition: textangle.C:4
TBranch::GetBasketBytes
Int_t * GetBasketBytes() const
Definition: TBranch.h:210
TTreeCloner::CompareEntry::operator()
Bool_t operator()(UInt_t i1, UInt_t i2)
Definition: TTreeCloner.cxx:56
TTreeCloner::fBasketBranchNum
UInt_t * fBasketBranchNum
[fMaxBaskets] Index of the branch(es) of the basket.
Definition: TTreeCloner.h:46
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:91
TTreeCloner::fToTree
TTree * fToTree
Definition: TTreeCloner.h:38
TObject::TestBit
R__ALWAYS_INLINE Bool_t TestBit(UInt_t f) const
Definition: TObject.h:187
TLeafL.h
snprintf
#define snprintf
Definition: civetweb.c:1540
TTreeCloner::CompareSeek::fObject
TTreeCloner * fObject
Definition: TTreeCloner.h:70
TObjArray
An array of TObjects.
Definition: TObjArray.h:37
TTreeCloner::~TTreeCloner
virtual ~TTreeCloner()
TTreeCloner destructor.
Definition: TTreeCloner.cxx:246
TTreeCloner::fMaxBaskets
UInt_t fMaxBaskets
Definition: TTreeCloner.h:45
Warning
void Warning(const char *location, const char *msgfmt,...)
Use this function in warning situations.
Definition: TError.cxx:231
f
#define f(i)
Definition: RSha256.hxx:104
Option_t
const char Option_t
Definition: RtypesCore.h:66
TStreamerInfo.h
TBranchRef.h
TTreeCloner::CollectBaskets
void CollectBaskets()
Collect the information about the on-file basket that need to be copied.
Definition: TTreeCloner.cxx:451
TBranchElement
A Branch for the case of an object.
Definition: TBranchElement.h:39
UShort_t
unsigned short UShort_t
Definition: RtypesCore.h:40
TTreeCloner::fNeedConversion
Bool_t fNeedConversion
True if the fast merge is not possible but a slow merge might possible.
Definition: TTreeCloner.h:35
TString::Data
const char * Data() const
Definition: TString.h:369
TFileCacheRead::GetBufferSize
virtual Int_t GetBufferSize() const
Definition: TFileCacheRead.h:88
TKey::ReadObjectAny
virtual void * ReadObjectAny(const TClass *expectedClass)
To read an object (non deriving from TObject) from the file.
Definition: TKey.cxx:1015
TLeafS.h
TTreeCloner::TTreeCloner
TTreeCloner(const TTreeCloner &)=delete
TBranch::GetListOfBranches
TObjArray * GetListOfBranches()
Definition: TBranch.h:242
TTree::GetReadCache
TTreeCache * GetReadCache(TFile *file) const
Find and return the TTreeCache registered with the file and which may contain branches for us.
Definition: TTree.cxx:6251
TBranch::AddBasket
virtual void AddBasket(TBasket &b, Bool_t ondisk, Long64_t startEntry)
Add the basket to this branch.
Definition: TBranch.cxx:543
TBranch.h
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TTreeCloner::CopyMemoryBaskets
void CopyMemoryBaskets()
Transfer the basket from the input file to the output file.
Definition: TTreeCloner.cxx:513
TTree
A TTree represents a columnar dataset.
Definition: TTree.h:79
TLeaf::GetTypeName
virtual const char * GetTypeName() const
Definition: TLeaf.h:138
TTreeCloner::RestoreCache
void RestoreCache()
Restore the TFileCacheRead to its previous value.
Definition: TTreeCloner.cxx:615
TBranch::GetListOfBaskets
TObjArray * GetListOfBaskets()
Definition: TBranch.h:241
TBranch::SetEntries
virtual void SetEntries(Long64_t entries)
Set the number of entries in this branch.
Definition: TBranch.cxx:2757
TProcessID
A TProcessID identifies a ROOT job in a unique way in time and space.
Definition: TProcessID.h:74
TClass::IsLoaded
Bool_t IsLoaded() const
Return true if the shared library of this class is currently in the a process's memory.
Definition: TClass.cxx:5896
TProcessID::IncrementCount
Int_t IncrementCount()
Increase the reference count to this object.
Definition: TProcessID.cxx:313
TDirectoryFile::cd
Bool_t cd(const char *path=nullptr) override
Change current directory to "this" directory.
Definition: TDirectoryFile.cxx:340
TLeafO.h
TString::Contains
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:624
TTree::fFlushedBytes
Long64_t fFlushedBytes
Number of auto-flushed bytes.
Definition: TTree.h:89
TDirectory::TContext
Small helper to keep current directory context.
Definition: TDirectory.h:52
TClass::GetStreamerInfo
TVirtualStreamerInfo * GetStreamerInfo(Int_t version=0, Bool_t isTransient=kFALSE) const
returns a pointer to the TVirtualStreamerInfo object for version If the object does not exist,...
Definition: TClass.cxx:4583
TDirectory::cd
virtual Bool_t cd(const char *path=nullptr)
Change current directory to "this" directory.
Definition: TDirectory.cxx:504
TTreeCloner::fBasketIndex
UInt_t * fBasketIndex
[fMaxBaskets] ordered list of basket indices to be written.
Definition: TTreeCloner.h:51
TBasket::CopyTo
Long64_t CopyTo(TFile *to)
Copy the basket of this branch onto the file to.
Definition: TBasket.cxx:146
TTreeCloner::ImportClusterRanges
void ImportClusterRanges()
Set the entries and import the cluster range of the.
Definition: TTreeCloner.cxx:626
TTreeCloner::fPrevCache
TFileCacheRead * fPrevCache
Cache that set before the TTreeCloner ctor for the 'from' TTree if any.
Definition: TTreeCloner.h:60
TObjArray::UncheckedAt
TObject * UncheckedAt(Int_t i) const
Definition: TObjArray.h:90
TObjArray::At
TObject * At(Int_t idx) const
Definition: TObjArray.h:166
TTree.h
TTreeCloner::fBasketEntry
Long64_t * fBasketEntry
[fMaxBaskets] list of basket start entries.
Definition: TTreeCloner.h:50
TFile::GetListOfProcessIDs
TObjArray * GetListOfProcessIDs() const
Definition: TFile.h:228
TTreeCloner::fFromTree
TTree * fFromTree
Definition: TTreeCloner.h:37
TBasket::GetNevBuf
Int_t GetNevBuf() const
Definition: TBasket.h:129
TString
Basic string class.
Definition: TString.h:136
TTreeCloner::CompareEntry
friend class CompareEntry
Definition: TTreeCloner.h:84
TBranch::GetBasketEntry
Long64_t * GetBasketEntry() const
Definition: TBranch.h:211
TObject::InheritsFrom
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
Definition: TObject.cxx:445
TBranchClones::fBranchCount
TBranch * fBranchCount
Branch with clones count.
Definition: TBranchClones.h:37
b
#define b(i)
Definition: RSha256.hxx:100
TFile.h
bool
TTreeCloner::CompareSeek
friend class CompareSeek
Definition: TTreeCloner.h:83
TClass::FindStreamerInfo
TVirtualStreamerInfo * FindStreamerInfo(TObjArray *arr, UInt_t checksum) const
Find the TVirtualStreamerInfo in the StreamerInfos corresponding to checksum.
Definition: TClass.cxx:7006
TDirectory::IsWritable
virtual Bool_t IsWritable() const
Definition: TDirectory.h:190
TTree::GetListOfBranches
virtual TObjArray * GetListOfBranches()
Definition: TTree.h:483
TObjArray::AddLast
virtual void AddLast(TObject *obj)
Add object in the next empty slot in the array.
Definition: TObjArray.cxx:178
TTree::ImportClusterRanges
void ImportClusterRanges(TTree *fromtree)
Appends the cluster range information stored in 'fromtree' to this tree, including the value of fAuto...
Definition: TTree.cxx:6305
TBranch::kDoNotUseBufferMap
@ kDoNotUseBufferMap
Definition: TBranch.h:108
TFile::GetStreamerInfoList
virtual TList * GetStreamerInfoList() final
Read the list of TStreamerInfo objects written to this file.
Definition: TFile.cxx:1361
TTreeCloner::CreateCache
void CreateCache()
Create a TFileCacheRead if it was requested.
Definition: TTreeCloner.cxx:596
TBasket::ReadBasketBytes
Int_t ReadBasketBytes(Long64_t pos, TFile *file)
Read basket buffers in memory and cleanup.
Definition: TBasket.cxx:698
TLeaf::IncludeRange
virtual Bool_t IncludeRange(TLeaf *)
Definition: TLeaf.h:145
TTreeCloner::kSortBasketsByEntry
@ kSortBasketsByEntry
Definition: TTreeCloner.h:66
TTree::GetTree
virtual TTree * GetTree() const
Definition: TTree.h:512
TTreeCloner::fToStartEntries
Long64_t fToStartEntries
Number of entries in the target tree before any addition.
Definition: TTreeCloner.h:56
TBranch
A TTree is a list of TBranches.
Definition: TBranch.h:89
TProcessID.h
TTreeCloner::fToFile
TFile * fToFile
Definition: TTreeCloner.h:40
TTree::GetCurrentFile
TFile * GetCurrentFile() const
Return pointer to the current file.
Definition: TTree.cxx:5411
TString::Form
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2309
TDirectoryFile::GetListOfKeys
TList * GetListOfKeys() const override
Definition: TDirectoryFile.h:93
TStreamerInfo::ForceWriteInfo
void ForceWriteInfo(TFile *file, Bool_t force=kFALSE)
Recursively mark streamer infos for writing to a file.
Definition: TStreamerInfo.cxx:3137
TTreeCache.h
TTreeCloner::CompareSeek::operator()
Bool_t operator()(UInt_t i1, UInt_t i2)
Definition: TTreeCloner.cxx:43
TBranchElement.h
TFileCacheRead.h
TBranch::GetFile
virtual TFile * GetFile(Int_t mode=0)
Return pointer to the file where branch buffers reside, returns 0 in case branch buffers reside in th...
Definition: TBranch.cxx:1793
TObject::ResetBit
void ResetBit(UInt_t f)
Definition: TObject.h:186
TBranch::GetBasketSeek
virtual Long64_t GetBasketSeek(Int_t basket) const
Return address of basket in the file.
Definition: TBranch.cxx:1300
TKey::GetClassName
virtual const char * GetClassName() const
Definition: TKey.h:76
TStreamerInfo
Describe Streamer information for one class version.
Definition: TStreamerInfo.h:44
TLeafI.h
TTreeCloner::fFileCache
TFileCacheRead * fFileCache
File Cache used to reduce the number of individual reads.
Definition: TTreeCloner.h:59
TLeaf
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition: TLeaf.h:57
TObject::SetBit
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:696
TBranchClones
A Branch for the case of an array of clone objects.
Definition: TBranchClones.h:29
TTreeCloner
Class implementing or helping the various TTree cloning method.
Definition: TTreeCloner.h:31
TObjArray::GetEntriesFast
Int_t GetEntriesFast() const
Definition: TObjArray.h:64
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TLeafC.h
TNamed::Clone
virtual TObject * Clone(const char *newname="") const
Make a clone of an object using the Streamer facility.
Definition: TNamed.cxx:74
TTree::GetBranchRef
virtual TBranchRef * GetBranchRef() const
Definition: TTree.h:445
TBranch::fBasketSeek
Long64_t * fBasketSeek
[fMaxBaskets] Addresses of baskets on file
Definition: TBranch.h:139
gDirectory
#define gDirectory
Definition: TDirectory.h:290
TTreeCloner::SortBaskets
void SortBaskets()
Sort the basket according to the user request.
Definition: TTreeCloner.cxx:670
TTreeCloner::IsInPlace
Bool_t IsInPlace() const
Definition: TTreeCloner.h:118
TObjArray::AddAtAndExpand
virtual void AddAtAndExpand(TObject *obj, Int_t idx)
Add object at position idx.
Definition: TObjArray.cxx:235
TBasket::LoadBasketBuffers
Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree=0)
Load basket buffers in memory without unziping.
Definition: TBasket.cxx:245
TTreeCloner::FillCache
UInt_t FillCache(UInt_t from)
Fill the file cache with the next set of basket.
Definition: TTreeCloner.cxx:698
gDebug
Int_t gDebug
Definition: TROOT.cxx:590
TTreeCloner.h
TTreeCloner::IsValid
Bool_t IsValid()
Definition: TTreeCloner.h:120
TClass::GetClass
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2946
TStreamerInfo::GetCheckSum
UInt_t GetCheckSum() const
Definition: TStreamerInfo.h:205
TFile::IncrementProcessIDs
virtual void IncrementProcessIDs()
Definition: TFile.h:249
TTreeCloner::fCloneMethod
UInt_t fCloneMethod
Indicates which cloning method was selected.
Definition: TTreeCloner.h:55
TLeafB.h
TKey::IncrementPidOffset
virtual void IncrementPidOffset(UShort_t offset)
Increment fPidOffset by 'offset'.
Definition: TKey.cxx:650
TBranchClones.h
TTreeCloner::CloseOutWriteBaskets
void CloseOutWriteBaskets()
Before we can start adding new basket, we need to flush to disk the partially filled baskets (the Wri...
Definition: TTreeCloner.cxx:262
TTree::SetDirectory
virtual void SetDirectory(TDirectory *dir)
Change the tree's directory.
Definition: TTree.cxx:8843
TFile
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format.
Definition: TFile.h:54
TTree::GetCacheAutoSize
Long64_t GetCacheAutoSize(Bool_t withDefault=kFALSE) const
Used for automatic sizing of the cache.
Definition: TTree.cxx:5349
unsigned int
TFileCacheRead
A cache when reading files over the network.
Definition: TFileCacheRead.h:22
TStreamerInfo::GetClassVersion
Int_t GetClassVersion() const
Definition: TStreamerInfo.h:207
TTree::GetDirectory
TDirectory * GetDirectory() const
Definition: TTree.h:457
TTreeCloner::fToDirectory
TDirectory * fToDirectory
Definition: TTreeCloner.h:39
TTreeCloner::kSortBasketsByBranch
@ kSortBasketsByBranch
Definition: TTreeCloner.h:64
TKey
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:28
TKey::GetSeekKey
virtual Long64_t GetSeekKey() const
Definition: TKey.h:90
TFile::GetNProcessIDs
virtual Int_t GetNProcessIDs() const
Definition: TFile.h:231
TTreeCloner::CopyStreamerInfos
void CopyStreamerInfos()
Make sure that all the needed TStreamerInfo are present in the output file.
Definition: TTreeCloner.cxx:472
TBranch::GetMother
TBranch * GetMother() const
Get our top-level parent branch in the tree.
Definition: TBranch.cxx:2046
TBasket
Manages buffers for branches of a Tree.
Definition: TBasket.h:34
TTreeCloner::fFromBranches
TObjArray fFromBranches
Definition: TTreeCloner.h:42
TTreeCloner::fIsValid
Bool_t fIsValid
Definition: TTreeCloner.h:34
TTreeCloner::fWarningMsg
TString fWarningMsg
Text of the error message lead to an 'invalid' state.
Definition: TTreeCloner.h:32
TTree::SetEntries
virtual Long64_t SetEntries(Long64_t n=-1)
Change number of entries in the tree.
Definition: TTree.cxx:8888
TClass::GetNew
ROOT::NewFunc_t GetNew() const
Return the wrapper around new ThisClass().
Definition: TClass.cxx:7383
Info
void Info(const char *location, const char *msgfmt,...)
Use this function for informational messages.
Definition: TError.cxx:220
TClass
TClass instances represent classes, structs and namespaces in the ROOT type system.
Definition: TClass.h:80
TTreeCloner::fBasketSeek
Long64_t * fBasketSeek
[fMaxBaskets] list of basket position to be read.
Definition: TTreeCloner.h:49
TObject::Write
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:798
TBranch::GetWriteBasket
Int_t GetWriteBasket() const
Definition: TBranch.h:234
TTreeCloner::CopyProcessIds
void CopyProcessIds()
Make sure that all the needed TStreamerInfo are present in the output file.
Definition: TTreeCloner.cxx:543
TDirectory::GetFile
virtual TFile * GetFile() const
Definition: TDirectory.h:174
name
char name[80]
Definition: TGX11.cxx:110
TBranch::AddLastBasket
virtual void AddLastBasket(Long64_t startEntry)
Add the start entry of the write basket (not yet created)
Definition: TBranch.cxx:616
TObjArray::IsEmpty
Bool_t IsEmpty() const
Definition: TObjArray.h:71
TTreeCloner::fCacheSize
Int_t fCacheSize
Requested size of the file cache.
Definition: TTreeCloner.h:58
TDirectory
Describe directory structure in memory.
Definition: TDirectory.h:45
TBranchElement::fMaximum
Int_t fMaximum
Maximum entries for a TClonesArray or variable array.
Definition: TBranchElement.h:80
TTree::BranchRef
virtual TBranch * BranchRef()
Build the optional branch supporting the TRefTable.
Definition: TTree.cxx:2308
TIter
Definition: TCollection.h:233
TTreeCloner::kIgnoreMissingTopLevel
@ kIgnoreMissingTopLevel
Definition: TTreeCloner.h:101
TTreeCloner::kNoFileCache
@ kNoFileCache
Definition: TTreeCloner.h:102
TTreeCloner::fToBranches
TObjArray fToBranches
Definition: TTreeCloner.h:43
TTreeCloner::WriteBaskets
void WriteBaskets()
Transfer the basket from the input file to the output file.
Definition: TTreeCloner.cxx:725
TBranch::FlushOneBasket
Int_t FlushOneBasket(UInt_t which)
If we have a write basket in memory and it contains some entries and has not yet been written to disk...
Definition: TBranch.cxx:1180
TNamed::GetName
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
TTreeCloner::kSortBasketsByOffset
@ kSortBasketsByOffset
Definition: TTreeCloner.h:65
kDefault
@ kDefault
Definition: TSystem.h:229
TBasket::SetBranch
void SetBranch(TBranch *branch)
Definition: TBasket.h:148
Class
void Class()
Definition: Class.C:29
TString::ToLower
void ToLower()
Change string to lower-case.
Definition: TString.cxx:1145
TBranch::GetListOfLeaves
TObjArray * GetListOfLeaves()
Definition: TBranch.h:243
TTreeCloner::fPidOffset
UShort_t fPidOffset
Offset to be added to the copied key/basket.
Definition: TTreeCloner.h:53
TBranch::GetBasket
TBasket * GetBasket(Int_t basket)
Definition: TBranch.h:209
TTree::GetEntries
virtual Long64_t GetEntries() const
Definition: TTree.h:458
TTreeCloner::CollectBranches
UInt_t CollectBranches()
Fill the array of branches, matching the branches of the 'from' and 'to' TTrees Returns the total num...
Definition: TTreeCloner.cxx:430
TTreeCloner::Exec
Bool_t Exec()
Execute the cloning.
Definition: TTreeCloner.cxx:222
TTreeCloner::SetCacheSize
void SetCacheSize(Int_t size)
Set the TFile cache size to be used.
Definition: TTreeCloner.cxx:652
TBranch::GetEntries
Long64_t GetEntries() const
Definition: TBranch.h:247
TBasket.h
TTreeCloner::fBasketNum
UInt_t * fBasketNum
[fMaxBaskets] index of the basket within the branch.
Definition: TTreeCloner.h:47
TList
A doubly linked list.
Definition: TList.h:44
TTreeCloner::kNoWarnings
@ kNoWarnings
Definition: TTreeCloner.h:100
TFile::kHasReferences
@ kHasReferences
Definition: TFile.h:184
int
Error
void Error(const char *location, const char *msgfmt,...)
Use this function in case an error occurred.
Definition: TError.cxx:187
TError.h
TTreeCloner::fOptions
UInt_t fOptions
Definition: TTreeCloner.h:36