Logo ROOT   6.10/09
Reference Guide
TTreeCloner.cxx
Go to the documentation of this file.
1 // @(#)root/tree:$Id$
2 // Author: Philippe Canal 07/11/2005
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /** \class TTreeCloner
13 \ingroup tree
14 
15 Class implementing or helping the various TTree cloning method
16 */
17 
18 #include "TBasket.h"
19 #include "TBranch.h"
20 #include "TBranchClones.h"
21 #include "TBranchElement.h"
22 #include "TStreamerInfo.h"
23 #include "TBranchRef.h"
24 #include "TError.h"
25 #include "TProcessID.h"
26 #include "TMath.h"
27 #include "TTree.h"
28 #include "TTreeCloner.h"
29 #include "TFile.h"
30 #include "TLeafB.h"
31 #include "TLeafI.h"
32 #include "TLeafL.h"
33 #include "TLeafS.h"
34 #include "TLeafO.h"
35 #include "TLeafC.h"
36 #include "TFileCacheRead.h"
37 
38 #include <algorithm>
39 
40 ////////////////////////////////////////////////////////////////////////////////
41 
43 {
44  if (fObject->fBasketSeek[i1] == fObject->fBasketSeek[i2]) {
45  if (fObject->fBasketEntry[i1] == fObject->fBasketEntry[i2]) {
46  return i1 < i2;
47  }
48  return fObject->fBasketEntry[i1] < fObject->fBasketEntry[i2];
49  }
50  return fObject->fBasketSeek[i1] < fObject->fBasketSeek[i2];
51 }
52 
53 ////////////////////////////////////////////////////////////////////////////////
54 
56 {
57  if (fObject->fBasketEntry[i1] == fObject->fBasketEntry[i2]) {
58  return i1 < i2;
59  }
60  return fObject->fBasketEntry[i1] < fObject->fBasketEntry[i2];
61 }
62 
63 ////////////////////////////////////////////////////////////////////////////////
64 /// Constructor. This object would transfer the data from
65 /// 'from' to 'to' using the method indicated in method.
66 ///
67 /// The value of the parameter 'method' determines in which
68 /// order the branches' baskets are written to the output file.
69 ///
70 /// When a TTree is filled the data is stored in the individual
71 /// branches' basket. Each basket is written individually to
72 /// the disk as soon as it is full. In consequence the baskets
73 /// of branches that contain 'large' data chunk are written to
74 /// the disk more often.
75 ///
76 /// There is currently 3 supported sorting order:
77 ///
78 /// SortBasketsByOffset (the default)
79 /// SortBasketsByBranch
80 /// SortBasketsByEntry
81 ///
82 /// When using SortBasketsByOffset the baskets are written in
83 /// the output file in the same order as in the original file
84 /// (i.e. the basket are sorted on their offset in the original
85 /// file; Usually this also means that the baskets are sorted
86 /// on the index/number of the _last_ entry they contain)
87 ///
88 /// When using SortBasketsByBranch all the baskets of each
89 /// individual branches are stored contiguously. This tends to
90 /// optimize reading speed when reading a small number (1->5) of
91 /// branches, since all their baskets will be clustered together
92 /// instead of being spread across the file. However it might
93 /// decrease the performance when reading more branches (or the full
94 /// entry).
95 ///
96 /// When using SortBasketsByEntry the baskets with the lowest
97 /// starting entry are written first. (i.e. the baskets are
98 /// sorted on the index/number of the first entry they contain).
99 /// This means that on the file the baskets will be in the order
100 /// in which they will be needed when reading the whole tree
101 /// sequentially.
102 
103 TTreeCloner::TTreeCloner(TTree *from, TTree *to, Option_t *method, UInt_t options) :
104  fWarningMsg(),
105  fIsValid(kTRUE),
107  fOptions(options),
108  fFromTree(from),
109  fToTree(to),
110  fMethod(method),
111  fFromBranches( from ? from->GetListOfLeaves()->GetEntries()+1 : 0),
112  fToBranches( to ? to->GetListOfLeaves()->GetEntries()+1 : 0),
119  fPidOffset(0),
121  fToStartEntries(0),
122  fCacheSize(0LL),
123  fFileCache(nullptr),
124  fPrevCache(nullptr)
125 {
126  TString opt(method);
127  opt.ToLower();
128  if (opt.Contains("sortbasketsbybranch")) {
129  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByBranch");
131  } else if (opt.Contains("sortbasketsbyentry")) {
132  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByEntry");
134  } else {
135  //::Info("TTreeCloner::TTreeCloner","use: kSortBasketsByOffset");
137  }
139 
140  if (fFromTree == nullptr) {
141  if (to)
142  fWarningMsg.Form("An input TTree is required (cloning to %s).",
143  to->GetName());
144  else
145  fWarningMsg.Form("An input and output TTree are required.");
146  if (!(fOptions & kNoWarnings)) {
147  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
148  }
149  fIsValid = kFALSE;
150  }
151  if (fToTree == nullptr) {
152  fWarningMsg.Form("An output TTree is required (cloning %s).",
153  from ? from->GetName() : "no tree");
154  if (!(fOptions & kNoWarnings)) {
155  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
156  }
157  fIsValid = kFALSE;
158  } else if (fToTree->GetDirectory() == nullptr) {
159  fWarningMsg.Form("The output TTree (%s) must be associated with a directory.",
160  fToTree->GetName());
161  if (!(fOptions & kNoWarnings)) {
162  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
163  }
164  fIsValid = kFALSE;
165  } else if (fToTree->GetCurrentFile() == nullptr) {
166  fWarningMsg.Form("The output TTree (%s) must be associated with a directory (%s) that is in a file.",
168  if (!(fOptions & kNoWarnings)) {
169  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
170  }
171  fIsValid = kFALSE;
172  } else if (! fToTree->GetDirectory()->IsWritable()) {
174  fWarningMsg.Form("The output TTree (%s) must be associated with a writable file (%s).",
176  } else {
177  fWarningMsg.Form("The output TTree (%s) must be associated with a writable directory (%s in %s).",
179  }
180  if (!(fOptions & kNoWarnings)) {
181  Warning("TTreeCloner::TTreeCloner", "%s", fWarningMsg.Data());
182  }
183  fIsValid = kFALSE;
184  }
185 
186  if (fIsValid && (!(fOptions & kNoFileCache))) {
188  }
189 }
190 
191 ////////////////////////////////////////////////////////////////////////////////
192 /// Execute the cloning.
193 
195 {
196  if (!IsValid()) {
197  return kFALSE;
198  }
199  CreateCache();
202  CopyProcessIds();
204  CollectBaskets();
205  SortBaskets();
206  WriteBaskets();
208  RestoreCache();
209 
210  return kTRUE;
211 }
212 
213 ////////////////////////////////////////////////////////////////////////////////
214 /// TTreeCloner destructor
215 
217 {
218  // The file cache was restored to its previous value at the end of Exec,
219  // we can safely delete our cache.
220  delete fFileCache;
221  delete [] fBasketBranchNum;
222  delete [] fBasketNum;
223  delete [] fBasketSeek;
224  delete [] fBasketEntry;
225  delete [] fBasketIndex;
226 }
227 
228 ////////////////////////////////////////////////////////////////////////////////
229 /// Before we can start adding new basket, we need to flush to
230 /// disk the partially filled baskets (the WriteBasket)
231 
233 {
234  for(Int_t i=0; i<fToBranches.GetEntries(); ++i) {
236  to->FlushOneBasket(to->GetWriteBasket());
237  }
238 }
239 
240 ////////////////////////////////////////////////////////////////////////////////
241 /// Fill the array of branches, adding the branch 'from' and 'to',
242 /// and matching the sub-branches of the 'from' and 'to' branches.
243 /// Returns the total number of baskets in all the from branch and
244 /// it sub-branches.
245 
247  // Since this is called from the constructor, this can not be a virtual function
248 
249  UInt_t numBaskets = 0;
250  if (from->InheritsFrom(TBranchClones::Class())) {
251  TBranchClones *fromclones = (TBranchClones*) from;
252  TBranchClones *toclones = (TBranchClones*) to;
253  numBaskets += CollectBranches(fromclones->fBranchCount, toclones->fBranchCount);
254 
255  } else if (from->InheritsFrom(TBranchElement::Class())) {
256  Int_t nb = from->GetListOfLeaves()->GetEntries();
257  Int_t fnb = to->GetListOfLeaves()->GetEntries();
258  if (nb != fnb && (nb == 0 || fnb == 0)) {
259  // We might be in the case where one branch is split
260  // while the other is not split. We must reject this match.
261  fWarningMsg.Form("The export branch and the import branch do not have the same split level. (The branch name is %s.)",
262  from->GetName());
263  if (!(fOptions & kNoWarnings)) {
264  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
265  }
267  fIsValid = kFALSE;
268  return 0;
269  }
270  if (((TBranchElement*) from)->GetStreamerType() != ((TBranchElement*) to)->GetStreamerType()) {
271  fWarningMsg.Form("The export branch and the import branch do not have the same streamer type. (The branch name is %s.)",
272  from->GetName());
273  if (!(fOptions & kNoWarnings)) {
274  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
275  }
276  fIsValid = kFALSE;
277  return 0;
278  }
279  TBranchElement *fromelem = (TBranchElement*) from;
280  TBranchElement *toelem = (TBranchElement*) to;
281  if (fromelem->fMaximum > toelem->fMaximum) toelem->fMaximum = fromelem->fMaximum;
282  } else {
283 
284  Int_t nb = from->GetListOfLeaves()->GetEntries();
285  Int_t fnb = to->GetListOfLeaves()->GetEntries();
286  if (nb != fnb) {
287  fWarningMsg.Form("The export branch and the import branch (%s) do not have the same number of leaves (%d vs %d)",
288  from->GetName(), fnb, nb);
289  if (!(fOptions & kNoWarnings)) {
290  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
291  }
292  fIsValid = kFALSE;
293  return 0;
294  }
295  for (Int_t i=0;i<nb;i++) {
296 
297  TLeaf *fromleaf_gen = (TLeaf*)from->GetListOfLeaves()->At(i);
298  TLeaf *toleaf_gen = (TLeaf*)to->GetListOfLeaves()->At(i);
299  if (toleaf_gen->IsA() != fromleaf_gen->IsA() ) {
300  // The data type do not match, we can not do a fast merge.
301  fWarningMsg.Form("The export leaf and the import leaf (%s.%s) do not have the data type (%s vs %s)",
302  from->GetName(),fromleaf_gen->GetName(),fromleaf_gen->GetTypeName(),toleaf_gen->GetTypeName());
303  if (! (fOptions & kNoWarnings) ) {
304  Warning("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
305  }
306  fIsValid = kFALSE;
308  return 0;
309  }
310  if (fromleaf_gen->IsA()==TLeafI::Class()) {
311  TLeafI *fromleaf = (TLeafI*)fromleaf_gen;
312  TLeafI *toleaf = (TLeafI*)toleaf_gen;
313  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
314  toleaf->SetMaximum( fromleaf->GetMaximum() );
315  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
316  toleaf->SetMinimum( fromleaf->GetMinimum() );
317  } else if (fromleaf_gen->IsA()==TLeafL::Class()) {
318  TLeafL *fromleaf = (TLeafL*)fromleaf_gen;
319  TLeafL *toleaf = (TLeafL*)toleaf_gen;
320  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
321  toleaf->SetMaximum( fromleaf->GetMaximum() );
322  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
323  toleaf->SetMinimum( fromleaf->GetMinimum() );
324  } else if (fromleaf_gen->IsA()==TLeafB::Class()) {
325  TLeafB *fromleaf = (TLeafB*)fromleaf_gen;
326  TLeafB *toleaf = (TLeafB*)toleaf_gen;
327  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
328  toleaf->SetMaximum( fromleaf->GetMaximum() );
329  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
330  toleaf->SetMinimum( fromleaf->GetMinimum() );
331  } else if (fromleaf_gen->IsA()==TLeafS::Class()) {
332  TLeafS *fromleaf = (TLeafS*)fromleaf_gen;
333  TLeafS *toleaf = (TLeafS*)toleaf_gen;
334  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
335  toleaf->SetMaximum( fromleaf->GetMaximum() );
336  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
337  toleaf->SetMinimum( fromleaf->GetMinimum() );
338  } else if (fromleaf_gen->IsA()==TLeafO::Class()) {
339  TLeafO *fromleaf = (TLeafO*)fromleaf_gen;
340  TLeafO *toleaf = (TLeafO*)toleaf_gen;
341  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
342  toleaf->SetMaximum( fromleaf->GetMaximum() );
343  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
344  toleaf->SetMinimum( fromleaf->GetMinimum() );
345  } else if (fromleaf_gen->IsA()==TLeafC::Class()) {
346  TLeafC *fromleaf = (TLeafC*)fromleaf_gen;
347  TLeafC *toleaf = (TLeafC*)toleaf_gen;
348  if (fromleaf->GetMaximum() > toleaf->GetMaximum())
349  toleaf->SetMaximum( fromleaf->GetMaximum() );
350  if (fromleaf->GetMinimum() < toleaf->GetMinimum())
351  toleaf->SetMinimum( fromleaf->GetMinimum() );
352  if (fromleaf->GetLenStatic() > toleaf->GetLenStatic())
353  toleaf->SetLen(fromleaf->GetLenStatic());
354  }
355  }
356 
357  }
358 
359  fFromBranches.AddLast(from);
360  if (!from->TestBit(TBranch::kDoNotUseBufferMap)) {
361  // Make sure that we reset the Buffer's map if needed.
363  }
364  fToBranches.AddLast(to);
365 
366  numBaskets += from->GetWriteBasket();
367  numBaskets += CollectBranches(from->GetListOfBranches(),to->GetListOfBranches());
368 
369  return numBaskets;
370 }
371 
372 ////////////////////////////////////////////////////////////////////////////////
373 /// Fill the array of branches, matching the branches of the 'from' and 'to' arrays.
374 /// Returns the total number of baskets in all the branches.
375 
377 {
378  // Since this is called from the constructor, this can not be a virtual function
379 
380  Int_t fnb = from->GetEntries();
381  Int_t tnb = to->GetEntries();
382  if (!fnb || !tnb) {
383  return 0;
384  }
385 
386  UInt_t numBasket = 0;
387  Int_t fi = 0;
388  Int_t ti = 0;
389  while (ti < tnb) {
390  TBranch* fb = (TBranch*) from->UncheckedAt(fi);
391  TBranch* tb = (TBranch*) to->UncheckedAt(ti);
392  Int_t firstfi = fi;
393  while (strcmp(fb->GetName(), tb->GetName())) {
394  ++fi;
395  if (fi >= fnb) {
396  // continue at the beginning
397  fi = 0;
398  }
399  if (fi==firstfi) {
400  // We tried all the branches and there is not match.
401  fb = 0;
402  break;
403  }
404  fb = (TBranch*) from->UncheckedAt(fi);
405  }
406  if (fb) {
407  numBasket += CollectBranches(fb, tb);
408  ++fi;
409  if (fi >= fnb) {
410  fi = 0;
411  }
412  } else {
413  if (tb->GetMother()==tb) {
414  // Top level branch.
415  if (!(fOptions & kIgnoreMissingTopLevel)) {
416  fWarningMsg.Form("One of the export top level branches (%s) is not present in the import TTree.",
417  tb->GetName());
418  if (!(fOptions & kNoWarnings)) {
419  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
420  }
421  fIsValid = kFALSE;
422  }
423  } else {
424  fWarningMsg.Form("One of the export sub-branches (%s) is not present in the import TTree.",
425  tb->GetName());
426  if (!(fOptions & kNoWarnings)) {
427  Error("TTreeCloner::CollectBranches", "%s", fWarningMsg.Data());
428  }
429  fIsValid = kFALSE;
430  }
431  }
432  ++ti;
433  }
434  return numBasket;
435 }
436 
437 ////////////////////////////////////////////////////////////////////////////////
438 /// Fill the array of branches, matching the branches of the 'from' and 'to' TTrees
439 /// Returns the total number of baskets in all the branches.
440 
442 {
443  // Since this is called from the constructor, this can not be a virtual function
444 
445  if (!fFromTree || !fToTree) {
446  return 0;
447  }
450 
451  if (fFromTree->GetBranchRef()) {
452  fToTree->BranchRef();
454  }
455  return numBasket;
456 }
457 
458 ////////////////////////////////////////////////////////////////////////////////
459 /// Collect the information about the on-file basket that need
460 /// to be copied.
461 
463 {
465 
466  for(UInt_t i=0,bi=0; i<len; ++i) {
468  for(Int_t b=0; b<from->GetWriteBasket(); ++b,++bi) {
469  fBasketBranchNum[bi] = i;
470  fBasketNum[bi] = b;
471  fBasketSeek[bi] = from->GetBasketSeek(b);
472  //fprintf(stderr,"For %s %d %lld\n",from->GetName(),bi,fBasketSeek[bi]);
473  fBasketEntry[bi] = from->GetBasketEntry()[b];
474  fBasketIndex[bi] = bi;
475  }
476  }
477 }
478 
479 ////////////////////////////////////////////////////////////////////////////////
480 /// Make sure that all the needed TStreamerInfo are
481 /// present in the output file
482 
484 {
485  TFile *fromFile = fFromTree->GetDirectory()->GetFile();
486  TFile *toFile = fToTree->GetDirectory()->GetFile();
487  TList *l = fromFile->GetStreamerInfoList();
488  TIter next(l);
489  TStreamerInfo *oldInfo;
490  while ( (oldInfo = (TStreamerInfo*)next()) ) {
491  if (oldInfo->IsA() != TStreamerInfo::Class()) {
492  continue;
493  }
494  TStreamerInfo *curInfo = 0;
495  TClass *cl = TClass::GetClass(oldInfo->GetName());
496 
497  if (!cl->IsLoaded() || cl->GetNew()) {
498  // Insure that the TStreamerInfo is loaded
499  curInfo = (TStreamerInfo*)cl->GetStreamerInfo(oldInfo->GetClassVersion());
500  if (oldInfo->GetClassVersion()==1) {
501  // We may have a Foreign class let's look using the
502  // checksum:
503  TStreamerInfo *matchInfo = (TStreamerInfo*)cl->FindStreamerInfo(oldInfo->GetCheckSum());
504  if (matchInfo) {
505  curInfo = matchInfo;
506  }
507  }
508  curInfo->ForceWriteInfo(toFile);
509  } else {
510  // If there is no default constructor the GetStreamerInfo
511  // will not work. It also means (hopefully) that an
512  // inheriting class has a streamerInfo in the list (which
513  // will induces the setting of this streamerInfo)
514 
515  oldInfo->ForceWriteInfo(toFile);
516  }
517  }
518  delete l;
519 }
520 
521 ////////////////////////////////////////////////////////////////////////////////
522 /// Transfer the basket from the input file to the output file
523 
525 {
526  TBasket *basket = 0;
527  for(Int_t i=0; i<fToBranches.GetEntries(); ++i) {
528  TBranch *from = (TBranch*)fFromBranches.UncheckedAt( i );
529  TBranch *to = (TBranch*)fToBranches.UncheckedAt( i );
530 
531  basket = from->GetListOfBaskets()->GetEntries() ? from->GetBasket(from->GetWriteBasket()) : 0;
532  if (basket) {
533  basket = (TBasket*)basket->Clone();
534  basket->SetBranch(to);
535  to->AddBasket(*basket, kFALSE, fToStartEntries+from->GetBasketEntry()[from->GetWriteBasket()]);
536  } else {
538  }
539  // In older files, if the branch is a TBranchElement non-terminal 'object' branch, it's basket will contain 0
540  // events, in newer file in the same case, the write basket will be missing.
541  if (from->GetEntries()!=0 && from->GetWriteBasket()==0 && (basket==0 || basket->GetNevBuf()==0)) {
542  to->SetEntries(to->GetEntries()+from->GetEntries());
543  }
544  }
545 }
546 
547 ////////////////////////////////////////////////////////////////////////////////
548 /// Make sure that all the needed TStreamerInfo are
549 /// present in the output file
550 
552 {
553  // NOTE: We actually need to merge the ProcessId somehow :(
554 
555  TFile *fromfile = fFromTree->GetDirectory()->GetFile();
556  TFile *tofile = fToTree->GetDirectory()->GetFile();
557 
558  fPidOffset = tofile->GetNProcessIDs();
559 
560  TIter next(fromfile->GetListOfKeys());
561  TKey *key;
562  TDirectory::TContext cur(fromfile);
563  while ((key = (TKey*)next())) {
564  if (!strcmp(key->GetClassName(),"TProcessID")) {
565  TProcessID *pid = (TProcessID*)key->ReadObjectAny(0);
566  if (!pid) continue;
567 
568  //UShort_t out = TProcessID::WriteProcessID(id,tofile);
569  UShort_t out = 0;
570  TObjArray *pids = tofile->GetListOfProcessIDs();
571  Int_t npids = tofile->GetNProcessIDs();
572  Bool_t wasIn = kFALSE;
573  for (Int_t i=0;i<npids;i++) {
574  if (pids->At(i) == pid) {out = (UShort_t)i; wasIn = kTRUE; break;}
575  }
576 
577  if (!wasIn) {
578  TDirectory *dirsav = gDirectory;
579  tofile->cd();
580  tofile->SetBit(TFile::kHasReferences);
581  pids->AddAtAndExpand(pid,npids);
582  pid->IncrementCount();
583  char name[32];
584  snprintf(name,32,"ProcessID%d",npids);
585  pid->Write(name);
586  tofile->IncrementProcessIDs();
587  if (gDebug > 0) {
588  Info("WriteProcessID", "name=%s, file=%s", name, tofile->GetName());
589  }
590  if (dirsav) dirsav->cd();
591  out = (UShort_t)npids;
592  }
593  if (out<fPidOffset) {
594  Error("CopyProcessIDs","Copied %s from %s might already exist!\n",
595  pid->GetName(),fromfile->GetName());
596  }
597  }
598  }
599 }
600 
601 ////////////////////////////////////////////////////////////////////////////////
602 /// Create a TFileCacheRead if it was requested.
603 
605 {
606  if (fCacheSize && fFromTree->GetCurrentFile()) {
608  auto prev = f->GetCacheRead(fFromTree);
609  if (fFileCache && prev == fFileCache) {
610  return;
611  }
612  fPrevCache = prev;
613  // Remove the previous cache if any.
614  if (prev) f->SetCacheRead(nullptr, fFromTree);
615  // The constructor attach the new cache.
617  }
618 }
619 
620 ////////////////////////////////////////////////////////////////////////////////
621 /// Restore the TFileCacheRead to its previous value.
622 
624  if (IsValid() && fFileCache && fFromTree->GetCurrentFile()) {
626  f->SetCacheRead(nullptr,fFromTree); // Remove our file cache.
628  }
629 }
630 
631 ////////////////////////////////////////////////////////////////////////////////
632 /// Set the entries and import the cluster range of the
633 
635 {
636  // First undo, the external call to SetEntries
637  // We could improve the interface to optional tell the TTreeCloner that the
638  // SetEntries was not done.
640 
642 
643  // This is only updated by TTree::Fill upon seeing a Flush event in TTree::Fill
644  // So we need to propagate (this has also the advantage of turning on the
645  // history recording feature of SetAutoFlush for the next iteration)
647 
649 }
650 
651 ////////////////////////////////////////////////////////////////////////////////
652 /// Set the TFile cache size to be used.
653 /// Note that the default is to use the same size as the default TTreeCache for
654 /// the input tree.
655 /// \param size Size of the cache. Zero disable the use of the cache.
656 
658 {
659  fCacheSize = size;
660  if (IsValid() && fFileCache) {
661  if (fCacheSize == 0 || fCacheSize != fFileCache->GetBufferSize()) {
663  f->SetCacheRead(nullptr,fFromTree);
664  delete fFileCache;
665  fFileCache = nullptr;
666  }
667  }
668  // Note if the TFile cache is needed, it will be created at the
669  // beginning of Exec.
670 }
671 
672 ////////////////////////////////////////////////////////////////////////////////
673 /// Sort the basket according to the user request.
674 
676 {
677  // Currently this sort __has to__ preserve the order
678  // of basket for each individual branch.
679 
680  switch (fCloneMethod) {
682  // nothing to do, it is already sorted.
683  break;
684  case kSortBasketsByEntry: {
685  for(UInt_t i = 0; i < fMaxBaskets; ++i) { fBasketIndex[i] = i; }
686  std::sort(fBasketIndex, fBasketIndex+fMaxBaskets, CompareEntry( this) );
687  break;
688  }
690  default: {
691  for(UInt_t i = 0; i < fMaxBaskets; ++i) { fBasketIndex[i] = i; }
692  std::sort(fBasketIndex, fBasketIndex+fMaxBaskets, CompareSeek( this) );
693  break;
694  }
695  }
696 }
697 
698 ////////////////////////////////////////////////////////////////////////////////
699 /// Fill the file cache with the next set of basket.
700 ///
701 /// \param from index of the first lement of fFromBranches to start caching
702 /// \return The index of first element of fFromBranches that is not in the cache
704 {
705  if (!fFileCache) return 0;
706  // Reset the cache
707  fFileCache->Prefetch(0, 0);
708  Long64_t size = 0;
709  for (UInt_t j = from; j < fMaxBaskets; ++j) {
711 
712 
713  Int_t index = fBasketNum[ fBasketIndex[j] ];
714  Long64_t pos = frombr->GetBasketSeek(index);
715  Int_t len = frombr->GetBasketBytes()[index];
716  if (pos && len) {
717  size += len;
718  if (size > fFileCache->GetBufferSize()) {
719  return j;
720  }
721  fFileCache->Prefetch(pos,len);
722  }
723  }
724  return fMaxBaskets;
725 }
726 
727 ////////////////////////////////////////////////////////////////////////////////
728 /// Transfer the basket from the input file to the output file
729 
731 {
732  TBasket *basket = new TBasket();
733  for(UInt_t j = 0, notCached = 0; j<fMaxBaskets; ++j) {
736 
737  TFile *tofile = to->GetFile(0);
738  TFile *fromfile = from->GetFile(0);
739 
740  Int_t index = fBasketNum[ fBasketIndex[j] ];
741 
742  Long64_t pos = from->GetBasketSeek(index);
743  if (pos!=0) {
744  if (fFileCache && j >= notCached) {
745  notCached = FillCache(notCached);
746  }
747  if (from->GetBasketBytes()[index] == 0) {
748  from->GetBasketBytes()[index] = basket->ReadBasketBytes(pos, fromfile);
749  }
750  Int_t len = from->GetBasketBytes()[index];
751 
752  basket->LoadBasketBuffers(pos,len,fromfile,fFromTree);
754  basket->CopyTo(tofile);
755  to->AddBasket(*basket,kTRUE,fToStartEntries + from->GetBasketEntry()[index]);
756  } else {
757  TBasket *frombasket = from->GetBasket( index );
758  if (frombasket && frombasket->GetNevBuf()>0) {
759  TBasket *tobasket = (TBasket*)frombasket->Clone();
760  tobasket->SetBranch(to);
761  to->AddBasket(*tobasket, kFALSE, fToStartEntries+from->GetBasketEntry()[index]);
762  to->FlushOneBasket(to->GetWriteBasket());
763  }
764  }
765  }
766  delete basket;
767 }
Describe Streamer information for one class version.
Definition: TStreamerInfo.h:43
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
TTreeCloner(const TTreeCloner &)=delete
Int_t ReadBasketBytes(Long64_t pos, TFile *file)
Read basket buffers in memory and cleanup.
Definition: TBasket.cxx:651
virtual void SetMaximum(Char_t max)
Definition: TLeafB.h:52
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
Definition: TObject.cxx:778
A TLeaf describes individual elements of a TBranch See TBranch structure in TTree.
Definition: TLeaf.h:32
A TLeaf for an 8 bit Integer data type.
Definition: TLeafB.h:26
UInt_t * fBasketIndex
[fMaxBaskets] ordered list of basket indices to be written.
Definition: TTreeCloner.h:56
An array of TObjects.
Definition: TObjArray.h:37
Long64_t * GetBasketEntry() const
Definition: TBranch.h:154
void ImportClusterRanges()
Set the entries and import the cluster range of the.
void RestoreCache()
Restore the TFileCacheRead to its previous value.
Long64_t * fBasketEntry
[fMaxBaskets] list of basket start entries.
Definition: TTreeCloner.h:55
long long Long64_t
Definition: RtypesCore.h:69
friend class CompareEntry
Definition: TTreeCloner.h:89
TObjArray * GetListOfBaskets()
Definition: TBranch.h:181
Int_t FlushOneBasket(UInt_t which)
If we have a write basket in memory and it contains some entries and has not yet been written to disk...
Definition: TBranch.cxx:1128
A cache when reading files over the network.
const char Option_t
Definition: RtypesCore.h:62
void WriteBaskets()
Transfer the basket from the input file to the output file.
virtual void SetMinimum(Int_t min)
Definition: TLeafC.h:54
virtual void SetCacheRead(TFileCacheRead *cache, TObject *tree=0, ECacheAction action=kDisconnect)
Set a pointer to the read cache.
Definition: TFile.cxx:2201
unsigned short UShort_t
Definition: RtypesCore.h:36
Bool_t TestBit(UInt_t f) const
Definition: TObject.h:159
virtual void SetMaximum(Bool_t max)
Definition: TLeafO.h:52
virtual void SetMinimum(Long64_t min)
Definition: TLeafL.h:56
virtual Int_t GetMaximum() const
Definition: TLeafS.h:41
virtual const char * GetTypeName() const
Definition: TLeaf.h:76
Long64_t fToStartEntries
Number of entries in the target tree before any addition.
Definition: TTreeCloner.h:61
TVirtualStreamerInfo * GetStreamerInfo(Int_t version=0) const
returns a pointer to the TVirtualStreamerInfo object for version If the object does not exist...
Definition: TClass.cxx:4360
TObjArray * GetListOfProcessIDs() const
Definition: TFile.h:207
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:46
virtual Int_t GetMaximum() const
Definition: TLeafB.h:41
void ForceWriteInfo(TFile *file, Bool_t force=kFALSE)
Recursively mark streamer infos for writing to a file.
virtual Int_t GetMinimum() const
Definition: TLeafI.h:44
Basic string class.
Definition: TString.h:129
TBranch * fBranchCount
Branch with clones count.
Definition: TBranchClones.h:37
void ToLower()
Change string to lower-case.
Definition: TString.cxx:1099
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
A TLeaf for a bool data type.
Definition: TLeafO.h:26
ROOT::NewFunc_t GetNew() const
TObject * At(Int_t idx) const
Definition: TObjArray.h:165
virtual void SetMinimum(Bool_t min)
Definition: TLeafO.h:53
A TLeaf for a variable length string.
Definition: TLeafC.h:26
virtual Int_t GetMinimum() const
Definition: TLeafC.h:42
void SetBit(UInt_t f, Bool_t set)
Set or unset the user status bits as specified in f.
Definition: TObject.cxx:687
void SetBranch(TBranch *branch)
Definition: TBasket.h:89
virtual void SetMaximum(Short_t max)
Definition: TLeafS.h:52
virtual Int_t GetMaximum() const
Definition: TLeafC.h:41
virtual TObjArray * GetListOfBranches()
Definition: TTree.h:405
virtual Int_t GetMaximum() const
Definition: TLeafL.h:43
UInt_t fMaxBaskets
Definition: TTreeCloner.h:50
Int_t * GetBasketBytes() const
Definition: TBranch.h:153
virtual void SetMinimum(Char_t min)
Definition: TLeafB.h:53
void ImportClusterRanges(TTree *fromtree)
Appends the cluster range information stored in &#39;fromtree&#39; to this tree, including the value of fAuto...
Definition: TTree.cxx:6023
void Class()
Definition: Class.C:29
virtual Bool_t IsWritable() const
Definition: TDirectory.h:161
virtual void SetLen(Int_t len=1)
Definition: TLeaf.h:97
Int_t fMaximum
Maximum entries for a TClonesArray or variable array.
Bool_t IsLoaded() const
Return true if the shared library of this class is currently in the a process&#39;s memory.
Definition: TClass.cxx:5549
void CloseOutWriteBaskets()
Before we can start adding new basket, we need to flush to disk the partially filled baskets (the Wri...
virtual Int_t GetMinimum() const
Definition: TLeafL.h:44
void Info(const char *location, const char *msgfmt,...)
Long64_t fFlushedBytes
Number of auto-flushed bytes.
Definition: TTree.h:86
TObjArray * GetListOfBranches()
Definition: TBranch.h:182
TBasket * GetBasket(Int_t basket)
Return pointer to basket basketnumber in this Branch.
Definition: TBranch.cxx:1169
void SortBaskets()
Sort the basket according to the user request.
Book space in a file, create I/O buffers, to fill them, (un)compress them.
Definition: TKey.h:24
A TProcessID identifies a ROOT job in a unique way in time and space.
Definition: TProcessID.h:69
TFileCacheRead * fPrevCache
Cache that set before the TTreeCloner ctor for the &#39;from&#39; TTree if any.
Definition: TTreeCloner.h:65
void Error(const char *location, const char *msgfmt,...)
virtual TFile * GetFile() const
Definition: TDirectory.h:145
Bool_t IsValid()
Definition: TTreeCloner.h:121
virtual TTree * GetTree() const
Definition: TTree.h:432
virtual void IncrementProcessIDs()
Definition: TFile.h:228
A doubly linked list.
Definition: TList.h:43
TTree * fFromTree
Definition: TTreeCloner.h:44
virtual void AddLastBasket(Long64_t startEntry)
Add the start entry of the write basket (not yet created)
Definition: TBranch.cxx:576
virtual void IncrementPidOffset(UShort_t offset)
Increment fPidOffset by &#39;offset&#39;.
Definition: TKey.cxx:642
virtual void AddAtAndExpand(TObject *obj, Int_t idx)
Add object at position idx.
Definition: TObjArray.cxx:222
TVirtualStreamerInfo * FindStreamerInfo(TObjArray *arr, UInt_t checksum) const
Find the TVirtualStreamerInfo in the StreamerInfos corresponding to checksum.
Definition: TClass.cxx:6617
UInt_t GetCheckSum() const
virtual void SetEntries(Long64_t entries)
Set the number of entries in this branch.
Definition: TBranch.cxx:2304
void CopyProcessIds()
Make sure that all the needed TStreamerInfo are present in the output file.
void CopyStreamerInfos()
Make sure that all the needed TStreamerInfo are present in the output file.
virtual void SetMinimum(Short_t min)
Definition: TLeafS.h:53
virtual Bool_t InheritsFrom(const char *classname) const
Returns kTRUE if object inherits from class "classname".
Definition: TObject.cxx:436
virtual Int_t GetMinimum() const
Definition: TLeafO.h:42
Int_t GetWriteBasket() const
Definition: TBranch.h:175
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2332
Int_t IncrementCount()
Increase the reference count to this object.
Definition: TProcessID.cxx:283
unsigned int UInt_t
Definition: RtypesCore.h:42
TFile * GetCurrentFile() const
Return pointer to the current file.
Definition: TTree.cxx:5163
Manages buffers for branches of a Tree.
Definition: TBasket.h:36
TObjArray fToBranches
Definition: TTreeCloner.h:48
TLine * l
Definition: textangle.C:4
The ROOT global object gROOT contains a list of all defined classes.
Definition: TClass.h:71
virtual TList * GetStreamerInfoList()
Read the list of TStreamerInfo objects written to this file.
Definition: TFile.cxx:1329
TTreeCloner * fObject
Definition: TTreeCloner.h:75
virtual TBranchRef * GetBranchRef() const
Definition: TTree.h:369
friend class CompareSeek
Definition: TTreeCloner.h:88
void Warning(const char *location, const char *msgfmt,...)
Bool_t operator()(UInt_t i1, UInt_t i2)
Definition: TTreeCloner.cxx:55
virtual Int_t GetLenStatic() const
Definition: TLeaf.h:69
A Branch for the case of an object.
virtual Long64_t GetBasketSeek(Int_t basket) const
Return address of basket in the file.
Definition: TBranch.cxx:1234
void CreateCache()
Create a TFileCacheRead if it was requested.
const Bool_t kFALSE
Definition: RtypesCore.h:92
virtual void SetMaximum(Long64_t max)
Definition: TLeafL.h:55
UInt_t CollectBranches()
Fill the array of branches, matching the branches of the &#39;from&#39; and &#39;to&#39; TTrees Returns the total num...
Bool_t Exec()
Execute the cloning.
TObject * UncheckedAt(Int_t i) const
Definition: TObjArray.h:89
double f(double x)
virtual Int_t GetMaximum() const
Definition: TLeafI.h:43
virtual Int_t GetMinimum() const
Definition: TLeafS.h:42
UInt_t * fBasketNum
[fMaxBaskets] index of the basket within the branch.
Definition: TTreeCloner.h:52
Long64_t GetCacheAutoSize(Bool_t withDefault=kFALSE) const
Used for automatic sizing of the cache.
Definition: TTree.cxx:5102
Bool_t operator()(UInt_t i1, UInt_t i2)
Definition: TTreeCloner.cxx:42
Describe directory structure in memory.
Definition: TDirectory.h:34
TDirectory * GetDirectory() const
Definition: TTree.h:380
virtual Int_t GetBufferSize() const
UInt_t fOptions
Definition: TTreeCloner.h:43
UInt_t FillCache(UInt_t from)
Fill the file cache with the next set of basket.
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:572
UShort_t fPidOffset
Offset to be added to the copied key/basket.
Definition: TTreeCloner.h:58
TObjArray * GetListOfLeaves()
Definition: TBranch.h:183
TFileCacheRead * fFileCache
File Cache used to reduce the number of individual reads.
Definition: TTreeCloner.h:64
static TClass * GetClass(const char *name, Bool_t load=kTRUE, Bool_t silent=kFALSE)
Static method returning pointer to TClass of the specified class name.
Definition: TClass.cxx:2885
virtual Long64_t GetEntries() const
Definition: TTree.h:381
UInt_t fCloneMethod
Indicates which cloning method was selected.
Definition: TTreeCloner.h:60
void SetCacheSize(Int_t size)
Set the TFile cache size to be used.
virtual TObject * Clone(const char *newname="") const
Make a clone of an object using the Streamer facility.
Definition: TNamed.cxx:65
virtual TBranch * BranchRef()
Build the optional branch supporting the TRefTable.
Definition: TTree.cxx:2147
UInt_t * fBasketBranchNum
[fMaxBaskets] Index of the branch(es) of the basket.
Definition: TTreeCloner.h:51
Int_t GetClassVersion() const
A TLeaf for a 16 bit Integer data type.
Definition: TLeafS.h:26
virtual void SetMaximum(Int_t max)
Definition: TLeafI.h:53
Long64_t GetEntries() const
Definition: TBranch.h:187
Int_t GetNevBuf() const
Definition: TBasket.h:77
virtual void SetMinimum(Int_t min)
Definition: TLeafI.h:54
Class implementing or helping the various TTree cloning method.
Definition: TTreeCloner.h:38
virtual Bool_t cd(const char *path=0)
Change current directory to "this" directory.
Definition: TDirectory.cxx:435
A Branch for the case of an array of clone objects.
Definition: TBranchClones.h:29
virtual TList * GetListOfKeys() const
virtual Int_t GetMaximum() const
Definition: TLeafO.h:41
virtual Int_t GetNProcessIDs() const
Definition: TFile.h:210
you should not use this method at all Int_t Int_t Double_t Double_t Double_t Int_t Double_t Double_t Double_t Double_t b
Definition: TRolke.cxx:630
virtual void AddBasket(TBasket &b, Bool_t ondisk, Long64_t startEntry)
Add the basket to this branch.
Definition: TBranch.cxx:510
#define snprintf
Definition: civetweb.c:822
A TLeaf for a 64 bit Integer data type.
Definition: TLeafL.h:27
Int_t GetEntries() const
Return the number of objects in array (i.e.
Definition: TObjArray.cxx:494
R__EXTERN Int_t gDebug
Definition: Rtypes.h:83
Bool_t fNeedConversion
True if the fast merge is not possible but a slow merge might possible.
Definition: TTreeCloner.h:42
TFileCacheRead * GetCacheRead(TObject *tree=0) const
Return a pointer to the current read cache.
Definition: TFile.cxx:1202
virtual void AddLast(TObject *obj)
Add object in the next empty slot in the array.
Definition: TObjArray.cxx:169
A TTree object has a header with a name and a title.
Definition: TTree.h:78
#define gDirectory
Definition: TDirectory.h:211
TString fWarningMsg
Text of the error message lead to an &#39;invalid&#39; state.
Definition: TTreeCloner.h:39
Long64_t CopyTo(TFile *to)
Copy the basket of this branch onto the file to.
Definition: TBasket.cxx:159
void ResetBit(UInt_t f)
Definition: TObject.h:158
TTree * fToTree
Definition: TTreeCloner.h:45
virtual Int_t GetMinimum() const
Definition: TLeafB.h:42
virtual TFile * GetFile(Int_t mode=0)
Return pointer to the file where branch buffers reside, returns 0 in case branch buffers reside in th...
Definition: TBranch.cxx:1474
virtual Long64_t SetEntries(Long64_t n=-1)
Change number of entries in the tree.
Definition: TTree.cxx:8484
A TTree is a list of TBranches.
Definition: TBranch.h:57
virtual void SetMaximum(Int_t max)
Definition: TLeafC.h:53
Long64_t * fBasketSeek
[fMaxBaskets] list of basket position to be read.
Definition: TTreeCloner.h:54
Option_t * fMethod
Definition: TTreeCloner.h:46
virtual ~TTreeCloner()
TTreeCloner destructor.
void CopyMemoryBaskets()
Transfer the basket from the input file to the output file.
const Bool_t kTRUE
Definition: RtypesCore.h:91
void CollectBaskets()
Collect the information about the on-file basket that need to be copied.
TBranch * GetMother() const
Get our top-level parent branch in the tree.
Definition: TBranch.cxx:1625
A TLeaf for an Integer data type.
Definition: TLeafI.h:27
Int_t fCacheSize
Requested size of the file cache.
Definition: TTreeCloner.h:63
TObjArray fFromBranches
Definition: TTreeCloner.h:47
Int_t LoadBasketBuffers(Long64_t pos, Int_t len, TFile *file, TTree *tree=0)
Load basket buffers in memory without unziping.
Definition: TBasket.cxx:221
Bool_t fIsValid
Definition: TTreeCloner.h:41
const char * Data() const
Definition: TString.h:347