Logo ROOT  
Reference Guide
TFileCacheRead.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Rene Brun 18/05/2006
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13  \class TFileCacheRead
14  \ingroup IO
15 
16  A cache when reading files over the network.
17 
18  A caching system to speed up network I/O, i.e. when there is
19  no operating system caching support (like the buffer cache for
20  local disk I/O). The cache makes sure that every I/O is done with
21  a (large) fixed length buffer thereby avoiding many small I/O's.
22  Currently the read cache system is used by the classes TNetFile,
23  TXNetFile and TWebFile (via TFile::ReadBuffers()).
24  When processing TTree, TChain, a specialized class TTreeCache that
25  derives from this class is automatically created.
26 */
27 
28 #include "TEnv.h"
29 #include "TFile.h"
30 #include "TFileCacheRead.h"
31 #include "TFileCacheWrite.h"
32 #include "TFilePrefetch.h"
33 #include "TMathBase.h"
34 
36 
37 ////////////////////////////////////////////////////////////////////////////////
38 /// Default Constructor.
39 
41 {
42  fBufferSizeMin = 0;
43  fBufferSize = 0;
44  fBufferLen = 0;
45  fBytesRead = 0;
47  fBytesReadExtra = 0;
48  fReadCalls = 0;
50  fNseek = 0;
51  fNtot = 0;
52  fNb = 0;
53  fSeekSize = 0;
54  fSeek = 0;
55  fSeekIndex = 0;
56  fSeekSort = 0;
57  fPos = 0;
58  fSeekLen = 0;
59  fSeekSortLen = 0;
60  fSeekPos = 0;
61  fLen = 0;
62  fFile = 0;
63  fBuffer = 0;
64  fIsSorted = kFALSE;
66 
67  //values for the second prefetched block
68  fBNseek = 0;
69  fBNtot = 0;
70  fBNb = 0;
71  fBSeekSize = 0;
72  fBSeek = 0;
73  fBSeekSort = 0;
74  fBSeekIndex = 0;
75  fBPos = 0;
76  fBSeekLen = 0;
77  fBSeekSortLen = 0;
78  fBSeekPos = 0;
79  fBLen = 0;
82 
85  fPrefetch = 0;
87 }
88 
89 ////////////////////////////////////////////////////////////////////////////////
90 /// Creates a TFileCacheRead data structure.
91 
93  : TObject()
94 {
95  if (buffersize <=10000) fBufferSize = 100000;
96  else fBufferSize = buffersize;
97 
99  fBufferLen = 0;
100  fBytesRead = 0;
101  fNoCacheBytesRead = 0;
102  fBytesReadExtra = 0;
103  fReadCalls = 0;
104  fNoCacheReadCalls = 0;
105  fNseek = 0;
106  fNtot = 0;
107  fNb = 0;
108  fSeekSize = 10000;
109  fSeek = new Long64_t[fSeekSize];
110  fSeekIndex = new Int_t[fSeekSize];
112  fPos = new Long64_t[fSeekSize];
113  fSeekLen = new Int_t[fSeekSize];
115  fSeekPos = new Int_t[fSeekSize];
116  fLen = new Int_t[fSeekSize];
117  fFile = file;
118 
119  //initialisation for the second block
120  fBNseek = 0;
121  fBNtot = 0;
122  fBNb = 0;
123  fBSeekSize = 10000;
124  fBSeek = new Long64_t[fBSeekSize];
127  fBPos = new Long64_t[fBSeekSize];
128  fBSeekLen = new Int_t[fBSeekSize];
130  fBSeekPos = new Int_t[fBSeekSize];
131  fBLen = new Int_t[fBSeekSize];
132 
133  fBuffer = 0;
134  fPrefetch = 0;
135  fPrefetchedBlocks = 0;
136 
137  //initialise the prefetch object and set the cache directory
138  // start the thread only if the file is not local
139  fEnablePrefetching = gEnv->GetValue("TFile.AsyncPrefetching", 0);
140 
141  if (fEnablePrefetching && file && strcmp(file->GetEndpointUrl()->GetProtocol(), "file")){
143  }
144  else { //disable the async pref for local files
146  }
147 
148  fIsSorted = kFALSE;
150  fBIsSorted = kFALSE;
152 
153  if (file) file->SetCacheRead(this, tree);
154 }
155 
156 ////////////////////////////////////////////////////////////////////////////////
157 /// Destructor.
158 
160 {
162  delete [] fSeek;
163  delete [] fSeekIndex;
164  delete [] fSeekSort;
165  delete [] fPos;
166  delete [] fSeekLen;
167  delete [] fSeekSortLen;
168  delete [] fSeekPos;
169  delete [] fLen;
170  if (fBuffer)
171  delete [] fBuffer;
172  delete [] fBSeek;
173  delete [] fBSeekIndex;
174  delete [] fBSeekSort;
175  delete [] fBPos;
176  delete [] fBSeekLen;
177  delete [] fBSeekSortLen;
178  delete [] fBSeekPos;
179  delete [] fBLen;
180 }
181 
182 ////////////////////////////////////////////////////////////////////////////////
183 /// Close out any threads or asynchronous fetches used by the underlying
184 /// implementation.
185 /// This is called by TFile::Close to prevent usage of the file handles
186 /// after the closing of the file.
187 
188 void TFileCacheRead::Close(Option_t * /* opt = "" */)
189 {
190  if (fPrefetch) {
191  delete fPrefetch;
192  fPrefetch = 0;
193  }
194 
195 }
196 
197 ////////////////////////////////////////////////////////////////////////////////
198 /// Add block of length len at position pos in the list of blocks to
199 /// be prefetched. If pos <= 0 the current blocks (if any) are reset.
200 
202 {
203  fIsSorted = kFALSE;
205  if (pos <= 0) {
206  fNseek = 0;
207  fNtot = 0;
208  return;
209  }
210  if (fNseek >= fSeekSize) {
211  //reallocate buffers
212  fSeekSize *= 2;
213  Long64_t *aSeek = new Long64_t[fSeekSize];
214  Int_t *aSeekIndex = new Int_t[fSeekSize];
215  Long64_t *aSeekSort = new Long64_t[fSeekSize];
216  Long64_t *aPos = new Long64_t[fSeekSize];
217  Int_t *aSeekLen = new Int_t[fSeekSize];
218  Int_t *aSeekSortLen = new Int_t[fSeekSize];
219  Int_t *aSeekPos = new Int_t[fSeekSize];
220  Int_t *aLen = new Int_t[fSeekSize];
221  for (Int_t i=0;i<fNseek;i++) {
222  aSeek[i] = fSeek[i];
223  aSeekIndex[i] = fSeekIndex[i];
224  aSeekSort[i] = fSeekSort[i];
225  aPos[i] = fPos[i];
226  aSeekLen[i] = fSeekLen[i];
227  aSeekSortLen[i] = fSeekSortLen[i];
228  aSeekPos[i] = fSeekPos[i];
229  aLen[i] = fLen[i];
230  }
231  delete [] fSeek;
232  delete [] fSeekIndex;
233  delete [] fSeekSort;
234  delete [] fPos;
235  delete [] fSeekLen;
236  delete [] fSeekSortLen;
237  delete [] fSeekPos;
238  delete [] fLen;
239  fSeek = aSeek;
240  fSeekIndex = aSeekIndex;
241  fSeekSort = aSeekSort;
242  fPos = aPos;
243  fSeekLen = aSeekLen;
244  fSeekSortLen = aSeekSortLen;
245  fSeekPos = aSeekPos;
246  fLen = aLen;
247  }
248 
249  fSeek[fNseek] = pos;
250  fSeekLen[fNseek] = len;
251  fNseek++;
252  fNtot += len;
253 }
254 
255 
256 ////////////////////////////////////////////////////////////////////////////////
257 
259  //add a new element and increase the size if necessary
260  fBIsSorted = kFALSE;
261  if (pos <= 0) {
262  fBNseek = 0;
263  fBNtot = 0;
264  return;
265  }
266  if (fBNseek >= fBSeekSize) {
267  //reallocate buffers
268  fBSeekSize *= 2;
269  Long64_t *aSeek = new Long64_t[fBSeekSize];
270  Int_t *aSeekIndex = new Int_t[fBSeekSize];
271  Long64_t *aSeekSort = new Long64_t[fBSeekSize];
272  Long64_t *aPos = new Long64_t[fBSeekSize];
273  Int_t *aSeekLen = new Int_t[fBSeekSize];
274  Int_t *aSeekSortLen = new Int_t[fBSeekSize];
275  Int_t *aSeekPos = new Int_t[fBSeekSize];
276  Int_t *aLen = new Int_t[fBSeekSize];
277  for (Int_t i=0;i<fBNseek;i++) {
278  aSeek[i] = fBSeek[i];
279  aSeekIndex[i] = fBSeekIndex[i];
280  aSeekSort[i] = fBSeekSort[i];
281  aPos[i] = fBPos[i];
282  aSeekLen[i] = fBSeekLen[i];
283  aSeekSortLen[i] = fBSeekSortLen[i];
284  aSeekPos[i] = fBSeekPos[i];
285  aLen[i] = fBLen[i];
286  }
287  delete [] fBSeek;
288  delete [] fBSeekIndex;
289  delete [] fBSeekSort;
290  delete [] fBPos;
291  delete [] fBSeekLen;
292  delete [] fBSeekSortLen;
293  delete [] fBSeekPos;
294  delete [] fBLen;
295  fBSeek = aSeek;
296  fBSeekIndex = aSeekIndex;
297  fBSeekSort = aSeekSort;
298  fBPos = aPos;
299  fBSeekLen = aSeekLen;
300  fBSeekSortLen = aSeekSortLen;
301  fBSeekPos = aSeekPos;
302  fBLen = aLen;
303  }
304 
305  fBSeek[fBNseek] = pos;
306  fBSeekLen[fBNseek] = len;
307  fBNseek++;
308  fBNtot += len;
309 }
310 
311 
312 ////////////////////////////////////////////////////////////////////////////////
313 /// Print cache statistics.
314 ///
315 /// The format is:
316 /// ******TreeCache statistics for file: cms2.root ******
317 /// Reading............................: 72761843 bytes in 7 transactions
318 /// Readahead..........................: 256000 bytes with overhead = 0 bytes
319 /// Average transaction................: 10394.549000 Kbytes
320 /// Number of blocks in current cache..: 210, total size: 6280352
321 ///
322 /// If option = "a" the list of blocks in the cache is printed
323 /// NB: this function is automatically called by TTreeCache::Print
324 
325 void TFileCacheRead::Print(Option_t *option) const
326 {
327  TString opt = option;
328  opt.ToLower();
329  printf("Cached Reading.....................: %lld bytes in %d transactions\n",this->GetBytesRead(), this->GetReadCalls());
330  printf("Reading............................: %lld bytes in %d uncached transactions\n",this->GetNoCacheBytesRead(), this->GetNoCacheReadCalls());
331  printf("Readahead..........................: %d bytes with overhead = %lld bytes\n",TFile::GetReadaheadSize(),this->GetBytesReadExtra());
332  if (this->GetReadCalls() > 0)
333  printf("Average transaction................: %f Kbytes\n",0.001*Double_t(this->GetBytesRead())/Double_t(this->GetReadCalls()));
334  else
335  printf("Average transaction................: No read calls yet\n");
336  printf("Number of blocks in current cache..: %d, total size: %d\n",fNseek,fNtot);
337  if (fPrefetch){
338  printf("Prefetching .......................: %lli blocks\n", fPrefetchedBlocks);
339  printf("Prefetching Wait Time..............: %f seconds\n", fPrefetch->GetWaitTime() / 1e+6);
340  }
341 
342  if (!opt.Contains("a")) return;
343  for (Int_t i=0;i<fNseek;i++) {
344  if (fIsSorted && !opt.Contains("s")) {
345  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeekSort[i],fSeekSort[i]+fSeekSortLen[i],fSeekSortLen[i]);
346  } else {
347  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeek[i],fSeek[i]+fSeekLen[i],fSeekLen[i]);
348  }
349  }
350  printf ("Number of long buffers = %d\n",fNb);
351  for (Int_t j=0;j<fNb;j++) {
352  printf("fPos[%d] = %lld, fLen = %d\n",j,fPos[j],fLen[j]);
353  }
354 }
355 
356 ////////////////////////////////////////////////////////////////////////////////
357 /// Read buffer at position pos.
358 ///
359 /// If pos is in the list of prefetched blocks read from fBuffer,
360 /// otherwise need to make a normal read from file. Returns -1 in case of
361 /// read error, 0 in case not in cache, 1 in case read from cache.
362 
364 {
365  Long64_t fileBytesRead0 = fFile->GetBytesRead();
366  Long64_t fileBytesReadExtra0 = fFile->GetBytesReadExtra();
367  Int_t fileReadCalls0 = fFile->GetReadCalls();
368 
369  Int_t loc = -1;
370  Int_t rc = ReadBufferExt(buf, pos, len, loc);
371 
372  fBytesRead += fFile->GetBytesRead() - fileBytesRead0;
373  fBytesReadExtra += fFile->GetBytesReadExtra() - fileBytesReadExtra0;
374  fReadCalls += fFile->GetReadCalls() - fileReadCalls0;
375 
376  return rc;
377 }
378 
379 ////////////////////////////////////////////////////////////////////////////////
380 
382 {
383  if (fEnablePrefetching)
384  return ReadBufferExtPrefetch(buf, pos, len, loc);
385  else
386  return ReadBufferExtNormal(buf, pos, len, loc);
387 }
388 
389 
390 ////////////////////////////////////////////////////////////////////////////////
391 ///prefetch the first block
392 
394 {
395  if (fNseek > 0 && !fIsSorted) {
396  Sort();
397  loc = -1;
401  }
402 
403  //try to prefetch the second block
404  if (fBNseek > 0 && !fBIsSorted) {
405  SecondSort();
406  loc = -1;
409  }
410 
411  // in case we are writing and reading to/from this file, we must check
412  // if this buffer is in the write cache (not yet written to the file)
413  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
414  if (cachew->ReadBuffer(buf,pos,len) == 0) {
415  fFile->SetOffset(pos+len);
416  return 1;
417  }
418  }
419 
420  // try to prefetch from the first block
421  if (loc < 0) {
423  }
424 
425  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
426  if (buf && fPrefetch){
427  // prefetch with the new method
428  fPrefetch->ReadBuffer(buf, pos, len);
429  return 1;
430  }
431  }
432  else if (buf && fPrefetch){
433  // try to preferch from the second block
435 
436  if (loc >= 0 && loc < fBNseek && pos == fBSeekSort[loc]){
437  if (fPrefetch->ReadBuffer(buf, pos, len)) {
438  return 1;
439  }
440  }
441  }
442 
443  return 0;
444 }
445 
446 
447 ////////////////////////////////////////////////////////////////////////////////
448 /// Base function for ReadBuffer.
449 ///
450 /// Also gives out the position of the block in the internal buffer.
451 /// This helps TTreeCacheUnzip to avoid doing twice the binary search.
452 
454 {
455  if (fNseek > 0 && !fIsSorted) {
456  Sort();
457  loc = -1;
458 
459  // If ReadBufferAsync is not supported by this implementation...
460  if (!fAsyncReading) {
461  // Then we use the vectored read to read everything now
463  return -1;
464  }
466  } else {
467  // In any case, we'll start to request the chunks.
468  // This implementation simply reads all the chunks in advance
469  // in the async way.
470 
471  // Use the async readv instead of single reads
472  fFile->ReadBuffers(0, 0, 0, 0); //Clear the XrdClient cache
473  if (fFile->ReadBuffers(0,fPos,fLen,fNb)) {
474  return -1;
475  }
477  }
478  }
479 
480  // in case we are writing and reading to/from this file, we much check
481  // if this buffer is in the write cache (not yet written to the file)
482  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
483  if (cachew->ReadBuffer(buf,pos,len) == 0) {
484  fFile->SetOffset(pos+len);
485  return 1;
486  }
487  }
488 
489  // If asynchronous reading is supported by this implementation...
490  if (fAsyncReading) {
491 
492  // Now we dont have to look for it in the local buffer
493  // if it's async, we expect that the communication library
494  // will handle it more efficiently than we can do here
495 
496  Int_t retval;
497  if (loc < 0)
499 
500  // We use the internal list just to notify if the list is to be reconstructed
501  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
502  // Block found, the caller will get it
503 
504  if (buf) {
505  // disable cache to avoid infinite recursion
506  if (fFile->ReadBuffer(buf, pos, len)) {
507  return -1;
508  }
509  fFile->SetOffset(pos+len);
510  }
511 
512  retval = 1;
513  } else {
514  // Block not found in the list, we report it as a miss
515  retval = 0;
516  }
517 
518  if (gDebug > 0)
519  Info("ReadBuffer","pos=%lld, len=%d, retval=%d, loc=%d, "
520  "fseekSort[loc]=%lld, fSeekLen[loc]=%d",
521  pos, len, retval, loc, fSeekSort[loc], fSeekLen[loc]);
522 
523  return retval;
524  } else {
525 
526  if (loc < 0)
528 
529  if (loc >= 0 && loc <fNseek && pos == fSeekSort[loc]) {
530  if (buf) {
531  memcpy(buf,&fBuffer[fSeekPos[loc]],len);
532  fFile->SetOffset(pos+len);
533  }
534  return 1;
535  }
536  }
537 
538  return 0;
539 }
540 
541 ////////////////////////////////////////////////////////////////////////////////
542 /// Set the file using this cache and reset the current blocks (if any).
543 
545 {
546  fFile = file;
547 
548  if (fAsyncReading) {
549  // If asynchronous reading is not supported by this TFile specialization
550  // we use sync primitives, hence we need the local buffer
551  if (file && file->ReadBufferAsync(0, 0)) {
553  fBuffer = new char[fBufferSize];
554  }
555  }
556 
557  if (action == TFile::kDisconnect)
558  Prefetch(0,0);
559 
560  if (fPrefetch) {
561  if (action == TFile::kDisconnect)
562  SecondPrefetch(0, 0);
563  fPrefetch->SetFile(file, action);
564  }
565 }
566 
567 ////////////////////////////////////////////////////////////////////////////////
568 /// Sort buffers to be prefetched in increasing order of positions.
569 /// Merge consecutive blocks if necessary.
570 
572 {
573  if (!fNseek) return;
575  Int_t i;
576  Int_t nb = 0;
577  Int_t effectiveNseek = 0;
578  for (i=0;i<fNseek;i++) {
579  // Skip duplicates
580  Int_t ind = fSeekIndex[i];
581  if (effectiveNseek!=0 && fSeek[ind]==fSeekSort[effectiveNseek-1])
582  {
583  if (fSeekSortLen[effectiveNseek-1] < fSeekLen[ind]) {
584  fSeekSortLen[effectiveNseek-1] = fSeekLen[ind];
585  }
586  continue;
587  }
588  fSeekSort[effectiveNseek] = fSeek[ind];
589  fSeekSortLen[effectiveNseek] = fSeekLen[ind];
590  ++effectiveNseek;
591  }
592  fNseek = effectiveNseek;
593  if (fNtot > fBufferSizeMin) {
594  fBufferSize = fNtot + 100;
595  delete [] fBuffer;
596  fBuffer = 0;
597  // If ReadBufferAsync is not supported by this implementation
598  // it means that we are using sync primitives, hence we need the local buffer
599  if (!fAsyncReading)
600  fBuffer = new char[fBufferSize];
601  }
602  fPos[0] = fSeekSort[0];
603  fLen[0] = fSeekSortLen[0];
604  fSeekPos[0] = 0;
605  for (i=1;i<fNseek;i++) {
606  fSeekPos[i] = fSeekPos[i-1] + fSeekSortLen[i-1];
607  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
608  //increasing this number must be done with care, as it may increase
609  //the job real time (mismatch with OS buffers)
610  if ((fSeekSort[i] != fSeekSort[i-1]+fSeekSortLen[i-1]) ||
611  (fLen[nb] > 16000000)) {
612  nb++;
613  fPos[nb] = fSeekSort[i];
614  fLen[nb] = fSeekSortLen[i];
615  } else {
616  fLen[nb] += fSeekSortLen[i];
617  }
618  }
619  fNb = nb+1;
620  fIsSorted = kTRUE;
621 }
622 
623 
624 ////////////////////////////////////////////////////////////////////////////////
625 /// Sort buffers to be prefetched in increasing order of positions.
626 ///
627 /// Merge consecutive blocks if necessary.
628 
630 {
631  if (!fBNseek) return;
633  Int_t i;
634  Int_t nb = 0;
635  Int_t effectiveNseek = 0;
636  for (i=0;i<fBNseek;i++) {
637  // Skip duplicates
638  Int_t ind = fBSeekIndex[i];
639  if (effectiveNseek!=0 && fBSeek[ind]==fBSeekSort[effectiveNseek-1])
640  {
641  if (fBSeekSortLen[effectiveNseek-1] < fBSeekLen[ind]) {
642  fBSeekSortLen[effectiveNseek-1] = fBSeekLen[ind];
643  }
644  continue;
645  }
646  fBSeekSort[effectiveNseek] = fBSeek[ind];
647  fBSeekSortLen[effectiveNseek] = fBSeekLen[ind];
648  ++effectiveNseek;
649  }
650  fBNseek = effectiveNseek;
651  if (fBNtot > fBufferSizeMin) {
652  fBufferSize = fBNtot + 100;
653  delete [] fBuffer;
654  fBuffer = 0;
655  // If ReadBufferAsync is not supported by this implementation
656  // it means that we are using sync primitives, hence we need the local buffer
657  if (!fAsyncReading)
658  fBuffer = new char[fBufferSize];
659  }
660  fBPos[0] = fBSeekSort[0];
661  fBLen[0] = fBSeekSortLen[0];
662  fBSeekPos[0] = 0;
663  for (i=1;i<fBNseek;i++) {
664  fBSeekPos[i] = fBSeekPos[i-1] + fBSeekSortLen[i-1];
665  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
666  //increasing this number must be done with care, as it may increase
667  //the job real time (mismatch with OS buffers)
668  if ((fBSeekSort[i] != fBSeekSort[i-1]+fBSeekSortLen[i-1]) ||
669  (fBLen[nb] > 16000000)) {
670  nb++;
671  fBPos[nb] = fBSeekSort[i];
672  fBLen[nb] = fBSeekSortLen[i];
673  } else {
674  fBLen[nb] += fBSeekSortLen[i];
675  }
676  }
677  fBNb = nb+1;
678  fBIsSorted = kTRUE;
679 }
680 
681 ////////////////////////////////////////////////////////////////////////////////
682 
684  return this->fPrefetch;
685 }
686 
687 
688 ////////////////////////////////////////////////////////////////////////////////
689 
691 {
692  if ( fEnablePrefetching && fPrefetch ) {
694  }
695 }
696 
697 
698 ////////////////////////////////////////////////////////////////////////////////
699 /// Sets the buffer size.
700 ///
701 /// If the current prefetch list is too large to fit in
702 /// the new buffer some or all of the prefetch blocks are dropped. The
703 /// requested buffersize must be greater than zero.
704 /// Return values:
705 /// - 0 if the prefetch block lists remain unchanged
706 /// - 1 if some or all blocks have been removed from the prefetch list
707 /// - -1 on error
708 
710 {
711  if (buffersize <= 0) return -1;
712  if (buffersize <=10000) buffersize = 100000;
713 
714  if (buffersize == fBufferSize) {
715  fBufferSizeMin = buffersize;
716  return 0;
717  }
718 
719  Bool_t inval = kFALSE;
720 
721  // the cached data is too large to fit in the new buffer size mark data unavailable
722  if (fNtot > buffersize) {
723  Prefetch(0, 0);
724  inval = kTRUE;
725  }
726  if (fBNtot > buffersize) {
727  SecondPrefetch(0, 0);
728  inval = kTRUE;
729  }
730 
731  char *np = 0;
733  char *pres = 0;
734  if (fIsTransferred) {
735  // will need to preserve buffer data
736  pres = fBuffer;
737  fBuffer = 0;
738  }
739  delete [] fBuffer;
740  fBuffer = 0;
741  np = new char[buffersize];
742  if (pres) {
743  memcpy(np, pres, fNtot);
744  }
745  delete [] pres;
746  }
747 
748  delete [] fBuffer;
749  fBuffer = np;
750  fBufferSizeMin = buffersize;
751  fBufferSize = buffersize;
752 
753  if (inval) {
754  return 1;
755  }
756 
757  return 0;
758 }
759 
760 
761 ////////////////////////////////////////////////////////////////////////////////
762 /// Set the prefetching mode of this file.
763 ///
764 /// If 'setPrefetching', enable the asynchronous prefetching
765 /// (using TFilePrefetch) and if the gEnv and rootrc
766 /// variable Cache.Directory is set, also enable the local
767 /// caching of the prefetched blocks.
768 /// if 'setPrefetching', the old prefetcher is enabled is
769 /// the gEnv and rootrc variable is TFile.AsyncReading
770 
772 {
773  SetEnablePrefetchingImpl(setPrefetching);
774 }
775 
776 ////////////////////////////////////////////////////////////////////////////////
777 /// TFileCacheRead implementation of SetEnablePrefetching.
778 ///
779 /// This function is called from the constructor and should not be virtual.
780 
782 {
783  fEnablePrefetching = setPrefetching;
784 
785  if (!fPrefetch && fEnablePrefetching) {
787  const char* cacheDir = gEnv->GetValue("Cache.Directory", "");
788  if (strcmp(cacheDir, ""))
789  if (!fPrefetch->SetCache((char*) cacheDir))
790  fprintf(stderr, "Error while trying to set the cache directory: %s.\n", cacheDir);
791  if (fPrefetch->ThreadStart()){
792  fprintf(stderr,"Error stating prefetching thread. Disabling prefetching.\n");
793  fEnablePrefetching = 0;
794  }
795  } else if (fPrefetch && !fEnablePrefetching) {
797  fPrefetch = NULL;
798  }
799 
800  //environment variable used to switch to the new method of reading asynchronously
801  if (fEnablePrefetching) {
803  }
804  else {
805  fAsyncReading = gEnv->GetValue("TFile.AsyncReading", 0);
806  if (fAsyncReading) {
807  // Check if asynchronous reading is supported by this TFile specialization
809  if (fFile && !(fFile->ReadBufferAsync(0, 0)))
811  }
812  if (!fAsyncReading && fBuffer == 0) {
813  // we use sync primitives, hence we need the local buffer
814  fBuffer = new char[fBufferSize];
815  }
816  }
817 }
818 
TFileCacheRead::Prefetch
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
Definition: TFileCacheRead.cxx:201
TFileCacheRead::fLen
Int_t * fLen
[fNb] Length of long buffers
Definition: TFileCacheRead.h:50
TFile::ReadBufferAsync
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
Definition: TFile.cxx:5048
TFileCacheRead::fNoCacheBytesRead
Long64_t fNoCacheBytesRead
Number of bytes read by basket to fill cached tree.
Definition: TFileCacheRead.h:33
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:91
e
#define e(i)
Definition: RSha256.hxx:121
TFileCacheRead::GetNoCacheBytesRead
virtual Long64_t GetNoCacheBytesRead() const
Definition: TFileCacheRead.h:90
TFileCacheRead::fSeek
Long64_t * fSeek
[fNseek] Position on file of buffers to be prefetched
Definition: TFileCacheRead.h:43
TFileCacheRead::fBSeekSize
Int_t fBSeekSize
Definition: TFileCacheRead.h:61
TFileCacheRead::fPrefetch
TFilePrefetch * fPrefetch
! Object that does the asynchronous reading in another thread
Definition: TFileCacheRead.h:25
gEnv
R__EXTERN TEnv * gEnv
Definition: TEnv.h:171
TFileCacheRead::GetNoCacheReadCalls
virtual Int_t GetNoCacheReadCalls() const
Definition: TFileCacheRead.h:96
TFileCacheRead::~TFileCacheRead
virtual ~TFileCacheRead()
Destructor.
Definition: TFileCacheRead.cxx:159
TFileCacheRead::fBNseek
Int_t fBNseek
Definition: TFileCacheRead.h:58
TFileCacheRead::TFileCacheRead
TFileCacheRead()
Default Constructor.
Definition: TFileCacheRead.cxx:40
TFile::ECacheAction
ECacheAction
TTreeCache flushing semantics.
Definition: TFile.h:71
TFileCacheRead::fBufferLen
Int_t fBufferLen
Current buffer length (<= fBufferSize)
Definition: TFileCacheRead.h:28
tree
Definition: tree.py:1
TFileCacheRead::ReadBufferExtNormal
virtual Int_t ReadBufferExtNormal(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Base function for ReadBuffer.
Definition: TFileCacheRead.cxx:453
ClassImp
#define ClassImp(name)
Definition: Rtypes.h:364
TFileCacheRead::fNb
Int_t fNb
Number of long buffers.
Definition: TFileCacheRead.h:41
TFileCacheRead::fPos
Long64_t * fPos
[fNb] start of long buffers
Definition: TFileCacheRead.h:46
TObject::Info
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:864
Long64_t
long long Long64_t
Definition: RtypesCore.h:73
TFilePrefetch::SetFile
void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Change the file.
Definition: TFilePrefetch.cxx:319
Int_t
int Int_t
Definition: RtypesCore.h:45
TFileCacheRead::ReadBufferExtPrefetch
virtual Int_t ReadBufferExtPrefetch(char *buf, Long64_t pos, Int_t len, Int_t &loc)
prefetch the first block
Definition: TFileCacheRead.cxx:393
TFileCacheWrite.h
TFileCacheRead::SecondPrefetch
virtual void SecondPrefetch(Long64_t, Int_t)
Definition: TFileCacheRead.cxx:258
TString::Contains
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:624
SafeDelete
#define SafeDelete(p)
Definition: RConfig.hxx:543
TFileCacheRead::fBufferSize
Int_t fBufferSize
Allocated size of fBuffer (at a given time)
Definition: TFileCacheRead.h:27
TFileCacheRead::fBNb
Int_t fBNb
Definition: TFileCacheRead.h:60
TEnv::GetValue
virtual Int_t GetValue(const char *name, Int_t dflt) const
Returns the integer value for a resource.
Definition: TEnv.cxx:491
TFileCacheRead::SetEnablePrefetching
virtual void SetEnablePrefetching(Bool_t setPrefetching=kFALSE)
Set the prefetching mode of this file.
Definition: TFileCacheRead.cxx:771
TFilePrefetch::SetCache
Bool_t SetCache(const char *)
Set the path of the cache directory.
Definition: TFilePrefetch.cxx:528
TMath::Sort
void Sort(Index n, const Element *a, Index *index, Bool_t down=kTRUE)
Definition: TMathBase.h:362
TEnv.h
TFileCacheRead::fBytesRead
Long64_t fBytesRead
Number of bytes read for this cache.
Definition: TFileCacheRead.h:30
TFile::ReadBuffer
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read a buffer from the file.
Definition: TFile.cxx:1634
TString
Definition: TString.h:136
TFile.h
TFileCacheRead::fBSeekIndex
Int_t * fBSeekIndex
[fBNseek]
Definition: TFileCacheRead.h:64
bool
TFileCacheRead::fBSeekSortLen
Int_t * fBSeekSortLen
[fBNseek]
Definition: TFileCacheRead.h:67
TFilePrefetch::ThreadStart
Int_t ThreadStart()
Used to start the consumer thread.
Definition: TFilePrefetch.cxx:351
TFileCacheRead::fBPos
Long64_t * fBPos
[fBNb]
Definition: TFileCacheRead.h:65
TFile::kDisconnect
@ kDisconnect
Definition: TFile.h:71
TFileCacheRead::GetReadCalls
virtual Int_t GetReadCalls() const
Definition: TFileCacheRead.h:95
TFileCacheRead::fBSeekLen
Int_t * fBSeekLen
[fBNseek]
Definition: TFileCacheRead.h:66
Option_t
const typedef char Option_t
Definition: RtypesCore.h:66
TFileCacheRead::fSeekIndex
Int_t * fSeekIndex
[fNseek] sorted index table of fSeek
Definition: TFileCacheRead.h:45
TFileCacheRead.h
TFileCacheRead::fSeekSize
Int_t fSeekSize
Allocated size of fSeek.
Definition: TFileCacheRead.h:42
TFileCacheRead::ReadBuffer
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
Definition: TFileCacheRead.cxx:363
TFile::GetBytesRead
virtual Long64_t GetBytesRead() const
Definition: TFile.h:233
TFileCacheRead::fBIsTransferred
Bool_t fBIsTransferred
Definition: TFileCacheRead.h:71
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
TFilePrefetch::ReadBlock
void ReadBlock(Long64_t *, Int_t *, Int_t)
Create a TFPBlock object or recycle one and add it to the prefetchBlocks list.
Definition: TFilePrefetch.cxx:216
TFile::GetReadCalls
virtual Int_t GetReadCalls() const
Definition: TFile.h:236
TFileCacheRead::fBLen
Int_t * fBLen
[fBNb]
Definition: TFileCacheRead.h:69
gDebug
R__EXTERN Int_t gDebug
Definition: RtypesCore.h:119
TFileCacheRead::fBuffer
char * fBuffer
[fBufferSize] buffer of contiguous prefetched blocks
Definition: TFileCacheRead.h:52
TFile::GetReadaheadSize
static Int_t GetReadaheadSize()
Static function returning the readahead buffer size.
Definition: TFile.cxx:4438
TFileCacheRead::SetBufferSize
virtual Int_t SetBufferSize(Int_t buffersize)
Sets the buffer size.
Definition: TFileCacheRead.cxx:709
TFileCacheRead::fReadCalls
Int_t fReadCalls
Number of read calls for this cache.
Definition: TFileCacheRead.h:32
TFileCacheRead::fBytesReadExtra
Long64_t fBytesReadExtra
Number of extra bytes (overhead) read by the readahead buffer.
Definition: TFileCacheRead.h:31
TFileCacheRead::fIsSorted
Bool_t fIsSorted
True if fSeek array is sorted.
Definition: TFileCacheRead.h:53
TFileCacheRead::Close
virtual void Close(Option_t *option="")
Close out any threads or asynchronous fetches used by the underlying implementation.
Definition: TFileCacheRead.cxx:188
TFileCacheRead::SecondSort
virtual void SecondSort()
Sort buffers to be prefetched in increasing order of positions.
Definition: TFileCacheRead.cxx:629
TFileCacheRead::GetBytesRead
virtual Long64_t GetBytesRead() const
Definition: TFileCacheRead.h:89
TFileCacheRead::fAsyncReading
Bool_t fAsyncReading
Definition: TFileCacheRead.h:36
TFileCacheRead::fSeekLen
Int_t * fSeekLen
[fNseek] Length of buffers to be prefetched
Definition: TFileCacheRead.h:47
TFile
Definition: TFile.h:54
TFileCacheRead::fBNtot
Int_t fBNtot
Definition: TFileCacheRead.h:59
TFileCacheRead
Definition: TFileCacheRead.h:22
TFileCacheRead::Sort
virtual void Sort()
Sort buffers to be prefetched in increasing order of positions.
Definition: TFileCacheRead.cxx:571
TMath::BinarySearch
Long64_t BinarySearch(Long64_t n, const T *array, T value)
Definition: TMathBase.h:278
TFileCacheRead::Print
virtual void Print(Option_t *option="") const
Print cache statistics.
Definition: TFileCacheRead.cxx:325
TFileCacheRead::fNtot
Int_t fNtot
Total size of prefetched blocks.
Definition: TFileCacheRead.h:40
TFileCacheRead::fBSeekSort
Long64_t * fBSeekSort
[fBNseek]
Definition: TFileCacheRead.h:63
TFilePrefetch
Definition: TFilePrefetch.h:33
TFile::ReadBuffers
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
Definition: TFile.cxx:1685
TFileCacheRead::ReadBufferExt
virtual Int_t ReadBufferExt(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Definition: TFileCacheRead.cxx:381
TFileCacheRead::GetPrefetchObj
virtual TFilePrefetch * GetPrefetchObj()
Definition: TFileCacheRead.cxx:683
TFileCacheRead::fSeekPos
Int_t * fSeekPos
[fNseek] Position of sorted blocks in fBuffer
Definition: TFileCacheRead.h:49
TFileCacheRead::fNoCacheReadCalls
Int_t fNoCacheReadCalls
Number of read calls by basket to fill cached tree.
Definition: TFileCacheRead.h:34
TFilePrefetch::GetWaitTime
Long64_t GetWaitTime()
Return the time spent wating for buffer to be read in microseconds.
Definition: TFilePrefetch.cxx:172
Double_t
double Double_t
Definition: RtypesCore.h:59
TFileCacheRead::fPrefetchedBlocks
Long64_t fPrefetchedBlocks
Number of blocks prefetched.
Definition: TFileCacheRead.h:55
TFileCacheRead::GetBytesReadExtra
virtual Long64_t GetBytesReadExtra() const
Definition: TFileCacheRead.h:91
file
Definition: file.py:1
TObject
Definition: TObject.h:37
TFileCacheRead::fSeekSortLen
Int_t * fSeekSortLen
[fNseek] Length of buffers to be prefetched (sorted)
Definition: TFileCacheRead.h:48
TFileCacheRead::fBSeek
Long64_t * fBSeek
[fBNseek]
Definition: TFileCacheRead.h:62
TFileCacheRead::SetEnablePrefetchingImpl
void SetEnablePrefetchingImpl(Bool_t setPrefetching=kFALSE)
TFileCacheRead implementation of SetEnablePrefetching.
Definition: TFileCacheRead.cxx:781
TFileCacheWrite
Definition: TFileCacheWrite.h:19
TFileCacheRead::fBIsSorted
Bool_t fBIsSorted
Definition: TFileCacheRead.h:70
TFileCacheRead::fSeekSort
Long64_t * fSeekSort
[fNseek] Position on file of buffers to be prefetched (sorted)
Definition: TFileCacheRead.h:44
TFileCacheRead::fEnablePrefetching
Bool_t fEnablePrefetching
reading by prefetching asynchronously
Definition: TFileCacheRead.h:37
TMathBase.h
TFileCacheRead::SetFile
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Set the file using this cache and reset the current blocks (if any).
Definition: TFileCacheRead.cxx:544
TFileCacheRead::fNseek
Int_t fNseek
Number of blocks to be prefetched.
Definition: TFileCacheRead.h:39
TFile::GetCacheWrite
TFileCacheWrite * GetCacheWrite() const
Return a pointer to the current write cache.
Definition: TFile.cxx:1196
TFilePrefetch.h
TFilePrefetch::ReadBuffer
Bool_t ReadBuffer(char *, Long64_t, Int_t)
Return a prefetched element.
Definition: TFilePrefetch.cxx:180
TFileCacheRead::fFile
TFile * fFile
Pointer to file.
Definition: TFileCacheRead.h:51
TFileCacheRead::WaitFinishPrefetch
virtual void WaitFinishPrefetch()
Definition: TFileCacheRead.cxx:690
TString::ToLower
void ToLower()
Change string to lower-case.
Definition: TString.cxx:1125
TFileCacheRead::fBSeekPos
Int_t * fBSeekPos
[fBNseek]
Definition: TFileCacheRead.h:68
TFile::GetBytesReadExtra
virtual Long64_t GetBytesReadExtra() const
Definition: TFile.h:234
TFilePrefetch::WaitFinishPrefetch
void WaitFinishPrefetch()
Killing the async prefetching thread.
Definition: TFilePrefetch.cxx:90
TFileCacheRead::fBufferSizeMin
Int_t fBufferSizeMin
Original size of fBuffer.
Definition: TFileCacheRead.h:26
TFile::SetOffset
virtual void SetOffset(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Definition: TFile.cxx:2117
TFileCacheRead::fIsTransferred
Bool_t fIsTransferred
True when fBuffer contains something valid.
Definition: TFileCacheRead.h:54
int