ROOT  6.06/09
Reference Guide
TFileCacheRead.cxx
Go to the documentation of this file.
1 // @(#)root/io:$Id$
2 // Author: Rene Brun 18/05/2006
3 
4 /*************************************************************************
5  * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers. *
6  * All rights reserved. *
7  * *
8  * For the licensing terms see $ROOTSYS/LICENSE. *
9  * For the list of contributors see $ROOTSYS/README/CREDITS. *
10  *************************************************************************/
11 
12 /**
13  \class TFileCacheRead
14  \ingroup IO
15 
16  A cache when reading files over the network.
17 
18  A caching system to speed up network I/O, i.e. when there is
19  no operating system caching support (like the buffer cache for
20  local disk I/O). The cache makes sure that every I/O is done with
21  a (large) fixed length buffer thereby avoiding many small I/O's.
22  Currently the read cache system is used by the classes TNetFile,
23  TXNetFile and TWebFile (via TFile::ReadBuffers()).
24  When processing TTree, TChain, a specialized class TTreeCache that
25  derives from this class is automatically created.
26 */
27 
28 #include "TEnv.h"
29 #include "TFile.h"
30 #include "TFileCacheRead.h"
31 #include "TFileCacheWrite.h"
32 #include "TFilePrefetch.h"
33 #include "TMath.h"
34 
36 
37 ////////////////////////////////////////////////////////////////////////////////
38 /// Default Constructor.
39 
41 {
42  fBufferSizeMin = 0;
43  fBufferSize = 0;
44  fBufferLen = 0;
45  fBytesRead = 0;
46  fNoCacheBytesRead = 0;
47  fBytesReadExtra = 0;
48  fReadCalls = 0;
49  fNoCacheReadCalls = 0;
50  fNseek = 0;
51  fNtot = 0;
52  fNb = 0;
53  fSeekSize = 0;
54  fSeek = 0;
55  fSeekIndex = 0;
56  fSeekSort = 0;
57  fPos = 0;
58  fSeekLen = 0;
59  fSeekSortLen = 0;
60  fSeekPos = 0;
61  fLen = 0;
62  fFile = 0;
63  fBuffer = 0;
64  fIsSorted = kFALSE;
65  fIsTransferred = kFALSE;
66 
67  //values for the second prefetched block
68  fBNseek = 0;
69  fBNtot = 0;
70  fBNb = 0;
71  fBSeekSize = 0;
72  fBSeek = 0;
73  fBSeekSort = 0;
74  fBSeekIndex = 0;
75  fBPos = 0;
76  fBSeekLen = 0;
77  fBSeekSortLen = 0;
78  fBSeekPos = 0;
79  fBLen = 0;
80  fBIsSorted = kFALSE;
81  fBIsTransferred=kFALSE;
82 
83  fAsyncReading = kFALSE;
84  fEnablePrefetching = kFALSE;
85  fPrefetch = 0;
86  fPrefetchedBlocks= 0;
87 }
88 
89 ////////////////////////////////////////////////////////////////////////////////
90 /// Creates a TFileCacheRead data structure.
91 
93  : TObject()
94 {
95  if (buffersize <=10000) fBufferSize = 100000;
96  else fBufferSize = buffersize;
97 
99  fBufferLen = 0;
100  fBytesRead = 0;
101  fNoCacheBytesRead = 0;
102  fBytesReadExtra = 0;
103  fReadCalls = 0;
104  fNoCacheReadCalls = 0;
105  fNseek = 0;
106  fNtot = 0;
107  fNb = 0;
108  fSeekSize = 10000;
109  fSeek = new Long64_t[fSeekSize];
110  fSeekIndex = new Int_t[fSeekSize];
112  fPos = new Long64_t[fSeekSize];
113  fSeekLen = new Int_t[fSeekSize];
115  fSeekPos = new Int_t[fSeekSize];
116  fLen = new Int_t[fSeekSize];
117  fFile = file;
118 
119  //initialisation for the second block
120  fBNseek = 0;
121  fBNtot = 0;
122  fBNb = 0;
123  fBSeekSize = 10000;
124  fBSeek = new Long64_t[fBSeekSize];
127  fBPos = new Long64_t[fBSeekSize];
128  fBSeekLen = new Int_t[fBSeekSize];
130  fBSeekPos = new Int_t[fBSeekSize];
131  fBLen = new Int_t[fBSeekSize];
132 
133  fBuffer = 0;
134  fPrefetch = 0;
135  fPrefetchedBlocks = 0;
136 
137  //initialise the prefetch object and set the cache directory
138  // start the thread only if the file is not local
139  fEnablePrefetching = gEnv->GetValue("TFile.AsyncPrefetching", 0);
140 
141  if (fEnablePrefetching && strcmp(file->GetEndpointUrl()->GetProtocol(), "file")){
143  }
144  else { //disable the async pref for local files
146  }
147 
148  fIsSorted = kFALSE;
150  fBIsSorted = kFALSE;
152 
153  if (file) file->SetCacheRead(this, tree);
154 }
155 
156 ////////////////////////////////////////////////////////////////////////////////
157 /// Destructor.
158 
160 {
162  delete [] fSeek;
163  delete [] fSeekIndex;
164  delete [] fSeekSort;
165  delete [] fPos;
166  delete [] fSeekLen;
167  delete [] fSeekSortLen;
168  delete [] fSeekPos;
169  delete [] fLen;
170  if (fBuffer)
171  delete [] fBuffer;
172  delete [] fBSeek;
173  delete [] fBSeekIndex;
174  delete [] fBSeekSort;
175  delete [] fBPos;
176  delete [] fBSeekLen;
177  delete [] fBSeekSortLen;
178  delete [] fBSeekPos;
179  delete [] fBLen;
180 }
181 
182 ////////////////////////////////////////////////////////////////////////////////
183 /// Close out any threads or asynchronous fetches used by the underlying
184 /// implementation.
185 /// This is called by TFile::Close to prevent usage of the file handles
186 /// after the closing of the file.
187 
188 void TFileCacheRead::Close(Option_t * /* opt = "" */)
189 {
190  if (fPrefetch) {
191  delete fPrefetch;
192  fPrefetch = 0;
193  }
194 
195 }
196 
197 ////////////////////////////////////////////////////////////////////////////////
198 /// Add block of length len at position pos in the list of blocks to
199 /// be prefetched. If pos <= 0 the current blocks (if any) are reset.
200 
202 {
203  fIsSorted = kFALSE;
205  if (pos <= 0) {
206  fNseek = 0;
207  fNtot = 0;
208  return;
209  }
210  if (fNseek >= fSeekSize) {
211  //reallocate buffers
212  fSeekSize *= 2;
213  Long64_t *aSeek = new Long64_t[fSeekSize];
214  Int_t *aSeekIndex = new Int_t[fSeekSize];
215  Long64_t *aSeekSort = new Long64_t[fSeekSize];
216  Long64_t *aPos = new Long64_t[fSeekSize];
217  Int_t *aSeekLen = new Int_t[fSeekSize];
218  Int_t *aSeekSortLen = new Int_t[fSeekSize];
219  Int_t *aSeekPos = new Int_t[fSeekSize];
220  Int_t *aLen = new Int_t[fSeekSize];
221  for (Int_t i=0;i<fNseek;i++) {
222  aSeek[i] = fSeek[i];
223  aSeekIndex[i] = fSeekIndex[i];
224  aSeekSort[i] = fSeekSort[i];
225  aPos[i] = fPos[i];
226  aSeekLen[i] = fSeekLen[i];
227  aSeekSortLen[i] = fSeekSortLen[i];
228  aSeekPos[i] = fSeekPos[i];
229  aLen[i] = fLen[i];
230  }
231  delete [] fSeek;
232  delete [] fSeekIndex;
233  delete [] fSeekSort;
234  delete [] fPos;
235  delete [] fSeekLen;
236  delete [] fSeekSortLen;
237  delete [] fSeekPos;
238  delete [] fLen;
239  fSeek = aSeek;
240  fSeekIndex = aSeekIndex;
241  fSeekSort = aSeekSort;
242  fPos = aPos;
243  fSeekLen = aSeekLen;
244  fSeekSortLen = aSeekSortLen;
245  fSeekPos = aSeekPos;
246  fLen = aLen;
247  }
248 
249  fSeek[fNseek] = pos;
250  fSeekLen[fNseek] = len;
251  fNseek++;
252  fNtot += len;
253 }
254 
255 
256 ////////////////////////////////////////////////////////////////////////////////
257 
259  //add a new element and increase the size if necessary
260  fBIsSorted = kFALSE;
261  if (pos <= 0) {
262  fBNseek = 0;
263  fBNtot = 0;
264  return;
265  }
266  if (fBNseek >= fBSeekSize) {
267  //reallocate buffers
268  fBSeekSize *= 2;
269  Long64_t *aSeek = new Long64_t[fBSeekSize];
270  Int_t *aSeekIndex = new Int_t[fBSeekSize];
271  Long64_t *aSeekSort = new Long64_t[fBSeekSize];
272  Long64_t *aPos = new Long64_t[fBSeekSize];
273  Int_t *aSeekLen = new Int_t[fBSeekSize];
274  Int_t *aSeekSortLen = new Int_t[fBSeekSize];
275  Int_t *aSeekPos = new Int_t[fBSeekSize];
276  Int_t *aLen = new Int_t[fBSeekSize];
277  for (Int_t i=0;i<fBNseek;i++) {
278  aSeek[i] = fBSeek[i];
279  aSeekIndex[i] = fBSeekIndex[i];
280  aSeekSort[i] = fBSeekSort[i];
281  aPos[i] = fBPos[i];
282  aSeekLen[i] = fBSeekLen[i];
283  aSeekSortLen[i] = fBSeekSortLen[i];
284  aSeekPos[i] = fBSeekPos[i];
285  aLen[i] = fBLen[i];
286  }
287  delete [] fBSeek;
288  delete [] fBSeekIndex;
289  delete [] fBSeekSort;
290  delete [] fBPos;
291  delete [] fBSeekLen;
292  delete [] fBSeekSortLen;
293  delete [] fBSeekPos;
294  delete [] fBLen;
295  fBSeek = aSeek;
296  fBSeekIndex = aSeekIndex;
297  fBSeekSort = aSeekSort;
298  fBPos = aPos;
299  fBSeekLen = aSeekLen;
300  fBSeekSortLen = aSeekSortLen;
301  fBSeekPos = aSeekPos;
302  fBLen = aLen;
303  }
304 
305  fBSeek[fBNseek] = pos;
306  fBSeekLen[fBNseek] = len;
307  fBNseek++;
308  fBNtot += len;
309 }
310 
311 
312 ////////////////////////////////////////////////////////////////////////////////
313 /// Print cache statistics.
314 ///
315 /// The format is:
316 /// ******TreeCache statistics for file: cms2.root ******
317 /// Reading............................: 72761843 bytes in 7 transactions
318 /// Readahead..........................: 256000 bytes with overhead = 0 bytes
319 /// Average transaction................: 10394.549000 Kbytes
320 /// Number of blocks in current cache..: 210, total size: 6280352
321 ///
322 /// If option = "a" the list of blocks in the cache is printed
323 /// NB: this function is automatically called by TTreeCache::Print
324 
325 void TFileCacheRead::Print(Option_t *option) const
326 {
327  TString opt = option;
328  opt.ToLower();
329  printf("Cached Reading.....................: %lld bytes in %d transactions\n",this->GetBytesRead(), this->GetReadCalls());
330  printf("Reading............................: %lld bytes in %d uncached transactions\n",this->GetNoCacheBytesRead(), this->GetNoCacheReadCalls());
331  printf("Readahead..........................: %d bytes with overhead = %lld bytes\n",TFile::GetReadaheadSize(),this->GetBytesReadExtra());
332  if (this->GetReadCalls() > 0)
333  printf("Average transaction................: %f Kbytes\n",0.001*Double_t(this->GetBytesRead())/Double_t(this->GetReadCalls()));
334  else
335  printf("Average transaction................: No read calls yet\n");
336  printf("Number of blocks in current cache..: %d, total size: %d\n",fNseek,fNtot);
337  if (fPrefetch){
338  printf("Prefetching .......................: %lli blocks\n", fPrefetchedBlocks);
339  printf("Prefetching Wait Time..............: %f seconds\n", fPrefetch->GetWaitTime() / 1e+6);
340  }
341 
342  if (!opt.Contains("a")) return;
343  for (Int_t i=0;i<fNseek;i++) {
344  if (fIsSorted && !opt.Contains("s")) {
345  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeekSort[i],fSeekSort[i]+fSeekSortLen[i],fSeekSortLen[i]);
346  } else {
347  printf("block: %5d, from: %lld to %lld, len = %d bytes\n",i,fSeek[i],fSeek[i]+fSeekLen[i],fSeekLen[i]);
348  }
349  }
350  printf ("Number of long buffers = %d\n",fNb);
351  for (Int_t j=0;j<fNb;j++) {
352  printf("fPos[%d] = %lld, fLen = %d\n",j,fPos[j],fLen[j]);
353  }
354 }
355 
356 ////////////////////////////////////////////////////////////////////////////////
357 /// Read buffer at position pos.
358 ///
359 /// If pos is in the list of prefetched blocks read from fBuffer,
360 /// otherwise need to make a normal read from file. Returns -1 in case of
361 /// read error, 0 in case not in cache, 1 in case read from cache.
362 
364 {
365  Long64_t fileBytesRead0 = fFile->GetBytesRead();
366  Long64_t fileBytesReadExtra0 = fFile->GetBytesReadExtra();
367  Int_t fileReadCalls0 = fFile->GetReadCalls();
368 
369  Int_t loc = -1;
370  Int_t rc = ReadBufferExt(buf, pos, len, loc);
371 
372  fBytesRead += fFile->GetBytesRead() - fileBytesRead0;
373  fBytesReadExtra += fFile->GetBytesReadExtra() - fileBytesReadExtra0;
374  fReadCalls += fFile->GetReadCalls() - fileReadCalls0;
375 
376  return rc;
377 }
378 
379 ////////////////////////////////////////////////////////////////////////////////
380 
382 {
383  if (fEnablePrefetching)
384  return ReadBufferExtPrefetch(buf, pos, len, loc);
385  else
386  return ReadBufferExtNormal(buf, pos, len, loc);
387 }
388 
389 
390 ////////////////////////////////////////////////////////////////////////////////
391 ///prefetch the first block
392 
394 {
395  if (fNseek > 0 && !fIsSorted) {
396  Sort();
397  loc = -1;
401  }
402 
403  //try to prefetch the second block
404  if (fBNseek > 0 && !fBIsSorted) {
405  SecondSort();
406  loc = -1;
409  }
410 
411  // in case we are writing and reading to/from this file, we must check
412  // if this buffer is in the write cache (not yet written to the file)
413  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
414  if (cachew->ReadBuffer(buf,pos,len) == 0) {
415  fFile->SetOffset(pos+len);
416  return 1;
417  }
418  }
419 
420  // try to prefetch from the first block
421  if (loc < 0) {
423  }
424 
425  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
426  if (buf && fPrefetch){
427  // prefetch with the new method
428  fPrefetch->ReadBuffer(buf, pos, len);
429  return 1;
430  }
431  }
432  else if (buf && fPrefetch){
433  // try to preferch from the second block
435 
436  if (loc >= 0 && loc < fBNseek && pos == fBSeekSort[loc]){
437  if (fPrefetch->ReadBuffer(buf, pos, len)) {
438  return 1;
439  }
440  }
441  }
442 
443  return 0;
444 }
445 
446 
447 ////////////////////////////////////////////////////////////////////////////////
448 /// Base function for ReadBuffer.
449 ///
450 /// Also gives out the position of the block in the internal buffer.
451 /// This helps TTreeCacheUnzip to avoid doing twice the binary search.
452 
454 {
455  if (fNseek > 0 && !fIsSorted) {
456  Sort();
457  loc = -1;
458 
459  // If ReadBufferAsync is not supported by this implementation...
460  if (!fAsyncReading) {
461  // Then we use the vectored read to read everything now
463  return -1;
464  }
466  } else {
467  // In any case, we'll start to request the chunks.
468  // This implementation simply reads all the chunks in advance
469  // in the async way.
470 
471  // Use the async readv instead of single reads
472  fFile->ReadBuffers(0, 0, 0, 0); //Clear the XrdClient cache
473  if (fFile->ReadBuffers(0,fPos,fLen,fNb)) {
474  return -1;
475  }
477  }
478  }
479 
480  // in case we are writing and reading to/from this file, we much check
481  // if this buffer is in the write cache (not yet written to the file)
482  if (TFileCacheWrite *cachew = fFile->GetCacheWrite()) {
483  if (cachew->ReadBuffer(buf,pos,len) == 0) {
484  fFile->SetOffset(pos+len);
485  return 1;
486  }
487  }
488 
489  // If asynchronous reading is supported by this implementation...
490  if (fAsyncReading) {
491 
492  // Now we dont have to look for it in the local buffer
493  // if it's async, we expect that the communication library
494  // will handle it more efficiently than we can do here
495 
496  Int_t retval;
497  if (loc < 0)
499 
500  // We use the internal list just to notify if the list is to be reconstructed
501  if (loc >= 0 && loc < fNseek && pos == fSeekSort[loc]) {
502  // Block found, the caller will get it
503 
504  if (buf) {
505  // disable cache to avoid infinite recursion
506  if (fFile->ReadBuffer(buf, pos, len)) {
507  return -1;
508  }
509  fFile->SetOffset(pos+len);
510  }
511 
512  retval = 1;
513  } else {
514  // Block not found in the list, we report it as a miss
515  retval = 0;
516  }
517 
518  if (gDebug > 0)
519  Info("ReadBuffer","pos=%lld, len=%d, retval=%d, loc=%d, "
520  "fseekSort[loc]=%lld, fSeekLen[loc]=%d",
521  pos, len, retval, loc, fSeekSort[loc], fSeekLen[loc]);
522 
523  return retval;
524  } else {
525 
526  if (loc < 0)
528 
529  if (loc >= 0 && loc <fNseek && pos == fSeekSort[loc]) {
530  if (buf) {
531  memcpy(buf,&fBuffer[fSeekPos[loc]],len);
532  fFile->SetOffset(pos+len);
533  }
534  return 1;
535  }
536  }
537 
538  return 0;
539 }
540 
541 ////////////////////////////////////////////////////////////////////////////////
542 /// Set the file using this cache and reset the current blocks (if any).
543 
545 {
546  fFile = file;
547 
548  if (fAsyncReading) {
549  // If asynchronous reading is not supported by this TFile specialization
550  // we use sync primitives, hence we need the local buffer
551  if (file && file->ReadBufferAsync(0, 0)) {
553  fBuffer = new char[fBufferSize];
554  }
555  }
556 
557  if (action == TFile::kDisconnect)
558  Prefetch(0,0);
559 
560  if (fPrefetch) {
561  if (action == TFile::kDisconnect)
562  SecondPrefetch(0, 0);
563  fPrefetch->SetFile(file);
564  }
565 }
566 
567 ////////////////////////////////////////////////////////////////////////////////
568 /// Sort buffers to be prefetched in increasing order of positions.
569 /// Merge consecutive blocks if necessary.
570 
572 {
573  if (!fNseek) return;
575  Int_t i;
576  Int_t nb = 0;
577  Int_t effectiveNseek = 0;
578  for (i=0;i<fNseek;i++) {
579  // Skip duplicates
580  Int_t ind = fSeekIndex[i];
581  if (effectiveNseek!=0 && fSeek[ind]==fSeekSort[effectiveNseek-1])
582  {
583  if (fSeekSortLen[effectiveNseek-1] < fSeekLen[ind]) {
584  fSeekSortLen[effectiveNseek-1] = fSeekLen[ind];
585  }
586  continue;
587  }
588  fSeekSort[effectiveNseek] = fSeek[ind];
589  fSeekSortLen[effectiveNseek] = fSeekLen[ind];
590  ++effectiveNseek;
591  }
592  fNseek = effectiveNseek;
593  if (fNtot > fBufferSizeMin) {
594  fBufferSize = fNtot + 100;
595  delete [] fBuffer;
596  fBuffer = 0;
597  // If ReadBufferAsync is not supported by this implementation
598  // it means that we are using sync primitives, hence we need the local buffer
599  if (!fAsyncReading)
600  fBuffer = new char[fBufferSize];
601  }
602  fPos[0] = fSeekSort[0];
603  fLen[0] = fSeekSortLen[0];
604  fSeekPos[0] = 0;
605  for (i=1;i<fNseek;i++) {
606  fSeekPos[i] = fSeekPos[i-1] + fSeekSortLen[i-1];
607  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
608  //increasing this number must be done with care, as it may increase
609  //the job real time (mismatch with OS buffers)
610  if ((fSeekSort[i] != fSeekSort[i-1]+fSeekSortLen[i-1]) ||
611  (fLen[nb] > 16000000)) {
612  nb++;
613  fPos[nb] = fSeekSort[i];
614  fLen[nb] = fSeekSortLen[i];
615  } else {
616  fLen[nb] += fSeekSortLen[i];
617  }
618  }
619  fNb = nb+1;
620  fIsSorted = kTRUE;
621 }
622 
623 
624 ////////////////////////////////////////////////////////////////////////////////
625 /// Sort buffers to be prefetched in increasing order of positions.
626 ///
627 /// Merge consecutive blocks if necessary.
628 
630 {
631  if (!fBNseek) return;
633  Int_t i;
634  Int_t nb = 0;
635  Int_t effectiveNseek = 0;
636  for (i=0;i<fBNseek;i++) {
637  // Skip duplicates
638  Int_t ind = fBSeekIndex[i];
639  if (effectiveNseek!=0 && fBSeek[ind]==fBSeekSort[effectiveNseek-1])
640  {
641  if (fBSeekSortLen[effectiveNseek-1] < fBSeekLen[ind]) {
642  fBSeekSortLen[effectiveNseek-1] = fBSeekLen[ind];
643  }
644  continue;
645  }
646  fBSeekSort[effectiveNseek] = fBSeek[ind];
647  fBSeekSortLen[effectiveNseek] = fBSeekLen[ind];
648  ++effectiveNseek;
649  }
650  fBNseek = effectiveNseek;
651  if (fBNtot > fBufferSizeMin) {
652  fBufferSize = fBNtot + 100;
653  delete [] fBuffer;
654  fBuffer = 0;
655  // If ReadBufferAsync is not supported by this implementation
656  // it means that we are using sync primitives, hence we need the local buffer
657  if (!fAsyncReading)
658  fBuffer = new char[fBufferSize];
659  }
660  fBPos[0] = fBSeekSort[0];
661  fBLen[0] = fBSeekSortLen[0];
662  fBSeekPos[0] = 0;
663  for (i=1;i<fBNseek;i++) {
664  fBSeekPos[i] = fBSeekPos[i-1] + fBSeekSortLen[i-1];
665  //in the test below 16 MBytes is pure empirirical and may depend on the file system.
666  //increasing this number must be done with care, as it may increase
667  //the job real time (mismatch with OS buffers)
668  if ((fBSeekSort[i] != fBSeekSort[i-1]+fBSeekSortLen[i-1]) ||
669  (fBLen[nb] > 16000000)) {
670  nb++;
671  fBPos[nb] = fBSeekSort[i];
672  fBLen[nb] = fBSeekSortLen[i];
673  } else {
674  fBLen[nb] += fBSeekSortLen[i];
675  }
676  }
677  fBNb = nb+1;
678  fBIsSorted = kTRUE;
679 }
680 
681 ////////////////////////////////////////////////////////////////////////////////
682 
684  return this->fPrefetch;
685 }
686 
687 
688 ////////////////////////////////////////////////////////////////////////////////
689 
691 {
692  if ( fEnablePrefetching && fPrefetch ) {
694  }
695 }
696 
697 
698 ////////////////////////////////////////////////////////////////////////////////
699 /// Sets the buffer size.
700 ///
701 /// If the current prefetch list is too large to fit in
702 /// the new buffer some or all of the prefetch blocks are dropped. The
703 /// requested buffersize must be greater than zero.
704 /// Return values:
705 /// - 0 if the prefetch block lists remain unchanged
706 /// - 1 if some or all blocks have been removed from the prefetch list
707 /// - -1 on error
708 
710 {
711  if (buffersize <= 0) return -1;
712  if (buffersize <=10000) buffersize = 100000;
713 
714  if (buffersize == fBufferSize) {
715  fBufferSizeMin = buffersize;
716  return 0;
717  }
718 
719  Bool_t inval = kFALSE;
720 
721  // the cached data is too large to fit in the new buffer size mark data unavailable
722  if (fNtot > buffersize) {
723  Prefetch(0, 0);
724  inval = kTRUE;
725  }
726  if (fBNtot > buffersize) {
727  SecondPrefetch(0, 0);
728  inval = kTRUE;
729  }
730 
731  char *np = 0;
733  char *pres = 0;
734  if (fIsTransferred) {
735  // will need to preserve buffer data
736  pres = fBuffer;
737  fBuffer = 0;
738  }
739  delete [] fBuffer;
740  fBuffer = 0;
741  np = new char[buffersize];
742  if (pres) {
743  memcpy(np, pres, fNtot);
744  }
745  delete [] pres;
746  }
747 
748  delete [] fBuffer;
749  fBuffer = np;
750  fBufferSizeMin = buffersize;
751  fBufferSize = buffersize;
752 
753  if (inval) {
754  return 1;
755  }
756 
757  return 0;
758 }
759 
760 
761 ////////////////////////////////////////////////////////////////////////////////
762 /// Set the prefetching mode of this file.
763 ///
764 /// If 'setPrefetching', enable the asynchronous prefetching
765 /// (using TFilePrefetch) and if the gEnv and rootrc
766 /// variable Cache.Directory is set, also enable the local
767 /// caching of the prefetched blocks.
768 /// if 'setPrefetching', the old prefetcher is enabled is
769 /// the gEnv and rootrc variable is TFile.AsyncReading
770 
772 {
773  SetEnablePrefetchingImpl(setPrefetching);
774 }
775 
776 ////////////////////////////////////////////////////////////////////////////////
777 /// TFileCacheRead implementation of SetEnablePrefetching.
778 ///
779 /// This function is called from the constructor and should not be virtual.
780 
782 {
783  fEnablePrefetching = setPrefetching;
784 
785  if (!fPrefetch && fEnablePrefetching) {
787  const char* cacheDir = gEnv->GetValue("Cache.Directory", "");
788  if (strcmp(cacheDir, ""))
789  if (!fPrefetch->SetCache((char*) cacheDir))
790  fprintf(stderr, "Error while trying to set the cache directory: %s.\n", cacheDir);
791  if (fPrefetch->ThreadStart()){
792  fprintf(stderr,"Error stating prefetching thread. Disabling prefetching.\n");
793  fEnablePrefetching = 0;
794  }
795  } else if (fPrefetch && !fEnablePrefetching) {
797  fPrefetch = NULL;
798  }
799 
800  //environment variable used to switch to the new method of reading asynchronously
801  if (fEnablePrefetching) {
803  }
804  else {
805  fAsyncReading = gEnv->GetValue("TFile.AsyncReading", 0);
806  if (fAsyncReading) {
807  // Check if asynchronous reading is supported by this TFile specialization
809  if (fFile && !(fFile->ReadBufferAsync(0, 0)))
811  }
812  if (!fAsyncReading && fBuffer == 0) {
813  // we use sync primitives, hence we need the local buffer
814  fBuffer = new char[fBufferSize];
815  }
816  }
817 }
818 
Int_t fNtot
Total size of prefetched blocks.
Long64_t * fBSeek
[fBNseek]
TFile * fFile
Pointer to file.
long long Long64_t
Definition: RtypesCore.h:69
char * fBuffer
[fBufferSize] buffer of contiguous prefetched blocks
virtual Long64_t GetBytesRead() const
virtual void SetEnablePrefetching(Bool_t setPrefetching=kFALSE)
Set the prefetching mode of this file.
A cache when reading files over the network.
const char Option_t
Definition: RtypesCore.h:62
virtual void SecondSort()
Sort buffers to be prefetched in increasing order of positions.
void SetEnablePrefetchingImpl(Bool_t setPrefetching=kFALSE)
TFileCacheRead implementation of SetEnablePrefetching.
virtual void SetCacheRead(TFileCacheRead *cache, TObject *tree=0, ECacheAction action=kDisconnect)
Set a pointer to the read cache.
Definition: TFile.cxx:2177
virtual void SetOffset(Long64_t offset, ERelativeTo pos=kBeg)
Set position from where to start reading.
Definition: TFile.cxx:2064
Int_t fBufferSize
Allocated size of fBuffer (at a given time)
const char * GetProtocol() const
Definition: TUrl.h:73
virtual void Info(const char *method, const char *msgfmt,...) const
Issue info message.
Definition: TObject.cxx:892
A ROOT file is a suite of consecutive data records (TKey instances) with a well defined format...
Definition: TFile.h:45
virtual Bool_t ReadBuffer(char *buf, Int_t len)
Read a buffer from the file.
Definition: TFile.cxx:1596
Long64_t fPrefetchedBlocks
Number of blocks prefetched.
virtual Int_t GetReadCalls() const
Long64_t GetWaitTime()
Return the time spent wating for buffer to be read in microseconds.
virtual void Sort()
Sort buffers to be prefetched in increasing order of positions.
virtual const TUrl * GetEndpointUrl() const
Definition: TFile.h:195
Basic string class.
Definition: TString.h:137
Bool_t fIsSorted
True if fSeek array is sorted.
void ToLower()
Change string to lower-case.
Definition: TString.cxx:1088
virtual void Prefetch(Long64_t pos, Int_t len)
Add block of length len at position pos in the list of blocks to be prefetched.
int Int_t
Definition: RtypesCore.h:41
bool Bool_t
Definition: RtypesCore.h:59
const Bool_t kFALSE
Definition: Rtypes.h:92
Int_t fReadCalls
Number of read calls for this cache.
Int_t ThreadStart()
Used to start the consumer thread.
virtual ~TFileCacheRead()
Destructor.
virtual TFilePrefetch * GetPrefetchObj()
#define SafeDelete(p)
Definition: RConfig.h:436
virtual void SecondPrefetch(Long64_t, Int_t)
Int_t * fBSeekPos
[fBNseek]
Long64_t * fBSeekSort
[fBNseek]
Int_t * fSeekLen
[fNseek] Length of buffers to be prefetched
Bool_t fIsTransferred
True when fBuffer contains something valid.
Int_t fBufferLen
Current buffer length (<= fBufferSize)
void Sort(Index n, const Element *a, Index *index, Bool_t down=kTRUE)
Definition: TMath.h:1002
Long64_t fBytesRead
Number of bytes read for this cache.
virtual Long64_t GetNoCacheBytesRead() const
void SetFile(TFile *)
Change the file.
Int_t * fBSeekSortLen
[fBNseek]
virtual Int_t ReadBufferExt(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Int_t * fBSeekIndex
[fBNseek]
virtual Int_t ReadBufferExtNormal(char *buf, Long64_t pos, Int_t len, Int_t &loc)
Base function for ReadBuffer.
Int_t fSeekSize
Allocated size of fSeek.
Int_t * fSeekIndex
[fNseek] sorted index table of fSeek
void WaitFinishPrefetch()
Killing the async prefetching thread.
virtual Int_t GetValue(const char *name, Int_t dflt)
Returns the integer value for a resource.
Definition: TEnv.cxx:494
virtual void Print(Option_t *option="") const
Print cache statistics.
virtual void SetFile(TFile *file, TFile::ECacheAction action=TFile::kDisconnect)
Set the file using this cache and reset the current blocks (if any).
virtual Int_t GetNoCacheReadCalls() const
Bool_t ReadBuffer(char *, Long64_t, Int_t)
Return a prefetched element.
Long64_t fBytesReadExtra
Number of extra bytes (overhead) read by the readahead buffer.
Long64_t fNoCacheBytesRead
Number of bytes read by basket to fill cached tree.
Bool_t fAsyncReading
virtual Int_t SetBufferSize(Int_t buffersize)
Sets the buffer size.
Long64_t * fPos
[fNb] start of long buffers
virtual Int_t GetReadCalls() const
Definition: TFile.h:204
Int_t * fBSeekLen
[fBNseek]
virtual Long64_t GetBytesRead() const
Definition: TFile.h:201
Int_t fNb
Number of long buffers.
ClassImp(TFileCacheRead) TFileCacheRead
Default Constructor.
Int_t fBufferSizeMin
Original size of fBuffer.
double Double_t
Definition: RtypesCore.h:55
static Int_t GetReadaheadSize()
Static function returning the readahead buffer size.
Definition: TFile.cxx:4344
virtual void WaitFinishPrefetch()
R__EXTERN TEnv * gEnv
Definition: TEnv.h:174
ClassImp(TMCParticle) void TMCParticle printf(": p=(%7.3f,%7.3f,%9.3f) ;", fPx, fPy, fPz)
The prefetching mechanism uses two classes (TFilePrefetch and TFPBlock) to prefetch in advance a bloc...
Definition: TFilePrefetch.h:53
Int_t * fSeekPos
[fNseek] Position of sorted blocks in fBuffer
Mother of all ROOT objects.
Definition: TObject.h:58
Long64_t * fSeekSort
[fNseek] Position on file of buffers to be prefetched (sorted)
virtual Int_t ReadBufferExtPrefetch(char *buf, Long64_t pos, Int_t len, Int_t &loc)
prefetch the first block
Bool_t fEnablePrefetching
reading by prefetching asynchronously
TFilePrefetch * fPrefetch
! Object that does the asynchronous reading in another thread
virtual Long64_t GetBytesReadExtra() const
Bool_t SetCache(const char *)
Set the path of the cache directory.
Int_t * fLen
[fNb] Length of long buffers
virtual Bool_t ReadBuffers(char *buf, Long64_t *pos, Int_t *len, Int_t nbuf)
Read the nbuf blocks described in arrays pos and len.
Definition: TFile.cxx:1647
Long64_t * fSeek
[fNseek] Position on file of buffers to be prefetched
Bool_t Contains(const char *pat, ECaseCompare cmp=kExact) const
Definition: TString.h:567
void ReadBlock(Long64_t *, Int_t *, Int_t)
Create a TFPBlock object or recycle one and add it to the prefetchBlocks list.
#define NULL
Definition: Rtypes.h:82
Long64_t * fBPos
[fBNb]
R__EXTERN Int_t gDebug
Definition: Rtypes.h:128
virtual Int_t ReadBuffer(char *buf, Long64_t pos, Int_t len)
Read buffer at position pos.
Int_t * fBLen
[fBNb]
virtual void Close(Option_t *option="")
Close out any threads or asynchronous fetches used by the underlying implementation.
const Bool_t kTRUE
Definition: Rtypes.h:91
Int_t * fSeekSortLen
[fNseek] Length of buffers to be prefetched (sorted)
Bool_t fBIsTransferred
TFileCacheWrite * GetCacheWrite() const
Return a pointer to the current write cache.
Definition: TFile.cxx:1213
Long64_t BinarySearch(Long64_t n, const T *array, T value)
Definition: TMath.h:944
Int_t fNoCacheReadCalls
Number of read calls by basket to fill cached tree.
virtual Long64_t GetBytesReadExtra() const
Definition: TFile.h:202
A cache when writing files over the network.
virtual Bool_t ReadBufferAsync(Long64_t offs, Int_t len)
Definition: TFile.cxx:4955
Int_t fNseek
Number of blocks to be prefetched.
ECacheAction
TTreeCache flushing semantics.
Definition: TFile.h:57