ROOT logo
// @(#)root/proof:$Id: TProofSuperMaster.cxx 27576 2009-02-23 12:15:26Z ganis $
// Author: Fons Rademakers   13/02/97

/*************************************************************************
 * Copyright (C) 1995-2000, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TProofSuperMaster                                                    //
//                                                                      //
// This class controls a Parallel ROOT Facility, PROOF, cluster.        //
// It fires the slave servers, it keeps track of how many slaves are    //
// running, it keeps track of the slaves running status, it broadcasts  //
// messages to all slaves, it collects results, etc.                    //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "TProofSuperMaster.h"
#include "TString.h"
#include "TObjString.h"
#include "TError.h"
#include "TList.h"
#include "TSortedList.h"
#include "TSlave.h"
#include "TMap.h"
#include "TProofServ.h"
#include "TSocket.h"
#include "TMonitor.h"
#include "TSemaphore.h"
#include "TDSet.h"
#include "TPluginManager.h"
#include "TVirtualProofPlayer.h"
#include "TMessage.h"
#include "TUrl.h"
#include "TProofResourcesStatic.h"
#include "TProofNodeInfo.h"
#include "TROOT.h"

ClassImp(TProofSuperMaster)

//______________________________________________________________________________
TProofSuperMaster::TProofSuperMaster(const char *masterurl, const char *conffile,
                                     const char *confdir, Int_t loglevel,
                                     const char *alias, TProofMgr *mgr)
{
   // Start super master PROOF session.

   // This may be needed during init
   fManager = mgr;

   fUrl = TUrl(masterurl);

   if (!conffile || strlen(conffile) == 0)
      conffile = kPROOF_ConfFile;
   else if (!strncasecmp(conffile, "sm:", 3))
      conffile+=3;
   if (!confdir  || strlen(confdir) == 0)
      confdir = kPROOF_ConfDir;

   // Instance type
   fMasterServ = kTRUE;
   ResetBit(TProof::kIsClient);
   SetBit(TProof::kIsMaster);

   Init(masterurl, conffile, confdir, loglevel, alias);

   // For Final cleanup
   gROOT->GetListOfProofs()->Add(this);
}

//______________________________________________________________________________
Bool_t TProofSuperMaster::StartSlaves(Bool_t)
{
   // Start up PROOF submasters.

   // If this is a supermaster server, find the config file and start
   // submaster servers as specified in the config file.
   // There is a difference in startup between a slave and a submaster
   // in which the submaster will issue a kPROOF_LOGFILE and
   // then a kPROOF_LOGDONE message (which must be collected)
   // while slaves do not.

   Int_t pc = 0;
   TList *submasterList = new TList;
   // Get list of workers
   if (gProofServ->GetWorkers(submasterList, pc) == TProofServ::kQueryStop) {
      Error("StartSlaves", "getting list of submaster nodes");
      return kFALSE;
   }
   fImage = gProofServ->GetImage();
   if (fImage.IsNull())
      fImage = Form("%s:%s", TUrl(gSystem->HostName()).GetHostFQDN(),
                             gProofServ->GetWorkDir());

   UInt_t nSubmasters = submasterList->GetSize();
   UInt_t nSubmastersDone = 0;
   Int_t ord = 0;
   TList validSubmasters;
   TList validPairs;
   validPairs.SetOwner();

   // Loop over all submasters and start them
   TListIter next(submasterList);
   TObject *to;
   TProofNodeInfo *submaster;
   while ((to = next())) {
      // Get the next submaster from the list
      submaster = (TProofNodeInfo *)to;
      const Char_t *conffile = submaster->GetConfig();
      const Char_t *image = submaster->GetImage();
      const Char_t *msd = submaster->GetMsd();
      Int_t sport = submaster->GetPort();
      if (sport == -1)
         sport = fUrl.GetPort();

      TString fullord = TString(gProofServ->GetOrdinal()) + "." + ((Long_t) ord);

      // create submaster server
      TUrl u(Form("%s:%d", submaster->GetNodeName().Data(), sport));
      // Add group info in the password firdl, if any
      if (strlen(gProofServ->GetGroup()) > 0) {
         // Set also the user, otherwise the password is not exported
         if (strlen(u.GetUser()) <= 0)
            u.SetUser(gProofServ->GetUser());
         u.SetPasswd(gProofServ->GetGroup());
      }
      TSlave *slave =
         CreateSubmaster(u.GetUrl(), fullord, image, msd);

      // Add to global list (we will add to the monitor list after
      // finalizing the server startup)
      Bool_t submasterOk = kTRUE;
      fSlaves->Add(slave);
      if (slave->IsValid()) {
         validPairs.Add(new TPair(slave, new TObjString(conffile)));
      } else {
         submasterOk = kFALSE;
         fBadSlaves->Add(slave);
      }

      PDB(kGlobal,3)
         Info("StartSlaves","submaster on host %s created and"
               " added to list", submaster->GetNodeName().Data());

      // Notify opening of connection
      nSubmastersDone++;
      TMessage m(kPROOF_SERVERSTARTED);
      m << TString("Opening connections to submasters") << nSubmasters
         << nSubmastersDone << submasterOk;
      gProofServ->GetSocket()->Send(m);

      ord++;

   } // end loop over all submasters

   // Cleanup
   SafeDelete(submasterList);

   nSubmastersDone = 0;

   // Here we finalize the server startup: in this way the bulk
   // of remote operations are almost parallelized
   TIter nxsc(&validPairs);
   TPair *sc = 0;
   while ((sc = (TPair *) nxsc())) {
      // Finalize setup of the server
      TSlave *sl = (TSlave *) sc->Key();
      TObjString *cf = (TObjString *) sc->Value();
      sl->SetupServ(TSlave::kMaster, cf->GetName());

      // Monitor good slaves
      Bool_t submasterOk = kTRUE;
      if (sl->IsValid()) {
         // check protocol compatability
         // protocol 1 is not supported anymore
         if (fProtocol == 1) {
            Error("StartSlaves", "master and submaster protocols"
                  " not compatible (%d and %d)",
                  kPROOF_Protocol, fProtocol);
            submasterOk = kFALSE;
            fBadSlaves->Add(sl);
         } else {
            fAllMonitor->Add(sl->GetSocket());
            validSubmasters.Add(sl);
         }
      } else {
         submasterOk = kFALSE;
         fBadSlaves->Add(sl);
      }

      // Notify end of startup operations
      nSubmastersDone++;
      TMessage m(kPROOF_SERVERSTARTED);
      m << TString("Setting up submasters") << nSubmasters
         << nSubmastersDone << submasterOk;
      gProofServ->GetSocket()->Send(m);
   }

   Collect(kAll); //Get kPROOF_LOGFILE and kPROOF_LOGDONE messages
   TIter nextSubmaster(&validSubmasters);
   while (TSlave* sl = dynamic_cast<TSlave*>(nextSubmaster())) {
      if (sl->GetStatus() == -99) {
         Error("StartSlaves", "not allowed to connect to PROOF master server");
         fBadSlaves->Add(sl);
         continue;
      }

      if (!sl->IsValid()) {
         Error("StartSlaves", "failed to setup connection with PROOF master server");
         fBadSlaves->Add(sl);
         continue;
      }
   }

   return kTRUE;
}

//______________________________________________________________________________
Long64_t TProofSuperMaster::Process(TDSet *set, const char *selector, Option_t *option,
                                    Long64_t nentries, Long64_t first)
{
   // Process a data set (TDSet) using the specified selector (.C) file.
   // Entry- or event-lists should be set in the data set object using
   // TDSet::SetEntryList.
   // The return value is -1 in case of error and TSelector::GetStatus() in
   // in case of success.

   if (!IsValid()) return -1;

   R__ASSERT(GetPlayer());

   if (GetProgressDialog())
      GetProgressDialog()->ExecPlugin(5, this, selector, set->GetListOfElements()->GetSize(),
                                  first, nentries);

   return GetPlayer()->Process(set, selector, option, nentries, first);
}

//______________________________________________________________________________
void TProofSuperMaster::ValidateDSet(TDSet *dset)
{
   // Validate a TDSet.

   if (dset->ElementsValid()) return;

   // We need to recheck after this
   dset->ResetBit(TDSet::kValidityChecked);
   dset->ResetBit(TDSet::kSomeInvalid);

   TList msds;
   msds.SetOwner();

   TList smholder;
   smholder.SetOwner();
   TList elemholder;
   elemholder.SetOwner();

   // build nodelist with slaves and elements
   TIter nextSubmaster(GetListOfActiveSlaves());
   while (TSlave *sl = dynamic_cast<TSlave*>(nextSubmaster())) {
      TList *smlist = 0;
      TPair *p = dynamic_cast<TPair*>(msds.FindObject(sl->GetMsd()));
      if (!p) {
         smlist = new TList;
         smlist->SetName(sl->GetMsd());

         smholder.Add(smlist);
         TList *elemlist = new TSortedList(kSortDescending);
         elemlist->SetName(TString(sl->GetMsd())+"_elem");
         elemholder.Add(elemlist);
         msds.Add(new TPair(smlist, elemlist));
      } else {
         smlist = dynamic_cast<TList*>(p->Key());
      }
      smlist->Add(sl);
   }

   TIter nextElem(dset->GetListOfElements());
   while (TDSetElement *elem = dynamic_cast<TDSetElement*>(nextElem())) {
      if (elem->GetValid()) continue;
      TPair *p = dynamic_cast<TPair*>(msds.FindObject(elem->GetMsd()));
      if (p) {
         dynamic_cast<TList*>(p->Value())->Add(elem);
      } else {
         Error("ValidateDSet", "no mass storage domain '%s' associated"
                               " with available submasters",
                               elem->GetMsd());
         return;
      }
   }

   // send to slaves
   TList usedsms;
   TIter nextSM(&msds);
   SetDSet(dset); // set dset to be validated in Collect()
   while (TPair *msd = dynamic_cast<TPair*>(nextSM())) {
      TList *sms = dynamic_cast<TList*>(msd->Key());
      TList *setelements = dynamic_cast<TList*>(msd->Value());

      // distribute elements over the slaves
      Int_t nsms = sms->GetSize();
      Int_t nelements = setelements->GetSize();
      for (Int_t i=0; i<nsms; i++) {

         TDSet set(dset->GetType(), dset->GetObjName(),
                   dset->GetDirectory());
         for (Int_t j = (i*nelements)/nsms;
                    j < ((i+1)*nelements)/nsms;
                    j++) {
            TDSetElement *elem =
               dynamic_cast<TDSetElement*>(setelements->At(j));
            set.Add(elem->GetFileName(), elem->GetObjName(),
                    elem->GetDirectory(), elem->GetFirst(),
                    elem->GetNum(), elem->GetMsd());
         }

         if (set.GetListOfElements()->GetSize()>0) {
            TMessage mesg(kPROOF_VALIDATE_DSET);
            mesg << &set;

            TSlave *sl = dynamic_cast<TSlave*>(sms->At(i));
            PDB(kGlobal,1)
               Info("ValidateDSet",
                    "Sending TDSet with %d elements to worker %s"
                    " to be validated", set.GetListOfElements()->GetSize(),
                                        sl->GetOrdinal());
            sl->GetSocket()->Send(mesg);
            usedsms.Add(sl);
         }
      }
   }

   PDB(kGlobal,1)
      Info("ValidateDSet","Calling Collect");
   Collect(&usedsms);
   SetDSet(0);
}

//______________________________________________________________________________
TVirtualProofPlayer *TProofSuperMaster::MakePlayer(const char *player, TSocket *s)
{
   // Construct a TProofPlayer object. The player string specifies which
   // player should be created: remote, slave, sm (supermaster) or base.
   // Default is sm. Socket is needed in case a slave player is created.

   if (!player)
      player = "sm";

   SetPlayer(TVirtualProofPlayer::Create(player, this, s));
   return GetPlayer();
}

 TProofSuperMaster.cxx:1
 TProofSuperMaster.cxx:2
 TProofSuperMaster.cxx:3
 TProofSuperMaster.cxx:4
 TProofSuperMaster.cxx:5
 TProofSuperMaster.cxx:6
 TProofSuperMaster.cxx:7
 TProofSuperMaster.cxx:8
 TProofSuperMaster.cxx:9
 TProofSuperMaster.cxx:10
 TProofSuperMaster.cxx:11
 TProofSuperMaster.cxx:12
 TProofSuperMaster.cxx:13
 TProofSuperMaster.cxx:14
 TProofSuperMaster.cxx:15
 TProofSuperMaster.cxx:16
 TProofSuperMaster.cxx:17
 TProofSuperMaster.cxx:18
 TProofSuperMaster.cxx:19
 TProofSuperMaster.cxx:20
 TProofSuperMaster.cxx:21
 TProofSuperMaster.cxx:22
 TProofSuperMaster.cxx:23
 TProofSuperMaster.cxx:24
 TProofSuperMaster.cxx:25
 TProofSuperMaster.cxx:26
 TProofSuperMaster.cxx:27
 TProofSuperMaster.cxx:28
 TProofSuperMaster.cxx:29
 TProofSuperMaster.cxx:30
 TProofSuperMaster.cxx:31
 TProofSuperMaster.cxx:32
 TProofSuperMaster.cxx:33
 TProofSuperMaster.cxx:34
 TProofSuperMaster.cxx:35
 TProofSuperMaster.cxx:36
 TProofSuperMaster.cxx:37
 TProofSuperMaster.cxx:38
 TProofSuperMaster.cxx:39
 TProofSuperMaster.cxx:40
 TProofSuperMaster.cxx:41
 TProofSuperMaster.cxx:42
 TProofSuperMaster.cxx:43
 TProofSuperMaster.cxx:44
 TProofSuperMaster.cxx:45
 TProofSuperMaster.cxx:46
 TProofSuperMaster.cxx:47
 TProofSuperMaster.cxx:48
 TProofSuperMaster.cxx:49
 TProofSuperMaster.cxx:50
 TProofSuperMaster.cxx:51
 TProofSuperMaster.cxx:52
 TProofSuperMaster.cxx:53
 TProofSuperMaster.cxx:54
 TProofSuperMaster.cxx:55
 TProofSuperMaster.cxx:56
 TProofSuperMaster.cxx:57
 TProofSuperMaster.cxx:58
 TProofSuperMaster.cxx:59
 TProofSuperMaster.cxx:60
 TProofSuperMaster.cxx:61
 TProofSuperMaster.cxx:62
 TProofSuperMaster.cxx:63
 TProofSuperMaster.cxx:64
 TProofSuperMaster.cxx:65
 TProofSuperMaster.cxx:66
 TProofSuperMaster.cxx:67
 TProofSuperMaster.cxx:68
 TProofSuperMaster.cxx:69
 TProofSuperMaster.cxx:70
 TProofSuperMaster.cxx:71
 TProofSuperMaster.cxx:72
 TProofSuperMaster.cxx:73
 TProofSuperMaster.cxx:74
 TProofSuperMaster.cxx:75
 TProofSuperMaster.cxx:76
 TProofSuperMaster.cxx:77
 TProofSuperMaster.cxx:78
 TProofSuperMaster.cxx:79
 TProofSuperMaster.cxx:80
 TProofSuperMaster.cxx:81
 TProofSuperMaster.cxx:82
 TProofSuperMaster.cxx:83
 TProofSuperMaster.cxx:84
 TProofSuperMaster.cxx:85
 TProofSuperMaster.cxx:86
 TProofSuperMaster.cxx:87
 TProofSuperMaster.cxx:88
 TProofSuperMaster.cxx:89
 TProofSuperMaster.cxx:90
 TProofSuperMaster.cxx:91
 TProofSuperMaster.cxx:92
 TProofSuperMaster.cxx:93
 TProofSuperMaster.cxx:94
 TProofSuperMaster.cxx:95
 TProofSuperMaster.cxx:96
 TProofSuperMaster.cxx:97
 TProofSuperMaster.cxx:98
 TProofSuperMaster.cxx:99
 TProofSuperMaster.cxx:100
 TProofSuperMaster.cxx:101
 TProofSuperMaster.cxx:102
 TProofSuperMaster.cxx:103
 TProofSuperMaster.cxx:104
 TProofSuperMaster.cxx:105
 TProofSuperMaster.cxx:106
 TProofSuperMaster.cxx:107
 TProofSuperMaster.cxx:108
 TProofSuperMaster.cxx:109
 TProofSuperMaster.cxx:110
 TProofSuperMaster.cxx:111
 TProofSuperMaster.cxx:112
 TProofSuperMaster.cxx:113
 TProofSuperMaster.cxx:114
 TProofSuperMaster.cxx:115
 TProofSuperMaster.cxx:116
 TProofSuperMaster.cxx:117
 TProofSuperMaster.cxx:118
 TProofSuperMaster.cxx:119
 TProofSuperMaster.cxx:120
 TProofSuperMaster.cxx:121
 TProofSuperMaster.cxx:122
 TProofSuperMaster.cxx:123
 TProofSuperMaster.cxx:124
 TProofSuperMaster.cxx:125
 TProofSuperMaster.cxx:126
 TProofSuperMaster.cxx:127
 TProofSuperMaster.cxx:128
 TProofSuperMaster.cxx:129
 TProofSuperMaster.cxx:130
 TProofSuperMaster.cxx:131
 TProofSuperMaster.cxx:132
 TProofSuperMaster.cxx:133
 TProofSuperMaster.cxx:134
 TProofSuperMaster.cxx:135
 TProofSuperMaster.cxx:136
 TProofSuperMaster.cxx:137
 TProofSuperMaster.cxx:138
 TProofSuperMaster.cxx:139
 TProofSuperMaster.cxx:140
 TProofSuperMaster.cxx:141
 TProofSuperMaster.cxx:142
 TProofSuperMaster.cxx:143
 TProofSuperMaster.cxx:144
 TProofSuperMaster.cxx:145
 TProofSuperMaster.cxx:146
 TProofSuperMaster.cxx:147
 TProofSuperMaster.cxx:148
 TProofSuperMaster.cxx:149
 TProofSuperMaster.cxx:150
 TProofSuperMaster.cxx:151
 TProofSuperMaster.cxx:152
 TProofSuperMaster.cxx:153
 TProofSuperMaster.cxx:154
 TProofSuperMaster.cxx:155
 TProofSuperMaster.cxx:156
 TProofSuperMaster.cxx:157
 TProofSuperMaster.cxx:158
 TProofSuperMaster.cxx:159
 TProofSuperMaster.cxx:160
 TProofSuperMaster.cxx:161
 TProofSuperMaster.cxx:162
 TProofSuperMaster.cxx:163
 TProofSuperMaster.cxx:164
 TProofSuperMaster.cxx:165
 TProofSuperMaster.cxx:166
 TProofSuperMaster.cxx:167
 TProofSuperMaster.cxx:168
 TProofSuperMaster.cxx:169
 TProofSuperMaster.cxx:170
 TProofSuperMaster.cxx:171
 TProofSuperMaster.cxx:172
 TProofSuperMaster.cxx:173
 TProofSuperMaster.cxx:174
 TProofSuperMaster.cxx:175
 TProofSuperMaster.cxx:176
 TProofSuperMaster.cxx:177
 TProofSuperMaster.cxx:178
 TProofSuperMaster.cxx:179
 TProofSuperMaster.cxx:180
 TProofSuperMaster.cxx:181
 TProofSuperMaster.cxx:182
 TProofSuperMaster.cxx:183
 TProofSuperMaster.cxx:184
 TProofSuperMaster.cxx:185
 TProofSuperMaster.cxx:186
 TProofSuperMaster.cxx:187
 TProofSuperMaster.cxx:188
 TProofSuperMaster.cxx:189
 TProofSuperMaster.cxx:190
 TProofSuperMaster.cxx:191
 TProofSuperMaster.cxx:192
 TProofSuperMaster.cxx:193
 TProofSuperMaster.cxx:194
 TProofSuperMaster.cxx:195
 TProofSuperMaster.cxx:196
 TProofSuperMaster.cxx:197
 TProofSuperMaster.cxx:198
 TProofSuperMaster.cxx:199
 TProofSuperMaster.cxx:200
 TProofSuperMaster.cxx:201
 TProofSuperMaster.cxx:202
 TProofSuperMaster.cxx:203
 TProofSuperMaster.cxx:204
 TProofSuperMaster.cxx:205
 TProofSuperMaster.cxx:206
 TProofSuperMaster.cxx:207
 TProofSuperMaster.cxx:208
 TProofSuperMaster.cxx:209
 TProofSuperMaster.cxx:210
 TProofSuperMaster.cxx:211
 TProofSuperMaster.cxx:212
 TProofSuperMaster.cxx:213
 TProofSuperMaster.cxx:214
 TProofSuperMaster.cxx:215
 TProofSuperMaster.cxx:216
 TProofSuperMaster.cxx:217
 TProofSuperMaster.cxx:218
 TProofSuperMaster.cxx:219
 TProofSuperMaster.cxx:220
 TProofSuperMaster.cxx:221
 TProofSuperMaster.cxx:222
 TProofSuperMaster.cxx:223
 TProofSuperMaster.cxx:224
 TProofSuperMaster.cxx:225
 TProofSuperMaster.cxx:226
 TProofSuperMaster.cxx:227
 TProofSuperMaster.cxx:228
 TProofSuperMaster.cxx:229
 TProofSuperMaster.cxx:230
 TProofSuperMaster.cxx:231
 TProofSuperMaster.cxx:232
 TProofSuperMaster.cxx:233
 TProofSuperMaster.cxx:234
 TProofSuperMaster.cxx:235
 TProofSuperMaster.cxx:236
 TProofSuperMaster.cxx:237
 TProofSuperMaster.cxx:238
 TProofSuperMaster.cxx:239
 TProofSuperMaster.cxx:240
 TProofSuperMaster.cxx:241
 TProofSuperMaster.cxx:242
 TProofSuperMaster.cxx:243
 TProofSuperMaster.cxx:244
 TProofSuperMaster.cxx:245
 TProofSuperMaster.cxx:246
 TProofSuperMaster.cxx:247
 TProofSuperMaster.cxx:248
 TProofSuperMaster.cxx:249
 TProofSuperMaster.cxx:250
 TProofSuperMaster.cxx:251
 TProofSuperMaster.cxx:252
 TProofSuperMaster.cxx:253
 TProofSuperMaster.cxx:254
 TProofSuperMaster.cxx:255
 TProofSuperMaster.cxx:256
 TProofSuperMaster.cxx:257
 TProofSuperMaster.cxx:258
 TProofSuperMaster.cxx:259
 TProofSuperMaster.cxx:260
 TProofSuperMaster.cxx:261
 TProofSuperMaster.cxx:262
 TProofSuperMaster.cxx:263
 TProofSuperMaster.cxx:264
 TProofSuperMaster.cxx:265
 TProofSuperMaster.cxx:266
 TProofSuperMaster.cxx:267
 TProofSuperMaster.cxx:268
 TProofSuperMaster.cxx:269
 TProofSuperMaster.cxx:270
 TProofSuperMaster.cxx:271
 TProofSuperMaster.cxx:272
 TProofSuperMaster.cxx:273
 TProofSuperMaster.cxx:274
 TProofSuperMaster.cxx:275
 TProofSuperMaster.cxx:276
 TProofSuperMaster.cxx:277
 TProofSuperMaster.cxx:278
 TProofSuperMaster.cxx:279
 TProofSuperMaster.cxx:280
 TProofSuperMaster.cxx:281
 TProofSuperMaster.cxx:282
 TProofSuperMaster.cxx:283
 TProofSuperMaster.cxx:284
 TProofSuperMaster.cxx:285
 TProofSuperMaster.cxx:286
 TProofSuperMaster.cxx:287
 TProofSuperMaster.cxx:288
 TProofSuperMaster.cxx:289
 TProofSuperMaster.cxx:290
 TProofSuperMaster.cxx:291
 TProofSuperMaster.cxx:292
 TProofSuperMaster.cxx:293
 TProofSuperMaster.cxx:294
 TProofSuperMaster.cxx:295
 TProofSuperMaster.cxx:296
 TProofSuperMaster.cxx:297
 TProofSuperMaster.cxx:298
 TProofSuperMaster.cxx:299
 TProofSuperMaster.cxx:300
 TProofSuperMaster.cxx:301
 TProofSuperMaster.cxx:302
 TProofSuperMaster.cxx:303
 TProofSuperMaster.cxx:304
 TProofSuperMaster.cxx:305
 TProofSuperMaster.cxx:306
 TProofSuperMaster.cxx:307
 TProofSuperMaster.cxx:308
 TProofSuperMaster.cxx:309
 TProofSuperMaster.cxx:310
 TProofSuperMaster.cxx:311
 TProofSuperMaster.cxx:312
 TProofSuperMaster.cxx:313
 TProofSuperMaster.cxx:314
 TProofSuperMaster.cxx:315
 TProofSuperMaster.cxx:316
 TProofSuperMaster.cxx:317
 TProofSuperMaster.cxx:318
 TProofSuperMaster.cxx:319
 TProofSuperMaster.cxx:320
 TProofSuperMaster.cxx:321
 TProofSuperMaster.cxx:322
 TProofSuperMaster.cxx:323
 TProofSuperMaster.cxx:324
 TProofSuperMaster.cxx:325
 TProofSuperMaster.cxx:326
 TProofSuperMaster.cxx:327
 TProofSuperMaster.cxx:328
 TProofSuperMaster.cxx:329
 TProofSuperMaster.cxx:330
 TProofSuperMaster.cxx:331
 TProofSuperMaster.cxx:332
 TProofSuperMaster.cxx:333
 TProofSuperMaster.cxx:334
 TProofSuperMaster.cxx:335
 TProofSuperMaster.cxx:336
 TProofSuperMaster.cxx:337
 TProofSuperMaster.cxx:338
 TProofSuperMaster.cxx:339
 TProofSuperMaster.cxx:340
 TProofSuperMaster.cxx:341
 TProofSuperMaster.cxx:342
 TProofSuperMaster.cxx:343
 TProofSuperMaster.cxx:344
 TProofSuperMaster.cxx:345
 TProofSuperMaster.cxx:346
 TProofSuperMaster.cxx:347
 TProofSuperMaster.cxx:348
 TProofSuperMaster.cxx:349
 TProofSuperMaster.cxx:350
 TProofSuperMaster.cxx:351
 TProofSuperMaster.cxx:352
 TProofSuperMaster.cxx:353
 TProofSuperMaster.cxx:354
 TProofSuperMaster.cxx:355
 TProofSuperMaster.cxx:356
 TProofSuperMaster.cxx:357