ROOT logo
// @(#)root/proofplayer:$Id: TPacketizerUnit.cxx 26875 2008-12-12 14:21:34Z ganis $
// Author: Long Tran-Thanh    22/07/07

/*************************************************************************
 * Copyright (C) 1995-2002, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TPacketizerUnit                                                      //
//                                                                      //
// This packetizer generates packets of generic units, representing the //
// number of times an operation cycle has to be repeated by the worker  //
// node, e.g. the number of Monte carlo events to be generated.         //
// Packets sizes are generated taking into account the performance of   //
// worker nodes, based on the time needed to process previous packets.  //
//                                                                      //
//////////////////////////////////////////////////////////////////////////


#include "TPacketizerUnit.h"

#include "Riostream.h"
#include "TDSet.h"
#include "TError.h"
#include "TEventList.h"
#include "TMap.h"
#include "TMessage.h"
#include "TMonitor.h"
#include "TNtupleD.h"
#include "TObject.h"
#include "TParameter.h"
#include "TPerfStats.h"
#include "TProofDebug.h"
#include "TProof.h"
#include "TProofPlayer.h"
#include "TProofServ.h"
#include "TSlave.h"
#include "TSocket.h"
#include "TStopwatch.h"
#include "TTimer.h"
#include "TUrl.h"
#include "TClass.h"
#include "TMath.h"
#include "TObjString.h"


using namespace TMath;
//
// The following utility class manage the state of the
// work to be performed and the slaves involved in the process.
//
// The list of TSlaveStat(s) keep track of the work (being) done
// by each slave
//

//------------------------------------------------------------------------------

class TPacketizerUnit::TSlaveStat : public TVirtualPacketizer::TVirtualSlaveStat {

friend class TPacketizerUnit;

private:
   Long64_t  fLastProcessed; // number of processed entries of the last packet
   Double_t  fSpeed;         // estimated current average speed of the processing slave
   Double_t  fTimeInstant;   // stores the time instant when the current packet started
   TNtupleD *fCircNtp;       // Keeps circular info for speed calculations
   Long_t    fCircLvl;       // Circularity level

public:
   TSlaveStat(TSlave *sl, TList *input);
   ~TSlaveStat();

   void        GetCurrentTime();

   const char *GetName() const { return fSlave->GetName(); }

   void        UpdatePerformance(Double_t time);
   TProofProgressStatus *AddProcessed(TProofProgressStatus *st);
};

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::TSlaveStat(TSlave *slave, TList *input)
                            : fLastProcessed(0),
                              fSpeed(0), fTimeInstant(0), fCircLvl(5)
{
   // Main constructor

   // Initialize the circularity ntple for speed calculations
   fCircNtp = new TNtupleD("Speed Circ Ntp", "Circular process info","tm:ev");
   TProof::GetParameter(input, "PROOF_TPacketizerUnitCircularity", fCircLvl);
   fCircLvl = (fCircLvl > 0) ? fCircLvl : 5;
   fCircNtp->SetCircular(fCircLvl);
   fSlave = slave;
   fStatus = new TProofProgressStatus();
}

//______________________________________________________________________________
TPacketizerUnit::TSlaveStat::~TSlaveStat()
{
   // Destructor

   SafeDelete(fCircNtp);
}

//______________________________________________________________________________
void TPacketizerUnit::TSlaveStat::UpdatePerformance(Double_t time)
{
   // Update the circular ntple

   Double_t ttot = time;
   Double_t *ar = fCircNtp->GetArgs();
   Int_t ne = fCircNtp->GetEntries();
   if (ne <= 0) {
      // First call: just fill one ref entry and return
      fCircNtp->Fill(0., 0);
      fSpeed = 0.;
      return;
   }
   // Fill the entry
   fCircNtp->GetEntry(ne-1);
   ttot = ar[0] + time;
   fCircNtp->Fill(ttot, GetEntriesProcessed());

   // Calculate the speed
   fCircNtp->GetEntry(0);
   Double_t dtime = (ttot > ar[0]) ? ttot - ar[0] : ne+1 ;
   Long64_t nevts = GetEntriesProcessed() - (Long64_t)ar[1];
   fSpeed = nevts / dtime;
   PDB(kPacketizer,2)
      Info("UpdatePerformance", "time:%f, dtime:%f, nevts:%lld, speed: %f",
                                time, dtime, nevts, fSpeed);

}

//______________________________________________________________________________
TProofProgressStatus *TPacketizerUnit::TSlaveStat::AddProcessed(TProofProgressStatus *st)
{
   // Update the status info to the 'st'.
   // return the difference (*st - *fStatus)

   if (st) {
      TProofProgressStatus *diff = new TProofProgressStatus(*st - *fStatus);
      *fStatus += *diff;
      return diff;
   } else {
      Error("AddProcessed", "status arg undefined");
      return 0;
   }
}

//------------------------------------------------------------------------------

ClassImp(TPacketizerUnit)

//______________________________________________________________________________
TPacketizerUnit::TPacketizerUnit(TList *slaves, Long64_t num, TList *input,
                                 TProofProgressStatus *st)
                : TVirtualPacketizer(input, st)
{
   // Constructor

   PDB(kPacketizer,1) Info("TPacketizerUnit", "enter (num %lld)", num);

   // Init pointer members
   fSlaveStats = 0;
   fPackets = 0;

   fTimeLimit = 1;
   TProof::GetParameter(input, "PROOF_PacketizerTimeLimit", fTimeLimit);
   PDB(kPacketizer,1)
      Info("TPacketizerUnit", "time limit is %lf", fTimeLimit);
   fProcessing = 0;
   fAssigned = 0;

   fStopwatch = new TStopwatch();

   fPackets = new TList;
   fPackets->SetOwner();

   fSlaveStats = new TMap;
   fSlaveStats->SetOwner(kFALSE);

   TSlave *slave;
   TIter si(slaves);
   while ((slave = (TSlave*) si.Next()))
      fSlaveStats->Add(slave, new TSlaveStat(slave, input));

   fTotalEntries = num;

   fStopwatch->Start();
   PDB(kPacketizer,1) Info("TPacketizerUnit", "return");
}

//______________________________________________________________________________
TPacketizerUnit::~TPacketizerUnit()
{
   // Destructor.

   if (fSlaveStats)
      fSlaveStats->DeleteValues();
   SafeDelete(fSlaveStats);
   SafeDelete(fPackets);
   SafeDelete(fStopwatch);
}

//______________________________________________________________________________
Double_t TPacketizerUnit::GetCurrentTime()
{
   // Get current time

   Double_t retValue = fStopwatch->RealTime();
   fStopwatch->Continue();
   return retValue;
}

//______________________________________________________________________________
TDSetElement *TPacketizerUnit::GetNextPacket(TSlave *sl, TMessage *r)
{
   // Get next packet

   if (!fValid)
      return 0;

   // Find slave
   TSlaveStat *slstat = (TSlaveStat*) fSlaveStats->GetValue(sl);
   R__ASSERT(slstat != 0);

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s: fAssigned %lld\t", sl->GetOrdinal(), fAssigned);

   // Update stats & free old element
   Double_t latency, proctime, proccpu;
   Long64_t bytesRead = -1;
   Long64_t totalEntries = -1; // used only to read an old message type
   Long64_t totev = 0;
   Long64_t numev = -1;

   TProofProgressStatus *status = 0;
   if (sl->GetProtocol() > 18) {
      (*r) >> latency;
      (*r) >> status;

      // Calculate the progress made in the last packet
      TProofProgressStatus *progress = 0;
      if (status) {
         // upadte the worker status
         numev = status->GetEntries() - slstat->GetEntriesProcessed();
         progress = slstat->AddProcessed(status);
         if (progress) {
            // (*fProgressStatus) += *progress;
            proctime = progress->GetProcTime();
            proccpu  = progress->GetCPUTime();
            totev  = status->GetEntries(); // for backward compatibility
            bytesRead  = progress->GetBytesRead();
            delete progress;
         }
         delete status;
      } else
          Error("GetNextPacket", "no status came in the kPROOF_GETPACKET message");
   } else {

      (*r) >> latency >> proctime >> proccpu;

      // only read new info if available
      if (r->BufferSize() > r->Length()) (*r) >> bytesRead;
      if (r->BufferSize() > r->Length()) (*r) >> totalEntries;
      if (r->BufferSize() > r->Length()) (*r) >> totev;

      numev = totev - slstat->GetEntriesProcessed();
      slstat->GetProgressStatus()->IncEntries(numev);
   }

   fProgressStatus->IncEntries(numev);

   fProcessing = 0;

   PDB(kPacketizer,2)
      Info("GetNextPacket","worker-%s (%s): %lld %7.3lf %7.3lf %7.3lf %lld",
                           sl->GetOrdinal(), sl->GetName(),
                           numev, latency, proctime, proccpu, bytesRead);

   if (gPerfStats != 0) {
      gPerfStats->PacketEvent(sl->GetOrdinal(), sl->GetName(), "", numev,
                              latency, proctime, proccpu, bytesRead);
   }

   if (fAssigned == fTotalEntries) {
      // Send last timer message
      HandleTimer(0);
      return 0;
   }

   if (fStop) {
      // Send last timer message
      HandleTimer(0);
      return 0;
   }


   Long64_t num;

   // Get the current time
   Double_t cTime = GetCurrentTime();

   if (slstat->fCircNtp->GetEntries() <= 0) {
      // The calibration phase
      PDB(kPacketizer,2) Info("GetNextPacket"," calibration: "
                              "total entries %lld, workers %d",
                              fTotalEntries, fSlaveStats->GetSize());
      Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
      num = (avg > 5) ? 5 : avg;

      // Create a reference entry
      slstat->UpdatePerformance(0.);

   } else {
      // Schedule tasks for workers based on the currently estimated processing speeds

      // Update performances
      // slstat->fStatus was updated before;
      slstat->UpdatePerformance(proctime);

      TMapIter *iter = (TMapIter *)fSlaveStats->MakeIterator();
      TSlave * tmpSlave;
      TSlaveStat * tmpStat;

      Double_t sumSpeed = 0;
      Double_t sumBusy = 0;

      // The basic idea is to estimate the optimal finishing time for the process, assuming
      // that the currently measured processing speeds of the slaves remain the same in the future.
      // The optTime can be calculated as follows.
      // Let s_i be the speed of worker i. We assume that worker i will be busy in the
      // next b_i time instant. Therefore we have the following equation:
      //                 SUM((optTime-b_i)*s_i) = (remaining_entries),
      // Hence optTime can be calculated easily:
      //                 optTime = ((remaining_entries) + SUM(b_i*s_i))/SUM(s_i)

      while ((tmpSlave = (TSlave *)iter->Next())) {
         tmpStat = (TSlaveStat *)fSlaveStats->GetValue(tmpSlave);
         // If the slave doesn't response for a long time, its service rate will be considered as 0
         if ((cTime - tmpStat->fTimeInstant) > 4*fTimeLimit)
            tmpStat->fSpeed = 0;
         PDB(kPacketizer,2)
            Info("GetNextPacket",
                 "worker-%s: speed %lf", tmpSlave->GetOrdinal(), tmpStat->fSpeed);
         if (tmpStat->fSpeed > 0) {
            // Calculating the SUM(s_i)
            sumSpeed += tmpStat->fSpeed;
            // There is nothing to do if slave_i is not calibrated or slave i is the current slave
            if ((tmpStat->fTimeInstant) && (cTime - tmpStat->fTimeInstant > 0)) {
               // Calculating the SUM(b_i*s_i)
               //      s_i = tmpStat->fSpeed
               //      b_i = tmpStat->fTimeInstant + tmpStat->fLastProcessed/s_i - cTime)
               //  b_i*s_i = (tmpStat->fTimeInstant - cTime)*s_i + tmpStat->fLastProcessed
               Double_t busyspeed =
                  ((tmpStat->fTimeInstant - cTime)*tmpStat->fSpeed + tmpStat->fLastProcessed);
               if (busyspeed > 0)
                  sumBusy += busyspeed;
            }
         }
      }
      PDB(kPacketizer,2)
         Info("GetNextPacket", "worker-%s: sum speed: %lf, sum busy: %f",
                               sl->GetOrdinal(), sumSpeed, sumBusy);
      // firstly the slave will try to get all of the remaining entries
      if (slstat->fSpeed > 0 &&
         (fTotalEntries - fAssigned)/(slstat->fSpeed) < fTimeLimit) {
         num = (fTotalEntries - fAssigned);
      } else {
         if (slstat->fSpeed > 0) {
            // calculating the optTime
            Double_t optTime = (fTotalEntries - fAssigned + sumBusy)/sumSpeed;
            // if optTime is greater than the official time limit, then the slave gets a number
            // of entries that still fit into the time limit, otherwise it uses the optTime as
            // a time limit
            num = (optTime > fTimeLimit) ? Nint(fTimeLimit*(slstat->fSpeed))
                                         : Nint(optTime*(slstat->fSpeed));
           PDB(kPacketizer,2)
              Info("GetNextPacket", "opTime %lf num %lld speed %lf", optTime, num, slstat->fSpeed);
         } else {
            Long64_t avg = fTotalEntries/(fSlaveStats->GetSize());
            num = (avg > 5) ? 5 : avg;
         }
      }
   }
   // Minimum packet size
   num = (num > 1) ? num : 1;
   fProcessing = (num < (fTotalEntries - fAssigned)) ? num
                                                     : (fTotalEntries - fAssigned);

   // Set the informations of the current slave
   slstat->fLastProcessed = fProcessing;
   // Set the start time of the current packet
   slstat->fTimeInstant = cTime;

   PDB(kPacketizer,2)
      Info("GetNextPacket", "worker-%s: num %lld, processing %lld, remaining %lld",sl->GetOrdinal(),
                            num, fProcessing, (fTotalEntries - fAssigned - fProcessing));
   TDSetElement *elem = new TDSetElement("", "", "", 0, fProcessing);
   elem->SetBit(TDSetElement::kEmpty);

   // Update the total counter
   fAssigned += slstat->fLastProcessed;

   return elem;
}
 TPacketizerUnit.cxx:1
 TPacketizerUnit.cxx:2
 TPacketizerUnit.cxx:3
 TPacketizerUnit.cxx:4
 TPacketizerUnit.cxx:5
 TPacketizerUnit.cxx:6
 TPacketizerUnit.cxx:7
 TPacketizerUnit.cxx:8
 TPacketizerUnit.cxx:9
 TPacketizerUnit.cxx:10
 TPacketizerUnit.cxx:11
 TPacketizerUnit.cxx:12
 TPacketizerUnit.cxx:13
 TPacketizerUnit.cxx:14
 TPacketizerUnit.cxx:15
 TPacketizerUnit.cxx:16
 TPacketizerUnit.cxx:17
 TPacketizerUnit.cxx:18
 TPacketizerUnit.cxx:19
 TPacketizerUnit.cxx:20
 TPacketizerUnit.cxx:21
 TPacketizerUnit.cxx:22
 TPacketizerUnit.cxx:23
 TPacketizerUnit.cxx:24
 TPacketizerUnit.cxx:25
 TPacketizerUnit.cxx:26
 TPacketizerUnit.cxx:27
 TPacketizerUnit.cxx:28
 TPacketizerUnit.cxx:29
 TPacketizerUnit.cxx:30
 TPacketizerUnit.cxx:31
 TPacketizerUnit.cxx:32
 TPacketizerUnit.cxx:33
 TPacketizerUnit.cxx:34
 TPacketizerUnit.cxx:35
 TPacketizerUnit.cxx:36
 TPacketizerUnit.cxx:37
 TPacketizerUnit.cxx:38
 TPacketizerUnit.cxx:39
 TPacketizerUnit.cxx:40
 TPacketizerUnit.cxx:41
 TPacketizerUnit.cxx:42
 TPacketizerUnit.cxx:43
 TPacketizerUnit.cxx:44
 TPacketizerUnit.cxx:45
 TPacketizerUnit.cxx:46
 TPacketizerUnit.cxx:47
 TPacketizerUnit.cxx:48
 TPacketizerUnit.cxx:49
 TPacketizerUnit.cxx:50
 TPacketizerUnit.cxx:51
 TPacketizerUnit.cxx:52
 TPacketizerUnit.cxx:53
 TPacketizerUnit.cxx:54
 TPacketizerUnit.cxx:55
 TPacketizerUnit.cxx:56
 TPacketizerUnit.cxx:57
 TPacketizerUnit.cxx:58
 TPacketizerUnit.cxx:59
 TPacketizerUnit.cxx:60
 TPacketizerUnit.cxx:61
 TPacketizerUnit.cxx:62
 TPacketizerUnit.cxx:63
 TPacketizerUnit.cxx:64
 TPacketizerUnit.cxx:65
 TPacketizerUnit.cxx:66
 TPacketizerUnit.cxx:67
 TPacketizerUnit.cxx:68
 TPacketizerUnit.cxx:69
 TPacketizerUnit.cxx:70
 TPacketizerUnit.cxx:71
 TPacketizerUnit.cxx:72
 TPacketizerUnit.cxx:73
 TPacketizerUnit.cxx:74
 TPacketizerUnit.cxx:75
 TPacketizerUnit.cxx:76
 TPacketizerUnit.cxx:77
 TPacketizerUnit.cxx:78
 TPacketizerUnit.cxx:79
 TPacketizerUnit.cxx:80
 TPacketizerUnit.cxx:81
 TPacketizerUnit.cxx:82
 TPacketizerUnit.cxx:83
 TPacketizerUnit.cxx:84
 TPacketizerUnit.cxx:85
 TPacketizerUnit.cxx:86
 TPacketizerUnit.cxx:87
 TPacketizerUnit.cxx:88
 TPacketizerUnit.cxx:89
 TPacketizerUnit.cxx:90
 TPacketizerUnit.cxx:91
 TPacketizerUnit.cxx:92
 TPacketizerUnit.cxx:93
 TPacketizerUnit.cxx:94
 TPacketizerUnit.cxx:95
 TPacketizerUnit.cxx:96
 TPacketizerUnit.cxx:97
 TPacketizerUnit.cxx:98
 TPacketizerUnit.cxx:99
 TPacketizerUnit.cxx:100
 TPacketizerUnit.cxx:101
 TPacketizerUnit.cxx:102
 TPacketizerUnit.cxx:103
 TPacketizerUnit.cxx:104
 TPacketizerUnit.cxx:105
 TPacketizerUnit.cxx:106
 TPacketizerUnit.cxx:107
 TPacketizerUnit.cxx:108
 TPacketizerUnit.cxx:109
 TPacketizerUnit.cxx:110
 TPacketizerUnit.cxx:111
 TPacketizerUnit.cxx:112
 TPacketizerUnit.cxx:113
 TPacketizerUnit.cxx:114
 TPacketizerUnit.cxx:115
 TPacketizerUnit.cxx:116
 TPacketizerUnit.cxx:117
 TPacketizerUnit.cxx:118
 TPacketizerUnit.cxx:119
 TPacketizerUnit.cxx:120
 TPacketizerUnit.cxx:121
 TPacketizerUnit.cxx:122
 TPacketizerUnit.cxx:123
 TPacketizerUnit.cxx:124
 TPacketizerUnit.cxx:125
 TPacketizerUnit.cxx:126
 TPacketizerUnit.cxx:127
 TPacketizerUnit.cxx:128
 TPacketizerUnit.cxx:129
 TPacketizerUnit.cxx:130
 TPacketizerUnit.cxx:131
 TPacketizerUnit.cxx:132
 TPacketizerUnit.cxx:133
 TPacketizerUnit.cxx:134
 TPacketizerUnit.cxx:135
 TPacketizerUnit.cxx:136
 TPacketizerUnit.cxx:137
 TPacketizerUnit.cxx:138
 TPacketizerUnit.cxx:139
 TPacketizerUnit.cxx:140
 TPacketizerUnit.cxx:141
 TPacketizerUnit.cxx:142
 TPacketizerUnit.cxx:143
 TPacketizerUnit.cxx:144
 TPacketizerUnit.cxx:145
 TPacketizerUnit.cxx:146
 TPacketizerUnit.cxx:147
 TPacketizerUnit.cxx:148
 TPacketizerUnit.cxx:149
 TPacketizerUnit.cxx:150
 TPacketizerUnit.cxx:151
 TPacketizerUnit.cxx:152
 TPacketizerUnit.cxx:153
 TPacketizerUnit.cxx:154
 TPacketizerUnit.cxx:155
 TPacketizerUnit.cxx:156
 TPacketizerUnit.cxx:157
 TPacketizerUnit.cxx:158
 TPacketizerUnit.cxx:159
 TPacketizerUnit.cxx:160
 TPacketizerUnit.cxx:161
 TPacketizerUnit.cxx:162
 TPacketizerUnit.cxx:163
 TPacketizerUnit.cxx:164
 TPacketizerUnit.cxx:165
 TPacketizerUnit.cxx:166
 TPacketizerUnit.cxx:167
 TPacketizerUnit.cxx:168
 TPacketizerUnit.cxx:169
 TPacketizerUnit.cxx:170
 TPacketizerUnit.cxx:171
 TPacketizerUnit.cxx:172
 TPacketizerUnit.cxx:173
 TPacketizerUnit.cxx:174
 TPacketizerUnit.cxx:175
 TPacketizerUnit.cxx:176
 TPacketizerUnit.cxx:177
 TPacketizerUnit.cxx:178
 TPacketizerUnit.cxx:179
 TPacketizerUnit.cxx:180
 TPacketizerUnit.cxx:181
 TPacketizerUnit.cxx:182
 TPacketizerUnit.cxx:183
 TPacketizerUnit.cxx:184
 TPacketizerUnit.cxx:185
 TPacketizerUnit.cxx:186
 TPacketizerUnit.cxx:187
 TPacketizerUnit.cxx:188
 TPacketizerUnit.cxx:189
 TPacketizerUnit.cxx:190
 TPacketizerUnit.cxx:191
 TPacketizerUnit.cxx:192
 TPacketizerUnit.cxx:193
 TPacketizerUnit.cxx:194
 TPacketizerUnit.cxx:195
 TPacketizerUnit.cxx:196
 TPacketizerUnit.cxx:197
 TPacketizerUnit.cxx:198
 TPacketizerUnit.cxx:199
 TPacketizerUnit.cxx:200
 TPacketizerUnit.cxx:201
 TPacketizerUnit.cxx:202
 TPacketizerUnit.cxx:203
 TPacketizerUnit.cxx:204
 TPacketizerUnit.cxx:205
 TPacketizerUnit.cxx:206
 TPacketizerUnit.cxx:207
 TPacketizerUnit.cxx:208
 TPacketizerUnit.cxx:209
 TPacketizerUnit.cxx:210
 TPacketizerUnit.cxx:211
 TPacketizerUnit.cxx:212
 TPacketizerUnit.cxx:213
 TPacketizerUnit.cxx:214
 TPacketizerUnit.cxx:215
 TPacketizerUnit.cxx:216
 TPacketizerUnit.cxx:217
 TPacketizerUnit.cxx:218
 TPacketizerUnit.cxx:219
 TPacketizerUnit.cxx:220
 TPacketizerUnit.cxx:221
 TPacketizerUnit.cxx:222
 TPacketizerUnit.cxx:223
 TPacketizerUnit.cxx:224
 TPacketizerUnit.cxx:225
 TPacketizerUnit.cxx:226
 TPacketizerUnit.cxx:227
 TPacketizerUnit.cxx:228
 TPacketizerUnit.cxx:229
 TPacketizerUnit.cxx:230
 TPacketizerUnit.cxx:231
 TPacketizerUnit.cxx:232
 TPacketizerUnit.cxx:233
 TPacketizerUnit.cxx:234
 TPacketizerUnit.cxx:235
 TPacketizerUnit.cxx:236
 TPacketizerUnit.cxx:237
 TPacketizerUnit.cxx:238
 TPacketizerUnit.cxx:239
 TPacketizerUnit.cxx:240
 TPacketizerUnit.cxx:241
 TPacketizerUnit.cxx:242
 TPacketizerUnit.cxx:243
 TPacketizerUnit.cxx:244
 TPacketizerUnit.cxx:245
 TPacketizerUnit.cxx:246
 TPacketizerUnit.cxx:247
 TPacketizerUnit.cxx:248
 TPacketizerUnit.cxx:249
 TPacketizerUnit.cxx:250
 TPacketizerUnit.cxx:251
 TPacketizerUnit.cxx:252
 TPacketizerUnit.cxx:253
 TPacketizerUnit.cxx:254
 TPacketizerUnit.cxx:255
 TPacketizerUnit.cxx:256
 TPacketizerUnit.cxx:257
 TPacketizerUnit.cxx:258
 TPacketizerUnit.cxx:259
 TPacketizerUnit.cxx:260
 TPacketizerUnit.cxx:261
 TPacketizerUnit.cxx:262
 TPacketizerUnit.cxx:263
 TPacketizerUnit.cxx:264
 TPacketizerUnit.cxx:265
 TPacketizerUnit.cxx:266
 TPacketizerUnit.cxx:267
 TPacketizerUnit.cxx:268
 TPacketizerUnit.cxx:269
 TPacketizerUnit.cxx:270
 TPacketizerUnit.cxx:271
 TPacketizerUnit.cxx:272
 TPacketizerUnit.cxx:273
 TPacketizerUnit.cxx:274
 TPacketizerUnit.cxx:275
 TPacketizerUnit.cxx:276
 TPacketizerUnit.cxx:277
 TPacketizerUnit.cxx:278
 TPacketizerUnit.cxx:279
 TPacketizerUnit.cxx:280
 TPacketizerUnit.cxx:281
 TPacketizerUnit.cxx:282
 TPacketizerUnit.cxx:283
 TPacketizerUnit.cxx:284
 TPacketizerUnit.cxx:285
 TPacketizerUnit.cxx:286
 TPacketizerUnit.cxx:287
 TPacketizerUnit.cxx:288
 TPacketizerUnit.cxx:289
 TPacketizerUnit.cxx:290
 TPacketizerUnit.cxx:291
 TPacketizerUnit.cxx:292
 TPacketizerUnit.cxx:293
 TPacketizerUnit.cxx:294
 TPacketizerUnit.cxx:295
 TPacketizerUnit.cxx:296
 TPacketizerUnit.cxx:297
 TPacketizerUnit.cxx:298
 TPacketizerUnit.cxx:299
 TPacketizerUnit.cxx:300
 TPacketizerUnit.cxx:301
 TPacketizerUnit.cxx:302
 TPacketizerUnit.cxx:303
 TPacketizerUnit.cxx:304
 TPacketizerUnit.cxx:305
 TPacketizerUnit.cxx:306
 TPacketizerUnit.cxx:307
 TPacketizerUnit.cxx:308
 TPacketizerUnit.cxx:309
 TPacketizerUnit.cxx:310
 TPacketizerUnit.cxx:311
 TPacketizerUnit.cxx:312
 TPacketizerUnit.cxx:313
 TPacketizerUnit.cxx:314
 TPacketizerUnit.cxx:315
 TPacketizerUnit.cxx:316
 TPacketizerUnit.cxx:317
 TPacketizerUnit.cxx:318
 TPacketizerUnit.cxx:319
 TPacketizerUnit.cxx:320
 TPacketizerUnit.cxx:321
 TPacketizerUnit.cxx:322
 TPacketizerUnit.cxx:323
 TPacketizerUnit.cxx:324
 TPacketizerUnit.cxx:325
 TPacketizerUnit.cxx:326
 TPacketizerUnit.cxx:327
 TPacketizerUnit.cxx:328
 TPacketizerUnit.cxx:329
 TPacketizerUnit.cxx:330
 TPacketizerUnit.cxx:331
 TPacketizerUnit.cxx:332
 TPacketizerUnit.cxx:333
 TPacketizerUnit.cxx:334
 TPacketizerUnit.cxx:335
 TPacketizerUnit.cxx:336
 TPacketizerUnit.cxx:337
 TPacketizerUnit.cxx:338
 TPacketizerUnit.cxx:339
 TPacketizerUnit.cxx:340
 TPacketizerUnit.cxx:341
 TPacketizerUnit.cxx:342
 TPacketizerUnit.cxx:343
 TPacketizerUnit.cxx:344
 TPacketizerUnit.cxx:345
 TPacketizerUnit.cxx:346
 TPacketizerUnit.cxx:347
 TPacketizerUnit.cxx:348
 TPacketizerUnit.cxx:349
 TPacketizerUnit.cxx:350
 TPacketizerUnit.cxx:351
 TPacketizerUnit.cxx:352
 TPacketizerUnit.cxx:353
 TPacketizerUnit.cxx:354
 TPacketizerUnit.cxx:355
 TPacketizerUnit.cxx:356
 TPacketizerUnit.cxx:357
 TPacketizerUnit.cxx:358
 TPacketizerUnit.cxx:359
 TPacketizerUnit.cxx:360
 TPacketizerUnit.cxx:361
 TPacketizerUnit.cxx:362
 TPacketizerUnit.cxx:363
 TPacketizerUnit.cxx:364
 TPacketizerUnit.cxx:365
 TPacketizerUnit.cxx:366
 TPacketizerUnit.cxx:367
 TPacketizerUnit.cxx:368
 TPacketizerUnit.cxx:369
 TPacketizerUnit.cxx:370
 TPacketizerUnit.cxx:371
 TPacketizerUnit.cxx:372
 TPacketizerUnit.cxx:373
 TPacketizerUnit.cxx:374
 TPacketizerUnit.cxx:375
 TPacketizerUnit.cxx:376
 TPacketizerUnit.cxx:377
 TPacketizerUnit.cxx:378
 TPacketizerUnit.cxx:379
 TPacketizerUnit.cxx:380
 TPacketizerUnit.cxx:381
 TPacketizerUnit.cxx:382
 TPacketizerUnit.cxx:383
 TPacketizerUnit.cxx:384
 TPacketizerUnit.cxx:385
 TPacketizerUnit.cxx:386
 TPacketizerUnit.cxx:387
 TPacketizerUnit.cxx:388
 TPacketizerUnit.cxx:389
 TPacketizerUnit.cxx:390
 TPacketizerUnit.cxx:391
 TPacketizerUnit.cxx:392
 TPacketizerUnit.cxx:393
 TPacketizerUnit.cxx:394
 TPacketizerUnit.cxx:395
 TPacketizerUnit.cxx:396
 TPacketizerUnit.cxx:397
 TPacketizerUnit.cxx:398
 TPacketizerUnit.cxx:399
 TPacketizerUnit.cxx:400
 TPacketizerUnit.cxx:401
 TPacketizerUnit.cxx:402
 TPacketizerUnit.cxx:403
 TPacketizerUnit.cxx:404
 TPacketizerUnit.cxx:405
 TPacketizerUnit.cxx:406
 TPacketizerUnit.cxx:407
 TPacketizerUnit.cxx:408
 TPacketizerUnit.cxx:409
 TPacketizerUnit.cxx:410
 TPacketizerUnit.cxx:411
 TPacketizerUnit.cxx:412