ROOT logo
// @(#)root/proofx:$Id: TXProofServ.cxx 31028 2009-11-09 07:37:35Z ganis $
// Author: Gerardo Ganis  12/12/2005

/*************************************************************************
 * Copyright (C) 1995-2005, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TXProofServ                                                          //
//                                                                      //
// TXProofServ is the XRD version of the PROOF server. It differs from  //
// TXProofServ only for the underlying connection technology            //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include "RConfigure.h"
#include "RConfig.h"
#include "Riostream.h"

#ifdef WIN32
   #include <io.h>
   typedef long off_t;
#endif
#include <sys/types.h>
#include <netinet/in.h>
#include <utime.h>

#include "TXProofServ.h"
#include "TObjString.h"
#include "TEnv.h"
#include "TError.h"
#include "TException.h"
#include "THashList.h"
#include "TInterpreter.h"
#include "TParameter.h"
#include "TProofDebug.h"
#include "TProof.h"
#include "TProofPlayer.h"
#include "TQueryResultManager.h"
#include "TRegexp.h"
#include "TClass.h"
#include "TROOT.h"
#include "TSystem.h"
#include "TPluginManager.h"
#include "TXSocketHandler.h"
#include "TXUnixSocket.h"
#include "compiledata.h"
#include "TProofNodeInfo.h"
#include "XProofProtocol.h"

#include <XrdClient/XrdClientConst.hh>
#include <XrdClient/XrdClientEnv.hh>


// debug hook
static volatile Int_t gProofServDebug = 1;

//----- SigPipe signal handler -------------------------------------------------
//______________________________________________________________________________
class TXProofServSigPipeHandler : public TSignalHandler {
   TXProofServ  *fServ;
public:
   TXProofServSigPipeHandler(TXProofServ *s) : TSignalHandler(kSigInterrupt, kFALSE)
      { fServ = s; }
   Bool_t  Notify();
};

//______________________________________________________________________________
Bool_t TXProofServSigPipeHandler::Notify()
{
   fServ->HandleSigPipe();
   return kTRUE;
}

//----- Termination signal handler ---------------------------------------------
//______________________________________________________________________________
class TXProofServTerminationHandler : public TSignalHandler {
   TXProofServ  *fServ;
public:
   TXProofServTerminationHandler(TXProofServ *s)
      : TSignalHandler(kSigTermination, kFALSE) { fServ = s; }
   Bool_t  Notify();
};

//______________________________________________________________________________
Bool_t TXProofServTerminationHandler::Notify()
{
   Printf("Received SIGTERM: terminating");

   fServ->HandleTermination();
   return kTRUE;
}

//----- Seg violation signal handler ---------------------------------------------
//______________________________________________________________________________
class TXProofServSegViolationHandler : public TSignalHandler {
   TXProofServ  *fServ;
public:
   TXProofServSegViolationHandler(TXProofServ *s)
      : TSignalHandler(kSigSegmentationViolation, kFALSE) { fServ = s; }
   Bool_t  Notify();
};

//______________________________________________________________________________
Bool_t TXProofServSegViolationHandler::Notify()
{
   Printf("**** ");
   Printf("**** Segmentation violation: terminating ****");
   Printf("**** ");
   fServ->HandleTermination();
   return kTRUE;
}

//----- Input handler for messages from parent or master -----------------------
//______________________________________________________________________________
class TXProofServInputHandler : public TFileHandler {
   TXProofServ  *fServ;
public:
   TXProofServInputHandler(TXProofServ *s, Int_t fd) : TFileHandler(fd, 1)
      { fServ = s; }
   Bool_t Notify();
   Bool_t ReadNotify() { return Notify(); }
};

//______________________________________________________________________________
Bool_t TXProofServInputHandler::Notify()
{
   fServ->HandleSocketInput();
   // This request has been completed: remove the client ID from the pipe
   ((TXUnixSocket *) fServ->GetSocket())->RemoveClientID();
   return kTRUE;
}

ClassImp(TXProofServ)

// Hook to the constructor. This is needed to avoid using the plugin manager
// which may create problems in multi-threaded environments.
extern "C" {
   TApplication *GetTXProofServ(Int_t *argc, char **argv, FILE *flog)
   { return new TXProofServ(argc, argv, flog); }
}

//______________________________________________________________________________
TXProofServ::TXProofServ(Int_t *argc, char **argv, FILE *flog)
            : TProofServ(argc, argv, flog)
{
   // Main constructor

   fInterruptHandler = 0;
   fInputHandler = 0;
   fTerminated = kFALSE;

   // TODO:
   //    Int_t useFIFO = 0;
/*   if (GetParameter(fProof->GetInputList(), "PROOF_UseFIFO", useFIFO) != 0) {
      if (useFIFO == 1)
         Info("", "enablig use of FIFO (if allowed by the server)");
      else
         Warning("", "unsupported strategy index (%d): ignore", strategy);
   }
*/
}

//______________________________________________________________________________
Int_t TXProofServ::CreateServer()
{
   // Finalize the server setup. If master, create the TProof instance to talk
   // the worker or submaster nodes.
   // Return 0 on success, -1 on error

   Bool_t xtest = (Argc() > 3 && !strcmp(Argv(3), "test")) ? kTRUE : kFALSE;

   if (gProofDebugLevel > 0)
      Info("CreateServer", "starting%s server creation", (xtest ? " test" : ""));

   // Get file descriptor for log file
   if (fLogFile) {
      // Use the file already open by pmain
      if ((fLogFileDes = fileno(fLogFile)) < 0) {
         Error("CreateServer", "resolving the log file description number");
         return -1;
      }
      // Hide the session start-up logs unless we are in verbose mode
      if (gProofDebugLevel <= 0)
         lseek(fLogFileDes, (off_t) 0, SEEK_END);
   }

   // Global location string in TXSocket
   TXSocket::SetLocation((IsMaster()) ? "master" : "slave");

   // Set debug level in XrdClient
   EnvPutInt(NAME_DEBUG, gEnv->GetValue("XNet.Debug", 0));

   // Get socket to be used to call back our xpd
   if (xtest) {
      // test session, just send the protocol version on the open pipe
      // and exit
      if (!(fSockPath = gSystem->Getenv("ROOTOPENSOCK"))) {
         Error("CreateServer", "Socket setup by xpd undefined");
         return -1;
      }
      Int_t fpw = (Int_t) strtol(fSockPath.Data(), 0, 10);
      int proto = htonl(kPROOF_Protocol);
      fSockPath = "";
      if (write(fpw, &proto, sizeof(proto)) != sizeof(proto)) {
         Error("CreateServer", "test: sending protocol number");
         return -1;
      }
      exit(0);
   } else {
      fSockPath = gEnv->GetValue("ProofServ.OpenSock", "");
      if (fSockPath.Length() <= 0) {
         Error("CreateServer", "Socket setup by xpd undefined");
         return -1;
      }
      TString entity = gEnv->GetValue("ProofServ.Entity", "");
      if (entity.Length() > 0)
         fSockPath.Insert(0,Form("%s/", entity.Data()));
   }

   // Get the sessions ID
   Int_t psid = gEnv->GetValue("ProofServ.SessionID", -1);
   if (psid < 0) {
     Error("CreateServer", "Session ID undefined");
     return -1;
   }

   // Call back the server
   fSocket = new TXUnixSocket(fSockPath, psid, -1, this);
   if (!fSocket || !(fSocket->IsValid())) {
      Error("CreateServer", "Failed to open connection to XrdProofd coordinator");
      return -1;
   }
   // Set compression level, if any
   fSocket->SetCompressionLevel(fCompressMsg);

   // Set the title for debugging
   TString tgt("client");
   if (fOrdinal != "0") {
      tgt = fOrdinal;
      if (tgt.Last('.') != kNPOS) tgt.Remove(tgt.Last('.'));
   }
   fSocket->SetTitle(tgt);

   // Set the this as reference of this socket
   ((TXSocket *)fSocket)->fReference = this;

   // Get socket descriptor
   Int_t sock = fSocket->GetDescriptor();

   // Install message input handlers
   fInputHandler =
      TXSocketHandler::GetSocketHandler(new TXProofServInputHandler(this, sock), fSocket);
   gSystem->AddFileHandler(fInputHandler);

   // Get the client ID
   Int_t cid = gEnv->GetValue("ProofServ.ClientID", -1);
   if (cid < 0) {
     Error("CreateServer", "Client ID undefined");
     SendLogFile();
     return -1;
   }
   ((TXSocket *)fSocket)->SetClientID(cid);

   // debug hooks
   if (IsMaster()) {
      // wait (loop) in master to allow debugger to connect
      if (gEnv->GetValue("Proof.GdbHook",0) == 1) {
         while (gProofServDebug)
            ;
      }
   } else {
      // wait (loop) in slave to allow debugger to connect
      if (gEnv->GetValue("Proof.GdbHook",0) == 2) {
         while (gProofServDebug)
            ;
      }
   }

   if (gProofDebugLevel > 0)
      Info("CreateServer", "Service: %s, ConfDir: %s, IsMaster: %d",
           fService.Data(), fConfDir.Data(), (Int_t)fMasterServ);

   if (Setup() == -1) {
      // Setup failure
      LogToMaster();
      SendLogFile();
      Terminate(0);
      return -1;
   }

   if (!fLogFile) {
      RedirectOutput();
      // If for some reason we failed setting a redirection file for the logs
      // we cannot continue
      if (!fLogFile || (fLogFileDes = fileno(fLogFile)) < 0) {
         LogToMaster();
         SendLogFile(-98);
         Terminate(0);
         return -1;
      }
   }

   // Send message of the day to the client
   if (IsMaster()) {
      if (CatMotd() == -1) {
         LogToMaster();
         SendLogFile(-99);
         Terminate(0);
         return -1;
      }
   }

   // Everybody expects iostream to be available, so load it...
   ProcessLine("#include <iostream>", kTRUE);
   ProcessLine("#include <_string>",kTRUE); // for std::string iostream.

   // Allow the usage of ClassDef and ClassImp in interpreted macros
   ProcessLine("#include <RtypesCint.h>", kTRUE);

   // Disallow the interpretation of Rtypes.h, TError.h and TGenericClassInfo.h
   ProcessLine("#define ROOT_Rtypes 0", kTRUE);
   ProcessLine("#define ROOT_TError 0", kTRUE);
   ProcessLine("#define ROOT_TGenericClassInfo 0", kTRUE);

   // Load user functions
   const char *logon;
   logon = gEnv->GetValue("Proof.Load", (char *)0);
   if (logon) {
      char *mac = gSystem->Which(TROOT::GetMacroPath(), logon, kReadPermission);
      if (mac)
         ProcessLine(Form(".L %s", logon), kTRUE);
      delete [] mac;
   }

   // Execute logon macro
   logon = gEnv->GetValue("Proof.Logon", (char *)0);
   if (logon && !NoLogOpt()) {
      char *mac = gSystem->Which(TROOT::GetMacroPath(), logon, kReadPermission);
      if (mac)
         ProcessFile(logon);
      delete [] mac;
   }

   // Save current interpreter context
   gInterpreter->SaveContext();
   gInterpreter->SaveGlobalsContext();

   // if master, start slave servers
   if (IsMaster()) {
      TString master = Form("proof://%s@__master__", fUser.Data());

      // Add port, if defined
      Int_t port = gEnv->GetValue("ProofServ.XpdPort", -1);
      if (port > -1) {
         master += ":";
         master += port;
      }

      // Make sure that parallel startup via threads is not active
      // (it is broken for xpd because of the locks on gCINTMutex)
      gEnv->SetValue("Proof.ParallelStartup", 0);

      // Get plugin manager to load appropriate TProof from
      TPluginManager *pm = gROOT->GetPluginManager();
      if (!pm) {
         Error("CreateServer", "no plugin manager found");
         SendLogFile(-99);
         Terminate(0);
         return -1;
      }

      // Find the appropriate handler
      TPluginHandler *h = pm->FindHandler("TProof", fConfFile);
      if (!h) {
         Error("CreateServer", "no plugin found for TProof with a"
                             " config file of '%s'", fConfFile.Data());
         SendLogFile(-99);
         Terminate(0);
         return -1;
      }

      // load the plugin
      if (h->LoadPlugin() == -1) {
         Error("CreateServer", "plugin for TProof could not be loaded");
         SendLogFile(-99);
         Terminate(0);
         return -1;
      }

      // make instance of TProof
      fProof = reinterpret_cast<TProof*>(h->ExecPlugin(5, master.Data(),
                                                          fConfFile.Data(),
                                                          fConfDir.Data(),
                                                          fLogLevel,
                                                          fTopSessionTag.Data()));
      if (!fProof || !fProof->IsValid()) {
         Error("CreateServer", "plugin for TProof could not be executed");
         FlushLogFile();
         delete fProof;
         fProof = 0;
         SendLogFile(-99);
         Terminate(0);
         return -1;
      }
      // Find out if we are a master in direct contact only with workers
      fEndMaster = fProof->IsEndMaster();

      // Save worker info
      fProof->SaveWorkerInfo();

      SendLogFile();
   }

   // Setup the shutdown timer
   if (!fShutdownTimer) {
      // Check activity on socket every 5 mins
      fShutdownTimer = new TShutdownTimer(this, 300000);
      fShutdownTimer->Start(-1, kFALSE);
   }

   // Check if schema evolution is effective: clients running versions <=17 do not
   // support that: send a warning message
   if (fProtocol <= 17) {
      TString msg;
      msg.Form("Warning: client version is too old: automatic schema evolution is ineffective.\n"
               "         This may generate compatibility problems between streamed objects.\n"
               "         The advise is to move to ROOT >= 5.21/02 .");
      SendAsynMessage(msg.Data());
   }

   // Done
   return 0;
}

//______________________________________________________________________________
TXProofServ::~TXProofServ()
{
   // Cleanup. Not really necessary since after this dtor there is no
   // live anyway.

   delete fSocket;
}

//______________________________________________________________________________
void TXProofServ::HandleUrgentData()
{
   // Handle high priority data sent by the master or client.

   // Real-time notification of messages
   TProofServLogHandlerGuard hg(fLogFile, fSocket, "", fRealTimeLog);

   // Get interrupt
   Bool_t fw = kFALSE;
   Int_t iLev = ((TXSocket *)fSocket)->GetInterrupt(fw);
   if (iLev < 0) {
      Error("HandleUrgentData", "error receiving interrupt");
      return;
   }

   PDB(kGlobal, 2)
      Info("HandleUrgentData", "got interrupt: %d\n", iLev);

   if (fProof)
      fProof->SetActive();

   switch (iLev) {

      case TProof::kPing:
         PDB(kGlobal, 2)
            Info("HandleUrgentData", "*** Ping");

         // If master server, propagate interrupt to slaves
         if (fw && IsMaster()) {
            Int_t nbad = fProof->fActiveSlaves->GetSize() - fProof->Ping();
            if (nbad > 0) {
               Info("HandleUrgentData","%d slaves did not reply to ping",nbad);
            }
         }

         // Touch the admin path to show we are alive
         if (fAdminPath.IsNull()) {
            fAdminPath = gEnv->GetValue("ProofServ.AdminPath", "");
         }

         if (!fAdminPath.IsNull()) {
            if (!fAdminPath.EndsWith(".status")) {
               // Update file time stamps
               if (utime(fAdminPath.Data(), 0) != 0)
                  Info("HandleUrgentData", "problems touching path: %s", fAdminPath.Data());
               else
                  PDB(kGlobal, 2)
                     Info("HandleUrgentData", "touching path: %s", fAdminPath.Data());
            } else {
               // Update the status in the file
               FILE *fs = fopen(fAdminPath.Data(), "w");
               if (fs) {
                  Int_t st = GetSessionStatus();
                  fprintf(fs, "%d", st);
                  fclose(fs);
                  PDB(kGlobal, 2)
                     Info("HandleUrgentData", "status (=%d) update in path: %s", st, fAdminPath.Data());
               } else {
                  Error("HandleUrgentData", "problems opening status path: %s (errno: %d)", fAdminPath.Data(), errno);
               }
            }
         } else {
            Info("HandleUrgentData", "admin path undefined");
         }

         break;

      case TProof::kHardInterrupt:
         Info("HandleUrgentData", "*** Hard Interrupt");

         // If master server, propagate interrupt to slaves
         if (fw && IsMaster())
            fProof->Interrupt(TProof::kHardInterrupt);

         // Flush input socket
         ((TXSocket *)fSocket)->Flush();

         if (IsMaster())
            SendLogFile();

         break;

      case TProof::kSoftInterrupt:
         Info("HandleUrgentData", "Soft Interrupt");

         // If master server, propagate interrupt to slaves
         if (fw && IsMaster())
            fProof->Interrupt(TProof::kSoftInterrupt);

         Interrupt();

         if (IsMaster())
            SendLogFile();

         break;


      case TProof::kShutdownInterrupt:
         Info("HandleUrgentData", "Shutdown Interrupt");

         // When returning for here connection are closed
         HandleTermination();

         break;

      default:
         Error("HandleUrgentData", "unexpected type: %d", iLev);
         break;
   }


   if (fProof) fProof->SetActive(kFALSE);
}

//______________________________________________________________________________
void TXProofServ::HandleSigPipe()
{
   // Called when the client is not alive anymore; terminate the session.

   // Real-time notification of messages

   Info("HandleSigPipe","got sigpipe ... do nothing");
}

//______________________________________________________________________________
void TXProofServ::HandleTermination()
{
   // Called when the client is not alive anymore; terminate the session.

   // If master server, propagate interrupt to slaves
   // (shutdown interrupt send internally).
   if (IsMaster()) {

      // If not idle, try first to stop processing
      if (!fIdle) {
         // Remove pending requests
         fWaitingQueries->Delete();
         // Interrupt the current monitor
         fProof->InterruptCurrentMonitor();
         // Do not wait for ever, but al least 20 seconds
         Long_t timeout = gEnv->GetValue("Proof.ShutdownTimeout", 60);
         timeout = (timeout > 20) ? timeout : 20;
         // Processing will be aborted
         fProof->StopProcess(kTRUE, (Long_t) (timeout / 2));
         // Receive end-of-processing messages, but do not wait for ever
         fProof->Collect(TProof::kActive, timeout);
         // Still not idle
         if (!fIdle)
            Warning("HandleTermination","processing could not be stopped");
      }
      // Close the session
      if (fProof)
         fProof->Close("S");
   }

   Terminate(0);  // will not return from here....
}

//______________________________________________________________________________
Int_t TXProofServ::Setup()
{
   // Print the ProofServ logo on standard output.
   // Return 0 on success, -1 on error

   char str[512];

   if (IsMaster()) {
      sprintf(str, "**** Welcome to the PROOF server @ %s ****", gSystem->HostName());
   } else {
      sprintf(str, "**** PROOF worker server @ %s started ****", gSystem->HostName());
   }

   if (fSocket->Send(str) != 1+static_cast<Int_t>(strlen(str))) {
      Error("Setup", "failed to send proof server startup message");
      return -1;
   }

   // Get client protocol
   if ((fProtocol = gEnv->GetValue("ProofServ.ClientVersion", -1)) < 0) {
      Error("Setup", "remote proof protocol missing");
      return -1;
   }

   // The local user
   fUser = gEnv->GetValue("ProofServ.Entity", "");
   if (fUser.Length() >= 0) {
      if (fUser.Contains(":"))
         fUser.Remove(fUser.Index(":"));
      if (fUser.Contains("@"))
         fUser.Remove(fUser.Index("@"));
   } else {
      UserGroup_t *pw = gSystem->GetUserInfo();
      if (pw) {
         fUser = pw->fUser;
         delete pw;
      }
   }

   // Work dir and ...
   if (IsMaster()) {
      TString cf = gEnv->GetValue("ProofServ.ProofConfFile", "");
      if (cf.Length() > 0)
         fConfFile = cf;
   }
   fWorkDir = gEnv->GetValue("ProofServ.Sandbox", Form("~/%s", kPROOF_WorkDir));

   // Get Session tag
   if ((fTopSessionTag = gEnv->GetValue("ProofServ.SessionTag", "-1")) == "-1") {
      Error("Setup", "Session tag missing");
      return -1;
   }
   fSessionTag = fTopSessionTag;
   // Make sure the process ID is in the tag
   TString spid = Form("-%d", gSystem->GetPid());
   if (!fSessionTag.EndsWith(spid)) {
      Int_t nd = 0;
      if ((nd = fSessionTag.CountChar('-')) >= 2) {
         Int_t id = fSessionTag.Index("-", fSessionTag.Index("-") + 1);
         if (id != kNPOS) fSessionTag.Remove(id);
      } else if (nd != 1) {
         Warning("Setup", "Wrong number of '-' in session tag: protocol error? %s", fSessionTag.Data());
      }
      // Add this process ID
      fSessionTag += spid;
   }
   if (gProofDebugLevel > 0)
      Info("Setup", "session tags: %s, %s", fTopSessionTag.Data(), fSessionTag.Data());

   // Get Session dir (sandbox)
   if ((fSessionDir = gEnv->GetValue("ProofServ.SessionDir", "-1")) == "-1") {
      Error("Setup", "Session dir missing");
      return -1;
   }

   // Goto to the main PROOF working directory
   char *workdir = gSystem->ExpandPathName(fWorkDir.Data());
   fWorkDir = workdir;
   delete [] workdir;
   if (gProofDebugLevel > 0)
      Info("Setup", "working directory set to %s", fWorkDir.Data());

   // Common setup
   if (SetupCommon() != 0) {
      Error("Setup", "common setup failed");
      return -1;
   }

   // Send packages off immediately to reduce latency
   fSocket->SetOption(kNoDelay, 1);

   // Check every two hours if client is still alive
   fSocket->SetOption(kKeepAlive, 1);

   // Install SigPipe handler to handle kKeepAlive failure
   gSystem->AddSignalHandler(new TXProofServSigPipeHandler(this));

   // Install Termination handler
   gSystem->AddSignalHandler(new TXProofServTerminationHandler(this));

   // Install seg violation handler
   gSystem->AddSignalHandler(new TXProofServSegViolationHandler(this));

   if (gProofDebugLevel > 0)
      Info("Setup", "successfully completed");

   // Done
   return 0;
}

//______________________________________________________________________________
TProofServ::EQueryAction TXProofServ::GetWorkers(TList *workers,
                                                 Int_t & /* prioritychange */,
                                                 Bool_t resume)
{
   // Get list of workers to be used from now on.
   // The list must be provided by the caller.

   TProofServ::EQueryAction rc = kQueryStop;

   // If user config files are enabled, check them first
   if (gEnv->GetValue("ProofServ.UseUserCfg", 0) != 0) {
      Int_t pc = 1;
      if ((rc = TProofServ::GetWorkers(workers, pc)) == kQueryOK)
         return rc;
   }

   // seqnum of the query for which we call getworkers
   Bool_t dynamicStartup = gEnv->GetValue("Proof.DynamicStartup", kFALSE);
   TString seqnum = (dynamicStartup) ? "" : XPD_GW_Static;
   if (!fWaitingQueries->IsEmpty()) {
      if (resume) {
         seqnum += ((TProofQueryResult *)(fWaitingQueries->First()))->GetSeqNum();
      } else {
         seqnum += ((TProofQueryResult *)(fWaitingQueries->Last()))->GetSeqNum();
      }
   }
   // Send request to the coordinator
   TObjString *os =
      ((TXSocket *)fSocket)->SendCoordinator(kGetWorkers, seqnum.Data());

   // The reply contains some information about the master (image, workdir)
   // followed by the information about the workers; the tokens for each node
   // are separated by '&'
   if (os) {
      TString fl(os->GetName());
      if (fl.BeginsWith(XPD_GW_QueryEnqueued)) {
         SendAsynMessage("+++ Query cannot be processed now: enqueued");
         return kQueryEnqueued;
      }

      // Honour a max number of workers request (typically when running in valgrind)
      Int_t nwrks = -1;
      Bool_t pernode = kFALSE;
      if (gSystem->Getenv("PROOF_NWORKERS")) {
         TString s(gSystem->Getenv("PROOF_NWORKERS"));
         if (s.EndsWith("x")) {
            pernode = kTRUE;
            s.ReplaceAll("x", "");
         }
         if (s.IsDigit()) {
            nwrks = s.Atoi();
            if (nwrks > 0) {
               // Notify
               TString msg;
               if (pernode) {
                  msg.Form("+++ Starting max %d workers per node following the setting of PROOF_NWORKERS", nwrks);
               } else {
                  msg.Form("+++ Starting max %d workers following the setting of PROOF_NWORKERS", nwrks);
               }
               SendAsynMessage(msg);
            } else {
               nwrks = -1;
            }
         } else {
            pernode = kFALSE;
         }
      }

      TString tok;
      Ssiz_t from = 0;
      TList *nodecnt = (pernode) ? new TList : 0 ;
      if (fl.Tokenize(tok, from, "&")) {
         if (!tok.IsNull()) {
            TProofNodeInfo *master = new TProofNodeInfo(tok);
            if (!master) {
               Error("GetWorkers", "no appropriate master line got from coordinator");
               return kQueryStop;
            } else {
               // Set image if not yet done and available
               if (fImage.IsNull() && strlen(master->GetImage()) > 0)
                  fImage = master->GetImage();
               SafeDelete(master);
            }
            // Now the workers
            while (fl.Tokenize(tok, from, "&") && (nwrks == -1 || nwrks > 0)) {
               if (!tok.IsNull()) {
                  // We have the minimal set of information to start
                  rc = kQueryOK;
                  if (pernode && nodecnt) {
                     TProofNodeInfo *ni = new TProofNodeInfo(tok);
                     TParameter<Int_t> *p = 0;
                     Int_t nw = 0;
                     if (!(p = (TParameter<Int_t> *) nodecnt->FindObject(ni->GetNodeName().Data()))) {
                        p = new TParameter<Int_t>(ni->GetNodeName().Data(), nw);
                        nodecnt->Add(p);
                     }
                     nw = p->GetVal();
                     if (gDebug > 0)
                        Info("GetWorkers","%p: name: %s (%s) val: %d (nwrks: %d)",
                                          p, p->GetName(), ni->GetNodeName().Data(),  nw, nwrks);
                     if (nw < nwrks) {
                        if (workers) workers->Add(ni);
                        nw++;
                        p->SetVal(nw);
                     } else {
                        // Two many workers on this machine already
                        SafeDelete(ni);
                     }
                  } else {
                     if (workers)
                        workers->Add(new TProofNodeInfo(tok));
                     // Count down
                     if (nwrks != -1) nwrks--;
                  }
               }
            }
         }
      }
      // Cleanup
      if (nodecnt) {
         nodecnt->SetOwner(kTRUE);
         SafeDelete(nodecnt);
      }
   }

   // We are done
   return rc;
}

//_____________________________________________________________________________
Bool_t TXProofServ::HandleError(const void *)
{
   // Handle error on the input socket

   // Try reconnection
   if (fSocket && !fSocket->IsValid()) {

      fSocket->Reconnect();
      if (fSocket && fSocket->IsValid()) {
         if (gDebug > 0)
            Info("HandleError",
               "%p: connection to local coordinator re-established", this);
         FlushLogFile();
         return kFALSE;
      }
   }
   Printf("TXProofServ::HandleError: %p: got called ...", this);

   // If master server, propagate interrupt to slaves
   // (shutdown interrupt send internally).
   if (IsMaster())
      fProof->Close("S");

   // Avoid communicating back anything to the coordinator (it is gone)
   ((TXSocket *)fSocket)->SetSessionID(-1);

   Terminate(0);

   Printf("TXProofServ::HandleError: %p: DONE ... ", this);

   // We are done
   return kTRUE;
}

//_____________________________________________________________________________
Bool_t TXProofServ::HandleInput(const void *in)
{
   // Handle asynchronous input on the input socket

   if (gDebug > 2)
      Printf("TXProofServ::HandleInput %p, in: %p", this, in);

   XHandleIn_t *hin = (XHandleIn_t *) in;
   Int_t acod = (hin) ? hin->fInt1 : kXPD_msg;

   // Act accordingly
   if (acod == kXPD_ping || acod == kXPD_interrupt) {
      // Interrupt or Ping
      HandleUrgentData();

   } else if (acod == kXPD_flush) {
      // Flush stdout, so that we can access the full log file
      Info("HandleInput","kXPD_flush: flushing log file (stdout)");
      fflush(stdout);

   } else if (acod == kXPD_urgent) {
      // Get type
      Int_t type = hin->fInt2;
      switch (type) {
      case TXSocket::kStopProcess:
         {
            // Abort or Stop ?
            Bool_t abort = (hin->fInt3 != 0) ? kTRUE : kFALSE;
            // Timeout
            Int_t timeout = hin->fInt4;
            // Act now
            if (fProof)
               fProof->StopProcess(abort, timeout);
            else
               if (fPlayer)
                  fPlayer->StopProcess(abort, timeout);
         }
         break;
      default:
         Info("HandleInput","kXPD_urgent: unknown type: %d", type);
      }

   } else if (acod == kXPD_inflate) {

      // Set inflate factor
      fInflateFactor = (hin->fInt2 >= 1000) ? hin->fInt2 : fInflateFactor;
      // Notify
      Info("HandleInput", "kXPD_inflate: inflate factor set to %f",
           (Float_t) fInflateFactor / 1000.);

   } else if (acod == kXPD_priority) {

      // The factor is the priority to be propagated
      fGroupPriority = hin->fInt2;
      if (fProof)
         fProof->BroadcastGroupPriority(fGroup, fGroupPriority);
      // Notify
      Info("HandleInput", "kXPD_priority: group %s priority set to %f",
           fGroup.Data(), (Float_t) fGroupPriority / 100.);

   } else if (acod == kXPD_clusterinfo) {

      // Information about the cluster status
      fTotSessions     = hin->fInt2;
      fActSessions     = hin->fInt3;
      fEffSessions     = (hin->fInt4)/1000.;
      // Notify
      Info("HandleInput", "kXPD_clusterinfo: tot: %d, act: %d, eff: %f",
           fTotSessions, fActSessions, fEffSessions);

   } else {
      // Standard socket input
      HandleSocketInput();
      // This request has been completed: remove the client ID from the pipe
      ((TXSocket *)fSocket)->RemoveClientID();
   }

   // We are done
   return kTRUE;
}

//______________________________________________________________________________
void TXProofServ::DisableTimeout()
{
   // Disable read timeout on the underlying socket

   if (fSocket)
     ((TXSocket *)fSocket)->DisableTimeout();
}

//______________________________________________________________________________
void TXProofServ::EnableTimeout()
{
   // Enable read timeout on the underlying socket

   if (fSocket)
     ((TXSocket *)fSocket)->EnableTimeout();
}

//______________________________________________________________________________
void TXProofServ::Terminate(Int_t status)
{
   // Terminate the proof server.
   if (fTerminated)
      // Avoid doubling the exit operations
      exit(1);
   fTerminated = kTRUE;

   // Notify
   Info("Terminate", "starting session termination operations ...");

   // Notify the memory footprint
   ProcInfo_t pi;
   if (!gSystem->GetProcInfo(&pi)){
      Info("Terminate", "process memory footprint: %ld kB virtual, %ld kB resident ",
                        pi.fMemVirtual, pi.fMemResident);
      if (fVirtMemHWM > 0 || fVirtMemMax > 0) {
         Info("Terminate", "process virtual memory limits: %ld kB HWM, %ld kB max ",
                           fVirtMemHWM, fVirtMemMax);
      }
   }

   // Deactivate current monitor, if any
   if (fProof)
      fProof->SetMonitor(0, kFALSE);

   // Cleanup session directory
   if (status == 0) {
      // make sure we remain in a "connected" directory
      gSystem->ChangeDirectory("/");
      // needed in case fSessionDir is on NFS ?!
      gSystem->MakeDirectory(fSessionDir+"/.delete");
      gSystem->Exec(Form("%s %s", kRM, fSessionDir.Data()));
   }

   // Cleanup queries directory if empty
   if (IsMaster()) {
      if (!(fQMgr && fQMgr->Queries() && fQMgr->Queries()->GetSize())) {
         // make sure we remain in a "connected" directory
         gSystem->ChangeDirectory("/");
         // needed in case fQueryDir is on NFS ?!
         gSystem->MakeDirectory(fQueryDir+"/.delete");
         gSystem->Exec(Form("%s %s", kRM, fQueryDir.Data()));
         // Remove lock file
         if (fQueryLock)
            gSystem->Unlink(fQueryLock->GetName());
       }

      // Unlock the query dir owned by this session
      if (fQueryLock)
         fQueryLock->Unlock();
   } else {
      // Try to stop processing if any
      Bool_t abort = (status == 0) ? kFALSE : kTRUE;
      if (!fIdle && fPlayer)
         fPlayer->StopProcess(abort,1);
      gSystem->Sleep(2000);
   }

   // Remove input and signal handlers to avoid spurious "signals"
   // for closing activities executed upon exit()
   gSystem->RemoveFileHandler(fInputHandler);

   // Stop processing events (set a flag to exit the event loop)
   gSystem->ExitLoop();

   // We post the pipe once to wake up the main thread which is waiting for
   // activity on this socket; this fake activity will make it return and
   // eventually exit the loop.
   TXSocket::fgPipe.Post((TXSocket *)fSocket);

   // Notify
   Printf("Terminate: termination operations ended: quitting!");
}

//______________________________________________________________________________
Int_t TXProofServ::LockSession(const char *sessiontag, TProofLockPath **lck)
{
   // Try locking query area of session tagged sessiontag.
   // The id of the locking file is returned in fid and must be
   // unlocked via UnlockQueryFile(fid).

   // We do not need to lock our own session
   if (strstr(sessiontag, fTopSessionTag))
      return 0;

   if (!lck) {
      Info("LockSession","locker space undefined");
      return -1;
   }
   *lck = 0;

   // Check the format
   TString stag = sessiontag;
   TRegexp re("session-.*-.*-.*");
   Int_t i1 = stag.Index(re);
   if (i1 == kNPOS) {
      Info("LockSession","bad format: %s", sessiontag);
      return -1;
   }
   stag.ReplaceAll("session-","");

   // Drop query number, if any
   Int_t i2 = stag.Index(":q");
   if (i2 != kNPOS)
      stag.Remove(i2);

   // Make sure that parent process does not exist anylonger
   TString parlog = fSessionDir;
   parlog = parlog.Remove(parlog.Index("master-")+strlen("master-"));
   parlog += stag;
   if (!gSystem->AccessPathName(parlog)) {
      Info("LockSession","parent still running: do nothing");
      return -1;
   }

   // Lock the query lock file
   TString qlock = fQueryLock->GetName();
   qlock.ReplaceAll(fTopSessionTag, stag);

   if (!gSystem->AccessPathName(qlock)) {
      *lck = new TProofLockPath(qlock);
      if (((*lck)->Lock()) < 0) {
         Info("LockSession","problems locking query lock file");
         SafeDelete(*lck);
         return -1;
      }
   }

   // We are done
   return 0;
}

//______________________________________________________________________________
void TXProofServ::ReleaseWorker(const char *ord)
{
   // Send message to intermediate coordinator to release worker of last ordinal
   // ord.

   Info("ReleaseWorker","releasing: %s", ord);

   ((TXSocket *)fSocket)->SendCoordinator(kReleaseWorker, ord);
}
 TXProofServ.cxx:1
 TXProofServ.cxx:2
 TXProofServ.cxx:3
 TXProofServ.cxx:4
 TXProofServ.cxx:5
 TXProofServ.cxx:6
 TXProofServ.cxx:7
 TXProofServ.cxx:8
 TXProofServ.cxx:9
 TXProofServ.cxx:10
 TXProofServ.cxx:11
 TXProofServ.cxx:12
 TXProofServ.cxx:13
 TXProofServ.cxx:14
 TXProofServ.cxx:15
 TXProofServ.cxx:16
 TXProofServ.cxx:17
 TXProofServ.cxx:18
 TXProofServ.cxx:19
 TXProofServ.cxx:20
 TXProofServ.cxx:21
 TXProofServ.cxx:22
 TXProofServ.cxx:23
 TXProofServ.cxx:24
 TXProofServ.cxx:25
 TXProofServ.cxx:26
 TXProofServ.cxx:27
 TXProofServ.cxx:28
 TXProofServ.cxx:29
 TXProofServ.cxx:30
 TXProofServ.cxx:31
 TXProofServ.cxx:32
 TXProofServ.cxx:33
 TXProofServ.cxx:34
 TXProofServ.cxx:35
 TXProofServ.cxx:36
 TXProofServ.cxx:37
 TXProofServ.cxx:38
 TXProofServ.cxx:39
 TXProofServ.cxx:40
 TXProofServ.cxx:41
 TXProofServ.cxx:42
 TXProofServ.cxx:43
 TXProofServ.cxx:44
 TXProofServ.cxx:45
 TXProofServ.cxx:46
 TXProofServ.cxx:47
 TXProofServ.cxx:48
 TXProofServ.cxx:49
 TXProofServ.cxx:50
 TXProofServ.cxx:51
 TXProofServ.cxx:52
 TXProofServ.cxx:53
 TXProofServ.cxx:54
 TXProofServ.cxx:55
 TXProofServ.cxx:56
 TXProofServ.cxx:57
 TXProofServ.cxx:58
 TXProofServ.cxx:59
 TXProofServ.cxx:60
 TXProofServ.cxx:61
 TXProofServ.cxx:62
 TXProofServ.cxx:63
 TXProofServ.cxx:64
 TXProofServ.cxx:65
 TXProofServ.cxx:66
 TXProofServ.cxx:67
 TXProofServ.cxx:68
 TXProofServ.cxx:69
 TXProofServ.cxx:70
 TXProofServ.cxx:71
 TXProofServ.cxx:72
 TXProofServ.cxx:73
 TXProofServ.cxx:74
 TXProofServ.cxx:75
 TXProofServ.cxx:76
 TXProofServ.cxx:77
 TXProofServ.cxx:78
 TXProofServ.cxx:79
 TXProofServ.cxx:80
 TXProofServ.cxx:81
 TXProofServ.cxx:82
 TXProofServ.cxx:83
 TXProofServ.cxx:84
 TXProofServ.cxx:85
 TXProofServ.cxx:86
 TXProofServ.cxx:87
 TXProofServ.cxx:88
 TXProofServ.cxx:89
 TXProofServ.cxx:90
 TXProofServ.cxx:91
 TXProofServ.cxx:92
 TXProofServ.cxx:93
 TXProofServ.cxx:94
 TXProofServ.cxx:95
 TXProofServ.cxx:96
 TXProofServ.cxx:97
 TXProofServ.cxx:98
 TXProofServ.cxx:99
 TXProofServ.cxx:100
 TXProofServ.cxx:101
 TXProofServ.cxx:102
 TXProofServ.cxx:103
 TXProofServ.cxx:104
 TXProofServ.cxx:105
 TXProofServ.cxx:106
 TXProofServ.cxx:107
 TXProofServ.cxx:108
 TXProofServ.cxx:109
 TXProofServ.cxx:110
 TXProofServ.cxx:111
 TXProofServ.cxx:112
 TXProofServ.cxx:113
 TXProofServ.cxx:114
 TXProofServ.cxx:115
 TXProofServ.cxx:116
 TXProofServ.cxx:117
 TXProofServ.cxx:118
 TXProofServ.cxx:119
 TXProofServ.cxx:120
 TXProofServ.cxx:121
 TXProofServ.cxx:122
 TXProofServ.cxx:123
 TXProofServ.cxx:124
 TXProofServ.cxx:125
 TXProofServ.cxx:126
 TXProofServ.cxx:127
 TXProofServ.cxx:128
 TXProofServ.cxx:129
 TXProofServ.cxx:130
 TXProofServ.cxx:131
 TXProofServ.cxx:132
 TXProofServ.cxx:133
 TXProofServ.cxx:134
 TXProofServ.cxx:135
 TXProofServ.cxx:136
 TXProofServ.cxx:137
 TXProofServ.cxx:138
 TXProofServ.cxx:139
 TXProofServ.cxx:140
 TXProofServ.cxx:141
 TXProofServ.cxx:142
 TXProofServ.cxx:143
 TXProofServ.cxx:144
 TXProofServ.cxx:145
 TXProofServ.cxx:146
 TXProofServ.cxx:147
 TXProofServ.cxx:148
 TXProofServ.cxx:149
 TXProofServ.cxx:150
 TXProofServ.cxx:151
 TXProofServ.cxx:152
 TXProofServ.cxx:153
 TXProofServ.cxx:154
 TXProofServ.cxx:155
 TXProofServ.cxx:156
 TXProofServ.cxx:157
 TXProofServ.cxx:158
 TXProofServ.cxx:159
 TXProofServ.cxx:160
 TXProofServ.cxx:161
 TXProofServ.cxx:162
 TXProofServ.cxx:163
 TXProofServ.cxx:164
 TXProofServ.cxx:165
 TXProofServ.cxx:166
 TXProofServ.cxx:167
 TXProofServ.cxx:168
 TXProofServ.cxx:169
 TXProofServ.cxx:170
 TXProofServ.cxx:171
 TXProofServ.cxx:172
 TXProofServ.cxx:173
 TXProofServ.cxx:174
 TXProofServ.cxx:175
 TXProofServ.cxx:176
 TXProofServ.cxx:177
 TXProofServ.cxx:178
 TXProofServ.cxx:179
 TXProofServ.cxx:180
 TXProofServ.cxx:181
 TXProofServ.cxx:182
 TXProofServ.cxx:183
 TXProofServ.cxx:184
 TXProofServ.cxx:185
 TXProofServ.cxx:186
 TXProofServ.cxx:187
 TXProofServ.cxx:188
 TXProofServ.cxx:189
 TXProofServ.cxx:190
 TXProofServ.cxx:191
 TXProofServ.cxx:192
 TXProofServ.cxx:193
 TXProofServ.cxx:194
 TXProofServ.cxx:195
 TXProofServ.cxx:196
 TXProofServ.cxx:197
 TXProofServ.cxx:198
 TXProofServ.cxx:199
 TXProofServ.cxx:200
 TXProofServ.cxx:201
 TXProofServ.cxx:202
 TXProofServ.cxx:203
 TXProofServ.cxx:204
 TXProofServ.cxx:205
 TXProofServ.cxx:206
 TXProofServ.cxx:207
 TXProofServ.cxx:208
 TXProofServ.cxx:209
 TXProofServ.cxx:210
 TXProofServ.cxx:211
 TXProofServ.cxx:212
 TXProofServ.cxx:213
 TXProofServ.cxx:214
 TXProofServ.cxx:215
 TXProofServ.cxx:216
 TXProofServ.cxx:217
 TXProofServ.cxx:218
 TXProofServ.cxx:219
 TXProofServ.cxx:220
 TXProofServ.cxx:221
 TXProofServ.cxx:222
 TXProofServ.cxx:223
 TXProofServ.cxx:224
 TXProofServ.cxx:225
 TXProofServ.cxx:226
 TXProofServ.cxx:227
 TXProofServ.cxx:228
 TXProofServ.cxx:229
 TXProofServ.cxx:230
 TXProofServ.cxx:231
 TXProofServ.cxx:232
 TXProofServ.cxx:233
 TXProofServ.cxx:234
 TXProofServ.cxx:235
 TXProofServ.cxx:236
 TXProofServ.cxx:237
 TXProofServ.cxx:238
 TXProofServ.cxx:239
 TXProofServ.cxx:240
 TXProofServ.cxx:241
 TXProofServ.cxx:242
 TXProofServ.cxx:243
 TXProofServ.cxx:244
 TXProofServ.cxx:245
 TXProofServ.cxx:246
 TXProofServ.cxx:247
 TXProofServ.cxx:248
 TXProofServ.cxx:249
 TXProofServ.cxx:250
 TXProofServ.cxx:251
 TXProofServ.cxx:252
 TXProofServ.cxx:253
 TXProofServ.cxx:254
 TXProofServ.cxx:255
 TXProofServ.cxx:256
 TXProofServ.cxx:257
 TXProofServ.cxx:258
 TXProofServ.cxx:259
 TXProofServ.cxx:260
 TXProofServ.cxx:261
 TXProofServ.cxx:262
 TXProofServ.cxx:263
 TXProofServ.cxx:264
 TXProofServ.cxx:265
 TXProofServ.cxx:266
 TXProofServ.cxx:267
 TXProofServ.cxx:268
 TXProofServ.cxx:269
 TXProofServ.cxx:270
 TXProofServ.cxx:271
 TXProofServ.cxx:272
 TXProofServ.cxx:273
 TXProofServ.cxx:274
 TXProofServ.cxx:275
 TXProofServ.cxx:276
 TXProofServ.cxx:277
 TXProofServ.cxx:278
 TXProofServ.cxx:279
 TXProofServ.cxx:280
 TXProofServ.cxx:281
 TXProofServ.cxx:282
 TXProofServ.cxx:283
 TXProofServ.cxx:284
 TXProofServ.cxx:285
 TXProofServ.cxx:286
 TXProofServ.cxx:287
 TXProofServ.cxx:288
 TXProofServ.cxx:289
 TXProofServ.cxx:290
 TXProofServ.cxx:291
 TXProofServ.cxx:292
 TXProofServ.cxx:293
 TXProofServ.cxx:294
 TXProofServ.cxx:295
 TXProofServ.cxx:296
 TXProofServ.cxx:297
 TXProofServ.cxx:298
 TXProofServ.cxx:299
 TXProofServ.cxx:300
 TXProofServ.cxx:301
 TXProofServ.cxx:302
 TXProofServ.cxx:303
 TXProofServ.cxx:304
 TXProofServ.cxx:305
 TXProofServ.cxx:306
 TXProofServ.cxx:307
 TXProofServ.cxx:308
 TXProofServ.cxx:309
 TXProofServ.cxx:310
 TXProofServ.cxx:311
 TXProofServ.cxx:312
 TXProofServ.cxx:313
 TXProofServ.cxx:314
 TXProofServ.cxx:315
 TXProofServ.cxx:316
 TXProofServ.cxx:317
 TXProofServ.cxx:318
 TXProofServ.cxx:319
 TXProofServ.cxx:320
 TXProofServ.cxx:321
 TXProofServ.cxx:322
 TXProofServ.cxx:323
 TXProofServ.cxx:324
 TXProofServ.cxx:325
 TXProofServ.cxx:326
 TXProofServ.cxx:327
 TXProofServ.cxx:328
 TXProofServ.cxx:329
 TXProofServ.cxx:330
 TXProofServ.cxx:331
 TXProofServ.cxx:332
 TXProofServ.cxx:333
 TXProofServ.cxx:334
 TXProofServ.cxx:335
 TXProofServ.cxx:336
 TXProofServ.cxx:337
 TXProofServ.cxx:338
 TXProofServ.cxx:339
 TXProofServ.cxx:340
 TXProofServ.cxx:341
 TXProofServ.cxx:342
 TXProofServ.cxx:343
 TXProofServ.cxx:344
 TXProofServ.cxx:345
 TXProofServ.cxx:346
 TXProofServ.cxx:347
 TXProofServ.cxx:348
 TXProofServ.cxx:349
 TXProofServ.cxx:350
 TXProofServ.cxx:351
 TXProofServ.cxx:352
 TXProofServ.cxx:353
 TXProofServ.cxx:354
 TXProofServ.cxx:355
 TXProofServ.cxx:356
 TXProofServ.cxx:357
 TXProofServ.cxx:358
 TXProofServ.cxx:359
 TXProofServ.cxx:360
 TXProofServ.cxx:361
 TXProofServ.cxx:362
 TXProofServ.cxx:363
 TXProofServ.cxx:364
 TXProofServ.cxx:365
 TXProofServ.cxx:366
 TXProofServ.cxx:367
 TXProofServ.cxx:368
 TXProofServ.cxx:369
 TXProofServ.cxx:370
 TXProofServ.cxx:371
 TXProofServ.cxx:372
 TXProofServ.cxx:373
 TXProofServ.cxx:374
 TXProofServ.cxx:375
 TXProofServ.cxx:376
 TXProofServ.cxx:377
 TXProofServ.cxx:378
 TXProofServ.cxx:379
 TXProofServ.cxx:380
 TXProofServ.cxx:381
 TXProofServ.cxx:382
 TXProofServ.cxx:383
 TXProofServ.cxx:384
 TXProofServ.cxx:385
 TXProofServ.cxx:386
 TXProofServ.cxx:387
 TXProofServ.cxx:388
 TXProofServ.cxx:389
 TXProofServ.cxx:390
 TXProofServ.cxx:391
 TXProofServ.cxx:392
 TXProofServ.cxx:393
 TXProofServ.cxx:394
 TXProofServ.cxx:395
 TXProofServ.cxx:396
 TXProofServ.cxx:397
 TXProofServ.cxx:398
 TXProofServ.cxx:399
 TXProofServ.cxx:400
 TXProofServ.cxx:401
 TXProofServ.cxx:402
 TXProofServ.cxx:403
 TXProofServ.cxx:404
 TXProofServ.cxx:405
 TXProofServ.cxx:406
 TXProofServ.cxx:407
 TXProofServ.cxx:408
 TXProofServ.cxx:409
 TXProofServ.cxx:410
 TXProofServ.cxx:411
 TXProofServ.cxx:412
 TXProofServ.cxx:413
 TXProofServ.cxx:414
 TXProofServ.cxx:415
 TXProofServ.cxx:416
 TXProofServ.cxx:417
 TXProofServ.cxx:418
 TXProofServ.cxx:419
 TXProofServ.cxx:420
 TXProofServ.cxx:421
 TXProofServ.cxx:422
 TXProofServ.cxx:423
 TXProofServ.cxx:424
 TXProofServ.cxx:425
 TXProofServ.cxx:426
 TXProofServ.cxx:427
 TXProofServ.cxx:428
 TXProofServ.cxx:429
 TXProofServ.cxx:430
 TXProofServ.cxx:431
 TXProofServ.cxx:432
 TXProofServ.cxx:433
 TXProofServ.cxx:434
 TXProofServ.cxx:435
 TXProofServ.cxx:436
 TXProofServ.cxx:437
 TXProofServ.cxx:438
 TXProofServ.cxx:439
 TXProofServ.cxx:440
 TXProofServ.cxx:441
 TXProofServ.cxx:442
 TXProofServ.cxx:443
 TXProofServ.cxx:444
 TXProofServ.cxx:445
 TXProofServ.cxx:446
 TXProofServ.cxx:447
 TXProofServ.cxx:448
 TXProofServ.cxx:449
 TXProofServ.cxx:450
 TXProofServ.cxx:451
 TXProofServ.cxx:452
 TXProofServ.cxx:453
 TXProofServ.cxx:454
 TXProofServ.cxx:455
 TXProofServ.cxx:456
 TXProofServ.cxx:457
 TXProofServ.cxx:458
 TXProofServ.cxx:459
 TXProofServ.cxx:460
 TXProofServ.cxx:461
 TXProofServ.cxx:462
 TXProofServ.cxx:463
 TXProofServ.cxx:464
 TXProofServ.cxx:465
 TXProofServ.cxx:466
 TXProofServ.cxx:467
 TXProofServ.cxx:468
 TXProofServ.cxx:469
 TXProofServ.cxx:470
 TXProofServ.cxx:471
 TXProofServ.cxx:472
 TXProofServ.cxx:473
 TXProofServ.cxx:474
 TXProofServ.cxx:475
 TXProofServ.cxx:476
 TXProofServ.cxx:477
 TXProofServ.cxx:478
 TXProofServ.cxx:479
 TXProofServ.cxx:480
 TXProofServ.cxx:481
 TXProofServ.cxx:482
 TXProofServ.cxx:483
 TXProofServ.cxx:484
 TXProofServ.cxx:485
 TXProofServ.cxx:486
 TXProofServ.cxx:487
 TXProofServ.cxx:488
 TXProofServ.cxx:489
 TXProofServ.cxx:490
 TXProofServ.cxx:491
 TXProofServ.cxx:492
 TXProofServ.cxx:493
 TXProofServ.cxx:494
 TXProofServ.cxx:495
 TXProofServ.cxx:496
 TXProofServ.cxx:497
 TXProofServ.cxx:498
 TXProofServ.cxx:499
 TXProofServ.cxx:500
 TXProofServ.cxx:501
 TXProofServ.cxx:502
 TXProofServ.cxx:503
 TXProofServ.cxx:504
 TXProofServ.cxx:505
 TXProofServ.cxx:506
 TXProofServ.cxx:507
 TXProofServ.cxx:508
 TXProofServ.cxx:509
 TXProofServ.cxx:510
 TXProofServ.cxx:511
 TXProofServ.cxx:512
 TXProofServ.cxx:513
 TXProofServ.cxx:514
 TXProofServ.cxx:515
 TXProofServ.cxx:516
 TXProofServ.cxx:517
 TXProofServ.cxx:518
 TXProofServ.cxx:519
 TXProofServ.cxx:520
 TXProofServ.cxx:521
 TXProofServ.cxx:522
 TXProofServ.cxx:523
 TXProofServ.cxx:524
 TXProofServ.cxx:525
 TXProofServ.cxx:526
 TXProofServ.cxx:527
 TXProofServ.cxx:528
 TXProofServ.cxx:529
 TXProofServ.cxx:530
 TXProofServ.cxx:531
 TXProofServ.cxx:532
 TXProofServ.cxx:533
 TXProofServ.cxx:534
 TXProofServ.cxx:535
 TXProofServ.cxx:536
 TXProofServ.cxx:537
 TXProofServ.cxx:538
 TXProofServ.cxx:539
 TXProofServ.cxx:540
 TXProofServ.cxx:541
 TXProofServ.cxx:542
 TXProofServ.cxx:543
 TXProofServ.cxx:544
 TXProofServ.cxx:545
 TXProofServ.cxx:546
 TXProofServ.cxx:547
 TXProofServ.cxx:548
 TXProofServ.cxx:549
 TXProofServ.cxx:550
 TXProofServ.cxx:551
 TXProofServ.cxx:552
 TXProofServ.cxx:553
 TXProofServ.cxx:554
 TXProofServ.cxx:555
 TXProofServ.cxx:556
 TXProofServ.cxx:557
 TXProofServ.cxx:558
 TXProofServ.cxx:559
 TXProofServ.cxx:560
 TXProofServ.cxx:561
 TXProofServ.cxx:562
 TXProofServ.cxx:563
 TXProofServ.cxx:564
 TXProofServ.cxx:565
 TXProofServ.cxx:566
 TXProofServ.cxx:567
 TXProofServ.cxx:568
 TXProofServ.cxx:569
 TXProofServ.cxx:570
 TXProofServ.cxx:571
 TXProofServ.cxx:572
 TXProofServ.cxx:573
 TXProofServ.cxx:574
 TXProofServ.cxx:575
 TXProofServ.cxx:576
 TXProofServ.cxx:577
 TXProofServ.cxx:578
 TXProofServ.cxx:579
 TXProofServ.cxx:580
 TXProofServ.cxx:581
 TXProofServ.cxx:582
 TXProofServ.cxx:583
 TXProofServ.cxx:584
 TXProofServ.cxx:585
 TXProofServ.cxx:586
 TXProofServ.cxx:587
 TXProofServ.cxx:588
 TXProofServ.cxx:589
 TXProofServ.cxx:590
 TXProofServ.cxx:591
 TXProofServ.cxx:592
 TXProofServ.cxx:593
 TXProofServ.cxx:594
 TXProofServ.cxx:595
 TXProofServ.cxx:596
 TXProofServ.cxx:597
 TXProofServ.cxx:598
 TXProofServ.cxx:599
 TXProofServ.cxx:600
 TXProofServ.cxx:601
 TXProofServ.cxx:602
 TXProofServ.cxx:603
 TXProofServ.cxx:604
 TXProofServ.cxx:605
 TXProofServ.cxx:606
 TXProofServ.cxx:607
 TXProofServ.cxx:608
 TXProofServ.cxx:609
 TXProofServ.cxx:610
 TXProofServ.cxx:611
 TXProofServ.cxx:612
 TXProofServ.cxx:613
 TXProofServ.cxx:614
 TXProofServ.cxx:615
 TXProofServ.cxx:616
 TXProofServ.cxx:617
 TXProofServ.cxx:618
 TXProofServ.cxx:619
 TXProofServ.cxx:620
 TXProofServ.cxx:621
 TXProofServ.cxx:622
 TXProofServ.cxx:623
 TXProofServ.cxx:624
 TXProofServ.cxx:625
 TXProofServ.cxx:626
 TXProofServ.cxx:627
 TXProofServ.cxx:628
 TXProofServ.cxx:629
 TXProofServ.cxx:630
 TXProofServ.cxx:631
 TXProofServ.cxx:632
 TXProofServ.cxx:633
 TXProofServ.cxx:634
 TXProofServ.cxx:635
 TXProofServ.cxx:636
 TXProofServ.cxx:637
 TXProofServ.cxx:638
 TXProofServ.cxx:639
 TXProofServ.cxx:640
 TXProofServ.cxx:641
 TXProofServ.cxx:642
 TXProofServ.cxx:643
 TXProofServ.cxx:644
 TXProofServ.cxx:645
 TXProofServ.cxx:646
 TXProofServ.cxx:647
 TXProofServ.cxx:648
 TXProofServ.cxx:649
 TXProofServ.cxx:650
 TXProofServ.cxx:651
 TXProofServ.cxx:652
 TXProofServ.cxx:653
 TXProofServ.cxx:654
 TXProofServ.cxx:655
 TXProofServ.cxx:656
 TXProofServ.cxx:657
 TXProofServ.cxx:658
 TXProofServ.cxx:659
 TXProofServ.cxx:660
 TXProofServ.cxx:661
 TXProofServ.cxx:662
 TXProofServ.cxx:663
 TXProofServ.cxx:664
 TXProofServ.cxx:665
 TXProofServ.cxx:666
 TXProofServ.cxx:667
 TXProofServ.cxx:668
 TXProofServ.cxx:669
 TXProofServ.cxx:670
 TXProofServ.cxx:671
 TXProofServ.cxx:672
 TXProofServ.cxx:673
 TXProofServ.cxx:674
 TXProofServ.cxx:675
 TXProofServ.cxx:676
 TXProofServ.cxx:677
 TXProofServ.cxx:678
 TXProofServ.cxx:679
 TXProofServ.cxx:680
 TXProofServ.cxx:681
 TXProofServ.cxx:682
 TXProofServ.cxx:683
 TXProofServ.cxx:684
 TXProofServ.cxx:685
 TXProofServ.cxx:686
 TXProofServ.cxx:687
 TXProofServ.cxx:688
 TXProofServ.cxx:689
 TXProofServ.cxx:690
 TXProofServ.cxx:691
 TXProofServ.cxx:692
 TXProofServ.cxx:693
 TXProofServ.cxx:694
 TXProofServ.cxx:695
 TXProofServ.cxx:696
 TXProofServ.cxx:697
 TXProofServ.cxx:698
 TXProofServ.cxx:699
 TXProofServ.cxx:700
 TXProofServ.cxx:701
 TXProofServ.cxx:702
 TXProofServ.cxx:703
 TXProofServ.cxx:704
 TXProofServ.cxx:705
 TXProofServ.cxx:706
 TXProofServ.cxx:707
 TXProofServ.cxx:708
 TXProofServ.cxx:709
 TXProofServ.cxx:710
 TXProofServ.cxx:711
 TXProofServ.cxx:712
 TXProofServ.cxx:713
 TXProofServ.cxx:714
 TXProofServ.cxx:715
 TXProofServ.cxx:716
 TXProofServ.cxx:717
 TXProofServ.cxx:718
 TXProofServ.cxx:719
 TXProofServ.cxx:720
 TXProofServ.cxx:721
 TXProofServ.cxx:722
 TXProofServ.cxx:723
 TXProofServ.cxx:724
 TXProofServ.cxx:725
 TXProofServ.cxx:726
 TXProofServ.cxx:727
 TXProofServ.cxx:728
 TXProofServ.cxx:729
 TXProofServ.cxx:730
 TXProofServ.cxx:731
 TXProofServ.cxx:732
 TXProofServ.cxx:733
 TXProofServ.cxx:734
 TXProofServ.cxx:735
 TXProofServ.cxx:736
 TXProofServ.cxx:737
 TXProofServ.cxx:738
 TXProofServ.cxx:739
 TXProofServ.cxx:740
 TXProofServ.cxx:741
 TXProofServ.cxx:742
 TXProofServ.cxx:743
 TXProofServ.cxx:744
 TXProofServ.cxx:745
 TXProofServ.cxx:746
 TXProofServ.cxx:747
 TXProofServ.cxx:748
 TXProofServ.cxx:749
 TXProofServ.cxx:750
 TXProofServ.cxx:751
 TXProofServ.cxx:752
 TXProofServ.cxx:753
 TXProofServ.cxx:754
 TXProofServ.cxx:755
 TXProofServ.cxx:756
 TXProofServ.cxx:757
 TXProofServ.cxx:758
 TXProofServ.cxx:759
 TXProofServ.cxx:760
 TXProofServ.cxx:761
 TXProofServ.cxx:762
 TXProofServ.cxx:763
 TXProofServ.cxx:764
 TXProofServ.cxx:765
 TXProofServ.cxx:766
 TXProofServ.cxx:767
 TXProofServ.cxx:768
 TXProofServ.cxx:769
 TXProofServ.cxx:770
 TXProofServ.cxx:771
 TXProofServ.cxx:772
 TXProofServ.cxx:773
 TXProofServ.cxx:774
 TXProofServ.cxx:775
 TXProofServ.cxx:776
 TXProofServ.cxx:777
 TXProofServ.cxx:778
 TXProofServ.cxx:779
 TXProofServ.cxx:780
 TXProofServ.cxx:781
 TXProofServ.cxx:782
 TXProofServ.cxx:783
 TXProofServ.cxx:784
 TXProofServ.cxx:785
 TXProofServ.cxx:786
 TXProofServ.cxx:787
 TXProofServ.cxx:788
 TXProofServ.cxx:789
 TXProofServ.cxx:790
 TXProofServ.cxx:791
 TXProofServ.cxx:792
 TXProofServ.cxx:793
 TXProofServ.cxx:794
 TXProofServ.cxx:795
 TXProofServ.cxx:796
 TXProofServ.cxx:797
 TXProofServ.cxx:798
 TXProofServ.cxx:799
 TXProofServ.cxx:800
 TXProofServ.cxx:801
 TXProofServ.cxx:802
 TXProofServ.cxx:803
 TXProofServ.cxx:804
 TXProofServ.cxx:805
 TXProofServ.cxx:806
 TXProofServ.cxx:807
 TXProofServ.cxx:808
 TXProofServ.cxx:809
 TXProofServ.cxx:810
 TXProofServ.cxx:811
 TXProofServ.cxx:812
 TXProofServ.cxx:813
 TXProofServ.cxx:814
 TXProofServ.cxx:815
 TXProofServ.cxx:816
 TXProofServ.cxx:817
 TXProofServ.cxx:818
 TXProofServ.cxx:819
 TXProofServ.cxx:820
 TXProofServ.cxx:821
 TXProofServ.cxx:822
 TXProofServ.cxx:823
 TXProofServ.cxx:824
 TXProofServ.cxx:825
 TXProofServ.cxx:826
 TXProofServ.cxx:827
 TXProofServ.cxx:828
 TXProofServ.cxx:829
 TXProofServ.cxx:830
 TXProofServ.cxx:831
 TXProofServ.cxx:832
 TXProofServ.cxx:833
 TXProofServ.cxx:834
 TXProofServ.cxx:835
 TXProofServ.cxx:836
 TXProofServ.cxx:837
 TXProofServ.cxx:838
 TXProofServ.cxx:839
 TXProofServ.cxx:840
 TXProofServ.cxx:841
 TXProofServ.cxx:842
 TXProofServ.cxx:843
 TXProofServ.cxx:844
 TXProofServ.cxx:845
 TXProofServ.cxx:846
 TXProofServ.cxx:847
 TXProofServ.cxx:848
 TXProofServ.cxx:849
 TXProofServ.cxx:850
 TXProofServ.cxx:851
 TXProofServ.cxx:852
 TXProofServ.cxx:853
 TXProofServ.cxx:854
 TXProofServ.cxx:855
 TXProofServ.cxx:856
 TXProofServ.cxx:857
 TXProofServ.cxx:858
 TXProofServ.cxx:859
 TXProofServ.cxx:860
 TXProofServ.cxx:861
 TXProofServ.cxx:862
 TXProofServ.cxx:863
 TXProofServ.cxx:864
 TXProofServ.cxx:865
 TXProofServ.cxx:866
 TXProofServ.cxx:867
 TXProofServ.cxx:868
 TXProofServ.cxx:869
 TXProofServ.cxx:870
 TXProofServ.cxx:871
 TXProofServ.cxx:872
 TXProofServ.cxx:873
 TXProofServ.cxx:874
 TXProofServ.cxx:875
 TXProofServ.cxx:876
 TXProofServ.cxx:877
 TXProofServ.cxx:878
 TXProofServ.cxx:879
 TXProofServ.cxx:880
 TXProofServ.cxx:881
 TXProofServ.cxx:882
 TXProofServ.cxx:883
 TXProofServ.cxx:884
 TXProofServ.cxx:885
 TXProofServ.cxx:886
 TXProofServ.cxx:887
 TXProofServ.cxx:888
 TXProofServ.cxx:889
 TXProofServ.cxx:890
 TXProofServ.cxx:891
 TXProofServ.cxx:892
 TXProofServ.cxx:893
 TXProofServ.cxx:894
 TXProofServ.cxx:895
 TXProofServ.cxx:896
 TXProofServ.cxx:897
 TXProofServ.cxx:898
 TXProofServ.cxx:899
 TXProofServ.cxx:900
 TXProofServ.cxx:901
 TXProofServ.cxx:902
 TXProofServ.cxx:903
 TXProofServ.cxx:904
 TXProofServ.cxx:905
 TXProofServ.cxx:906
 TXProofServ.cxx:907
 TXProofServ.cxx:908
 TXProofServ.cxx:909
 TXProofServ.cxx:910
 TXProofServ.cxx:911
 TXProofServ.cxx:912
 TXProofServ.cxx:913
 TXProofServ.cxx:914
 TXProofServ.cxx:915
 TXProofServ.cxx:916
 TXProofServ.cxx:917
 TXProofServ.cxx:918
 TXProofServ.cxx:919
 TXProofServ.cxx:920
 TXProofServ.cxx:921
 TXProofServ.cxx:922
 TXProofServ.cxx:923
 TXProofServ.cxx:924
 TXProofServ.cxx:925
 TXProofServ.cxx:926
 TXProofServ.cxx:927
 TXProofServ.cxx:928
 TXProofServ.cxx:929
 TXProofServ.cxx:930
 TXProofServ.cxx:931
 TXProofServ.cxx:932
 TXProofServ.cxx:933
 TXProofServ.cxx:934
 TXProofServ.cxx:935
 TXProofServ.cxx:936
 TXProofServ.cxx:937
 TXProofServ.cxx:938
 TXProofServ.cxx:939
 TXProofServ.cxx:940
 TXProofServ.cxx:941
 TXProofServ.cxx:942
 TXProofServ.cxx:943
 TXProofServ.cxx:944
 TXProofServ.cxx:945
 TXProofServ.cxx:946
 TXProofServ.cxx:947
 TXProofServ.cxx:948
 TXProofServ.cxx:949
 TXProofServ.cxx:950
 TXProofServ.cxx:951
 TXProofServ.cxx:952
 TXProofServ.cxx:953
 TXProofServ.cxx:954
 TXProofServ.cxx:955
 TXProofServ.cxx:956
 TXProofServ.cxx:957
 TXProofServ.cxx:958
 TXProofServ.cxx:959
 TXProofServ.cxx:960
 TXProofServ.cxx:961
 TXProofServ.cxx:962
 TXProofServ.cxx:963
 TXProofServ.cxx:964
 TXProofServ.cxx:965
 TXProofServ.cxx:966
 TXProofServ.cxx:967
 TXProofServ.cxx:968
 TXProofServ.cxx:969
 TXProofServ.cxx:970
 TXProofServ.cxx:971
 TXProofServ.cxx:972
 TXProofServ.cxx:973
 TXProofServ.cxx:974
 TXProofServ.cxx:975
 TXProofServ.cxx:976
 TXProofServ.cxx:977
 TXProofServ.cxx:978
 TXProofServ.cxx:979
 TXProofServ.cxx:980
 TXProofServ.cxx:981
 TXProofServ.cxx:982
 TXProofServ.cxx:983
 TXProofServ.cxx:984
 TXProofServ.cxx:985
 TXProofServ.cxx:986
 TXProofServ.cxx:987
 TXProofServ.cxx:988
 TXProofServ.cxx:989
 TXProofServ.cxx:990
 TXProofServ.cxx:991
 TXProofServ.cxx:992
 TXProofServ.cxx:993
 TXProofServ.cxx:994
 TXProofServ.cxx:995
 TXProofServ.cxx:996
 TXProofServ.cxx:997
 TXProofServ.cxx:998
 TXProofServ.cxx:999
 TXProofServ.cxx:1000
 TXProofServ.cxx:1001
 TXProofServ.cxx:1002
 TXProofServ.cxx:1003
 TXProofServ.cxx:1004
 TXProofServ.cxx:1005
 TXProofServ.cxx:1006
 TXProofServ.cxx:1007
 TXProofServ.cxx:1008
 TXProofServ.cxx:1009
 TXProofServ.cxx:1010
 TXProofServ.cxx:1011
 TXProofServ.cxx:1012
 TXProofServ.cxx:1013
 TXProofServ.cxx:1014
 TXProofServ.cxx:1015
 TXProofServ.cxx:1016
 TXProofServ.cxx:1017
 TXProofServ.cxx:1018
 TXProofServ.cxx:1019
 TXProofServ.cxx:1020
 TXProofServ.cxx:1021
 TXProofServ.cxx:1022
 TXProofServ.cxx:1023
 TXProofServ.cxx:1024
 TXProofServ.cxx:1025
 TXProofServ.cxx:1026
 TXProofServ.cxx:1027
 TXProofServ.cxx:1028
 TXProofServ.cxx:1029
 TXProofServ.cxx:1030
 TXProofServ.cxx:1031
 TXProofServ.cxx:1032
 TXProofServ.cxx:1033
 TXProofServ.cxx:1034
 TXProofServ.cxx:1035
 TXProofServ.cxx:1036
 TXProofServ.cxx:1037
 TXProofServ.cxx:1038
 TXProofServ.cxx:1039
 TXProofServ.cxx:1040
 TXProofServ.cxx:1041
 TXProofServ.cxx:1042
 TXProofServ.cxx:1043
 TXProofServ.cxx:1044
 TXProofServ.cxx:1045
 TXProofServ.cxx:1046
 TXProofServ.cxx:1047
 TXProofServ.cxx:1048
 TXProofServ.cxx:1049
 TXProofServ.cxx:1050
 TXProofServ.cxx:1051
 TXProofServ.cxx:1052
 TXProofServ.cxx:1053
 TXProofServ.cxx:1054
 TXProofServ.cxx:1055
 TXProofServ.cxx:1056
 TXProofServ.cxx:1057
 TXProofServ.cxx:1058
 TXProofServ.cxx:1059
 TXProofServ.cxx:1060
 TXProofServ.cxx:1061
 TXProofServ.cxx:1062
 TXProofServ.cxx:1063
 TXProofServ.cxx:1064
 TXProofServ.cxx:1065
 TXProofServ.cxx:1066
 TXProofServ.cxx:1067
 TXProofServ.cxx:1068
 TXProofServ.cxx:1069
 TXProofServ.cxx:1070
 TXProofServ.cxx:1071
 TXProofServ.cxx:1072
 TXProofServ.cxx:1073
 TXProofServ.cxx:1074
 TXProofServ.cxx:1075
 TXProofServ.cxx:1076
 TXProofServ.cxx:1077
 TXProofServ.cxx:1078
 TXProofServ.cxx:1079
 TXProofServ.cxx:1080
 TXProofServ.cxx:1081
 TXProofServ.cxx:1082
 TXProofServ.cxx:1083
 TXProofServ.cxx:1084
 TXProofServ.cxx:1085
 TXProofServ.cxx:1086
 TXProofServ.cxx:1087
 TXProofServ.cxx:1088
 TXProofServ.cxx:1089
 TXProofServ.cxx:1090
 TXProofServ.cxx:1091
 TXProofServ.cxx:1092
 TXProofServ.cxx:1093
 TXProofServ.cxx:1094
 TXProofServ.cxx:1095
 TXProofServ.cxx:1096
 TXProofServ.cxx:1097
 TXProofServ.cxx:1098
 TXProofServ.cxx:1099
 TXProofServ.cxx:1100
 TXProofServ.cxx:1101
 TXProofServ.cxx:1102
 TXProofServ.cxx:1103
 TXProofServ.cxx:1104
 TXProofServ.cxx:1105
 TXProofServ.cxx:1106
 TXProofServ.cxx:1107
 TXProofServ.cxx:1108
 TXProofServ.cxx:1109
 TXProofServ.cxx:1110
 TXProofServ.cxx:1111
 TXProofServ.cxx:1112
 TXProofServ.cxx:1113
 TXProofServ.cxx:1114
 TXProofServ.cxx:1115
 TXProofServ.cxx:1116
 TXProofServ.cxx:1117
 TXProofServ.cxx:1118
 TXProofServ.cxx:1119
 TXProofServ.cxx:1120
 TXProofServ.cxx:1121
 TXProofServ.cxx:1122
 TXProofServ.cxx:1123
 TXProofServ.cxx:1124
 TXProofServ.cxx:1125
 TXProofServ.cxx:1126
 TXProofServ.cxx:1127
 TXProofServ.cxx:1128