ROOT logo
// @(#)root/proof:$Id: TCondor.cxx 29145 2009-06-22 15:22:22Z brun $
// Author: Maarten Ballintijn   06/12/03

/*************************************************************************
 * Copyright (C) 1995-2001, Rene Brun and Fons Rademakers.               *
 * All rights reserved.                                                  *
 *                                                                       *
 * For the licensing terms see $ROOTSYS/LICENSE.                         *
 * For the list of contributors see $ROOTSYS/README/CREDITS.             *
 *************************************************************************/

//////////////////////////////////////////////////////////////////////////
//                                                                      //
// TCondor                                                              //
//                                                                      //
// Interface to the Condor system. TCondor provides a (partial) API for //
// querying and controlling the Condor system, including experimental   //
// extensions like COD (computing on demand)                            //
//                                                                      //
//////////////////////////////////////////////////////////////////////////

#include <stdlib.h>

#include "TCondor.h"
#include "TList.h"
#include "TSystem.h"
#include "TObjString.h"
#include "TRegexp.h"
#include "TProofDebug.h"
#include "Riostream.h"
#include "TEnv.h"
#include "TClass.h"

ClassImp(TCondorSlave)
ClassImp(TCondor)


//______________________________________________________________________________
TCondor::TCondor(const char *pool) : fPool(pool), fState(kFree)
{
   // Create Condor interface object. Uses Condor apps since there is no
   // API yet.

   fClaims = new TList;

   // Setup Condor

   TString condorHome = gEnv->GetValue("Proof.CondorHome", (char*)0);
   if (condorHome != "") {
      TString path = gSystem->Getenv("PATH");
      path = condorHome + "/bin:" + path;
      gSystem->Setenv("PATH",path);
   }

   TString condorConf = gEnv->GetValue("Proof.CondorConfig", (char*)0);
   if (condorConf != "") {
      gSystem->Setenv("CONDOR_CONFIG",condorConf);
   }

   const char *loc = gSystem->Which(gSystem->Getenv("PATH"), "condor_cod",
                                    kExecutePermission);
   fValid = (loc != 0);
}


//______________________________________________________________________________
TCondor::~TCondor()
{
   // Cleanup Condor interface.

   PDB(kCondor,1) Info("~TCondor","fState %d", fState );

   if (fState != kFree) {
      Release();
   }
   delete fClaims;
}


//______________________________________________________________________________
void TCondor::Print(Option_t * opt) const
{
   // Print master status

   cout << "OBJ: " << IsA()->GetName()
      << "\tPool: \"" << fPool << "\""
      << "\tState: " << fState << endl;
   fClaims->Print(opt);
}


//______________________________________________________________________________
TCondorSlave *TCondor::ClaimVM(const char *vm, const char *cmd)
{
   // Claim a VirtualMachine for PROOF usage.

//    TString reinitCmd = "KRB5CCNAME=FILE:/tmp/condor.$$ && /usr/krb5/bin/kinit -F -k -t /etc/cdfcaf.keytab cafuser/cdf/h2caf@FNAL.GOV";
//    gSystem->Exec(reinitCmd.Data());
   Int_t port = 0;

   TString claimCmd = Form("condor_cod request -name %s -timeout 10 2>>%s/condor.proof.%d",
                           vm, gSystem->TempDirectory(), gSystem->GetUid() );

   PDB(kCondor,2) Info("ClaimVM","command: %s", claimCmd.Data());
   FILE  *pipe = gSystem->OpenPipe(claimCmd, "r");

   if (!pipe) {
      SysError("ClaimVM","cannot run command: %s", claimCmd.Data());
      return 0;
   }

   TString claimId;
   TString line;
   while (line.Gets(pipe)) {
      PDB(kCondor,3) Info("ClaimVM","line = %s", line.Data());

      if (line.BeginsWith("ClaimId = \"")) {
         line.Remove(0, line.Index("\"")+1);
         line.Chop(); // remove trailing "
         claimId = line;
         PDB(kCondor,1) Info("ClaimVM","claim = '%s'", claimId.Data());
         TRegexp r("[0-9]*$");
         TString num = line(r);
         port = 37000 + atoi(num.Data());
         PDB(kCondor,1) Info("ClaimVM","port = %d", port);
      }
   }

   Int_t r = gSystem->ClosePipe(pipe);
   if (r) {
      Error("ClaimVM","command: %s returned %d", claimCmd.Data(), r);
      return 0;
   } else {
      PDB(kCondor,1) Info("ClaimVM","command: %s returned %d", claimCmd.Data(), r);
   }

   TString jobad("jobad");
   FILE *jf = gSystem->TempFileName(jobad);

   if (jf == 0) return 0;

   TString str(cmd);
   str.ReplaceAll("$(Port)", Form("%d", port));
   fputs(str, jf);

   fclose(jf);

   TString activateCmd = Form("condor_cod activate -id '%s' -jobad %s",
                              claimId.Data(), jobad.Data() );

   PDB(kCondor,2) Info("ClaimVM","command: %s", activateCmd.Data());
   pipe = gSystem->OpenPipe(activateCmd, "r");

   if (!pipe) {
      SysError("ClaimVM","cannot run command: %s", activateCmd.Data());
      return 0;
   }

   while (line.Gets(pipe)) {
      PDB(kCondor,3) Info("ClaimVM","Activate: line = %s", line.Data());
   }

   r = gSystem->ClosePipe(pipe);
   if (r) {
      Error("ClaimVM","command: %s returned %d", activateCmd.Data(), r);
   } else {
      PDB(kCondor,1) Info("ClaimVM","command: %s returned %d", activateCmd.Data(), r);
   }

   gSystem->Unlink(jobad);

   // TODO: get info at the start for all nodes ...
   TCondorSlave *claim = new TCondorSlave;
   claim->fClaimID = claimId;
   TString node(vm);
   node = node.Remove(0, node.Index("@")+1);
   claim->fHostname = node;
   claim->fPort = port;
   claim->fPerfIdx = 100; //set performance index to 100 by default
   claim->fImage = node; //set image to hostname by default

   return claim;
}


//______________________________________________________________________________
TList *TCondor::GetVirtualMachines() const
{
   // Get the names of the virtual machines in the pool.
   // Return a TList of TObjString or 0 in case of failure

   TString poolopt = fPool ? "" : Form("-pool %s", fPool.Data());
   TString cmd = Form("condor_status %s -format \"%%s\\n\" Name", poolopt.Data());

   PDB(kCondor,2) Info("GetVirtualMachines","command: %s", cmd.Data());

   FILE  *pipe = gSystem->OpenPipe(cmd, "r");

   if (!pipe) {
      SysError("GetVirtualMachines","cannot run command: %s", cmd.Data());
      return 0;
   }

   TString line;
   TList *l = new TList;
   while (line.Gets(pipe)) {
      PDB(kCondor,3) Info("GetVirtualMachines","line = %s", line.Data());
      if (line != "") l->Add(new TObjString(line));
   }

   Int_t r = gSystem->ClosePipe(pipe);
   if (r) {
      delete l;
      Error("GetVirtualMachines","command: %s returned %d", cmd.Data(), r);
      return 0;
   } else {
      PDB(kCondor,1) Info("GetVirtualMachines","command: %s returned %d", cmd.Data(), r);
   }

   return l;
}


//______________________________________________________________________________
TList *TCondor::Claim(Int_t n, const char *cmd)
{
   // Claim n virtual machines
   // This function figures out the image and performance index before returning
   // the list of condor slaves

   if (fState != kFree) {
      Error("Claim","not in state Free");
      return 0;
   }

   TList *vms = GetVirtualMachines();
   TIter next(vms);
   TObjString *vm;
   for(Int_t i=0; i < n && (vm = (TObjString*) next()) != 0; i++ ) {
      TCondorSlave *claim = ClaimVM(vm->GetName(), cmd);
      if (claim != 0) {
         if ( !GetVmInfo(vm->GetName(), claim->fImage, claim->fPerfIdx) ) {
            // assume vm is gone
            delete claim;
         } else {
            fClaims->Add(claim);
            fState = kActive;
         }
      }
   }

   return fClaims;
}


//______________________________________________________________________________
TCondorSlave *TCondor::Claim(const char *vmname, const char *cmd)
{
   // Claim virtual machine with name vmname
   // This function does not figure out the image and performance index before
   // returning the condor slave

   if (fState != kFree && fState != kActive) {
      Error("Claim","not in state Free or Active");
      return 0;
   }

   TCondorSlave *claim = ClaimVM(vmname, cmd);
   if (claim != 0) {
      fClaims->Add(claim);
      fState = kActive;
   }

   return claim;
}


//______________________________________________________________________________
Bool_t TCondor::SetState(EState state)
{
   // Set the state of workers

   PDB(kCondor,1) Info("SetState","state: %s (%ld)",
                       state == kSuspended ? "kSuspended" : "kActive", long(gSystem->Now()));
   TIter next(fClaims);
   TCondorSlave *claim;
   while((claim = (TCondorSlave*) next()) != 0) {
      TString cmd = Form("condor_cod %s -id '%s'",
                         state == kSuspended ? "suspend" : "resume",
                         claim->fClaimID.Data());

      PDB(kCondor,2) Info("SetState","command: %s", cmd.Data());
      FILE  *pipe = gSystem->OpenPipe(cmd, "r");

      if (!pipe) {
         SysError("SetState","cannot run command: %s", cmd.Data());
         return kFALSE;
      }

      TString line;
      while (line.Gets(pipe)) {
         PDB(kCondor,3) Info("SetState","line = %s", line.Data());
      }

      Int_t r = gSystem->ClosePipe(pipe);
      if (r) {
         Error("SetState","command: %s returned %d", cmd.Data(), r);
         return kFALSE;
      } else {
         PDB(kCondor,1) Info("SetState","command: %s returned %d", cmd.Data(), r);
      }
   }

   fState = state;
   return kTRUE;
}


//______________________________________________________________________________
Bool_t TCondor::Suspend()
{
   // Suspend worker

   if (fState != kActive) {
      Error("Suspend","not in state Active");
      return kFALSE;
   }

   return SetState(kSuspended);
}


//______________________________________________________________________________
Bool_t TCondor::Resume()
{
   // Resume worker

   if (fState != kSuspended) {
      Error("Suspend","not in state Suspended");
      return kFALSE;
   }

   return SetState(kActive);
}


//______________________________________________________________________________
Bool_t TCondor::Release()
{
   // Release worker

   if (fState == kFree) {
      Error("Suspend","not in state Active or Suspended");
      return kFALSE;
   }

   TCondorSlave *claim;
   while((claim = (TCondorSlave*) fClaims->First()) != 0) {
      TString cmd = Form("condor_cod release -id '%s'", claim->fClaimID.Data());

      PDB(kCondor,2) Info("SetState","command: %s", cmd.Data());
      FILE  *pipe = gSystem->OpenPipe(cmd, "r");

      if (!pipe) {
         SysError("Release","cannot run command: %s", cmd.Data());
         return kFALSE;
      }

      TString line;
      while (line.Gets(pipe)) {
         PDB(kCondor,3) Info("Release","line = %s", line.Data());
      }

      Int_t r = gSystem->ClosePipe(pipe);
      if (r) {
         Error("Release","command: %s returned %d", cmd.Data(), r);
         return kFALSE;
      } else {
         PDB(kCondor,1) Info("Release","command: %s returned %d", cmd.Data(), r);
      }

      fClaims->Remove(claim);
      delete claim;
   }

   fState = kFree;
   return kTRUE;
}


//______________________________________________________________________________
Bool_t TCondor::GetVmInfo(const char *vm, TString &image, Int_t &perfidx) const
{
   // Get info about worker status

   TString cmd = Form("condor_status -format \"%%d:\" Mips -format \"%%s\\n\" FileSystemDomain "
                      "-const 'Name==\"%s\"'", vm);

   PDB(kCondor,2) Info("GetVmInfo","command: %s", cmd.Data());
   FILE  *pipe = gSystem->OpenPipe(cmd, "r");

   if (!pipe) {
      SysError("GetVmInfo","cannot run command: %s", cmd.Data());
      return kFALSE;
   }

   TString line;
   while (line.Gets(pipe)) {
      PDB(kCondor,3) Info("GetVmInfo","line = %s", line.Data());
      if (line != "") {
         TString amips = line(TRegexp("^[0-9]*"));
         perfidx = atoi(amips);
         image = line(TRegexp("[^:]+$"));
         break;
      }
   }

   Int_t r = gSystem->ClosePipe(pipe);
   if (r) {
      Error("GetVmInfo","command: %s returned %d", cmd.Data(), r);
      return kFALSE;
   } else {
      PDB(kCondor,1) Info("GetVmInfo","command: %s returned %d", cmd.Data(), r);
   }

   return kTRUE;
}


//______________________________________________________________________________
TString TCondor::GetImage(const char *host) const
{
   // Get image of the worker

   TString cmd = Form("condor_status -direct %s -format \"Image:%%s\\n\" "
                      "FileSystemDomain", host);

   PDB(kCondor,2) Info("GetImage","command: %s", cmd.Data());

   FILE  *pipe = gSystem->OpenPipe(cmd, "r");

   if (!pipe) {
      SysError("GetImage","cannot run command: %s", cmd.Data());
      return "";
   }

   TString image;
   TString line;
   while (line.Gets(pipe)) {
      PDB(kCondor,3) Info("GetImage","line = %s", line.Data());
      if (line != "") {
         image = line(TRegexp("[^:]+$"));
         break;
      }
   }

   Int_t r = gSystem->ClosePipe(pipe);
   if (r) {
      Error("GetImage","command: %s returned %d", cmd.Data(), r);
      return "";
   } else {
      PDB(kCondor,1) Info("GetImage","command: %s returned %d", cmd.Data(), r);
   }

   return image;
}


//______________________________________________________________________________
void TCondorSlave::Print(Option_t * /*opt*/ ) const
{
   // Print worker status

   cout << "OBJ: " << IsA()->GetName()
      << " " << fHostname << ":" << fPort
      << "  Perf: " << fPerfIdx
      << "  Image: " << fImage << endl;
}
 TCondor.cxx:1
 TCondor.cxx:2
 TCondor.cxx:3
 TCondor.cxx:4
 TCondor.cxx:5
 TCondor.cxx:6
 TCondor.cxx:7
 TCondor.cxx:8
 TCondor.cxx:9
 TCondor.cxx:10
 TCondor.cxx:11
 TCondor.cxx:12
 TCondor.cxx:13
 TCondor.cxx:14
 TCondor.cxx:15
 TCondor.cxx:16
 TCondor.cxx:17
 TCondor.cxx:18
 TCondor.cxx:19
 TCondor.cxx:20
 TCondor.cxx:21
 TCondor.cxx:22
 TCondor.cxx:23
 TCondor.cxx:24
 TCondor.cxx:25
 TCondor.cxx:26
 TCondor.cxx:27
 TCondor.cxx:28
 TCondor.cxx:29
 TCondor.cxx:30
 TCondor.cxx:31
 TCondor.cxx:32
 TCondor.cxx:33
 TCondor.cxx:34
 TCondor.cxx:35
 TCondor.cxx:36
 TCondor.cxx:37
 TCondor.cxx:38
 TCondor.cxx:39
 TCondor.cxx:40
 TCondor.cxx:41
 TCondor.cxx:42
 TCondor.cxx:43
 TCondor.cxx:44
 TCondor.cxx:45
 TCondor.cxx:46
 TCondor.cxx:47
 TCondor.cxx:48
 TCondor.cxx:49
 TCondor.cxx:50
 TCondor.cxx:51
 TCondor.cxx:52
 TCondor.cxx:53
 TCondor.cxx:54
 TCondor.cxx:55
 TCondor.cxx:56
 TCondor.cxx:57
 TCondor.cxx:58
 TCondor.cxx:59
 TCondor.cxx:60
 TCondor.cxx:61
 TCondor.cxx:62
 TCondor.cxx:63
 TCondor.cxx:64
 TCondor.cxx:65
 TCondor.cxx:66
 TCondor.cxx:67
 TCondor.cxx:68
 TCondor.cxx:69
 TCondor.cxx:70
 TCondor.cxx:71
 TCondor.cxx:72
 TCondor.cxx:73
 TCondor.cxx:74
 TCondor.cxx:75
 TCondor.cxx:76
 TCondor.cxx:77
 TCondor.cxx:78
 TCondor.cxx:79
 TCondor.cxx:80
 TCondor.cxx:81
 TCondor.cxx:82
 TCondor.cxx:83
 TCondor.cxx:84
 TCondor.cxx:85
 TCondor.cxx:86
 TCondor.cxx:87
 TCondor.cxx:88
 TCondor.cxx:89
 TCondor.cxx:90
 TCondor.cxx:91
 TCondor.cxx:92
 TCondor.cxx:93
 TCondor.cxx:94
 TCondor.cxx:95
 TCondor.cxx:96
 TCondor.cxx:97
 TCondor.cxx:98
 TCondor.cxx:99
 TCondor.cxx:100
 TCondor.cxx:101
 TCondor.cxx:102
 TCondor.cxx:103
 TCondor.cxx:104
 TCondor.cxx:105
 TCondor.cxx:106
 TCondor.cxx:107
 TCondor.cxx:108
 TCondor.cxx:109
 TCondor.cxx:110
 TCondor.cxx:111
 TCondor.cxx:112
 TCondor.cxx:113
 TCondor.cxx:114
 TCondor.cxx:115
 TCondor.cxx:116
 TCondor.cxx:117
 TCondor.cxx:118
 TCondor.cxx:119
 TCondor.cxx:120
 TCondor.cxx:121
 TCondor.cxx:122
 TCondor.cxx:123
 TCondor.cxx:124
 TCondor.cxx:125
 TCondor.cxx:126
 TCondor.cxx:127
 TCondor.cxx:128
 TCondor.cxx:129
 TCondor.cxx:130
 TCondor.cxx:131
 TCondor.cxx:132
 TCondor.cxx:133
 TCondor.cxx:134
 TCondor.cxx:135
 TCondor.cxx:136
 TCondor.cxx:137
 TCondor.cxx:138
 TCondor.cxx:139
 TCondor.cxx:140
 TCondor.cxx:141
 TCondor.cxx:142
 TCondor.cxx:143
 TCondor.cxx:144
 TCondor.cxx:145
 TCondor.cxx:146
 TCondor.cxx:147
 TCondor.cxx:148
 TCondor.cxx:149
 TCondor.cxx:150
 TCondor.cxx:151
 TCondor.cxx:152
 TCondor.cxx:153
 TCondor.cxx:154
 TCondor.cxx:155
 TCondor.cxx:156
 TCondor.cxx:157
 TCondor.cxx:158
 TCondor.cxx:159
 TCondor.cxx:160
 TCondor.cxx:161
 TCondor.cxx:162
 TCondor.cxx:163
 TCondor.cxx:164
 TCondor.cxx:165
 TCondor.cxx:166
 TCondor.cxx:167
 TCondor.cxx:168
 TCondor.cxx:169
 TCondor.cxx:170
 TCondor.cxx:171
 TCondor.cxx:172
 TCondor.cxx:173
 TCondor.cxx:174
 TCondor.cxx:175
 TCondor.cxx:176
 TCondor.cxx:177
 TCondor.cxx:178
 TCondor.cxx:179
 TCondor.cxx:180
 TCondor.cxx:181
 TCondor.cxx:182
 TCondor.cxx:183
 TCondor.cxx:184
 TCondor.cxx:185
 TCondor.cxx:186
 TCondor.cxx:187
 TCondor.cxx:188
 TCondor.cxx:189
 TCondor.cxx:190
 TCondor.cxx:191
 TCondor.cxx:192
 TCondor.cxx:193
 TCondor.cxx:194
 TCondor.cxx:195
 TCondor.cxx:196
 TCondor.cxx:197
 TCondor.cxx:198
 TCondor.cxx:199
 TCondor.cxx:200
 TCondor.cxx:201
 TCondor.cxx:202
 TCondor.cxx:203
 TCondor.cxx:204
 TCondor.cxx:205
 TCondor.cxx:206
 TCondor.cxx:207
 TCondor.cxx:208
 TCondor.cxx:209
 TCondor.cxx:210
 TCondor.cxx:211
 TCondor.cxx:212
 TCondor.cxx:213
 TCondor.cxx:214
 TCondor.cxx:215
 TCondor.cxx:216
 TCondor.cxx:217
 TCondor.cxx:218
 TCondor.cxx:219
 TCondor.cxx:220
 TCondor.cxx:221
 TCondor.cxx:222
 TCondor.cxx:223
 TCondor.cxx:224
 TCondor.cxx:225
 TCondor.cxx:226
 TCondor.cxx:227
 TCondor.cxx:228
 TCondor.cxx:229
 TCondor.cxx:230
 TCondor.cxx:231
 TCondor.cxx:232
 TCondor.cxx:233
 TCondor.cxx:234
 TCondor.cxx:235
 TCondor.cxx:236
 TCondor.cxx:237
 TCondor.cxx:238
 TCondor.cxx:239
 TCondor.cxx:240
 TCondor.cxx:241
 TCondor.cxx:242
 TCondor.cxx:243
 TCondor.cxx:244
 TCondor.cxx:245
 TCondor.cxx:246
 TCondor.cxx:247
 TCondor.cxx:248
 TCondor.cxx:249
 TCondor.cxx:250
 TCondor.cxx:251
 TCondor.cxx:252
 TCondor.cxx:253
 TCondor.cxx:254
 TCondor.cxx:255
 TCondor.cxx:256
 TCondor.cxx:257
 TCondor.cxx:258
 TCondor.cxx:259
 TCondor.cxx:260
 TCondor.cxx:261
 TCondor.cxx:262
 TCondor.cxx:263
 TCondor.cxx:264
 TCondor.cxx:265
 TCondor.cxx:266
 TCondor.cxx:267
 TCondor.cxx:268
 TCondor.cxx:269
 TCondor.cxx:270
 TCondor.cxx:271
 TCondor.cxx:272
 TCondor.cxx:273
 TCondor.cxx:274
 TCondor.cxx:275
 TCondor.cxx:276
 TCondor.cxx:277
 TCondor.cxx:278
 TCondor.cxx:279
 TCondor.cxx:280
 TCondor.cxx:281
 TCondor.cxx:282
 TCondor.cxx:283
 TCondor.cxx:284
 TCondor.cxx:285
 TCondor.cxx:286
 TCondor.cxx:287
 TCondor.cxx:288
 TCondor.cxx:289
 TCondor.cxx:290
 TCondor.cxx:291
 TCondor.cxx:292
 TCondor.cxx:293
 TCondor.cxx:294
 TCondor.cxx:295
 TCondor.cxx:296
 TCondor.cxx:297
 TCondor.cxx:298
 TCondor.cxx:299
 TCondor.cxx:300
 TCondor.cxx:301
 TCondor.cxx:302
 TCondor.cxx:303
 TCondor.cxx:304
 TCondor.cxx:305
 TCondor.cxx:306
 TCondor.cxx:307
 TCondor.cxx:308
 TCondor.cxx:309
 TCondor.cxx:310
 TCondor.cxx:311
 TCondor.cxx:312
 TCondor.cxx:313
 TCondor.cxx:314
 TCondor.cxx:315
 TCondor.cxx:316
 TCondor.cxx:317
 TCondor.cxx:318
 TCondor.cxx:319
 TCondor.cxx:320
 TCondor.cxx:321
 TCondor.cxx:322
 TCondor.cxx:323
 TCondor.cxx:324
 TCondor.cxx:325
 TCondor.cxx:326
 TCondor.cxx:327
 TCondor.cxx:328
 TCondor.cxx:329
 TCondor.cxx:330
 TCondor.cxx:331
 TCondor.cxx:332
 TCondor.cxx:333
 TCondor.cxx:334
 TCondor.cxx:335
 TCondor.cxx:336
 TCondor.cxx:337
 TCondor.cxx:338
 TCondor.cxx:339
 TCondor.cxx:340
 TCondor.cxx:341
 TCondor.cxx:342
 TCondor.cxx:343
 TCondor.cxx:344
 TCondor.cxx:345
 TCondor.cxx:346
 TCondor.cxx:347
 TCondor.cxx:348
 TCondor.cxx:349
 TCondor.cxx:350
 TCondor.cxx:351
 TCondor.cxx:352
 TCondor.cxx:353
 TCondor.cxx:354
 TCondor.cxx:355
 TCondor.cxx:356
 TCondor.cxx:357
 TCondor.cxx:358
 TCondor.cxx:359
 TCondor.cxx:360
 TCondor.cxx:361
 TCondor.cxx:362
 TCondor.cxx:363
 TCondor.cxx:364
 TCondor.cxx:365
 TCondor.cxx:366
 TCondor.cxx:367
 TCondor.cxx:368
 TCondor.cxx:369
 TCondor.cxx:370
 TCondor.cxx:371
 TCondor.cxx:372
 TCondor.cxx:373
 TCondor.cxx:374
 TCondor.cxx:375
 TCondor.cxx:376
 TCondor.cxx:377
 TCondor.cxx:378
 TCondor.cxx:379
 TCondor.cxx:380
 TCondor.cxx:381
 TCondor.cxx:382
 TCondor.cxx:383
 TCondor.cxx:384
 TCondor.cxx:385
 TCondor.cxx:386
 TCondor.cxx:387
 TCondor.cxx:388
 TCondor.cxx:389
 TCondor.cxx:390
 TCondor.cxx:391
 TCondor.cxx:392
 TCondor.cxx:393
 TCondor.cxx:394
 TCondor.cxx:395
 TCondor.cxx:396
 TCondor.cxx:397
 TCondor.cxx:398
 TCondor.cxx:399
 TCondor.cxx:400
 TCondor.cxx:401
 TCondor.cxx:402
 TCondor.cxx:403
 TCondor.cxx:404
 TCondor.cxx:405
 TCondor.cxx:406
 TCondor.cxx:407
 TCondor.cxx:408
 TCondor.cxx:409
 TCondor.cxx:410
 TCondor.cxx:411
 TCondor.cxx:412
 TCondor.cxx:413
 TCondor.cxx:414
 TCondor.cxx:415
 TCondor.cxx:416
 TCondor.cxx:417
 TCondor.cxx:418
 TCondor.cxx:419
 TCondor.cxx:420
 TCondor.cxx:421
 TCondor.cxx:422
 TCondor.cxx:423
 TCondor.cxx:424
 TCondor.cxx:425
 TCondor.cxx:426
 TCondor.cxx:427
 TCondor.cxx:428
 TCondor.cxx:429
 TCondor.cxx:430
 TCondor.cxx:431
 TCondor.cxx:432
 TCondor.cxx:433
 TCondor.cxx:434
 TCondor.cxx:435
 TCondor.cxx:436
 TCondor.cxx:437
 TCondor.cxx:438
 TCondor.cxx:439
 TCondor.cxx:440
 TCondor.cxx:441
 TCondor.cxx:442
 TCondor.cxx:443
 TCondor.cxx:444
 TCondor.cxx:445
 TCondor.cxx:446
 TCondor.cxx:447
 TCondor.cxx:448
 TCondor.cxx:449
 TCondor.cxx:450
 TCondor.cxx:451
 TCondor.cxx:452
 TCondor.cxx:453
 TCondor.cxx:454
 TCondor.cxx:455
 TCondor.cxx:456
 TCondor.cxx:457
 TCondor.cxx:458
 TCondor.cxx:459
 TCondor.cxx:460
 TCondor.cxx:461
 TCondor.cxx:462
 TCondor.cxx:463
 TCondor.cxx:464
 TCondor.cxx:465
 TCondor.cxx:466
 TCondor.cxx:467
 TCondor.cxx:468
 TCondor.cxx:469
 TCondor.cxx:470
 TCondor.cxx:471
 TCondor.cxx:472
 TCondor.cxx:473
 TCondor.cxx:474
 TCondor.cxx:475
 TCondor.cxx:476
 TCondor.cxx:477
 TCondor.cxx:478