Logo ROOT  
Reference Guide
HypoTestResult.cxx
Go to the documentation of this file.
1 // @(#)root/roostats:$Id$
2 // Author: Kyle Cranmer, Lorenzo Moneta, Gregory Schott, Wouter Verkerke, Sven Kreiss
3 /*************************************************************************
4  * Copyright (C) 1995-2008, Rene Brun and Fons Rademakers. *
5  * All rights reserved. *
6  * *
7  * For the licensing terms see $ROOTSYS/LICENSE. *
8  * For the list of contributors see $ROOTSYS/README/CREDITS. *
9  *************************************************************************/
10 
11 /*****************************************************************************
12  * Project: RooStats
13  * Package: RooFit/RooStats
14  * @(#)root/roofit/roostats:$Id$
15  * Authors:
16  * Kyle Cranmer, Lorenzo Moneta, Gregory Schott, Wouter Verkerke, Sven Kreiss
17  *
18  *****************************************************************************/
19 
20 
21 /** \class RooStats::HypoTestResult
22  \ingroup Roostats
23 
24 HypoTestResult is a base class for results from hypothesis tests.
25 Any tool inheriting from HypoTestCalculator can return a HypoTestResult.
26 As such, it stores a p-value for the null-hypothesis (eg. background-only)
27 and an alternate hypothesis (eg. signal+background).
28 The p-values can also be transformed into confidence levels
29 (\f$CL_{b}\f$, \f$CL_{s+b}\f$) in a trivial way.
30 The ratio of the \f$CL_{s+b}\f$ to \f$CL_{b}\f$ is often called
31 \f$CL_{s}\f$, and is considered useful, though it is not a probability.
32 Finally, the p-value of the null can be transformed into a number of
33 equivalent Gaussian sigma using the Significance method.
34 
35 The p-value of the null for a given test statistic is rigorously defined and
36 this is the starting point for the following conventions.
37 
38 ### Conventions used in this class
39 
40 The p-value for the null and alternate are on the **same side** of the
41 observed value of the test statistic. This is the more standard
42 convention and avoids confusion when doing inverted tests.
43 
44 For exclusion, we also want the formula \f$CL_{s} = CL_{s+b} / CL_{b}\f$
45 to hold which therefore defines our conventions for \f$CL_{s+b}\f$ and
46 \f$CL_{b}\f$. \f$CL_{s}\f$ was specifically invented for exclusion
47 and therefore all quantities need be related through the assignments
48 as they are for exclusion: \f$CL_{s+b} = p_{s+b}\f$; \f$CL_{b} = p_{b}\f$. This
49 is derived by considering the scenarios of a powerful and not powerful
50 inverted test, where for the not so powerful test, \f$CL_{s}\f$ must be
51 close to one.
52 
53 For results of Hypothesis tests,
54 \f$CL_{s}\f$ has no similar direct interpretation as for exclusion and can
55 be larger than one.
56 
57 */
58 
61 #include "RooAbsReal.h"
62 
63 #include "RooStats/RooStatsUtils.h"
64 
65 #include <limits>
66 #define NaN numeric_limits<float>::quiet_NaN()
67 #define IsNaN(a) TMath::IsNaN(a)
68 
70 
71 using namespace RooStats;
72 using namespace std;
73 
74 ////////////////////////////////////////////////////////////////////////////////
75 /// Default constructor
76 
78  TNamed(name,name),
79  fNullPValue(NaN), fAlternatePValue(NaN),
80  fNullPValueError(0), fAlternatePValueError(0),
81  fTestStatisticData(NaN),
82  fAllTestStatisticsData(NULL),
83  fNullDistr(NULL), fAltDistr(NULL),
84  fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
85  fPValueIsRightTail(kTRUE),
86  fBackgroundIsAlt(kFALSE)
87 {
88 }
89 
90 ////////////////////////////////////////////////////////////////////////////////
91 /// Alternate constructor
92 
94  TNamed(name,name),
95  fNullPValue(nullp), fAlternatePValue(altp),
96  fNullPValueError(0), fAlternatePValueError(0),
97  fTestStatisticData(NaN),
98  fAllTestStatisticsData(NULL),
99  fNullDistr(NULL), fAltDistr(NULL),
100  fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
101  fPValueIsRightTail(kTRUE),
102  fBackgroundIsAlt(kFALSE)
103 {
104 }
105 
106 ////////////////////////////////////////////////////////////////////////////////
107 /// copy constructor
108 
110  TNamed(other),
111  fNullPValue(NaN), fAlternatePValue(NaN),
112  fNullPValueError(0), fAlternatePValueError(0),
113  fTestStatisticData(NaN),
114  fAllTestStatisticsData(NULL),
115  fNullDistr(NULL), fAltDistr(NULL),
116  fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
117  fPValueIsRightTail( other.GetPValueIsRightTail() ),
118  fBackgroundIsAlt( other.GetBackGroundIsAlt() )
119 {
120  this->Append( &other );
121 }
122 
123 ////////////////////////////////////////////////////////////////////////////////
124 /// Destructor
125 
127 {
128  if( fNullDistr ) delete fNullDistr;
129  if( fAltDistr ) delete fAltDistr;
130 
133 
135 }
136 
137 ////////////////////////////////////////////////////////////////////////////////
138 /// assignment operator
139 
141  if (this == &other) return *this;
142  SetName(other.GetName());
143  SetTitle(other.GetTitle());
144  fNullPValue = other.fNullPValue;
149 
151  fAllTestStatisticsData = NULL;
152  if( fNullDistr ) { delete fNullDistr; fNullDistr = NULL; }
153  if( fAltDistr ) { delete fAltDistr; fAltDistr = NULL; }
156  fFitInfo = nullptr;
157 
160 
161  this->Append( &other );
162 
163  return *this;
164 }
165 
166 ////////////////////////////////////////////////////////////////////////////////
167 /// Add additional toy-MC experiments to the current results.
168 /// Use the data test statistics of the added object if it is not already
169 /// set (otherwise, ignore the new one).
170 
172  if(fNullDistr)
174  else
176 
177  if(fAltDistr)
178  fAltDistr->Add(other->GetAltDistribution());
179  else
181 
182 
183  if( fNullDetailedOutput ) {
185  }else{
187  }
188 
189  if( fAltDetailedOutput ) {
191  }else{
193  }
194 
195  if( fFitInfo ) {
196  if( other->GetFitInfo() ) fFitInfo->append( *other->GetFitInfo() );
197  }else{
198  if( other->GetFitInfo() ) fFitInfo.reset(new RooDataSet( *other->GetFitInfo() ));
199  }
200 
201  // if no data is present use the other HypoTestResult's data
203 
206 }
207 
208 ////////////////////////////////////////////////////////////////////////////////
209 
211  fAltDistr = alt;
213 }
214 
215 ////////////////////////////////////////////////////////////////////////////////
216 
218  fNullDistr = null;
220 }
221 
222 ////////////////////////////////////////////////////////////////////////////////
223 
225  fTestStatisticData = tsd;
226 
229 }
230 
231 ////////////////////////////////////////////////////////////////////////////////
232 
235  delete fAllTestStatisticsData;
237  }
238  if (tsd) fAllTestStatisticsData = (const RooArgList*)tsd->snapshot();
239 
242  if( firstTS ) SetTestStatisticData( firstTS->getVal() );
243  }
244 }
245 
246 ////////////////////////////////////////////////////////////////////////////////
247 
249  fPValueIsRightTail = pr;
250 
253 }
254 
255 ////////////////////////////////////////////////////////////////////////////////
256 
258  return !IsNaN(fTestStatisticData);
259 }
260 
261 ////////////////////////////////////////////////////////////////////////////////
262 
264  // compute error on Null pvalue
265  return fNullPValueError;
266 }
267 
268 ////////////////////////////////////////////////////////////////////////////////
269 /// compute \f$CL_{b}\f$ error
270 /// \f$CL_{b}\f$ = 1 - NullPValue()
271 /// must use opposite condition that routine above
272 
275 }
276 
277 ////////////////////////////////////////////////////////////////////////////////
278 
281 }
282 
283 ////////////////////////////////////////////////////////////////////////////////
284 /// Taylor expansion series approximation for standard deviation (error propagation)
285 
288 }
289 
290 ////////////////////////////////////////////////////////////////////////////////
291 /// Returns an estimate of the error on \f$CL_{s}\f$ through combination of the
292 /// errors on \f$CL_{b}\f$ and \f$CL_{s+b}\f$:
293 /// \f[
294 /// \sigma_{CL_s} = CL_s
295 /// \sqrt{\left( \frac{\sigma_{CL_{s+b}}}{CL_{s+b}} \right)^2 + \left( \frac{\sigma_{CL_{b}}}{CL_{b}} \right)^2}
296 /// \f]
297 
299  if(!fAltDistr || !fNullDistr) return 0.0;
300 
301  // unsigned const int n_b = fNullDistr->GetSamplingDistribution().size();
302  // unsigned const int n_sb = fAltDistr->GetSamplingDistribution().size();
303 
304  // if CLb() == 0 CLs = -1 so return a -1 error
305  if (CLb() == 0 ) return -1;
306 
307  double cl_b_err2 = pow(CLbError(),2);
308  double cl_sb_err2 = pow(CLsplusbError(),2);
309 
310  return TMath::Sqrt(cl_sb_err2 + cl_b_err2 * pow(CLs(),2))/CLb();
311 }
312 
313 ////////////////////////////////////////////////////////////////////////////////
314 /// updates the pvalue if sufficient data is available
315 
316 void HypoTestResult::UpdatePValue(const SamplingDistribution* distr, Double_t &pvalue, Double_t &perror, Bool_t /*isNull*/) {
317  if(IsNaN(fTestStatisticData)) return;
318  if(!distr) return;
319 
320  /* Got to be careful for discrete distributions:
321  * To get the right behaviour for limits, the p-value must
322  * include the value of fTestStatistic both for Alt and Null cases
323  */
324  if(fPValueIsRightTail) {
325  pvalue = distr->IntegralAndError(perror, fTestStatisticData, RooNumber::infinity(), kTRUE,
326  kTRUE , kTRUE ); // always closed interval [ fTestStatistic, inf ]
327 
328  }else{
329  pvalue = distr->IntegralAndError(perror, -RooNumber::infinity(), fTestStatisticData, kTRUE,
330  kTRUE, kTRUE ); // // always closed [ -inf, fTestStatistic ]
331  }
332 }
333 
334 ////////////////////////////////////////////////////////////////////////////////
335 /// Print out some information about the results
336 /// Note: use Alt/Null labels for the hypotheses here as the Null
337 /// might be the s+b hypothesis.
338 
340 {
341  bool fromToys = (fAltDistr || fNullDistr);
342 
343  std::cout << std::endl << "Results " << GetName() << ": " << endl;
344  std::cout << " - Null p-value = " << NullPValue();
345  if (fromToys) std::cout << " +/- " << NullPValueError();
346  std::cout << std::endl;
347  std::cout << " - Significance = " << Significance();
348  if (fromToys) std::cout << " +/- " << SignificanceError() << " sigma";
349  std::cout << std::endl;
350  if(fAltDistr)
351  std::cout << " - Number of Alt toys: " << fAltDistr->GetSize() << std::endl;
352  if(fNullDistr)
353  std::cout << " - Number of Null toys: " << fNullDistr->GetSize() << std::endl;
354 
355  if (HasTestStatisticData() ) std::cout << " - Test statistic evaluated on data: " << fTestStatisticData << std::endl;
356  std::cout << " - CL_b: " << CLb();
357  if (fromToys) std::cout << " +/- " << CLbError();
358  std::cout << std::endl;
359  std::cout << " - CL_s+b: " << CLsplusb();
360  if (fromToys) std::cout << " +/- " << CLsplusbError();
361  std::cout << std::endl;
362  std::cout << " - CL_s: " << CLs();
363  if (fromToys) std::cout << " +/- " << CLsError();
364  std::cout << std::endl;
365 
366  return;
367 }
RooStats::HypoTestResult::NullPValueError
Double_t NullPValueError() const
The error on the Null p-value.
Definition: HypoTestResult.cxx:263
RooStats::HypoTestResult::Significance
virtual Double_t Significance() const
familiar name for the Null p-value in terms of 1-sided Gaussian significance
Definition: HypoTestResult.h:68
RooStats::HypoTestResult::SetTestStatisticData
void SetTestStatisticData(const Double_t tsd)
Definition: HypoTestResult.cxx:224
RooStats::HypoTestResult::~HypoTestResult
virtual ~HypoTestResult()
destructor
Definition: HypoTestResult.cxx:126
kTRUE
const Bool_t kTRUE
Definition: RtypesCore.h:91
RooAbsReal.h
TNamed::SetName
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition: TNamed.cxx:140
Option_t
const char Option_t
Definition: RtypesCore.h:66
RooStats::SamplingDistribution
This class simply holds a sampling distribution of some test statistic.
Definition: SamplingDistribution.h:28
RooStats::HypoTestResult::SetAllTestStatisticsData
void SetAllTestStatisticsData(const RooArgList *tsd)
Definition: HypoTestResult.cxx:233
RooStats::HypoTestResult::HypoTestResult
HypoTestResult(const char *name=0)
default constructor
Definition: HypoTestResult.cxx:77
RooStats::HypoTestResult::GetNullDistribution
SamplingDistribution * GetNullDistribution(void) const
Definition: HypoTestResult.h:70
RooStats::HypoTestResult::Print
void Print(const Option_t *="") const
Print out some information about the results Note: use Alt/Null labels for the hypotheses here as the...
Definition: HypoTestResult.cxx:339
RooStats::HypoTestResult::fAltDistr
SamplingDistribution * fAltDistr
Definition: HypoTestResult.h:124
ClassImp
#define ClassImp(name)
Definition: Rtypes.h:364
TNamed::GetTitle
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
RooStats::HypoTestResult::fPValueIsRightTail
Bool_t fPValueIsRightTail
Definition: HypoTestResult.h:128
RooStats::HypoTestResult::NullPValue
virtual Double_t NullPValue() const
Return p-value for null hypothesis.
Definition: HypoTestResult.h:45
RooStats::HypoTestResult::GetTestStatisticData
Double_t GetTestStatisticData(void) const
Definition: HypoTestResult.h:75
RooArgList
RooArgList is a container object that can hold multiple RooAbsArg objects.
Definition: RooArgList.h:21
TMath::Sqrt
Double_t Sqrt(Double_t x)
Definition: TMath.h:691
RooStats::HypoTestResult::GetPValueIsRightTail
Bool_t GetPValueIsRightTail(void) const
Definition: HypoTestResult.h:88
RooAbsReal::getVal
Double_t getVal(const RooArgSet *normalisationSet=nullptr) const
Evaluate object.
Definition: RooAbsReal.h:91
RooStats::HypoTestResult::GetAltDetailedOutput
RooDataSet * GetAltDetailedOutput(void) const
Definition: HypoTestResult.h:73
RooDataSet::append
void append(RooDataSet &data)
Add all data points of given data set to this data set.
Definition: RooDataSet.cxx:1368
RooArgList::at
RooAbsArg * at(Int_t idx) const
Return object at given index, or nullptr if index is out of range.
Definition: RooArgList.h:74
HypoTestResult.h
bool
RooStats::HypoTestResult::SetPValueIsRightTail
void SetPValueIsRightTail(Bool_t pr)
Definition: HypoTestResult.cxx:248
NaN
#define NaN
Definition: HypoTestResult.cxx:66
RooStats::HypoTestResult::SetAltDistribution
void SetAltDistribution(SamplingDistribution *alt)
Definition: HypoTestResult.cxx:210
RooStats::HypoTestResult::fAlternatePValueError
Double_t fAlternatePValueError
Definition: HypoTestResult.h:120
RooStats::HypoTestResult::GetNullDetailedOutput
RooDataSet * GetNullDetailedOutput(void) const
Definition: HypoTestResult.h:72
RooStats::HypoTestResult::operator=
HypoTestResult & operator=(const HypoTestResult &other)
assignment operator
Definition: HypoTestResult.cxx:140
RooStats::HypoTestResult::CLsplusbError
Double_t CLsplusbError() const
The error on the "confidence level" of the alternative hypothesis.
Definition: HypoTestResult.cxx:279
RooStats::SamplingDistribution::Add
void Add(const SamplingDistribution *other)
merge two sampling distributions
Definition: SamplingDistribution.cxx:153
ROOT::Math::normal_pdf
double normal_pdf(double x, double sigma=1, double x0=0)
Probability density function of the normal (Gaussian) distribution.
Definition: PdfFuncMathCore.h:501
RooStats::HypoTestResult::fAlternatePValue
Double_t fAlternatePValue
Definition: HypoTestResult.h:118
RooStats::HypoTestResult::fNullDetailedOutput
RooDataSet * fNullDetailedOutput
Definition: HypoTestResult.h:125
RooStats::HypoTestResult::Append
virtual void Append(const HypoTestResult *other)
add values from another HypoTestResult
Definition: HypoTestResult.cxx:171
RooStats::HypoTestResult::fTestStatisticData
Double_t fTestStatisticData
Definition: HypoTestResult.h:121
TNamed
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
RooStats::HypoTestResult::fNullDistr
SamplingDistribution * fNullDistr
Definition: HypoTestResult.h:123
kFALSE
const Bool_t kFALSE
Definition: RtypesCore.h:92
RooStats::HypoTestResult::fAllTestStatisticsData
const RooArgList * fAllTestStatisticsData
Definition: HypoTestResult.h:122
RooStats::HypoTestResult::fNullPValueError
Double_t fNullPValueError
Definition: HypoTestResult.h:119
RooStats::HypoTestResult::UpdatePValue
void UpdatePValue(const SamplingDistribution *distr, Double_t &pvalue, Double_t &perror, Bool_t pIsRightTail)
updates the pvalue if sufficient data is available
Definition: HypoTestResult.cxx:316
RooStats::HypoTestResult::fAltDetailedOutput
RooDataSet * fAltDetailedOutput
Definition: HypoTestResult.h:126
RooStats::HypoTestResult::CLsError
Double_t CLsError() const
The error on the ratio .
Definition: HypoTestResult.cxx:298
TNamed::SetTitle
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
Definition: TNamed.cxx:164
RooStats::SamplingDistribution::GetSize
Int_t GetSize() const
size of samples
Definition: SamplingDistribution.h:62
RooStats::HypoTestResult::CLbError
Double_t CLbError() const
The error on the "confidence level" of the null hypothesis.
Definition: HypoTestResult.cxx:273
RooStats::HypoTestResult::GetAltDistribution
SamplingDistribution * GetAltDistribution(void) const
Definition: HypoTestResult.h:71
RooStats::HypoTestResult::fBackgroundIsAlt
Bool_t fBackgroundIsAlt
Definition: HypoTestResult.h:129
SamplingDistribution.h
RooStatsUtils.h
RooNumber::infinity
static Double_t infinity()
Return internal infinity representation.
Definition: RooNumber.cxx:49
Double_t
double Double_t
Definition: RtypesCore.h:59
RooStats::HypoTestResult::HasTestStatisticData
Bool_t HasTestStatisticData(void) const
Definition: HypoTestResult.cxx:257
RooStats::HypoTestResult::GetFitInfo
RooDataSet * GetFitInfo() const
Definition: HypoTestResult.h:74
RooStats
Namespace for the RooStats classes.
Definition: Asimov.h:19
RooStats::HypoTestResult::CLs
virtual Double_t CLs() const
is simply (not a method, but a quantity)
Definition: HypoTestResult.h:57
RooStats::HypoTestResult
HypoTestResult is a base class for results from hypothesis tests.
Definition: HypoTestResult.h:22
name
char name[80]
Definition: TGX11.cxx:110
RooDataSet
RooDataSet is a container class to hold unbinned data.
Definition: RooDataSet.h:33
TNamed::GetName
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
RooRealVar
RooRealVar represents a variable that can be changed from the outside.
Definition: RooRealVar.h:36
RooStats::HypoTestResult::GetBackGroundIsAlt
Bool_t GetBackGroundIsAlt(void) const
Definition: HypoTestResult.h:91
pow
double pow(double, double)
RooStats::SamplingDistribution::IntegralAndError
Double_t IntegralAndError(Double_t &error, Double_t low, Double_t high, Bool_t normalize=kTRUE, Bool_t lowClosed=kTRUE, Bool_t highClosed=kFALSE) const
numerical integral in these limits including error estimation
Definition: SamplingDistribution.cxx:238
RooStats::HypoTestResult::fFitInfo
std::unique_ptr< RooDataSet > fFitInfo
Definition: HypoTestResult.h:127
RooStats::HypoTestResult::CLsplusb
virtual Double_t CLsplusb() const
Convert AlternatePValue into a "confidence level".
Definition: HypoTestResult.h:54
RooAbsCollection::getSize
Int_t getSize() const
Definition: RooAbsCollection.h:171
RooStats::HypoTestResult::fNullPValue
Double_t fNullPValue
Definition: HypoTestResult.h:117
IsNaN
#define IsNaN(a)
Definition: HypoTestResult.cxx:67
RooAbsCollection::snapshot
RooAbsCollection * snapshot(Bool_t deepCopy=kTRUE) const
Take a snap shot of current collection contents.
Definition: RooAbsCollection.cxx:201
RooStats::HypoTestResult::SignificanceError
Double_t SignificanceError() const
The error on the significance, computed from NullPValueError via error propagation.
Definition: HypoTestResult.cxx:286
RooStats::HypoTestResult::SetNullDistribution
void SetNullDistribution(SamplingDistribution *null)
Definition: HypoTestResult.cxx:217
RooStats::HypoTestResult::CLb
virtual Double_t CLb() const
Convert NullPValue into a "confidence level".
Definition: HypoTestResult.h:51