Logo ROOT  
Reference Guide
HypoTestResult.cxx
Go to the documentation of this file.
1// @(#)root/roostats:$Id$
2// Author: Kyle Cranmer, Lorenzo Moneta, Gregory Schott, Wouter Verkerke, Sven Kreiss
3/*************************************************************************
4 * Copyright (C) 1995-2008, Rene Brun and Fons Rademakers. *
5 * All rights reserved. *
6 * *
7 * For the licensing terms see $ROOTSYS/LICENSE. *
8 * For the list of contributors see $ROOTSYS/README/CREDITS. *
9 *************************************************************************/
10
11/*****************************************************************************
12 * Project: RooStats
13 * Package: RooFit/RooStats
14 * @(#)root/roofit/roostats:$Id$
15 * Authors:
16 * Kyle Cranmer, Lorenzo Moneta, Gregory Schott, Wouter Verkerke, Sven Kreiss
17 *
18 *****************************************************************************/
19
20
21/** \class RooStats::HypoTestResult
22 \ingroup Roostats
23
24HypoTestResult is a base class for results from hypothesis tests.
25Any tool inheriting from HypoTestCalculator can return a HypoTestResult.
26As such, it stores a p-value for the null-hypothesis (eg. background-only)
27and an alternate hypothesis (eg. signal+background).
28The p-values can also be transformed into confidence levels
29(\f$CL_{b}\f$, \f$CL_{s+b}\f$) in a trivial way.
30The ratio of the \f$CL_{s+b}\f$ to \f$CL_{b}\f$ is often called
31\f$CL_{s}\f$, and is considered useful, though it is not a probability.
32Finally, the p-value of the null can be transformed into a number of
33equivalent Gaussian sigma using the Significance method.
34
35The p-value of the null for a given test statistic is rigorously defined and
36this is the starting point for the following conventions.
37
38### Conventions used in this class
39
40The p-value for the null and alternate are on the **same side** of the
41observed value of the test statistic. This is the more standard
42convention and avoids confusion when doing inverted tests.
43
44For exclusion, we also want the formula \f$CL_{s} = CL_{s+b} / CL_{b}\f$
45to hold which therefore defines our conventions for \f$CL_{s+b}\f$ and
46\f$CL_{b}\f$. \f$CL_{s}\f$ was specifically invented for exclusion
47and therefore all quantities need be related through the assignments
48as they are for exclusion: \f$CL_{s+b} = p_{s+b}\f$; \f$CL_{b} = p_{b}\f$. This
49is derived by considering the scenarios of a powerful and not powerful
50inverted test, where for the not so powerful test, \f$CL_{s}\f$ must be
51close to one.
52
53For results of Hypothesis tests,
54\f$CL_{s}\f$ has no similar direct interpretation as for exclusion and can
55be larger than one.
56
57*/
58
61#include "RooAbsReal.h"
62
64
65#include <limits>
66#define NaN numeric_limits<float>::quiet_NaN()
67#define IsNaN(a) TMath::IsNaN(a)
68
70
71using namespace RooStats;
72using namespace std;
73
74////////////////////////////////////////////////////////////////////////////////
75/// Default constructor
76
77HypoTestResult::HypoTestResult(const char* name) :
79 fNullPValue(NaN), fAlternatePValue(NaN),
80 fNullPValueError(0), fAlternatePValueError(0),
81 fTestStatisticData(NaN),
82 fAllTestStatisticsData(NULL),
83 fNullDistr(NULL), fAltDistr(NULL),
84 fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
85 fPValueIsRightTail(kTRUE),
86 fBackgroundIsAlt(kFALSE)
87{
88}
89
90////////////////////////////////////////////////////////////////////////////////
91/// Alternate constructor
92
95 fNullPValue(nullp), fAlternatePValue(altp),
96 fNullPValueError(0), fAlternatePValueError(0),
97 fTestStatisticData(NaN),
98 fAllTestStatisticsData(NULL),
99 fNullDistr(NULL), fAltDistr(NULL),
100 fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
101 fPValueIsRightTail(kTRUE),
102 fBackgroundIsAlt(kFALSE)
103{
104}
105
106////////////////////////////////////////////////////////////////////////////////
107/// copy constructor
108
110 TNamed(other),
111 fNullPValue(NaN), fAlternatePValue(NaN),
112 fNullPValueError(0), fAlternatePValueError(0),
113 fTestStatisticData(NaN),
114 fAllTestStatisticsData(NULL),
115 fNullDistr(NULL), fAltDistr(NULL),
116 fNullDetailedOutput(NULL), fAltDetailedOutput(NULL),
117 fPValueIsRightTail( other.GetPValueIsRightTail() ),
118 fBackgroundIsAlt( other.GetBackGroundIsAlt() )
119{
120 this->Append( &other );
121}
122
123////////////////////////////////////////////////////////////////////////////////
124/// Destructor
125
127{
128 if( fNullDistr ) delete fNullDistr;
129 if( fAltDistr ) delete fAltDistr;
130
133
135}
136
137////////////////////////////////////////////////////////////////////////////////
138/// assignment operator
139
141 if (this == &other) return *this;
142 SetName(other.GetName());
143 SetTitle(other.GetTitle());
144 fNullPValue = other.fNullPValue;
149
152 if( fNullDistr ) { delete fNullDistr; fNullDistr = NULL; }
153 if( fAltDistr ) { delete fAltDistr; fAltDistr = NULL; }
156 fFitInfo = nullptr;
157
160
161 this->Append( &other );
162
163 return *this;
164}
165
166////////////////////////////////////////////////////////////////////////////////
167/// Add additional toy-MC experiments to the current results.
168/// Use the data test statistics of the added object if it is not already
169/// set (otherwise, ignore the new one).
170
172 if(fNullDistr)
174 else
176
177 if(fAltDistr)
179 else
181
182
183 if( fNullDetailedOutput ) {
185 }else{
187 }
188
189 if( fAltDetailedOutput ) {
191 }else{
193 }
194
195 if( fFitInfo ) {
196 if( other->GetFitInfo() ) fFitInfo->append( *other->GetFitInfo() );
197 }else{
198 if( other->GetFitInfo() ) fFitInfo.reset(new RooDataSet( *other->GetFitInfo() ));
199 }
200
201 // if no data is present use the other HypoTestResult's data
203
206}
207
208////////////////////////////////////////////////////////////////////////////////
209
211 fAltDistr = alt;
213}
214
215////////////////////////////////////////////////////////////////////////////////
216
220}
221
222////////////////////////////////////////////////////////////////////////////////
223
225 fTestStatisticData = tsd;
226
229}
230
231////////////////////////////////////////////////////////////////////////////////
232
237 }
238 if (tsd) fAllTestStatisticsData = (const RooArgList*)tsd->snapshot();
239
242 if( firstTS ) SetTestStatisticData( firstTS->getVal() );
243 }
244}
245
246////////////////////////////////////////////////////////////////////////////////
247
250
253}
254
255////////////////////////////////////////////////////////////////////////////////
256
258 return !IsNaN(fTestStatisticData);
259}
260
261////////////////////////////////////////////////////////////////////////////////
262
264 // compute error on Null pvalue
265 return fNullPValueError;
266}
267
268////////////////////////////////////////////////////////////////////////////////
269/// compute \f$CL_{b}\f$ error
270/// \f$CL_{b}\f$ = 1 - NullPValue()
271/// must use opposite condition that routine above
272
275}
276
277////////////////////////////////////////////////////////////////////////////////
278
281}
282
283////////////////////////////////////////////////////////////////////////////////
284/// Taylor expansion series approximation for standard deviation (error propagation)
285
288}
289
290////////////////////////////////////////////////////////////////////////////////
291/// Returns an estimate of the error on \f$CL_{s}\f$ through combination of the
292/// errors on \f$CL_{b}\f$ and \f$CL_{s+b}\f$:
293/// \f[
294/// \sigma_{CL_s} = CL_s
295/// \sqrt{\left( \frac{\sigma_{CL_{s+b}}}{CL_{s+b}} \right)^2 + \left( \frac{\sigma_{CL_{b}}}{CL_{b}} \right)^2}
296/// \f]
297
299 if(!fAltDistr || !fNullDistr) return 0.0;
300
301 // unsigned const int n_b = fNullDistr->GetSamplingDistribution().size();
302 // unsigned const int n_sb = fAltDistr->GetSamplingDistribution().size();
303
304 // if CLb() == 0 CLs = -1 so return a -1 error
305 if (CLb() == 0 ) return -1;
306
307 double cl_b_err2 = pow(CLbError(),2);
308 double cl_sb_err2 = pow(CLsplusbError(),2);
309
310 return TMath::Sqrt(cl_sb_err2 + cl_b_err2 * pow(CLs(),2))/CLb();
311}
312
313////////////////////////////////////////////////////////////////////////////////
314/// updates the pvalue if sufficient data is available
315
316void HypoTestResult::UpdatePValue(const SamplingDistribution* distr, Double_t &pvalue, Double_t &perror, Bool_t /*isNull*/) {
317 if(IsNaN(fTestStatisticData)) return;
318 if(!distr) return;
319
320 /* Got to be careful for discrete distributions:
321 * To get the right behaviour for limits, the p-value must
322 * include the value of fTestStatistic both for Alt and Null cases
323 */
326 kTRUE , kTRUE ); // always closed interval [ fTestStatistic, inf ]
327
328 }else{
330 kTRUE, kTRUE ); // // always closed [ -inf, fTestStatistic ]
331 }
332}
333
334////////////////////////////////////////////////////////////////////////////////
335/// Print out some information about the results
336/// Note: use Alt/Null labels for the hypotheses here as the Null
337/// might be the s+b hypothesis.
338
340{
341 bool fromToys = (fAltDistr || fNullDistr);
342
343 std::cout << std::endl << "Results " << GetName() << ": " << endl;
344 std::cout << " - Null p-value = " << NullPValue();
345 if (fromToys) std::cout << " +/- " << NullPValueError();
346 std::cout << std::endl;
347 std::cout << " - Significance = " << Significance();
348 if (fromToys) std::cout << " +/- " << SignificanceError() << " sigma";
349 std::cout << std::endl;
350 if(fAltDistr)
351 std::cout << " - Number of Alt toys: " << fAltDistr->GetSize() << std::endl;
352 if(fNullDistr)
353 std::cout << " - Number of Null toys: " << fNullDistr->GetSize() << std::endl;
354
355 if (HasTestStatisticData() ) std::cout << " - Test statistic evaluated on data: " << fTestStatisticData << std::endl;
356 std::cout << " - CL_b: " << CLb();
357 if (fromToys) std::cout << " +/- " << CLbError();
358 std::cout << std::endl;
359 std::cout << " - CL_s+b: " << CLsplusb();
360 if (fromToys) std::cout << " +/- " << CLsplusbError();
361 std::cout << std::endl;
362 std::cout << " - CL_s: " << CLs();
363 if (fromToys) std::cout << " +/- " << CLsError();
364 std::cout << std::endl;
365
366 return;
367}
#define NaN
#define IsNaN(a)
const Bool_t kFALSE
Definition: RtypesCore.h:88
bool Bool_t
Definition: RtypesCore.h:59
double Double_t
Definition: RtypesCore.h:55
const Bool_t kTRUE
Definition: RtypesCore.h:87
const char Option_t
Definition: RtypesCore.h:62
#define ClassImp(name)
Definition: Rtypes.h:365
char name[80]
Definition: TGX11.cxx:109
double pow(double, double)
Int_t getSize() const
RooAbsCollection * snapshot(Bool_t deepCopy=kTRUE) const
Take a snap shot of current collection contents.
Double_t getVal(const RooArgSet *normalisationSet=nullptr) const
Evaluate object.
Definition: RooAbsReal.h:87
RooArgList is a container object that can hold multiple RooAbsArg objects.
Definition: RooArgList.h:21
RooAbsArg * at(Int_t idx) const
Return object at given index, or nullptr if index is out of range.
Definition: RooArgList.h:74
RooDataSet is a container class to hold unbinned data.
Definition: RooDataSet.h:31
void append(RooDataSet &data)
Add all data points of given data set to this data set.
static Double_t infinity()
Return internal infinity representation.
Definition: RooNumber.cxx:49
RooRealVar represents a variable that can be changed from the outside.
Definition: RooRealVar.h:35
HypoTestResult is a base class for results from hypothesis tests.
RooDataSet * GetFitInfo() const
RooDataSet * fAltDetailedOutput
HypoTestResult & operator=(const HypoTestResult &other)
assignment operator
virtual ~HypoTestResult()
destructor
HypoTestResult(const char *name=0)
default constructor
Double_t CLbError() const
The error on the "confidence level" of the null hypothesis.
void SetAllTestStatisticsData(const RooArgList *tsd)
Double_t NullPValueError() const
The error on the Null p-value.
Bool_t GetPValueIsRightTail(void) const
Double_t GetTestStatisticData(void) const
virtual void Append(const HypoTestResult *other)
add values from another HypoTestResult
virtual Double_t CLb() const
Convert NullPValue into a "confidence level".
virtual Double_t CLsplusb() const
Convert AlternatePValue into a "confidence level".
virtual Double_t Significance() const
familiar name for the Null p-value in terms of 1-sided Gaussian significance
RooDataSet * GetNullDetailedOutput(void) const
void SetPValueIsRightTail(Bool_t pr)
RooDataSet * fNullDetailedOutput
SamplingDistribution * fAltDistr
virtual Double_t NullPValue() const
Return p-value for null hypothesis.
void SetTestStatisticData(const Double_t tsd)
void SetNullDistribution(SamplingDistribution *null)
Double_t CLsplusbError() const
The error on the "confidence level" of the alternative hypothesis.
virtual Double_t CLs() const
is simply (not a method, but a quantity)
Bool_t GetBackGroundIsAlt(void) const
void SetAltDistribution(SamplingDistribution *alt)
RooDataSet * GetAltDetailedOutput(void) const
const RooArgList * fAllTestStatisticsData
void UpdatePValue(const SamplingDistribution *distr, Double_t &pvalue, Double_t &perror, Bool_t pIsRightTail)
updates the pvalue if sufficient data is available
SamplingDistribution * GetNullDistribution(void) const
Bool_t HasTestStatisticData(void) const
Double_t SignificanceError() const
The error on the significance, computed from NullPValueError via error propagation.
std::unique_ptr< RooDataSet > fFitInfo
Double_t CLsError() const
The error on the ratio .
void Print(const Option_t *="") const
Print out some information about the results Note: use Alt/Null labels for the hypotheses here as the...
SamplingDistribution * fNullDistr
SamplingDistribution * GetAltDistribution(void) const
This class simply holds a sampling distribution of some test statistic.
Int_t GetSize() const
size of samples
Double_t IntegralAndError(Double_t &error, Double_t low, Double_t high, Bool_t normalize=kTRUE, Bool_t lowClosed=kTRUE, Bool_t highClosed=kFALSE) const
numerical integral in these limits including error estimation
void Add(const SamplingDistribution *other)
merge two sampling distributions
The TNamed class is the base class for all named ROOT classes.
Definition: TNamed.h:29
virtual void SetTitle(const char *title="")
Set the title of the TNamed.
Definition: TNamed.cxx:164
virtual void SetName(const char *name)
Set the name of the TNamed.
Definition: TNamed.cxx:140
virtual const char * GetTitle() const
Returns title of object.
Definition: TNamed.h:48
virtual const char * GetName() const
Returns name of object.
Definition: TNamed.h:47
double normal_pdf(double x, double sigma=1, double x0=0)
Probability density function of the normal (Gaussian) distribution.
Namespace for the RooStats classes.
Definition: Asimov.h:20
null_t< F > null()
Double_t Sqrt(Double_t x)
Definition: TMath.h:681