63 fTuneParameters(tuneParameters),
65 fOptimizationFitType(optimizationFitType),
72 std::string
name =
"OptimizeConfigParameters_";
76 Log() << kFATAL <<
" ERROR: Sorry, Regression is not yet implement for automatic parameter optimization"
77 <<
" --> exit" <<
Endl;
80 Log() << kINFO <<
"Automatic optimisation of tuning parameters in "
83 std::map<TString,TMVA::Interval*>::iterator it;
85 Log() << kINFO << it->first
86 <<
" in range from: " << it->second->GetMin()
87 <<
" to: " << it->second->GetMax()
88 <<
" in : " << it->second->GetNbins() <<
" steps"
108 y[i] = fFOMvsIter[i];
114 h->SetXTitle(
"#iteration "+fOptimizationFitType);
115 h->SetYTitle(fFOMType);
130 if (fOptimizationFitType ==
"Scan" ) this->optimizeScan();
131 else if (fOptimizationFitType ==
"FitGA" || fOptimizationFitType ==
"Minuit" ) this->optimizeFit();
133 Log() << kFATAL <<
"You have chosen as optimization type " << fOptimizationFitType
134 <<
" that is not (yet) coded --> exit()" <<
Endl;
137 Log() << kINFO <<
"For " <<
GetMethod()->GetName() <<
" the optimized Parameters are: " <<
Endl;
138 std::map<TString,Double_t>::iterator it;
139 for(it=fTunedParameters.begin(); it!= fTunedParameters.end(); ++it){
140 Log() << kINFO << it->first <<
" = " << it->second <<
Endl;
142 return fTunedParameters;
151 std::vector < int > indices;
152 for (
UInt_t i=0; i< base.size(); i++){
153 indices.push_back(val % base[i] );
154 val =
int(
floor(
float(val) /
float(base[i]) ) );
168 Double_t bestFOM=-1000000, currentFOM;
170 std::map<TString,Double_t> currentParameters;
171 std::map<TString,TMVA::Interval*>::iterator it;
175 currentParameters.clear();
176 fTunedParameters.clear();
178 for (it=fTuneParameters.begin(); it!=fTuneParameters.end(); ++it){
179 currentParameters.insert(std::pair<TString,Double_t>(it->first,it->second->GetMin()));
180 fTunedParameters.insert(std::pair<TString,Double_t>(it->first,it->second->GetMin()));
187 std::vector< std::vector <Double_t> >
v;
188 for (it=fTuneParameters.begin(); it!=fTuneParameters.end(); ++it){
189 std::vector< Double_t > tmp;
190 for (
Int_t k=0; k<it->second->GetNbins(); k++){
191 tmp.push_back(it->second->GetElement(k));
196 std::vector< int > Nindividual;
197 for (
UInt_t i=0; i<
v.size(); i++) {
199 Nindividual.push_back(
v[i].size());
203 for (
int i=0; i<Ntot; i++){
205 std::vector<int> indices = GetScanIndices(i, Nindividual );
206 for (it=fTuneParameters.begin(), index=0; index< indices.size(); ++index, ++it){
207 currentParameters[it->first] =
v[index][indices[index]];
209 Log() << kINFO <<
"--------------------------" <<
Endl;
210 Log() << kINFO <<
"Settings being evaluated:" <<
Endl;
211 for (std::map<TString,Double_t>::iterator it_print=currentParameters.begin();
212 it_print!=currentParameters.end(); ++it_print){
213 Log() << kINFO <<
" " << it_print->first <<
" = " << it_print->second <<
Endl;
217 GetMethod()->SetTuneParameters(currentParameters);
220 if (i==0)
GetMethod()->GetTransformationHandler().CalcTransformations(
221 GetMethod()->Data()->GetEventCollection());
225 currentFOM = GetFOM();
226 Log() << kINFO <<
"FOM was found : " << currentFOM <<
"; current best is " << bestFOM <<
Endl;
228 if (currentFOM > bestFOM) {
229 bestFOM = currentFOM;
230 for (std::map<TString,Double_t>::iterator iter=currentParameters.begin();
231 iter != currentParameters.end(); ++iter){
232 fTunedParameters[iter->first]=iter->second;
238 GetMethod()->SetTuneParameters(fTunedParameters);
246 std::vector<TMVA::Interval*> ranges;
247 std::map<TString, TMVA::Interval*>::iterator it;
248 std::vector<Double_t> pars;
250 for (it=fTuneParameters.begin(); it != fTuneParameters.end(); ++it){
252 pars.push_back( (it->second)->GetMean() );
258 GetMethod()->GetTransformationHandler().CalcTransformations(
GetMethod()->Data()->GetEventCollection());
264 if ( fOptimizationFitType ==
"Minuit" ) {
265 TString opt=
"FitStrategy=0:UseImprove=False:UseMinos=False:Tolerance=100";
269 "FitterMinuit_BDTOptimize",
271 }
else if ( fOptimizationFitType ==
"FitGA" ) {
272 TString opt=
"PopSize=20:Steps=30:Cycles=3:ConvCrit=0.01:SaveBestCycle=5";
274 "FitterGA_BDTOptimize",
277 Log() << kWARNING <<
" you did not specify a valid OptimizationFitType "
278 <<
" will use the default (FitGA) " <<
Endl;
279 TString opt=
"PopSize=20:Steps=30:Cycles=3:ConvCrit=0.01:SaveBestCycle=5";
281 "FitterGA_BDTOptimize",
291 for (
UInt_t ipar=0; ipar<ranges.size(); ipar++)
delete ranges[ipar];
295 fTunedParameters.clear();
297 for (it=fTuneParameters.begin(); it!=fTuneParameters.end(); ++it){
298 fTunedParameters.insert(std::pair<TString,Double_t>(it->first,pars[jcount++]));
301 GetMethod()->SetTuneParameters(fTunedParameters);
310 std::map< std::vector<Double_t> ,
Double_t>::const_iterator iter;
311 iter = fAlreadyTrainedParCombination.find(pars);
313 if (iter != fAlreadyTrainedParCombination.end()) {
319 std::map<TString,Double_t> currentParameters;
322 std::map<TString, TMVA::Interval*>::iterator it;
323 for (it=fTuneParameters.begin(); it!=fTuneParameters.end(); ++it){
324 currentParameters[it->first] = pars[icount++];
327 GetMethod()->SetTuneParameters(currentParameters);
332 CalcTransformations(
GetMethod()->Data()->GetEventCollection());
342 fAlreadyTrainedParCombination.insert(std::make_pair(pars,-currentFOM));
360 return percent.
Atof();
362 Log() << kFATAL <<
" ERROR, " << percent <<
" in " << fFOMType
363 <<
" is not a valid floating point number" <<
Endl;
369 if (fMethod->DoRegression()){
370 std::cout <<
" ERROR: Sorry, Regression is not yet implement for automatic parameter optimisation"
371 <<
" --> exit" << std::endl;
374 if (fFOMType ==
"Separation") fom = GetSeparation();
375 else if (fFOMType ==
"ROCIntegral") fom = GetROCIntegral();
376 else if (fFOMType.BeginsWith(
"SigEffAtBkgEff0")) fom = GetSigEffAtBkgEff(parsePercent(fFOMType));
377 else if (fFOMType.BeginsWith(
"BkgRejAtSigEff0")) fom = GetBkgRejAtSigEff(parsePercent(fFOMType));
378 else if (fFOMType.BeginsWith(
"BkgEffAtSigEff0")) fom = GetBkgEffAtSigEff(parsePercent(fFOMType));
380 Log()<< kFATAL <<
" ERROR, you've specified as Figure of Merit in the "
381 <<
" parameter optimisation " << fFOMType <<
" which has not"
382 <<
" been implemented yet!! ---> exit " <<
Endl;
386 fFOMvsIter.push_back(fom);
396 if (fMvaSig) fMvaSig->Delete();
397 if (fMvaBkg) fMvaBkg->Delete();
398 if (fMvaSigFineBin) fMvaSigFineBin->Delete();
399 if (fMvaBkgFineBin) fMvaBkgFineBin->Delete();
407 fMvaSig =
new TH1D(
"fMvaSig",
"",100,-1.5,1.5);
408 fMvaBkg =
new TH1D(
"fMvaBkg",
"",100,-1.5,1.5);
409 fMvaSigFineBin =
new TH1D(
"fMvaSigFineBin",
"",100000,-1.5,1.5);
410 fMvaBkgFineBin =
new TH1D(
"fMvaBkgFineBin",
"",100000,-1.5,1.5);
412 const std::vector< Event*> events=fMethod->Data()->GetEventCollection(
Types::kTesting);
414 UInt_t signalClassNr = fMethod->DataInfo().GetClassInfo(
"Signal")->GetNumber();
418 for (
UInt_t iev=0; iev < events.size() ; iev++){
422 if (events[iev]->
GetClass() == signalClassNr) {
423 fMvaSig->Fill(fMethod->GetMvaValue(events[iev]),events[iev]->GetWeight());
424 fMvaSigFineBin->Fill(fMethod->GetMvaValue(events[iev]),events[iev]->GetWeight());
426 fMvaBkg->Fill(fMethod->GetMvaValue(events[iev]),events[iev]->GetWeight());
427 fMvaBkgFineBin->Fill(fMethod->GetMvaValue(events[iev]),events[iev]->GetWeight());
443 std::cout <<
"Separation calculation via histograms (not PDFs) seems to give still strange results!! Don't do that, check!!"<<std::endl;
474 for (
UInt_t i=0; i<nsteps; i++){
481 if ( (fMvaSigFineBin->GetXaxis()->GetXmin() != fMvaBkgFineBin->GetXaxis()->GetXmin()) ||
482 (fMvaSigFineBin->GetNbinsX() != fMvaBkgFineBin->GetNbinsX()) ){
483 std::cout <<
" Error in OptimizeConfigParameters GetROCIntegral, unequal histograms for sig and bkg.." << std::endl;
487 Double_t *cumulator = fMvaBkgFineBin->GetIntegral();
488 Int_t nbins = fMvaSigFineBin->GetNbinsX();
493 for (
Int_t ibin=1; ibin<=nbins; ibin++){
494 sigIntegral += fMvaSigFineBin->GetBinContent(ibin) * fMvaSigFineBin->GetBinWidth(ibin);
498 for (
Int_t ibin=1; ibin <= nbins; ibin++){
499 integral += (cumulator[ibin]) * fMvaSigFineBin->GetBinContent(ibin)/sigIntegral * fMvaSigFineBin->GetBinWidth(ibin) ;
516 if ( (fMvaSigFineBin->GetXaxis()->GetXmin() != fMvaBkgFineBin->GetXaxis()->GetXmin()) ||
517 (fMvaSigFineBin->GetNbinsX() != fMvaBkgFineBin->GetNbinsX()) ){
518 std::cout <<
" Error in OptimizeConfigParameters GetSigEffAt, unequal histograms for sig and bkg.." << std::endl;
521 Double_t *bkgCumulator = fMvaBkgFineBin->GetIntegral();
522 Double_t *sigCumulator = fMvaSigFineBin->GetIntegral();
524 Int_t nbins=fMvaBkgFineBin->GetNbinsX();
533 while (bkgCumulator[nbins-ibin] > (1-bkgEff)) {
534 sigEff = sigCumulator[nbins]-sigCumulator[nbins-ibin];
553 if ( (fMvaSigFineBin->GetXaxis()->GetXmin() != fMvaBkgFineBin->GetXaxis()->GetXmin()) ||
554 (fMvaSigFineBin->GetNbinsX() != fMvaBkgFineBin->GetNbinsX()) ){
555 std::cout <<
" Error in OptimizeConfigParameters GetBkgEffAt, unequal histograms for sig and bkg.." << std::endl;
559 Double_t *bkgCumulator = fMvaBkgFineBin->GetIntegral();
560 Double_t *sigCumulator = fMvaSigFineBin->GetIntegral();
562 Int_t nbins=fMvaBkgFineBin->GetNbinsX();
571 while ( sigCumulator[nbins]-sigCumulator[nbins-ibin] < sigEff) {
572 bkgEff = bkgCumulator[nbins]-bkgCumulator[nbins-ibin];
590 if ( (fMvaSigFineBin->GetXaxis()->GetXmin() != fMvaBkgFineBin->GetXaxis()->GetXmin()) ||
591 (fMvaSigFineBin->GetNbinsX() != fMvaBkgFineBin->GetNbinsX()) ){
592 std::cout <<
" Error in OptimizeConfigParameters GetBkgEffAt, unequal histograms for sig and bkg.." << std::endl;
596 Double_t *bkgCumulator = fMvaBkgFineBin->GetIntegral();
597 Double_t *sigCumulator = fMvaSigFineBin->GetIntegral();
599 Int_t nbins=fMvaBkgFineBin->GetNbinsX();
608 while ( sigCumulator[nbins]-sigCumulator[nbins-ibin] < sigEff) {
609 bkgRej = bkgCumulator[nbins-ibin];
A TGraph is an object made of two arrays X and Y with npoints each.
virtual void SetName(const char *name="")
Set graph name.
1-D histogram with a double per channel (see TH1 documentation)}
2-D histogram with a double per channel (see TH1 documentation)}
void CheckForUnusedOptions() const
checks for unused options in option string
static void SetIsTraining(Bool_t)
when this static function is called, it sets the flag whether events with negative event weight shoul...
Base class for TMVA fitters.
Double_t Run()
estimator function interface for fitting
Fitter using a Genetic Algorithm.
The TMVA::Interval Class.
Virtual base Class for all MVA method.
const char * GetName() const
Bool_t DoRegression() const
ostringstream derivative to redirect and format output
std::vector< int > GetScanIndices(int val, std::vector< int > base)
helper function to scan through the all the combinations in the parameter space
Double_t GetBkgRejAtSigEff(Double_t sigEff=0.5)
calculate the background rejection for a given signal efficiency
virtual ~OptimizeConfigParameters()
the destructor (delete the OptimizeConfigParameters, store the graph and .. delete it)
Double_t GetBkgEffAtSigEff(Double_t sigEff=0.5)
calculate the background efficiency for a given signal efficiency
void optimizeScan()
do the actual optimization using a simple scan method, i.e.
OptimizeConfigParameters(MethodBase *const method, std::map< TString, TMVA::Interval * > tuneParameters, TString fomType="Separation", TString optimizationType="GA")
Constructor which sets either "Classification or Regression".
std::map< TString, TMVA::Interval * > fTuneParameters
MethodBase *const fMethod
TString fOptimizationFitType
std::map< TString, Double_t > optimize()
Double_t GetSeparation()
return the separation between the signal and background MVA ouput distribution
Double_t GetFOM()
Return the Figure of Merit (FOM) used in the parameter optimization process.
Double_t GetSigEffAtBkgEff(Double_t bkgEff=0.1)
calculate the signal efficiency for a given background efficiency
Double_t GetROCIntegral()
calculate the area (integral) under the ROC curve as a overall quality measure of the classification
void GetMVADists()
fill the private histograms with the mva distributions for sig/bkg
Double_t EstimatorFunction(std::vector< Double_t > &)
return the estimator (from current FOM) for the fitting interface
PDF wrapper for histograms; uses user-defined spline interpolation.
Double_t GetVal(Double_t x) const
returns value PDF(x)
Double_t GetIntegral(Double_t xmin, Double_t xmax)
computes PDF integral within given ranges
virtual Int_t Write(const char *name=0, Int_t option=0, Int_t bufsize=0)
Write this object to the current directory.
TString & Insert(Ssiz_t pos, const char *s)
Double_t Atof() const
Return floating-point value contained in string.
Bool_t IsFloat() const
Returns kTRUE if string contains a floating point or integer number.
Int_t CountChar(Int_t c) const
Return number of times character c occurs in the string.
RPY_EXPORTED TCppMethod_t GetMethod(TCppScope_t scope, TCppIndex_t imeth)
MsgLogger & Endl(MsgLogger &ml)
Short_t Max(Short_t a, Short_t b)
Short_t Min(Short_t a, Short_t b)