Logo ROOT  
Reference Guide
Go to the documentation of this file.
1// @(#)root/tmva $Id$
2// Author: Krzysztof Danielowski, Andreas Hoecker, Matt Jachowski, Kamil Kraszewski, Maciej Kruk, Peter Speckmayer, Joerg Stelzer, Eckhard von Toerne, Jan Therhaag, Jiahang Zhong
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : MethodMLP *
8 * Web : http://tmva.sourceforge.net *
9 * *
10 * Description: *
11 * ANN Multilayer Perceptron class for the discrimination of signal *
12 * from background. BFGS implementation based on TMultiLayerPerceptron *
13 * class from ROOT (http://root.cern.ch). *
14 * *
15 * Authors (alphabetical): *
16 * Krzysztof Danielowski <danielow@cern.ch> - IFJ & AGH, Poland *
17 * Andreas Hoecker <Andreas.Hocker@cern.ch> - CERN, Switzerland *
18 * Matt Jachowski <jachowski@stanford.edu> - Stanford University, USA *
19 * Kamil Kraszewski <kalq@cern.ch> - IFJ & UJ, Poland *
20 * Maciej Kruk <mkruk@cern.ch> - IFJ & AGH, Poland *
21 * Peter Speckmayer <peter.speckmayer@cern.ch> - CERN, Switzerland *
22 * Joerg Stelzer <stelzer@cern.ch> - DESY, Germany *
23 * Jan Therhaag <Jan.Therhaag@cern.ch> - U of Bonn, Germany *
24 * Eckhard v. Toerne <evt@uni-bonn.de> - U of Bonn, Germany *
25 * Jiahang Zhong <Jiahang.Zhong@cern.ch> - Academia Sinica, Taipei *
26 * *
27 * Copyright (c) 2005-2011: *
28 * CERN, Switzerland *
29 * U. of Victoria, Canada *
30 * MPI-K Heidelberg, Germany *
31 * U. of Bonn, Germany *
32 * *
33 * Redistribution and use in source and binary forms, with or without *
34 * modification, are permitted according to the terms listed in LICENSE *
35 * (http://tmva.sourceforge.net/LICENSE) *
36 **********************************************************************************/
38#ifndef ROOT_TMVA_MethodMLP
39#define ROOT_TMVA_MethodMLP
42// //
43// MethodMLP //
44// //
45// Multilayer Perceptron built off of MethodANNBase //
46// //
49#include <vector>
50#include <utility>
51#include "TString.h"
52#include "TTree.h"
53#include "TRandom3.h"
54#include "TH1F.h"
55#include "TMatrixDfwd.h"
57#include "TMVA/IFitterTarget.h"
58#include "TMVA/MethodBase.h"
59#include "TMVA/MethodANNBase.h"
60#include "TMVA/TNeuron.h"
61#include "TMVA/TActivation.h"
64#define MethodMLP_UseMinuit__
65#undef MethodMLP_UseMinuit__
67namespace TMVA {
69 class MethodMLP : public MethodANNBase, public IFitterTarget, public ConvergenceTest {
71 public:
73 // standard constructors
74 MethodMLP( const TString& jobName,
75 const TString& methodTitle,
76 DataSetInfo& theData,
77 const TString& theOption );
79 MethodMLP( DataSetInfo& theData,
80 const TString& theWeightFile );
82 virtual ~MethodMLP();
84 virtual Bool_t HasAnalysisType( Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets );
86 void Train();
87 // for GA
88 Double_t ComputeEstimator ( std::vector<Double_t>& parameters );
89 Double_t EstimatorFunction( std::vector<Double_t>& parameters );
95 Double_t GetMvaValue( Double_t* err=0, Double_t* errUpper=0 );
97 protected:
99 // make ROOT-independent C++ class for classifier response (classifier-specific implementation)
100 void MakeClassSpecific( std::ostream&, const TString& ) const;
102 // get help message text
103 void GetHelpMessage() const;
106 private:
108 // the option handling methods
109 void DeclareOptions();
110 void ProcessOptions();
112 // general helper functions
113 void Train( Int_t nEpochs );
114 void Init();
115 void InitializeLearningRates(); // although this is only needed by backprop
117 // used as a measure of success in all minimization techniques
120 // BFGS functions
121 void BFGSMinimize( Int_t nEpochs );
122 void SetGammaDelta( TMatrixD &Gamma, TMatrixD &Delta, std::vector<Double_t> &Buffer );
123 void SteepestDir( TMatrixD &Dir );
124 Bool_t GetHessian( TMatrixD &Hessian, TMatrixD &Gamma, TMatrixD &Delta );
125 void SetDir( TMatrixD &Hessian, TMatrixD &Dir );
126 Double_t DerivDir( TMatrixD &Dir );
127 Bool_t LineSearch( TMatrixD &Dir, std::vector<Double_t> &Buffer, Double_t* dError=0 ); ///< zjh
128 void ComputeDEDw();
129 void SimulateEvent( const Event* ev );
130 void SetDirWeights( std::vector<Double_t> &Origin, TMatrixD &Dir, Double_t alpha );
132 Double_t GetMSEErr( const Event* ev, UInt_t index = 0 ); ///< zjh
133 Double_t GetCEErr( const Event* ev, UInt_t index = 0 ); ///< zjh
135 // backpropagation functions
136 void BackPropagationMinimize( Int_t nEpochs );
137 void TrainOneEpoch();
138 void Shuffle( Int_t* index, Int_t n );
139 void DecaySynapseWeights(Bool_t lateEpoch );
140 void TrainOneEvent( Int_t ievt);
141 Double_t GetDesiredOutput( const Event* ev );
142 void UpdateNetwork( Double_t desired, Double_t eventWeight=1.0 );
143 void UpdateNetwork(const std::vector<Float_t>& desired, Double_t eventWeight=1.0);
145 void UpdateSynapses();
148 // faster backpropagation
149 void TrainOneEventFast( Int_t ievt, Float_t*& branchVar, Int_t& type );
151 // genetic algorithm functions
152 void GeneticMinimize();
155#ifdef MethodMLP_UseMinuit__
156 // minuit functions -- commented out because they rely on a static pointer
157 void MinuitMinimize();
158 static MethodMLP* GetThisPtr();
159 static void IFCN( Int_t& npars, Double_t* grad, Double_t &f, Double_t* fitPars, Int_t ifl );
160 void FCN( Int_t& npars, Double_t* grad, Double_t &f, Double_t* fitPars, Int_t ifl );
163 // general
164 bool fUseRegulator; ///< zjh
165 bool fCalculateErrors; ///< compute inverse hessian matrix at the end of the training
166 Double_t fPrior; ///< zjh
167 std::vector<Double_t> fPriorDev; ///< zjh
168 void GetApproxInvHessian ( TMatrixD& InvHessian, bool regulate=true ); ///< rank-1 approximation, neglect 2nd derivatives. //zjh
169 void UpdateRegulators(); ///< zjh
170 void UpdatePriors(); ///< zjh
173 ETrainingMethod fTrainingMethod; ///< method of training, BP or GA
174 TString fTrainMethodS; ///< training method option param
176 Float_t fSamplingFraction; ///< fraction of events which is sampled for training
177 Float_t fSamplingEpoch; ///< fraction of epochs where sampling is used
178 Float_t fSamplingWeight; ///< changing factor for event weights when sampling is turned on
179 Bool_t fSamplingTraining; ///< The training sample is sampled
180 Bool_t fSamplingTesting; ///< The testing sample is sampled
182 // BFGS variables
183 Double_t fLastAlpha; ///< line search variable
184 Double_t fTau; ///< line search variable
185 Int_t fResetStep; ///< reset time (how often we clear hessian matrix)
187 // back propagation variable
188 Double_t fLearnRate; ///< learning rate for synapse weight adjustments
189 Double_t fDecayRate; ///< decay rate for above learning rate
190 EBPTrainingMode fBPMode; ///< backprop learning mode (sequential or batch)
191 TString fBpModeS; ///< backprop learning mode option string (sequential or batch)
192 Int_t fBatchSize; ///< batch size, only matters if in batch learning mode
193 Int_t fTestRate; ///< test for overtraining performed at each #th epochs
194 Bool_t fEpochMon; ///< create and fill epoch-wise monitoring histograms (makes outputfile big!)
196 // genetic algorithm variables
197 Int_t fGA_nsteps; ///< GA settings: number of steps
198 Int_t fGA_preCalc; ///< GA settings: number of pre-calc steps
199 Int_t fGA_SC_steps; ///< GA settings: SC_steps
200 Int_t fGA_SC_rate; ///< GA settings: SC_rate
201 Double_t fGA_SC_factor; ///< GA settings: SC_factor
203 // regression, storage of deviations
204 std::vector<std::pair<Float_t,Float_t> >* fDeviationsFromTargets; ///< deviation from the targets, event weight
206 Float_t fWeightRange; ///< suppress outliers for the estimator calculation
208#ifdef MethodMLP_UseMinuit__
209 // minuit variables -- commented out because they rely on a static pointer
210 Int_t fNumberOfWeights; ///< Minuit: number of weights
211 static MethodMLP* fgThis; ///< Minuit: this pointer
214 // debugging flags
215 static const Int_t fgPRINT_ESTIMATOR_INC = 10; ///< debug flags
216 static const Bool_t fgPRINT_SEQ = kFALSE; ///< debug flags
217 static const Bool_t fgPRINT_BATCH = kFALSE; ///< debug flags
219 ClassDef(MethodMLP,0); // Multi-layer perceptron implemented specifically for TMVA
220 };
222} // namespace TMVA
#define f(i)
Definition: RSha256.hxx:104
bool Bool_t
Definition: RtypesCore.h:63
const Bool_t kFALSE
Definition: RtypesCore.h:101
unsigned int UInt_t
Definition: RtypesCore.h:46
float Float_t
Definition: RtypesCore.h:57
double Double_t
Definition: RtypesCore.h:59
#define ClassDef(name, id)
Definition: Rtypes.h:335
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t index
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
Check for convergence.
Class that contains all the data information.
Definition: DataSetInfo.h:62
Interface for a fitter 'target'.
Definition: IFitterTarget.h:44
Base class for all TMVA methods using artificial neural networks.
Definition: MethodANNBase.h:62
Multilayer Perceptron class built off of MethodANNBase.
Definition: MethodMLP.h:69
Int_t fResetStep
reset time (how often we clear hessian matrix)
Definition: MethodMLP.h:185
bool fCalculateErrors
compute inverse hessian matrix at the end of the training
Definition: MethodMLP.h:165
std::vector< std::pair< Float_t, Float_t > > * fDeviationsFromTargets
deviation from the targets, event weight
Definition: MethodMLP.h:204
Bool_t LineSearch(TMatrixD &Dir, std::vector< Double_t > &Buffer, Double_t *dError=0)
Definition: MethodMLP.cxx:840
Double_t fTau
line search variable
Definition: MethodMLP.h:184
Int_t fGA_SC_rate
GA settings: SC_rate.
Definition: MethodMLP.h:200
Float_t fWeightRange
suppress outliers for the estimator calculation
Definition: MethodMLP.h:206
Float_t fSamplingWeight
changing factor for event weights when sampling is turned on
Definition: MethodMLP.h:178
Int_t fBatchSize
batch size, only matters if in batch learning mode
Definition: MethodMLP.h:192
void GetHelpMessage() const
get help message text
Definition: MethodMLP.cxx:1715
void BackPropagationMinimize(Int_t nEpochs)
minimize estimator / train network with back propagation algorithm
Definition: MethodMLP.cxx:1037
Double_t GetMSEErr(const Event *ev, UInt_t index=0)
Definition: MethodMLP.cxx:1003
bool fUseRegulator
Definition: MethodMLP.h:164
Double_t GetMvaValue(Double_t *err=0, Double_t *errUpper=0)
get the mva value generated by the NN
Definition: MethodMLP.cxx:1549
void MakeClassSpecific(std::ostream &, const TString &) const
write specific classifier response
Definition: MethodMLP.cxx:1704
void AdjustSynapseWeights()
just adjust the synapse weights (should be called in batch mode)
Definition: MethodMLP.cxx:1434
std::vector< Double_t > fPriorDev
Definition: MethodMLP.h:167
bool HasInverseHessian()
Definition: MethodMLP.h:94
TString fBpModeS
backprop learning mode option string (sequential or batch)
Definition: MethodMLP.h:191
void SteepestDir(TMatrixD &Dir)
Definition: MethodMLP.cxx:774
void TrainOneEpoch()
train network over a single epoch/cycle of events
Definition: MethodMLP.cxx:1142
virtual Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets)
MLP can handle classification with 2 classes and regression with one regression-target.
Definition: MethodMLP.cxx:152
TString fTrainMethodS
training method option param
Definition: MethodMLP.h:174
Int_t fGA_nsteps
GA settings: number of steps.
Definition: MethodMLP.h:197
Double_t fPrior
Definition: MethodMLP.h:166
Bool_t GetHessian(TMatrixD &Hessian, TMatrixD &Gamma, TMatrixD &Delta)
Definition: MethodMLP.cxx:787
Double_t ComputeEstimator(std::vector< Double_t > &parameters)
this function is called by GeneticANN for GA optimization
Definition: MethodMLP.cxx:1393
Int_t fUpdateLimit
Definition: MethodMLP.h:171
static const Bool_t fgPRINT_BATCH
debug flags
Definition: MethodMLP.h:217
void InitializeLearningRates()
initialize learning rates of synapses, used only by back propagation
Definition: MethodMLP.cxx:278
Int_t fGA_preCalc
GA settings: number of pre-calc steps.
Definition: MethodMLP.h:198
void CalculateNeuronDeltas()
have each neuron calculate its delta by back propagation
Definition: MethodMLP.cxx:1328
Int_t fTestRate
test for overtraining performed at each #th epochs
Definition: MethodMLP.h:193
Double_t fDecayRate
decay rate for above learning rate
Definition: MethodMLP.h:189
ETrainingMethod fTrainingMethod
method of training, BP or GA
Definition: MethodMLP.h:173
EBPTrainingMode fBPMode
backprop learning mode (sequential or batch)
Definition: MethodMLP.h:190
Double_t DerivDir(TMatrixD &Dir)
Definition: MethodMLP.cxx:825
Double_t fGA_SC_factor
GA settings: SC_factor.
Definition: MethodMLP.h:201
Double_t GetCEErr(const Event *ev, UInt_t index=0)
Definition: MethodMLP.cxx:1020
virtual ~MethodMLP()
destructor nothing to be done
Definition: MethodMLP.cxx:138
Int_t fGA_SC_steps
GA settings: SC_steps.
Definition: MethodMLP.h:199
void SetDir(TMatrixD &Hessian, TMatrixD &Dir)
Definition: MethodMLP.cxx:808
void Shuffle(Int_t *index, Int_t n)
Definition: MethodMLP.cxx:1190
Bool_t fSamplingTraining
The training sample is sampled.
Definition: MethodMLP.h:179
void SimulateEvent(const Event *ev)
Definition: MethodMLP.cxx:734
void SetDirWeights(std::vector< Double_t > &Origin, TMatrixD &Dir, Double_t alpha)
Definition: MethodMLP.cxx:950
void SetGammaDelta(TMatrixD &Gamma, TMatrixD &Delta, std::vector< Double_t > &Buffer)
Definition: MethodMLP.cxx:672
Double_t EstimatorFunction(std::vector< Double_t > &parameters)
interface to the estimate
Definition: MethodMLP.cxx:1385
Double_t fLearnRate
learning rate for synapse weight adjustments
Definition: MethodMLP.h:188
void UpdatePriors()
Definition: MethodMLP.cxx:1454
void GetApproxInvHessian(TMatrixD &InvHessian, bool regulate=true)
rank-1 approximation, neglect 2nd derivatives. //zjh
Definition: MethodMLP.cxx:1508
void BFGSMinimize(Int_t nEpochs)
train network with BFGS algorithm
Definition: MethodMLP.cxx:487
void UpdateSynapses()
update synapse error fields and adjust the weights (if in sequential mode)
Definition: MethodMLP.cxx:1412
void Init()
default initializations
Definition: MethodMLP.cxx:164
static const Int_t fgPRINT_ESTIMATOR_INC
debug flags
Definition: MethodMLP.h:215
void ProcessOptions()
process user options
Definition: MethodMLP.cxx:247
Float_t fSamplingFraction
fraction of events which is sampled for training
Definition: MethodMLP.h:176
void TrainOneEvent(Int_t ievt)
train network over a single event this uses the new event model
Definition: MethodMLP.cxx:1258
Double_t GetDesiredOutput(const Event *ev)
get the desired output of this event
Definition: MethodMLP.cxx:1277
void GeneticMinimize()
create genetics class similar to GeneticCut give it vector of parameter ranges (parameters = weights)...
Definition: MethodMLP.cxx:1356
Bool_t fSamplingTesting
The testing sample is sampled.
Definition: MethodMLP.h:180
Double_t GetError()
Definition: MethodMLP.cxx:966
Double_t fLastAlpha
line search variable
Definition: MethodMLP.h:183
Float_t fSamplingEpoch
fraction of epochs where sampling is used
Definition: MethodMLP.h:177
void DecaySynapseWeights(Bool_t lateEpoch)
decay synapse weights in last 10 epochs, lower learning rate even more to find a good minimum
Definition: MethodMLP.cxx:1208
void TrainOneEventFast(Int_t ievt, Float_t *&branchVar, Int_t &type)
fast per-event training
Definition: MethodMLP.cxx:1222
Bool_t fEpochMon
create and fill epoch-wise monitoring histograms (makes outputfile big!)
Definition: MethodMLP.h:194
void ComputeDEDw()
Definition: MethodMLP.cxx:697
void UpdateNetwork(Double_t desired, Double_t eventWeight=1.0)
update the network based on how closely the output matched the desired output
Definition: MethodMLP.cxx:1286
MethodMLP(const TString &jobName, const TString &methodTitle, DataSetInfo &theData, const TString &theOption)
standard constructor
Definition: MethodMLP.cxx:90
void UpdateRegulators()
Definition: MethodMLP.cxx:1468
void DeclareOptions()
define the options (their key words) that can be set in the option string
Definition: MethodMLP.cxx:195
Double_t CalculateEstimator(Types::ETreeType treeType=Types::kTraining, Int_t iEpoch=-1)
calculate the estimator that training is attempting to minimize
Definition: MethodMLP.cxx:292
static const Bool_t fgPRINT_SEQ
debug flags
Definition: MethodMLP.h:216
Definition: Types.h:126
@ kTraining
Definition: Types.h:143
Basic string class.
Definition: TString.h:136
const Int_t n
Definition: legend1.C:16
create variable transformations