Logo ROOT  
Reference Guide
MethodMLP.h
Go to the documentation of this file.
1// @(#)root/tmva $Id$
2// Author: Krzysztof Danielowski, Andreas Hoecker, Matt Jachowski, Kamil Kraszewski, Maciej Kruk, Peter Speckmayer, Joerg Stelzer, Eckhard von Toerne, Jan Therhaag, Jiahang Zhong
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : MethodMLP *
8 * Web : http://tmva.sourceforge.net *
9 * *
10 * Description: *
11 * ANN Multilayer Perceptron class for the discrimination of signal *
12 * from background. BFGS implementation based on TMultiLayerPerceptron *
13 * class from ROOT (http://root.cern.ch). *
14 * *
15 * Authors (alphabetical): *
16 * Krzysztof Danielowski <danielow@cern.ch> - IFJ & AGH, Poland *
17 * Andreas Hoecker <Andreas.Hocker@cern.ch> - CERN, Switzerland *
18 * Matt Jachowski <jachowski@stanford.edu> - Stanford University, USA *
19 * Kamil Kraszewski <kalq@cern.ch> - IFJ & UJ, Poland *
20 * Maciej Kruk <mkruk@cern.ch> - IFJ & AGH, Poland *
21 * Peter Speckmayer <peter.speckmayer@cern.ch> - CERN, Switzerland *
22 * Joerg Stelzer <stelzer@cern.ch> - DESY, Germany *
23 * Jan Therhaag <Jan.Therhaag@cern.ch> - U of Bonn, Germany *
24 * Eckhard v. Toerne <evt@uni-bonn.de> - U of Bonn, Germany *
25 * Jiahang Zhong <Jiahang.Zhong@cern.ch> - Academia Sinica, Taipei *
26 * *
27 * Copyright (c) 2005-2011: *
28 * CERN, Switzerland *
29 * U. of Victoria, Canada *
30 * MPI-K Heidelberg, Germany *
31 * U. of Bonn, Germany *
32 * *
33 * Redistribution and use in source and binary forms, with or without *
34 * modification, are permitted according to the terms listed in LICENSE *
35 * (http://tmva.sourceforge.net/LICENSE) *
36 **********************************************************************************/
37
38#ifndef ROOT_TMVA_MethodMLP
39#define ROOT_TMVA_MethodMLP
40
41//////////////////////////////////////////////////////////////////////////
42// //
43// MethodMLP //
44// //
45// Multilayer Perceptron built off of MethodANNBase //
46// //
47//////////////////////////////////////////////////////////////////////////
48
49#include <vector>
50#include "TString.h"
51#include "TTree.h"
52#include "TRandom3.h"
53#include "TH1F.h"
54#include "TMatrixDfwd.h"
55
56#include "TMVA/IFitterTarget.h"
57#include "TMVA/MethodBase.h"
58#include "TMVA/MethodANNBase.h"
59#include "TMVA/TNeuron.h"
60#include "TMVA/TActivation.h"
62
63#define MethodMLP_UseMinuit__
64#undef MethodMLP_UseMinuit__
65
66namespace TMVA {
67
68 class MethodMLP : public MethodANNBase, public IFitterTarget, public ConvergenceTest {
69
70 public:
71
72 // standard constructors
73 MethodMLP( const TString& jobName,
74 const TString& methodTitle,
75 DataSetInfo& theData,
76 const TString& theOption );
77
78 MethodMLP( DataSetInfo& theData,
79 const TString& theWeightFile );
80
81 virtual ~MethodMLP();
82
83 virtual Bool_t HasAnalysisType( Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets );
84
85 void Train();
86 // for GA
87 Double_t ComputeEstimator ( std::vector<Double_t>& parameters );
88 Double_t EstimatorFunction( std::vector<Double_t>& parameters );
89
92
94 Double_t GetMvaValue( Double_t* err=0, Double_t* errUpper=0 );
95
96 protected:
97
98 // make ROOT-independent C++ class for classifier response (classifier-specific implementation)
99 void MakeClassSpecific( std::ostream&, const TString& ) const;
100
101 // get help message text
102 void GetHelpMessage() const;
103
104
105 private:
106
107 // the option handling methods
108 void DeclareOptions();
109 void ProcessOptions();
110
111 // general helper functions
112 void Train( Int_t nEpochs );
113 void Init();
114 void InitializeLearningRates(); // although this is only needed by backprop
115
116 // used as a measure of success in all minimization techniques
118
119 // BFGS functions
120 void BFGSMinimize( Int_t nEpochs );
121 void SetGammaDelta( TMatrixD &Gamma, TMatrixD &Delta, std::vector<Double_t> &Buffer );
122 void SteepestDir( TMatrixD &Dir );
123 Bool_t GetHessian( TMatrixD &Hessian, TMatrixD &Gamma, TMatrixD &Delta );
124 void SetDir( TMatrixD &Hessian, TMatrixD &Dir );
125 Double_t DerivDir( TMatrixD &Dir );
126 Bool_t LineSearch( TMatrixD &Dir, std::vector<Double_t> &Buffer, Double_t* dError=0 ); //zjh
127 void ComputeDEDw();
128 void SimulateEvent( const Event* ev );
129 void SetDirWeights( std::vector<Double_t> &Origin, TMatrixD &Dir, Double_t alpha );
131 Double_t GetMSEErr( const Event* ev, UInt_t index = 0 ); //zjh
132 Double_t GetCEErr( const Event* ev, UInt_t index = 0 ); //zjh
133
134 // backpropagation functions
135 void BackPropagationMinimize( Int_t nEpochs );
136 void TrainOneEpoch();
137 void Shuffle( Int_t* index, Int_t n );
138 void DecaySynapseWeights(Bool_t lateEpoch );
139 void TrainOneEvent( Int_t ievt);
140 Double_t GetDesiredOutput( const Event* ev );
141 void UpdateNetwork( Double_t desired, Double_t eventWeight=1.0 );
142 void UpdateNetwork(const std::vector<Float_t>& desired, Double_t eventWeight=1.0);
144 void UpdateSynapses();
146
147 // faster backpropagation
148 void TrainOneEventFast( Int_t ievt, Float_t*& branchVar, Int_t& type );
149
150 // genetic algorithm functions
151 void GeneticMinimize();
152
153
154#ifdef MethodMLP_UseMinuit__
155 // minuit functions -- commented out because they rely on a static pointer
156 void MinuitMinimize();
157 static MethodMLP* GetThisPtr();
158 static void IFCN( Int_t& npars, Double_t* grad, Double_t &f, Double_t* fitPars, Int_t ifl );
159 void FCN( Int_t& npars, Double_t* grad, Double_t &f, Double_t* fitPars, Int_t ifl );
160#endif
161
162 // general
163 bool fUseRegulator; // zjh
164 bool fCalculateErrors; // compute inverse hessian matrix at the end of the training
166 std::vector<Double_t> fPriorDev; // zjh
167 void GetApproxInvHessian ( TMatrixD& InvHessian, bool regulate=true ); //rank-1 approximation, neglect 2nd derivatives. //zjh
168 void UpdateRegulators(); // zjh
169 void UpdatePriors(); // zjh
171
172 ETrainingMethod fTrainingMethod; // method of training, BP or GA
173 TString fTrainMethodS; // training method option param
174
175 Float_t fSamplingFraction; // fraction of events which is sampled for training
176 Float_t fSamplingEpoch; // fraction of epochs where sampling is used
177 Float_t fSamplingWeight; // changing factor for event weights when sampling is turned on
178 Bool_t fSamplingTraining; // The training sample is sampled
179 Bool_t fSamplingTesting; // The testing sample is sampled
180
181 // BFGS variables
182 Double_t fLastAlpha; // line search variable
183 Double_t fTau; // line search variable
184 Int_t fResetStep; // reset time (how often we clear hessian matrix)
185
186 // back propagation variable
187 Double_t fLearnRate; // learning rate for synapse weight adjustments
188 Double_t fDecayRate; // decay rate for above learning rate
189 EBPTrainingMode fBPMode; // backprop learning mode (sequential or batch)
190 TString fBpModeS; // backprop learning mode option string (sequential or batch)
191 Int_t fBatchSize; // batch size, only matters if in batch learning mode
192 Int_t fTestRate; // test for overtraining performed at each #th epochs
193 Bool_t fEpochMon; // create and fill epoch-wise monitoring histograms (makes outputfile big!)
194
195 // genetic algorithm variables
196 Int_t fGA_nsteps; // GA settings: number of steps
197 Int_t fGA_preCalc; // GA settings: number of pre-calc steps
198 Int_t fGA_SC_steps; // GA settings: SC_steps
199 Int_t fGA_SC_rate; // GA settings: SC_rate
200 Double_t fGA_SC_factor; // GA settings: SC_factor
201
202 // regression, storage of deviations
203 std::vector<std::pair<Float_t,Float_t> >* fDeviationsFromTargets; // deviation from the targets, event weight
204
205 Float_t fWeightRange; // suppress outliers for the estimator calculation
206
207#ifdef MethodMLP_UseMinuit__
208 // minuit variables -- commented out because they rely on a static pointer
209 Int_t fNumberOfWeights; // Minuit: number of weights
210 static MethodMLP* fgThis; // Minuit: this pointer
211#endif
212
213 // debugging flags
214 static const Int_t fgPRINT_ESTIMATOR_INC = 10; // debug flags
215 static const Bool_t fgPRINT_SEQ = kFALSE; // debug flags
216 static const Bool_t fgPRINT_BATCH = kFALSE; // debug flags
217
218 ClassDef(MethodMLP,0); // Multi-layer perceptron implemented specifically for TMVA
219 };
220
221} // namespace TMVA
222
223#endif
#define f(i)
Definition: RSha256.hxx:104
int Int_t
Definition: RtypesCore.h:43
unsigned int UInt_t
Definition: RtypesCore.h:44
const Bool_t kFALSE
Definition: RtypesCore.h:90
bool Bool_t
Definition: RtypesCore.h:61
double Double_t
Definition: RtypesCore.h:57
float Float_t
Definition: RtypesCore.h:55
#define ClassDef(name, id)
Definition: Rtypes.h:322
int type
Definition: TGX11.cxx:120
Check for convergence.
Class that contains all the data information.
Definition: DataSetInfo.h:60
Interface for a fitter 'target'.
Definition: IFitterTarget.h:44
Base class for all TMVA methods using artificial neural networks.
Definition: MethodANNBase.h:62
Multilayer Perceptron class built off of MethodANNBase.
Definition: MethodMLP.h:68
Int_t fResetStep
Definition: MethodMLP.h:184
bool fCalculateErrors
Definition: MethodMLP.h:164
std::vector< std::pair< Float_t, Float_t > > * fDeviationsFromTargets
Definition: MethodMLP.h:203
Bool_t LineSearch(TMatrixD &Dir, std::vector< Double_t > &Buffer, Double_t *dError=0)
Definition: MethodMLP.cxx:842
Double_t fTau
Definition: MethodMLP.h:183
Int_t fGA_SC_rate
Definition: MethodMLP.h:199
Float_t fWeightRange
Definition: MethodMLP.h:205
Float_t fSamplingWeight
Definition: MethodMLP.h:177
Int_t fBatchSize
Definition: MethodMLP.h:191
void GetHelpMessage() const
get help message text
Definition: MethodMLP.cxx:1717
void BackPropagationMinimize(Int_t nEpochs)
minimize estimator / train network with back propagation algorithm
Definition: MethodMLP.cxx:1039
Double_t GetMSEErr(const Event *ev, UInt_t index=0)
Definition: MethodMLP.cxx:1005
bool fUseRegulator
Definition: MethodMLP.h:163
Double_t GetMvaValue(Double_t *err=0, Double_t *errUpper=0)
get the mva value generated by the NN
Definition: MethodMLP.cxx:1551
void MakeClassSpecific(std::ostream &, const TString &) const
write specific classifier response
Definition: MethodMLP.cxx:1706
void AdjustSynapseWeights()
just adjust the synapse weights (should be called in batch mode)
Definition: MethodMLP.cxx:1436
std::vector< Double_t > fPriorDev
Definition: MethodMLP.h:166
bool HasInverseHessian()
Definition: MethodMLP.h:93
TString fBpModeS
Definition: MethodMLP.h:190
void SteepestDir(TMatrixD &Dir)
Definition: MethodMLP.cxx:776
void TrainOneEpoch()
train network over a single epoch/cycle of events
Definition: MethodMLP.cxx:1144
virtual Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t numberTargets)
MLP can handle classification with 2 classes and regression with one regression-target.
Definition: MethodMLP.cxx:154
TString fTrainMethodS
Definition: MethodMLP.h:173
Int_t fGA_nsteps
Definition: MethodMLP.h:196
Double_t fPrior
Definition: MethodMLP.h:165
Bool_t GetHessian(TMatrixD &Hessian, TMatrixD &Gamma, TMatrixD &Delta)
Definition: MethodMLP.cxx:789
Double_t ComputeEstimator(std::vector< Double_t > &parameters)
this function is called by GeneticANN for GA optimization
Definition: MethodMLP.cxx:1395
Int_t fUpdateLimit
Definition: MethodMLP.h:170
static const Bool_t fgPRINT_BATCH
Definition: MethodMLP.h:216
void InitializeLearningRates()
initialize learning rates of synapses, used only by back propagation
Definition: MethodMLP.cxx:280
Int_t fGA_preCalc
Definition: MethodMLP.h:197
void CalculateNeuronDeltas()
have each neuron calculate its delta by back propagation
Definition: MethodMLP.cxx:1330
Double_t fDecayRate
Definition: MethodMLP.h:188
ETrainingMethod fTrainingMethod
Definition: MethodMLP.h:172
EBPTrainingMode fBPMode
Definition: MethodMLP.h:189
Double_t DerivDir(TMatrixD &Dir)
Definition: MethodMLP.cxx:827
Double_t fGA_SC_factor
Definition: MethodMLP.h:200
Double_t GetCEErr(const Event *ev, UInt_t index=0)
Definition: MethodMLP.cxx:1022
virtual ~MethodMLP()
destructor nothing to be done
Definition: MethodMLP.cxx:140
Int_t fGA_SC_steps
Definition: MethodMLP.h:198
void SetDir(TMatrixD &Hessian, TMatrixD &Dir)
Definition: MethodMLP.cxx:810
void Shuffle(Int_t *index, Int_t n)
Input:
Definition: MethodMLP.cxx:1192
Bool_t fSamplingTraining
Definition: MethodMLP.h:178
void SimulateEvent(const Event *ev)
Definition: MethodMLP.cxx:736
void SetDirWeights(std::vector< Double_t > &Origin, TMatrixD &Dir, Double_t alpha)
Definition: MethodMLP.cxx:952
void SetGammaDelta(TMatrixD &Gamma, TMatrixD &Delta, std::vector< Double_t > &Buffer)
Definition: MethodMLP.cxx:674
Double_t EstimatorFunction(std::vector< Double_t > &parameters)
interface to the estimate
Definition: MethodMLP.cxx:1387
Double_t fLearnRate
Definition: MethodMLP.h:187
void GetApproxInvHessian(TMatrixD &InvHessian, bool regulate=true)
Definition: MethodMLP.cxx:1510
void BFGSMinimize(Int_t nEpochs)
train network with BFGS algorithm
Definition: MethodMLP.cxx:489
void UpdateSynapses()
update synapse error fields and adjust the weights (if in sequential mode)
Definition: MethodMLP.cxx:1414
void Init()
default initializations
Definition: MethodMLP.cxx:166
static const Int_t fgPRINT_ESTIMATOR_INC
Definition: MethodMLP.h:214
void ProcessOptions()
process user options
Definition: MethodMLP.cxx:249
Float_t fSamplingFraction
Definition: MethodMLP.h:175
void TrainOneEvent(Int_t ievt)
train network over a single event this uses the new event model
Definition: MethodMLP.cxx:1260
Double_t GetDesiredOutput(const Event *ev)
get the desired output of this event
Definition: MethodMLP.cxx:1279
void GeneticMinimize()
create genetics class similar to GeneticCut give it vector of parameter ranges (parameters = weights)...
Definition: MethodMLP.cxx:1358
Bool_t fSamplingTesting
Definition: MethodMLP.h:179
Double_t GetError()
Definition: MethodMLP.cxx:968
Double_t fLastAlpha
Definition: MethodMLP.h:182
Float_t fSamplingEpoch
Definition: MethodMLP.h:176
void DecaySynapseWeights(Bool_t lateEpoch)
decay synapse weights in last 10 epochs, lower learning rate even more to find a good minimum
Definition: MethodMLP.cxx:1210
void TrainOneEventFast(Int_t ievt, Float_t *&branchVar, Int_t &type)
fast per-event training
Definition: MethodMLP.cxx:1224
Bool_t fEpochMon
Definition: MethodMLP.h:193
void ComputeDEDw()
Definition: MethodMLP.cxx:699
void UpdateNetwork(Double_t desired, Double_t eventWeight=1.0)
update the network based on how closely the output matched the desired output
Definition: MethodMLP.cxx:1288
MethodMLP(const TString &jobName, const TString &methodTitle, DataSetInfo &theData, const TString &theOption)
standard constructor
Definition: MethodMLP.cxx:92
void UpdateRegulators()
Definition: MethodMLP.cxx:1470
void DeclareOptions()
define the options (their key words) that can be set in the option string
Definition: MethodMLP.cxx:197
Double_t CalculateEstimator(Types::ETreeType treeType=Types::kTraining, Int_t iEpoch=-1)
calculate the estimator that training is attempting to minimize
Definition: MethodMLP.cxx:294
static const Bool_t fgPRINT_SEQ
Definition: MethodMLP.h:215
EAnalysisType
Definition: Types.h:127
@ kTraining
Definition: Types.h:144
Basic string class.
Definition: TString.h:131
const Int_t n
Definition: legend1.C:16
create variable transformations
Double_t Gamma(Double_t z)
Computation of gamma(z) for all z.
Definition: TMath.cxx:348