Logo ROOT   6.08/07
Reference Guide
MethodPyKeras.cxx
Go to the documentation of this file.
1 // @(#)root/tmva/pymva $Id$
2 // Author: Stefan Wunsch, 2016
3 
4 #include <Python.h>
5 #include "TMVA/MethodPyKeras.h"
6 
7 #define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
8 #include <numpy/arrayobject.h>
9 
10 #include "TMVA/Types.h"
11 #include "TMVA/Config.h"
12 #include "TMVA/ClassifierFactory.h"
13 #include "TMVA/Results.h"
16 
17 using namespace TMVA;
18 
19 REGISTER_METHOD(PyKeras)
20 
22 
23 MethodPyKeras::MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption)
24  : PyMethodBase(jobName, Types::kPyKeras, methodTitle, dsi, theOption) {
25  fVerbose = 1;
26  fContinueTraining = false;
27  fSaveBestOnly = true;
28  fTriesEarlyStopping = -1;
29  fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
30  fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
31 }
32 
33 MethodPyKeras::MethodPyKeras(DataSetInfo &theData, const TString &theWeightFile)
34  : PyMethodBase(Types::kPyKeras, theData, theWeightFile) {
35  fVerbose = 1;
36  fContinueTraining = false;
37  fSaveBestOnly = true;
39  fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
40  fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
41 }
42 
44 }
45 
47  if (type == Types::kRegression) return kTRUE;
48  if (type == Types::kClassification && numberClasses == 2) return kTRUE;
49  if (type == Types::kMulticlass && numberClasses >= 2) return kTRUE;
50  return kFALSE;
51 }
52 
54  DeclareOptionRef(fFilenameModel, "FilenameModel", "Filename of the initial Keras model");
55  DeclareOptionRef(fFilenameTrainedModel, "FilenameTrainedModel", "Filename of the trained output Keras model");
56  DeclareOptionRef(fBatchSize, "BatchSize", "Training batch size");
57  DeclareOptionRef(fNumEpochs, "NumEpochs", "Number of training epochs");
58  DeclareOptionRef(fVerbose, "Verbose", "Keras verbosity during training");
59  DeclareOptionRef(fContinueTraining, "ContinueTraining", "Load weights from previous training");
60  DeclareOptionRef(fSaveBestOnly, "SaveBestOnly", "Store only weights with smallest validation loss");
61  DeclareOptionRef(fTriesEarlyStopping, "TriesEarlyStopping", "Number of epochs with no improvement in validation loss after which training will be stopped. The default or a negative number deactivates this option.");
62  DeclareOptionRef(fLearningRateSchedule, "LearningRateSchedule", "Set new learning rate during training at specific epochs, e.g., \"50,0.01;70,0.005\"");
63 }
64 
66  // Set default filename for trained model if option is not used
67  if (fFilenameTrainedModel.IsNull()) {
68  fFilenameTrainedModel = GetWeightFileDir() + "/TrainedModel_" + GetName() + ".h5";
69  }
70  // Setup model, either the initial model from `fFilenameModel` or
71  // the trained model from `fFilenameTrainedModel`
72  if (fContinueTraining) Log() << kINFO << "Continue training with trained model" << Endl;
74 }
75 
76 void MethodPyKeras::SetupKerasModel(bool loadTrainedModel) {
77  /*
78  * Load Keras model from file
79  */
80 
81  // Load initial model or already trained model
82  TString filenameLoadModel;
83  if (loadTrainedModel) {
84  filenameLoadModel = fFilenameTrainedModel;
85  }
86  else {
87  filenameLoadModel = fFilenameModel;
88  }
89  PyRunString("model = keras.models.load_model('"+filenameLoadModel+"')",
90  "Failed to load Keras model from file: "+filenameLoadModel);
91  Log() << kINFO << "Load model from file: " << filenameLoadModel << Endl;
92 
93  /*
94  * Init variables and weights
95  */
96 
97  // Get variables, classes and target numbers
101  else Log() << kFATAL << "Selected analysis type is not implemented" << Endl;
102 
103  // Init evaluation (needed for getMvaValue)
104  fVals = new float[fNVars]; // holds values used for classification and regression
105  npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars};
106  PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals);
107  PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals);
108 
109  fOutput.resize(fNOutputs); // holds classification probabilities or regression output
110  npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs};
111  PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)&fOutput[0]);
112  PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput);
113 
114  // Mark the model as setup
115  fModelIsSetup = true;
116 }
117 
119  if (!PyIsInitialized()) {
120  Log() << kFATAL << "Python is not initialized" << Endl;
121  }
122  _import_array(); // required to use numpy arrays
123 
124  // Import Keras
125  PyRunString("import keras", "Import Keras failed");
126 
127  // Set flag that model is not setup
128  fModelIsSetup = false;
129 }
130 
132  if(!fModelIsSetup) Log() << kFATAL << "Model is not setup for training" << Endl;
133 
134  /*
135  * Load training data to numpy array
136  */
137 
138  UInt_t nTrainingEvents = Data()->GetNTrainingEvents();
139 
140  float* trainDataX = new float[nTrainingEvents*fNVars];
141  float* trainDataY = new float[nTrainingEvents*fNOutputs];
142  float* trainDataWeights = new float[nTrainingEvents];
143  for (UInt_t i=0; i<nTrainingEvents; i++) {
144  const TMVA::Event* e = GetTrainingEvent(i);
145  // Fill variables
146  for (UInt_t j=0; j<fNVars; j++) {
147  trainDataX[j + i*fNVars] = e->GetValue(j);
148  }
149  // Fill targets
150  // NOTE: For classification, convert class number in one-hot vector,
151  // e.g., 1 -> [0, 1] or 0 -> [1, 0] for binary classification
153  for (UInt_t j=0; j<fNOutputs; j++) {
154  trainDataY[j + i*fNOutputs] = 0;
155  }
156  trainDataY[e->GetClass() + i*fNOutputs] = 1;
157  }
158  else if (GetAnalysisType() == Types::kRegression) {
159  for (UInt_t j=0; j<fNOutputs; j++) {
160  trainDataY[j + i*fNOutputs] = e->GetTarget(j);
161  }
162  }
163  else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
164  // Fill weights
165  // NOTE: If no weight branch is given, this defaults to ones for all events
166  trainDataWeights[i] = e->GetWeight();
167  }
168 
169  npy_intp dimsTrainX[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNVars};
170  npy_intp dimsTrainY[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNOutputs};
171  npy_intp dimsTrainWeights[1] = {(npy_intp)nTrainingEvents};
172  PyArrayObject* pTrainDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainX, NPY_FLOAT, (void*)trainDataX);
173  PyArrayObject* pTrainDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainY, NPY_FLOAT, (void*)trainDataY);
174  PyArrayObject* pTrainDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsTrainWeights, NPY_FLOAT, (void*)trainDataWeights);
175  PyDict_SetItemString(fLocalNS, "trainX", (PyObject*)pTrainDataX);
176  PyDict_SetItemString(fLocalNS, "trainY", (PyObject*)pTrainDataY);
177  PyDict_SetItemString(fLocalNS, "trainWeights", (PyObject*)pTrainDataWeights);
178 
179  /*
180  * Load validation data to numpy array
181  */
182 
183  // NOTE: In TMVA, test data is actually validation data
184 
185  UInt_t nValEvents = Data()->GetNTestEvents();
186 
187  float* valDataX = new float[nValEvents*fNVars];
188  float* valDataY = new float[nValEvents*fNOutputs];
189  float* valDataWeights = new float[nValEvents];
190  for (UInt_t i=0; i<nValEvents; i++) {
191  const TMVA::Event* e = GetTestingEvent(i);
192  // Fill variables
193  for (UInt_t j=0; j<fNVars; j++) {
194  valDataX[j + i*fNVars] = e->GetValue(j);
195  }
196  // Fill targets
198  for (UInt_t j=0; j<fNOutputs; j++) {
199  valDataY[j + i*fNOutputs] = 0;
200  }
201  valDataY[e->GetClass() + i*fNOutputs] = 1;
202  }
203  else if (GetAnalysisType() == Types::kRegression) {
204  for (UInt_t j=0; j<fNOutputs; j++) {
205  valDataY[j + i*fNOutputs] = e->GetTarget(j);
206  }
207  }
208  else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
209  // Fill weights
210  valDataWeights[i] = e->GetWeight();
211  }
212 
213  npy_intp dimsValX[2] = {(npy_intp)nValEvents, (npy_intp)fNVars};
214  npy_intp dimsValY[2] = {(npy_intp)nValEvents, (npy_intp)fNOutputs};
215  npy_intp dimsValWeights[1] = {(npy_intp)nValEvents};
216  PyArrayObject* pValDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValX, NPY_FLOAT, (void*)valDataX);
217  PyArrayObject* pValDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValY, NPY_FLOAT, (void*)valDataY);
218  PyArrayObject* pValDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsValWeights, NPY_FLOAT, (void*)valDataWeights);
219  PyDict_SetItemString(fLocalNS, "valX", (PyObject*)pValDataX);
220  PyDict_SetItemString(fLocalNS, "valY", (PyObject*)pValDataY);
221  PyDict_SetItemString(fLocalNS, "valWeights", (PyObject*)pValDataWeights);
222 
223  /*
224  * Train Keras model
225  */
226 
227  // Setup parameters
228 
229  PyObject* pBatchSize = PyLong_FromLong(fBatchSize);
230  PyObject* pNumEpochs = PyLong_FromLong(fNumEpochs);
231  PyObject* pVerbose = PyLong_FromLong(fVerbose);
232  PyDict_SetItemString(fLocalNS, "batchSize", pBatchSize);
233  PyDict_SetItemString(fLocalNS, "numEpochs", pNumEpochs);
234  PyDict_SetItemString(fLocalNS, "verbose", pVerbose);
235 
236  // Setup training callbacks
237  PyRunString("callbacks = []");
238 
239  // Callback: Save only weights with smallest validation loss
240  if (fSaveBestOnly) {
241  PyRunString("callbacks.append(keras.callbacks.ModelCheckpoint('"+fFilenameTrainedModel+"', monitor='val_loss', verbose=verbose, save_best_only=True, mode='auto'))", "Failed to setup training callback: SaveBestOnly");
242  Log() << kINFO << "Option SaveBestOnly: Only model weights with smallest validation loss will be stored" << Endl;
243  }
244 
245  // Callback: Stop training early if no improvement in validation loss is observed
246  if (fTriesEarlyStopping>=0) {
247  TString tries;
248  tries.Form("%i", fTriesEarlyStopping);
249  PyRunString("callbacks.append(keras.callbacks.EarlyStopping(monitor='val_loss', patience="+tries+", verbose=verbose, mode='auto'))", "Failed to setup training callback: TriesEarlyStopping");
250  Log() << kINFO << "Option TriesEarlyStopping: Training will stop after " << tries << " number of epochs with no improvement of validation loss" << Endl;
251  }
252 
253  // Callback: Learning rate scheduler
254  if (fLearningRateSchedule!="") {
255  // Setup a python dictionary with the desired learning rate steps
256  PyRunString("strScheduleSteps = '"+fLearningRateSchedule+"'\n"
257  "schedulerSteps = {}\n"
258  "for c in strScheduleSteps.split(';'):\n"
259  " x = c.split(',')\n"
260  " schedulerSteps[int(x[0])] = float(x[1])\n",
261  "Failed to setup steps for scheduler function from string: "+fLearningRateSchedule,
262  Py_file_input);
263  // Set scheduler function as piecewise function with given steps
264  PyRunString("def schedule(epoch, model=model, schedulerSteps=schedulerSteps):\n"
265  " if epoch in schedulerSteps: return float(schedulerSteps[epoch])\n"
266  " else: return float(model.optimizer.lr.get_value())\n",
267  "Failed to setup scheduler function with string: "+fLearningRateSchedule,
268  Py_file_input);
269  // Setup callback
270  PyRunString("callbacks.append(keras.callbacks.LearningRateScheduler(schedule))",
271  "Failed to setup training callback: LearningRateSchedule");
272  Log() << kINFO << "Option LearningRateSchedule: Set learning rate during training: " << fLearningRateSchedule << Endl;
273  }
274 
275  // Train model
276  PyRunString("history = model.fit(trainX, trainY, sample_weight=trainWeights, batch_size=batchSize, nb_epoch=numEpochs, verbose=verbose, validation_data=(valX, valY, valWeights), callbacks=callbacks)",
277  "Failed to train model");
278 
279  /*
280  * Store trained model to file (only if option 'SaveBestOnly' is NOT activated,
281  * because we do not want to override the best model checkpoint)
282  */
283 
284  if (!fSaveBestOnly) {
285  PyRunString("model.save('"+fFilenameTrainedModel+"', overwrite=True)",
286  "Failed to save trained model: "+fFilenameTrainedModel);
287  Log() << kINFO << "Trained model written to file: " << fFilenameTrainedModel << Endl;
288  }
289 
290  /*
291  * Clean-up
292  */
293 
294  delete[] trainDataX;
295  delete[] trainDataY;
296  delete[] trainDataWeights;
297  delete[] valDataX;
298  delete[] valDataY;
299  delete[] valDataWeights;
300 }
301 
304 }
305 
307  // Cannot determine error
308  NoErrorCalc(errLower, errUpper);
309 
310  // Check whether the model is setup
311  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
312  if (!fModelIsSetup) {
313  // Setup the trained model
314  SetupKerasModel(true);
315  }
316 
317  // Get signal probability (called mvaValue here)
318  const TMVA::Event* e = GetEvent();
319  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
320  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
321  "Failed to get predictions");
322 
324 }
325 
326 std::vector<Double_t> MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t) {
327  // Check whether the model is setup
328  // NOTE: Unfortunately this is needed because during evaluation ProcessOptions is not called again
329  if (!fModelIsSetup) {
330  // Setup the trained model
331  SetupKerasModel(true);
332  }
333 
334  // Load data to numpy array
336  if (firstEvt > lastEvt || lastEvt > nEvents) lastEvt = nEvents;
337  if (firstEvt < 0) firstEvt = 0;
338  nEvents = lastEvt-firstEvt;
339 
340  float* data = new float[nEvents*fNVars];
341  for (UInt_t i=0; i<nEvents; i++) {
342  Data()->SetCurrentEvent(i);
343  const TMVA::Event *e = GetEvent();
344  for (UInt_t j=0; j<fNVars; j++) {
345  data[j + i*fNVars] = e->GetValue(j);
346  }
347  }
348 
349  npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)fNVars};
350  PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (void*)data);
351  if (pDataMvaValues==0) Log() << "Failed to load data to Python array" << Endl;
352 
353  // Get prediction for all events
354  PyObject* pModel = PyDict_GetItemString(fLocalNS, "model");
355  if (pModel==0) Log() << kFATAL << "Failed to get model Python object" << Endl;
356  PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallMethod(pModel, (char*)"predict", (char*)"O", pDataMvaValues);
357  if (pPredictions==0) Log() << kFATAL << "Failed to get predictions" << Endl;
358  delete[] data;
359 
360  // Load predictions to double vector
361  // NOTE: The signal probability is given at the output
362  std::vector<double> mvaValues(nEvents);
363  float* predictionsData = (float*) PyArray_DATA(pPredictions);
364  for (UInt_t i=0; i<nEvents; i++) {
365  mvaValues[i] = (double) predictionsData[i*fNOutputs + TMVA::Types::kSignal];
366  }
367 
368  return mvaValues;
369 }
370 
371 std::vector<Float_t>& MethodPyKeras::GetRegressionValues() {
372  // Check whether the model is setup
373  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
374  if (!fModelIsSetup){
375  // Setup the model and load weights
376  SetupKerasModel(true);
377  }
378 
379  // Get regression values
380  const TMVA::Event* e = GetEvent();
381  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
382  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
383  "Failed to get predictions");
384 
385  // Use inverse transformation of targets to get final regression values
386  Event * eTrans = new Event(*e);
387  for (UInt_t i=0; i<fNOutputs; ++i) {
388  eTrans->SetTarget(i,fOutput[i]);
389  }
390 
391  const Event* eTrans2 = GetTransformationHandler().InverseTransform(eTrans);
392  for (UInt_t i=0; i<fNOutputs; ++i) {
393  fOutput[i] = eTrans2->GetTarget(i);
394  }
395 
396  return fOutput;
397 }
398 
399 std::vector<Float_t>& MethodPyKeras::GetMulticlassValues() {
400  // Check whether the model is setup
401  // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
402  if (!fModelIsSetup){
403  // Setup the model and load weights
404  SetupKerasModel(true);
405  }
406 
407  // Get class probabilites
408  const TMVA::Event* e = GetEvent();
409  for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
410  PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
411  "Failed to get predictions");
412 
413  return fOutput;
414 }
415 
417 }
418 
420 // typical length of text line:
421 // "|--------------------------------------------------------------|"
422  Log() << Endl;
423  Log() << "Keras is a high-level API for the Theano and Tensorflow packages." << Endl;
424  Log() << "This method wraps the training and predictions steps of the Keras" << Endl;
425  Log() << "Python package for TMVA, so that dataloading, preprocessing and" << Endl;
426  Log() << "evaluation can be done within the TMVA system. To use this Keras" << Endl;
427  Log() << "interface, you have to generate a model with Keras first. Then," << Endl;
428  Log() << "this model can be loaded and trained in TMVA." << Endl;
429  Log() << Endl;
430 }
Double_t GetMvaValue(Double_t *errLower, Double_t *errUpper)
void SetCurrentEvent(Long64_t ievt) const
Definition: DataSet.h:113
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:162
long long Long64_t
Definition: RtypesCore.h:69
MsgLogger & Log() const
Definition: Configurable.h:128
OptionBase * DeclareOptionRef(T &ref, const TString &name, const TString &desc="")
EAnalysisType
Definition: Types.h:129
TransformationHandler & GetTransformationHandler(Bool_t takeReroutedIfAvailable=true)
Definition: MethodBase.h:390
bool Bool_t
Definition: RtypesCore.h:59
const Bool_t kFALSE
Definition: Rtypes.h:92
UInt_t GetNClasses() const
Definition: DataSetInfo.h:154
static int PyIsInitialized()
const TString & GetWeightFileDir() const
Definition: MethodBase.h:486
std::vector< Float_t > & GetRegressionValues()
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
void GetHelpMessage() const
const Event * GetEvent() const
Definition: MethodBase.h:745
DataSet * Data() const
Definition: MethodBase.h:405
TString fFilenameTrainedModel
Definition: MethodPyKeras.h:88
UInt_t GetClass() const
Definition: Event.h:89
DataSetInfo & DataInfo() const
Definition: MethodBase.h:406
Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
Double_t GetWeight() const
return the event weight - depending on whether the flag IgnoreNegWeightsInTraining is or not...
Definition: Event.cxx:378
Long64_t GetNTrainingEvents() const
Definition: DataSet.h:93
const Event * GetTrainingEvent(Long64_t ievt) const
Definition: MethodBase.h:765
TString fLearningRateSchedule
Definition: MethodPyKeras.h:81
const int nEvents
Definition: testRooFit.cxx:42
const Event * GetTestingEvent(Long64_t ievt) const
Definition: MethodBase.h:771
Float_t GetTarget(UInt_t itgt) const
Definition: Event.h:104
UInt_t GetNTargets() const
Definition: DataSetInfo.h:129
const char * GetName() const
Definition: MethodBase.h:330
unsigned int UInt_t
Definition: RtypesCore.h:42
const Event * InverseTransform(const Event *, Bool_t suppressIfNoTargets=true) const
void SetTarget(UInt_t itgt, Float_t value)
set the target value (dimension itgt) to value
Definition: Event.cxx:356
std::vector< Float_t > & GetMulticlassValues()
Long64_t GetNTestEvents() const
Definition: DataSet.h:94
UInt_t GetNVariables() const
Definition: MethodBase.h:341
Float_t GetValue(UInt_t ivar) const
return value of i&#39;th variable
Definition: Event.cxx:233
void SetupKerasModel(Bool_t loadTrainedModel)
static PyObject * fLocalNS
Definition: PyMethodBase.h:144
#define ClassImp(name)
Definition: Rtypes.h:279
double Double_t
Definition: RtypesCore.h:55
std::vector< Double_t > GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress)
get all the MVA values for the events of the current Data type
int type
Definition: TGX11.cxx:120
virtual void TestClassification()
initialization
you should not use this method at all Int_t Int_t Double_t Double_t Double_t e
Definition: TRolke.cxx:630
#define REGISTER_METHOD(CLASS)
for example
Abstract ClassifierFactory template that handles arbitrary types.
Long64_t GetNEvents(Types::ETreeType type=Types::kMaxTreeType) const
Definition: DataSet.h:229
Types::EAnalysisType GetAnalysisType() const
Definition: MethodBase.h:433
MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
const Bool_t kTRUE
Definition: Rtypes.h:91
virtual void TestClassification()
initialization
std::vector< float > fOutput
Definition: MethodPyKeras.h:85
_object PyObject
Definition: TPyArg.h:22
void NoErrorCalc(Double_t *const err, Double_t *const errUpper)
Definition: MethodBase.cxx:819