Logo ROOT  
Reference Guide
MethodPyKeras.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Author: Stefan Wunsch, 2016
3
4#include <Python.h>
6
7#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
8#include <numpy/arrayobject.h>
9
10#include "TMVA/Types.h"
11#include "TMVA/Config.h"
13#include "TMVA/Results.h"
16#include "TMVA/Tools.h"
17#include "TMVA/Timer.h"
18#include "TSystem.h"
19
20using namespace TMVA;
21
22namespace TMVA {
23namespace Internal {
24class PyGILRAII {
25 PyGILState_STATE m_GILState;
26
27public:
28 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
29 ~PyGILRAII() { PyGILState_Release(m_GILState); }
30};
31} // namespace Internal
32} // namespace TMVA
33
34REGISTER_METHOD(PyKeras)
35
37
38MethodPyKeras::MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption)
39 : PyMethodBase(jobName, Types::kPyKeras, methodTitle, dsi, theOption) {
40 fNumEpochs = 10;
41 fBatchSize = 100;
42 fVerbose = 1;
43 fContinueTraining = false;
44 fSaveBestOnly = true;
46 fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
47 fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
48 fTensorBoard = ""; // empty string deactivates TensorBoard callback
49}
50
51MethodPyKeras::MethodPyKeras(DataSetInfo &theData, const TString &theWeightFile)
52 : PyMethodBase(Types::kPyKeras, theData, theWeightFile) {
53 fNumEpochs = 10;
54 fNumThreads = 0;
55 fBatchSize = 100;
56 fVerbose = 1;
57 fContinueTraining = false;
58 fSaveBestOnly = true;
60 fLearningRateSchedule = ""; // empty string deactivates learning rate scheduler
61 fFilenameTrainedModel = ""; // empty string sets output model filename to default (in weights/)
62 fTensorBoard = ""; // empty string deactivates TensorBoard callback
63}
64
66}
67
69 if (type == Types::kRegression) return kTRUE;
70 if (type == Types::kClassification && numberClasses == 2) return kTRUE;
71 if (type == Types::kMulticlass && numberClasses >= 2) return kTRUE;
72 return kFALSE;
73}
74
75///////////////////////////////////////////////////////////////////////////////
76
78 DeclareOptionRef(fFilenameModel, "FilenameModel", "Filename of the initial Keras model");
79 DeclareOptionRef(fFilenameTrainedModel, "FilenameTrainedModel", "Filename of the trained output Keras model");
80 DeclareOptionRef(fBatchSize, "BatchSize", "Training batch size");
81 DeclareOptionRef(fNumEpochs, "NumEpochs", "Number of training epochs");
82 DeclareOptionRef(fNumThreads, "NumThreads", "Number of CPU threads (only for Tensorflow backend)");
83 DeclareOptionRef(fGpuOptions, "GpuOptions", "GPU options for tensorflow, such as allow_growth");
84 DeclareOptionRef(fUseTFKeras, "tf.keras", "Use tensorflow from Keras");
85 DeclareOptionRef(fUseTFKeras, "tfkeras", "Use tensorflow from Keras");
86 DeclareOptionRef(fVerbose, "Verbose", "Keras verbosity during training");
87 DeclareOptionRef(fContinueTraining, "ContinueTraining", "Load weights from previous training");
88 DeclareOptionRef(fSaveBestOnly, "SaveBestOnly", "Store only weights with smallest validation loss");
89 DeclareOptionRef(fTriesEarlyStopping, "TriesEarlyStopping", "Number of epochs with no improvement in validation loss after which training will be stopped. The default or a negative number deactivates this option.");
90 DeclareOptionRef(fLearningRateSchedule, "LearningRateSchedule", "Set new learning rate during training at specific epochs, e.g., \"50,0.01;70,0.005\"");
91 DeclareOptionRef(fTensorBoard, "TensorBoard",
92 "Write a log during training to visualize and monitor the training performance with TensorBoard");
93
94 DeclareOptionRef(fNumValidationString = "20%", "ValidationSize", "Part of the training data to use for validation. "
95 "Specify as 0.2 or 20% to use a fifth of the data set as validation set. "
96 "Specify as 100 to use exactly 100 events. (Default: 20%)");
97 DeclareOptionRef(fUserCodeName = "", "UserCode",
98 "Optional python code provided by the user to be executed before loading the Keras model");
99}
100
101////////////////////////////////////////////////////////////////////////////////
102/// Validation of the ValidationSize option. Allowed formats are 20%, 0.2 and
103/// 100 etc.
104/// - 20% and 0.2 selects 20% of the training set as validation data.
105/// - 100 selects 100 events as the validation data.
106///
107/// @return number of samples in validation set
108///
110{
111 Int_t nValidationSamples = 0;
112 UInt_t trainingSetSize = GetEventCollection(Types::kTraining).size();
113
114 // Parsing + Validation
115 // --------------------
116 if (fNumValidationString.EndsWith("%")) {
117 // Relative spec. format 20%
118 TString intValStr = TString(fNumValidationString.Strip(TString::kTrailing, '%'));
119
120 if (intValStr.IsFloat()) {
121 Double_t valSizeAsDouble = fNumValidationString.Atof() / 100.0;
122 nValidationSamples = GetEventCollection(Types::kTraining).size() * valSizeAsDouble;
123 } else {
124 Log() << kFATAL << "Cannot parse number \"" << fNumValidationString
125 << "\". Expected string like \"20%\" or \"20.0%\"." << Endl;
126 }
127 } else if (fNumValidationString.IsFloat()) {
128 Double_t valSizeAsDouble = fNumValidationString.Atof();
129
130 if (valSizeAsDouble < 1.0) {
131 // Relative spec. format 0.2
132 nValidationSamples = GetEventCollection(Types::kTraining).size() * valSizeAsDouble;
133 } else {
134 // Absolute spec format 100 or 100.0
135 nValidationSamples = valSizeAsDouble;
136 }
137 } else {
138 Log() << kFATAL << "Cannot parse number \"" << fNumValidationString << "\". Expected string like \"0.2\" or \"100\"."
139 << Endl;
140 }
141
142 // Value validation
143 // ----------------
144 if (nValidationSamples < 0) {
145 Log() << kFATAL << "Validation size \"" << fNumValidationString << "\" is negative." << Endl;
146 }
147
148 if (nValidationSamples == 0) {
149 Log() << kFATAL << "Validation size \"" << fNumValidationString << "\" is zero." << Endl;
150 }
151
152 if (nValidationSamples >= (Int_t)trainingSetSize) {
153 Log() << kFATAL << "Validation size \"" << fNumValidationString
154 << "\" is larger than or equal in size to training set (size=\"" << trainingSetSize << "\")." << Endl;
155 }
156
157 return nValidationSamples;
158}
159
160/// Function processing the options
161/// This is called only when creating the method before training not when
162/// readinf from XML file. Called from MethodBase::ProcessSetup
163/// that is called from Factory::BookMethod
165
166 // Set default filename for trained model if option is not used
168 fFilenameTrainedModel = GetWeightFileDir() + "/TrainedModel_" + GetName() + ".h5";
169 }
170
171 // Setup model, either the initial model from `fFilenameModel` or
172 // the trained model from `fFilenameTrainedModel`
173 if (fContinueTraining) Log() << kINFO << "Continue training with trained model" << Endl;
175}
176
177void MethodPyKeras::SetupKerasModel(bool loadTrainedModel) {
178
179 // initialize first Keras. This is done only here when class has
180 // all state variable set from options or read from XML file
181 // Import Keras
182
183 if (fUseTFKeras)
184 Log() << kINFO << "Setting up tf.keras" << Endl;
185 else
186 Log() << kINFO << "Setting up keras with " << gSystem->Getenv("KERAS_BACKEND") << " backend" << Endl;
187
188 bool useTFBackend = kFALSE;
189 bool kerasIsCompatible = kTRUE;
190 bool kerasIsPresent = kFALSE;
191
192 if (!fUseTFKeras) {
193 auto ret = PyRun_String("import keras", Py_single_input, fGlobalNS, fLocalNS);
194 // need importing also in global namespace
195 if (ret != nullptr) ret = PyRun_String("import keras", Py_single_input, fGlobalNS, fGlobalNS);
196 if (ret != nullptr)
197 kerasIsPresent = kTRUE;
198 if (kerasIsPresent) {
199 // check compatibility with tensorflow
200 if (GetKerasBackend() == kTensorFlow ) {
201 useTFBackend = kTRUE;
202
203 PyRunString("keras_major_version = int(keras.__version__.split('.')[0])");
204 PyRunString("keras_minor_version = int(keras.__version__.split('.')[1])");
205 PyObject *pyKerasMajorVersion = PyDict_GetItemString(fLocalNS, "keras_major_version");
206 PyObject *pyKerasMinorVersion = PyDict_GetItemString(fLocalNS, "keras_minor_version");
207 int kerasMajorVersion = PyLong_AsLong(pyKerasMajorVersion);
208 int kerasMinorVersion = PyLong_AsLong(pyKerasMinorVersion);
209 Log() << kINFO << "Using Keras version " << kerasMajorVersion << "." << kerasMinorVersion << Endl;
210 // only version 2.3 is latest multi-backend version.
211 // version 2.4 is just tf.keras and should not be used in standalone and will not work in this workflow
212 // see https://github.com/keras-team/keras/releases/tag/2.4.0
213 // for example variable keras.backend.tensorflow_backend will not exist anymore in keras 2.4
214 kerasIsCompatible = (kerasMajorVersion >= 2 && kerasMinorVersion == 3);
215
216 }
217 } else {
218 // Keras is not found. try tyo use tf.keras
219 Log() << kINFO << "Keras is not found. Trying using tf.keras" << Endl;
220 fUseTFKeras = 1;
221 }
222 }
223
224 // import Tensoprflow (if requested or because is keras backend)
225 if (fUseTFKeras || useTFBackend) {
226 auto ret = PyRun_String("import tensorflow as tf", Py_single_input, fGlobalNS, fLocalNS);
227 if (ret != nullptr) ret = PyRun_String("import tensorflow as tf", Py_single_input, fGlobalNS, fGlobalNS);
228 if (ret == nullptr) {
229 Log() << kFATAL << "Importing TensorFlow failed" << Endl;
230 }
231 // check tensorflow version
232 PyRunString("tf_major_version = int(tf.__version__.split('.')[0])");
233 PyObject *pyTfVersion = PyDict_GetItemString(fLocalNS, "tf_major_version");
234 int tfVersion = PyLong_AsLong(pyTfVersion);
235 Log() << kINFO << "Using TensorFlow version " << tfVersion << Endl;
236
237 if (tfVersion < 2) {
238 if (fUseTFKeras == 1) {
239 Log() << kWARNING << "Using TensorFlow version 1.x which does not contain tf.keras - use then TensorFlow as Keras backend" << Endl;
241 // case when Keras was not found
242 if (!kerasIsPresent) {
243 Log() << kFATAL << "Keras is not present and not a suitable TensorFlow version is found " << Endl;
244 return;
245 }
246 }
247 }
248 else {
249 // using version larger than 2.0 - can use tf.keras
250 if (!kerasIsCompatible) {
251 Log() << kWARNING << "The Keras version is not compatible with TensorFlow 2. Use instead tf.keras" << Endl;
252 fUseTFKeras = 1;
253 }
254 }
255
256 // if keras 2.3 and tensorflow 2 are found. Use tf.keras or keras ?
257 // at the moment default is tf.keras=false to keep compatibility
258 // but this might change in future releases
259 if (fUseTFKeras) {
260 Log() << kINFO << "Use Keras version from TensorFlow : tf.keras" << Endl;
261 fKerasString = "tf.keras";
262 PyRunString("K = tf.keras.backend");
263 PyRun_String("K = tf.keras.backend", Py_single_input, fGlobalNS, fGlobalNS);
264 }
265 else {
266 Log() << kINFO << "Use TensorFlow as Keras backend" << Endl;
267 fKerasString = "keras";
268 PyRunString("from keras.backend import tensorflow_backend as K");
269 PyRun_String("from keras.backend import tensorflow_backend as K", Py_single_input, fGlobalNS, fGlobalNS);
270 }
271
272 // extra options for tensorflow
273 // use different naming in tf2 for ConfigProto and Session
274 TString configProto = (tfVersion >= 2) ? "tf.compat.v1.ConfigProto" : "tf.ConfigProto";
275 TString session = (tfVersion >= 2) ? "tf.compat.v1.Session" : "tf.Session";
276
277 // in case specify number of threads
278 int num_threads = fNumThreads;
279 if (num_threads > 0) {
280 Log() << kINFO << "Setting the CPU number of threads = " << num_threads << Endl;
281
283 TString::Format("session_conf = %s(intra_op_parallelism_threads=%d,inter_op_parallelism_threads=%d)",
284 configProto.Data(), num_threads, num_threads));
285 } else
286 PyRunString(TString::Format("session_conf = %s()", configProto.Data()));
287
288 // applying GPU options such as allow_growth=True to avoid allocating all memory on GPU
289 // that prevents running later TMVA-GPU
290 // Also new Nvidia RTX cards (e.g. RTX 2070) require this option
291 if (!fGpuOptions.IsNull()) {
292 TObjArray *optlist = fGpuOptions.Tokenize(",");
293 for (int item = 0; item < optlist->GetEntries(); ++item) {
294 Log() << kINFO << "Applying GPU option: gpu_options." << optlist->At(item)->GetName() << Endl;
295 PyRunString(TString::Format("session_conf.gpu_options.%s", optlist->At(item)->GetName()));
296 }
297 }
298 PyRunString(TString::Format("sess = %s(config=session_conf)", session.Data()));
299
300 if (tfVersion < 2) {
301 PyRunString("K.set_session(sess)");
302 } else {
303 PyRunString("tf.compat.v1.keras.backend.set_session(sess)");
304 }
305 }
306 // case not using a Tensorflow backend
307 else {
308 fKerasString = "keras";
309 if (fNumThreads > 0)
310 Log() << kWARNING << "Cannot set the given " << fNumThreads << " threads when not using tensorflow as backend"
311 << Endl;
312 if (!fGpuOptions.IsNull()) {
313 Log() << kWARNING << "Cannot set the given GPU option " << fGpuOptions
314 << " when not using tensorflow as backend" << Endl;
315 }
316 }
317
318 /*
319 * Load Keras model from file
320 */
321
322 Log() << kINFO << " Loading Keras Model " << Endl;
323
324 PyRunString("load_model_custom_objects=None");
325
326
327
328 if (!fUserCodeName.IsNull()) {
329 Log() << kINFO << " Executing user initialization code from " << fUserCodeName << Endl;
330
331
332 // run some python code provided by user for model initialization if needed
333 TString cmd = "exec(open('" + fUserCodeName + "').read())";
334 TString errmsg = "Error executing the provided user code";
335 PyRunString(cmd, errmsg);
336
337 PyRunString("print('custom objects for loading model : ',load_model_custom_objects)");
338 }
339
340 // Load initial model or already trained model
341 TString filenameLoadModel;
342 if (loadTrainedModel) {
343 filenameLoadModel = fFilenameTrainedModel;
344 }
345 else {
346 filenameLoadModel = fFilenameModel;
347 }
348
349 PyRunString("model = " + fKerasString + ".models.load_model('" + filenameLoadModel +
350 "', custom_objects=load_model_custom_objects)", "Failed to load Keras model from file: " + filenameLoadModel);
351
352 Log() << kINFO << "Loaded model from file: " << filenameLoadModel << Endl;
353
354
355 /*
356 * Init variables and weights
357 */
358
359 // Get variables, classes and target numbers
363 else Log() << kFATAL << "Selected analysis type is not implemented" << Endl;
364
365 // Init evaluation (needed for getMvaValue)
366 fVals = new float[fNVars]; // holds values used for classification and regression
367 npy_intp dimsVals[2] = {(npy_intp)1, (npy_intp)fNVars};
368 PyArrayObject* pVals = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsVals, NPY_FLOAT, (void*)fVals);
369 PyDict_SetItemString(fLocalNS, "vals", (PyObject*)pVals);
370
371 fOutput.resize(fNOutputs); // holds classification probabilities or regression output
372 npy_intp dimsOutput[2] = {(npy_intp)1, (npy_intp)fNOutputs};
373 PyArrayObject* pOutput = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsOutput, NPY_FLOAT, (void*)&fOutput[0]);
374 PyDict_SetItemString(fLocalNS, "output", (PyObject*)pOutput);
375
376 // Mark the model as setup
377 fModelIsSetup = true;
378}
379
380/// Initialization function called from MethodBase::SetupMethod()
381/// Note that option string are not yet filled with their values.
382/// This is done before ProcessOption method or after reading from XML file
384
386
387 if (!PyIsInitialized()) {
388 Log() << kFATAL << "Python is not initialized" << Endl;
389 }
390 _import_array(); // required to use numpy arrays
391
392 // NOTE: sys.argv has to be cleared because otherwise TensorFlow breaks
393 PyRunString("import sys; sys.argv = ['']", "Set sys.argv failed");
394
395 // Set flag that model is not setup
396 fModelIsSetup = false;
397}
398
400
401 if(!fModelIsSetup) Log() << kFATAL << "Model is not setup for training" << Endl;
402
403 /*
404 * Load training data to numpy array
405 */
406
407 UInt_t nAllEvents = Data()->GetNTrainingEvents();
408 UInt_t nValEvents = GetNumValidationSamples();
409 UInt_t nTrainingEvents = nAllEvents - nValEvents;
410
411 Log() << kINFO << "Split TMVA training data in " << nTrainingEvents << " training events and "
412 << nValEvents << " validation events" << Endl;
413
414 float* trainDataX = new float[nTrainingEvents*fNVars];
415 float* trainDataY = new float[nTrainingEvents*fNOutputs];
416 float* trainDataWeights = new float[nTrainingEvents];
417 for (UInt_t i=0; i<nTrainingEvents; i++) {
418 const TMVA::Event* e = GetTrainingEvent(i);
419 // Fill variables
420 for (UInt_t j=0; j<fNVars; j++) {
421 trainDataX[j + i*fNVars] = e->GetValue(j);
422 }
423 // Fill targets
424 // NOTE: For classification, convert class number in one-hot vector,
425 // e.g., 1 -> [0, 1] or 0 -> [1, 0] for binary classification
427 for (UInt_t j=0; j<fNOutputs; j++) {
428 trainDataY[j + i*fNOutputs] = 0;
429 }
430 trainDataY[e->GetClass() + i*fNOutputs] = 1;
431 }
432 else if (GetAnalysisType() == Types::kRegression) {
433 for (UInt_t j=0; j<fNOutputs; j++) {
434 trainDataY[j + i*fNOutputs] = e->GetTarget(j);
435 }
436 }
437 else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
438 // Fill weights
439 // NOTE: If no weight branch is given, this defaults to ones for all events
440 trainDataWeights[i] = e->GetWeight();
441 }
442
443 npy_intp dimsTrainX[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNVars};
444 npy_intp dimsTrainY[2] = {(npy_intp)nTrainingEvents, (npy_intp)fNOutputs};
445 npy_intp dimsTrainWeights[1] = {(npy_intp)nTrainingEvents};
446 PyArrayObject* pTrainDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainX, NPY_FLOAT, (void*)trainDataX);
447 PyArrayObject* pTrainDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsTrainY, NPY_FLOAT, (void*)trainDataY);
448 PyArrayObject* pTrainDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsTrainWeights, NPY_FLOAT, (void*)trainDataWeights);
449 PyDict_SetItemString(fLocalNS, "trainX", (PyObject*)pTrainDataX);
450 PyDict_SetItemString(fLocalNS, "trainY", (PyObject*)pTrainDataY);
451 PyDict_SetItemString(fLocalNS, "trainWeights", (PyObject*)pTrainDataWeights);
452
453 /*
454 * Load validation data to numpy array
455 */
456
457 // NOTE: from TMVA, we get the validation data as a subset of all the training data
458 // we will not use test data for validation. They will be used for the real testing
459
460
461 float* valDataX = new float[nValEvents*fNVars];
462 float* valDataY = new float[nValEvents*fNOutputs];
463 float* valDataWeights = new float[nValEvents];
464 //validation events follows the trainig one in the TMVA training vector
465 for (UInt_t i=0; i< nValEvents ; i++) {
466 UInt_t ievt = nTrainingEvents + i; // TMVA event index
467 const TMVA::Event* e = GetTrainingEvent(ievt);
468 // Fill variables
469 for (UInt_t j=0; j<fNVars; j++) {
470 valDataX[j + i*fNVars] = e->GetValue(j);
471 }
472 // Fill targets
474 for (UInt_t j=0; j<fNOutputs; j++) {
475 valDataY[j + i*fNOutputs] = 0;
476 }
477 valDataY[e->GetClass() + i*fNOutputs] = 1;
478 }
479 else if (GetAnalysisType() == Types::kRegression) {
480 for (UInt_t j=0; j<fNOutputs; j++) {
481 valDataY[j + i*fNOutputs] = e->GetTarget(j);
482 }
483 }
484 else Log() << kFATAL << "Can not fill target vector because analysis type is not known" << Endl;
485 // Fill weights
486 valDataWeights[i] = e->GetWeight();
487 }
488
489 npy_intp dimsValX[2] = {(npy_intp)nValEvents, (npy_intp)fNVars};
490 npy_intp dimsValY[2] = {(npy_intp)nValEvents, (npy_intp)fNOutputs};
491 npy_intp dimsValWeights[1] = {(npy_intp)nValEvents};
492 PyArrayObject* pValDataX = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValX, NPY_FLOAT, (void*)valDataX);
493 PyArrayObject* pValDataY = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsValY, NPY_FLOAT, (void*)valDataY);
494 PyArrayObject* pValDataWeights = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsValWeights, NPY_FLOAT, (void*)valDataWeights);
495 PyDict_SetItemString(fLocalNS, "valX", (PyObject*)pValDataX);
496 PyDict_SetItemString(fLocalNS, "valY", (PyObject*)pValDataY);
497 PyDict_SetItemString(fLocalNS, "valWeights", (PyObject*)pValDataWeights);
498
499 /*
500 * Train Keras model
501 */
502 Log() << kINFO << "Training Model Summary" << Endl;
503 PyRunString("model.summary()");
504
505 // Setup parameters
506
507 PyObject* pBatchSize = PyLong_FromLong(fBatchSize);
508 PyObject* pNumEpochs = PyLong_FromLong(fNumEpochs);
509 PyObject* pVerbose = PyLong_FromLong(fVerbose);
510 PyDict_SetItemString(fLocalNS, "batchSize", pBatchSize);
511 PyDict_SetItemString(fLocalNS, "numEpochs", pNumEpochs);
512 PyDict_SetItemString(fLocalNS, "verbose", pVerbose);
513
514 // Setup training callbacks
515 PyRunString("callbacks = []");
516
517 // Callback: Save only weights with smallest validation loss
518 if (fSaveBestOnly) {
519 PyRunString("callbacks.append(" + fKerasString +".callbacks.ModelCheckpoint('"+fFilenameTrainedModel+"', monitor='val_loss', verbose=verbose, save_best_only=True, mode='auto'))", "Failed to setup training callback: SaveBestOnly");
520 Log() << kINFO << "Option SaveBestOnly: Only model weights with smallest validation loss will be stored" << Endl;
521 }
522
523 // Callback: Stop training early if no improvement in validation loss is observed
524 if (fTriesEarlyStopping>=0) {
525 TString tries;
526 tries.Form("%i", fTriesEarlyStopping);
527 PyRunString("callbacks.append(" + fKerasString + ".callbacks.EarlyStopping(monitor='val_loss', patience="+tries+", verbose=verbose, mode='auto'))", "Failed to setup training callback: TriesEarlyStopping");
528 Log() << kINFO << "Option TriesEarlyStopping: Training will stop after " << tries << " number of epochs with no improvement of validation loss" << Endl;
529 }
530
531 // Callback: Learning rate scheduler
532 if (fLearningRateSchedule!="") {
533 // Setup a python dictionary with the desired learning rate steps
534 PyRunString("strScheduleSteps = '"+fLearningRateSchedule+"'\n"
535 "schedulerSteps = {}\n"
536 "for c in strScheduleSteps.split(';'):\n"
537 " x = c.split(',')\n"
538 " schedulerSteps[int(x[0])] = float(x[1])\n",
539 "Failed to setup steps for scheduler function from string: "+fLearningRateSchedule,
540 Py_file_input);
541 // Set scheduler function as piecewise function with given steps
542 PyRunString("def schedule(epoch, model=model, schedulerSteps=schedulerSteps):\n"
543 " if epoch in schedulerSteps: return float(schedulerSteps[epoch])\n"
544 " else: return float(model.optimizer.lr.get_value())\n",
545 "Failed to setup scheduler function with string: "+fLearningRateSchedule,
546 Py_file_input);
547 // Setup callback
548 PyRunString("callbacks.append(" + fKerasString + ".callbacks.LearningRateScheduler(schedule))",
549 "Failed to setup training callback: LearningRateSchedule");
550 Log() << kINFO << "Option LearningRateSchedule: Set learning rate during training: " << fLearningRateSchedule << Endl;
551 }
552
553 // Callback: TensorBoard
554 if (fTensorBoard != "") {
555 TString logdir = TString("'") + fTensorBoard + TString("'");
557 "callbacks.append(" + fKerasString + ".callbacks.TensorBoard(log_dir=" + logdir +
558 ", histogram_freq=0, batch_size=batchSize, write_graph=True, write_grads=False, write_images=False))",
559 "Failed to setup training callback: TensorBoard");
560 Log() << kINFO << "Option TensorBoard: Log files for training monitoring are stored in: " << logdir << Endl;
561 }
562
563 // Train model
564 PyRunString("history = model.fit(trainX, trainY, sample_weight=trainWeights, batch_size=batchSize, epochs=numEpochs, verbose=verbose, validation_data=(valX, valY, valWeights), callbacks=callbacks)",
565 "Failed to train model");
566
567
568 std::vector<float> fHistory; // Hold training history (val_acc or loss etc)
569 fHistory.resize(fNumEpochs); // holds training loss or accuracy output
570 npy_intp dimsHistory[1] = { (npy_intp)fNumEpochs};
571 PyArrayObject* pHistory = (PyArrayObject*)PyArray_SimpleNewFromData(1, dimsHistory, NPY_FLOAT, (void*)&fHistory[0]);
572 PyDict_SetItemString(fLocalNS, "HistoryOutput", (PyObject*)pHistory);
573
574 // Store training history data
575 Int_t iHis=0;
576 PyRunString("number_of_keys=len(history.history.keys())");
577 PyObject* PyNkeys=PyDict_GetItemString(fLocalNS, "number_of_keys");
578 int nkeys=PyLong_AsLong(PyNkeys);
579 for (iHis=0; iHis<nkeys; iHis++) {
580
581 PyRunString(TString::Format("copy_string=str(list(history.history.keys())[%d])",iHis));
582 PyObject* stra=PyDict_GetItemString(fLocalNS, "copy_string");
583 if(!stra) break;
584#if PY_MAJOR_VERSION < 3 // for Python2
585 const char *stra_name = PyBytes_AsString(stra);
586 // need to add string delimiter for Python2
587 TString sname = TString::Format("'%s'",stra_name);
588 const char * name = sname.Data();
589#else // for Python3
590 PyObject* repr = PyObject_Repr(stra);
591 PyObject* str = PyUnicode_AsEncodedString(repr, "utf-8", "~E~");
592 const char *name = PyBytes_AsString(str);
593#endif
594
595 Log() << kINFO << "Getting training history for item:" << iHis << " name = " << name << Endl;
596 PyRunString(TString::Format("for i,p in enumerate(history.history[%s]):\n HistoryOutput[i]=p\n",name),
597 TString::Format("Failed to get %s from training history",name));
598 for (size_t i=0; i<fHistory.size(); i++)
599 fTrainHistory.AddValue(name,i+1,fHistory[i]);
600
601 }
602//#endif
603
604 /*
605 * Store trained model to file (only if option 'SaveBestOnly' is NOT activated,
606 * because we do not want to override the best model checkpoint)
607 */
608
609 if (!fSaveBestOnly) {
610 PyRunString("model.save('"+fFilenameTrainedModel+"', overwrite=True)",
611 "Failed to save trained model: "+fFilenameTrainedModel);
612 Log() << kINFO << "Trained model written to file: " << fFilenameTrainedModel << Endl;
613 }
614
615 /*
616 * Clean-up
617 */
618
619 delete[] trainDataX;
620 delete[] trainDataY;
621 delete[] trainDataWeights;
622 delete[] valDataX;
623 delete[] valDataY;
624 delete[] valDataWeights;
625}
626
629}
630
632 // Cannot determine error
633 NoErrorCalc(errLower, errUpper);
634
635 // Check whether the model is setup
636 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
637 if (!fModelIsSetup) {
638 // Setup the trained model
639 SetupKerasModel(true);
640 }
641
642 // Get signal probability (called mvaValue here)
643 const TMVA::Event* e = GetEvent();
644 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
645 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
646 "Failed to get predictions");
647
649}
650
651std::vector<Double_t> MethodPyKeras::GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress) {
652 // Check whether the model is setup
653 // NOTE: Unfortunately this is needed because during evaluation ProcessOptions is not called again
654 if (!fModelIsSetup) {
655 // Setup the trained model
656 SetupKerasModel(true);
657 }
658
659 // Load data to numpy array
660 Long64_t nEvents = Data()->GetNEvents();
661 if (firstEvt > lastEvt || lastEvt > nEvents) lastEvt = nEvents;
662 if (firstEvt < 0) firstEvt = 0;
663 nEvents = lastEvt-firstEvt;
664
665 // use timer
666 Timer timer( nEvents, GetName(), kTRUE );
667
668 if (logProgress)
669 Log() << kHEADER << Form("[%s] : ",DataInfo().GetName())
670 << "Evaluation of " << GetMethodName() << " on "
671 << (Data()->GetCurrentType() == Types::kTraining ? "training" : "testing")
672 << " sample (" << nEvents << " events)" << Endl;
673
674 float* data = new float[nEvents*fNVars];
675 for (UInt_t i=0; i<nEvents; i++) {
676 Data()->SetCurrentEvent(i);
677 const TMVA::Event *e = GetEvent();
678 for (UInt_t j=0; j<fNVars; j++) {
679 data[j + i*fNVars] = e->GetValue(j);
680 }
681 }
682
683 npy_intp dimsData[2] = {(npy_intp)nEvents, (npy_intp)fNVars};
684 PyArrayObject* pDataMvaValues = (PyArrayObject*)PyArray_SimpleNewFromData(2, dimsData, NPY_FLOAT, (void*)data);
685 if (pDataMvaValues==0) Log() << "Failed to load data to Python array" << Endl;
686
687 // Get prediction for all events
688 PyObject* pModel = PyDict_GetItemString(fLocalNS, "model");
689 if (pModel==0) Log() << kFATAL << "Failed to get model Python object" << Endl;
690 PyArrayObject* pPredictions = (PyArrayObject*) PyObject_CallMethod(pModel, (char*)"predict", (char*)"O", pDataMvaValues);
691 if (pPredictions==0) Log() << kFATAL << "Failed to get predictions" << Endl;
692 delete[] data;
693
694 // Load predictions to double vector
695 // NOTE: The signal probability is given at the output
696 std::vector<double> mvaValues(nEvents);
697 float* predictionsData = (float*) PyArray_DATA(pPredictions);
698 for (UInt_t i=0; i<nEvents; i++) {
699 mvaValues[i] = (double) predictionsData[i*fNOutputs + TMVA::Types::kSignal];
700 }
701
702 if (logProgress) {
703 Log() << kINFO
704 << "Elapsed time for evaluation of " << nEvents << " events: "
705 << timer.GetElapsedTime() << " " << Endl;
706 }
707
708
709 return mvaValues;
710}
711
712std::vector<Float_t>& MethodPyKeras::GetRegressionValues() {
713 // Check whether the model is setup
714 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
715 if (!fModelIsSetup){
716 // Setup the model and load weights
717 SetupKerasModel(true);
718 }
719
720 // Get regression values
721 const TMVA::Event* e = GetEvent();
722 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
723 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
724 "Failed to get predictions");
725
726 // Use inverse transformation of targets to get final regression values
727 Event * eTrans = new Event(*e);
728 for (UInt_t i=0; i<fNOutputs; ++i) {
729 eTrans->SetTarget(i,fOutput[i]);
730 }
731
732 const Event* eTrans2 = GetTransformationHandler().InverseTransform(eTrans);
733 for (UInt_t i=0; i<fNOutputs; ++i) {
734 fOutput[i] = eTrans2->GetTarget(i);
735 }
736
737 return fOutput;
738}
739
740std::vector<Float_t>& MethodPyKeras::GetMulticlassValues() {
741 // Check whether the model is setup
742 // NOTE: unfortunately this is needed because during evaluation ProcessOptions is not called again
743 if (!fModelIsSetup){
744 // Setup the model and load weights
745 SetupKerasModel(true);
746 }
747
748 // Get class probabilites
749 const TMVA::Event* e = GetEvent();
750 for (UInt_t i=0; i<fNVars; i++) fVals[i] = e->GetValue(i);
751 PyRunString("for i,p in enumerate(model.predict(vals)): output[i]=p\n",
752 "Failed to get predictions");
753
754 return fOutput;
755}
756
758}
759
761// typical length of text line:
762// "|--------------------------------------------------------------|"
763 Log() << Endl;
764 Log() << "Keras is a high-level API for the Theano and Tensorflow packages." << Endl;
765 Log() << "This method wraps the training and predictions steps of the Keras" << Endl;
766 Log() << "Python package for TMVA, so that dataloading, preprocessing and" << Endl;
767 Log() << "evaluation can be done within the TMVA system. To use this Keras" << Endl;
768 Log() << "interface, you have to generate a model with Keras first. Then," << Endl;
769 Log() << "this model can be loaded and trained in TMVA." << Endl;
770 Log() << Endl;
771}
772
774 // get the keras backend
775
776 // in case we use tf.keras backend is tensorflow
777 if (UseTFKeras()) return kTensorFlow;
778
779 // check first if using tensorflow backend
780 PyRunString("keras_backend_is_set = keras.backend.backend() == \"tensorflow\"");
781 PyObject * keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
782 if (keras_backend != nullptr && keras_backend == Py_True)
783 return kTensorFlow;
784
785 PyRunString("keras_backend_is_set = keras.backend.backend() == \"theano\"");
786 keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
787 if (keras_backend != nullptr && keras_backend == Py_True)
788 return kTheano;
789
790 PyRunString("keras_backend_is_set = keras.backend.backend() == \"cntk\"");
791 keras_backend = PyDict_GetItemString(fLocalNS,"keras_backend_is_set");
792 if (keras_backend != nullptr && keras_backend == Py_True)
793 return kCNTK;
794
795 return kUndefined;
796}
797
799 // get the keras backend name
801 if (type == kTensorFlow) return "TensorFlow";
802 if (type == kTheano) return "Theano";
803 if (type == kCNTK) return "CNTK";
804 return "Undefined";
805}
#define PyBytes_AsString
Definition: CPyCppyy.h:86
#define REGISTER_METHOD(CLASS)
for example
_object PyObject
Definition: PyMethodBase.h:43
#define Py_single_input
Definition: PyMethodBase.h:44
#define e(i)
Definition: RSha256.hxx:103
const Bool_t kFALSE
Definition: RtypesCore.h:101
long long Long64_t
Definition: RtypesCore.h:80
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:375
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void data
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t Int_t Int_t Window_t TString Int_t GCValues_t GetPrimarySelectionOwner GetDisplay GetScreen GetColormap GetNativeEvent const char const char dpyName wid window const char font_name cursor keysym reg const char only_if_exist regb h Point_t winding char text const char depth char const char Int_t count const char ColorStruct_t color const char Pixmap_t Pixmap_t PictureAttributes_t attr const char char ret_data h unsigned char height h Atom_t Int_t ULong_t ULong_t unsigned char prop_list Atom_t Atom_t Atom_t Time_t type
char name[80]
Definition: TGX11.cxx:110
char * Form(const char *fmt,...)
Formats a string in a circular formatting buffer.
Definition: TString.cxx:2468
R__EXTERN TSystem * gSystem
Definition: TSystem.h:560
OptionBase * DeclareOptionRef(T &ref, const TString &name, const TString &desc="")
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
UInt_t GetNClasses() const
Definition: DataSetInfo.h:155
UInt_t GetNTargets() const
Definition: DataSetInfo.h:128
Types::ETreeType GetCurrentType() const
Definition: DataSet.h:194
Long64_t GetNEvents(Types::ETreeType type=Types::kMaxTreeType) const
Definition: DataSet.h:206
Long64_t GetNTrainingEvents() const
Definition: DataSet.h:68
void SetCurrentEvent(Long64_t ievt) const
Definition: DataSet.h:88
void SetTarget(UInt_t itgt, Float_t value)
set the target value (dimension itgt) to value
Definition: Event.cxx:367
Float_t GetTarget(UInt_t itgt) const
Definition: Event.h:102
const char * GetName() const
Definition: MethodBase.h:334
Types::EAnalysisType GetAnalysisType() const
Definition: MethodBase.h:437
const TString & GetWeightFileDir() const
Definition: MethodBase.h:492
const TString & GetMethodName() const
Definition: MethodBase.h:331
const Event * GetEvent() const
Definition: MethodBase.h:751
DataSetInfo & DataInfo() const
Definition: MethodBase.h:410
virtual void TestClassification()
initialization
UInt_t GetNVariables() const
Definition: MethodBase.h:345
TransformationHandler & GetTransformationHandler(Bool_t takeReroutedIfAvailable=true)
Definition: MethodBase.h:394
void NoErrorCalc(Double_t *const err, Double_t *const errUpper)
Definition: MethodBase.cxx:836
TrainingHistory fTrainHistory
Definition: MethodBase.h:425
DataSet * Data() const
Definition: MethodBase.h:409
const Event * GetTrainingEvent(Long64_t ievt) const
Definition: MethodBase.h:771
void GetHelpMessage() const
void Init()
Initialization function called from MethodBase::SetupMethod() Note that option string are not yet fil...
std::vector< float > fOutput
virtual void TestClassification()
initialization
void ProcessOptions()
Function processing the options This is called only when creating the method before training not when...
Bool_t UseTFKeras() const
Definition: MethodPyKeras.h:80
EBackendType
enumeration defining the used Keras backend
Definition: MethodPyKeras.h:74
void SetupKerasModel(Bool_t loadTrainedModel)
std::vector< Float_t > & GetMulticlassValues()
UInt_t GetNumValidationSamples()
Validation of the ValidationSize option.
Double_t GetMvaValue(Double_t *errLower, Double_t *errUpper)
std::vector< Float_t > & GetRegressionValues()
TString fNumValidationString
Definition: MethodPyKeras.h:95
Bool_t HasAnalysisType(Types::EAnalysisType type, UInt_t numberClasses, UInt_t)
TString GetKerasBackendName()
MethodPyKeras(const TString &jobName, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
TString fLearningRateSchedule
Definition: MethodPyKeras.h:93
EBackendType GetKerasBackend()
Get the Keras backend (can be: TensorFlow, Theano or CNTK)
TString fFilenameTrainedModel
std::vector< Double_t > GetMvaValues(Long64_t firstEvt, Long64_t lastEvt, Bool_t logProgress)
get all the MVA values for the events of the current Data type
static int PyIsInitialized()
Check Python interpreter initialization status.
static PyObject * fGlobalNS
Definition: PyMethodBase.h:136
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=256)
Execute Python code from string.
PyObject * fLocalNS
Definition: PyMethodBase.h:137
Timing information for training and evaluation of MVA methods.
Definition: Timer.h:58
TString GetElapsedTime(Bool_t Scientific=kTRUE)
returns pretty string with elapsed time
Definition: Timer.cxx:146
void AddValue(TString Property, Int_t stage, Double_t value)
const Event * InverseTransform(const Event *, Bool_t suppressIfNoTargets=true) const
Singleton class for Global types used by TMVA.
Definition: Types.h:71
@ kSignal
Never change this number - it is elsewhere assumed to be zero !
Definition: Types.h:135
EAnalysisType
Definition: Types.h:126
@ kMulticlass
Definition: Types.h:129
@ kClassification
Definition: Types.h:127
@ kRegression
Definition: Types.h:128
@ kTraining
Definition: Types.h:143
@ kHEADER
Definition: Types.h:63
@ kINFO
Definition: Types.h:58
@ kWARNING
Definition: Types.h:59
@ kFATAL
Definition: Types.h:61
An array of TObjects.
Definition: TObjArray.h:31
Int_t GetEntries() const override
Return the number of objects in array (i.e.
Definition: TObjArray.cxx:523
TObject * At(Int_t idx) const override
Definition: TObjArray.h:164
virtual const char * GetName() const
Returns name of object.
Definition: TObject.cxx:439
Basic string class.
Definition: TString.h:136
Bool_t IsFloat() const
Returns kTRUE if string contains a floating point or integer number.
Definition: TString.cxx:1837
const char * Data() const
Definition: TString.h:369
@ kTrailing
Definition: TString.h:267
TObjArray * Tokenize(const TString &delim) const
This function is used to isolate sequential tokens in a TString.
Definition: TString.cxx:2243
Bool_t IsNull() const
Definition: TString.h:407
static TString Format(const char *fmt,...)
Static method which formats a string using a printf style format descriptor and return a TString.
Definition: TString.cxx:2357
void Form(const char *fmt,...)
Formats a string using a printf style format descriptor.
Definition: TString.cxx:2335
virtual const char * Getenv(const char *env)
Get environment variable.
Definition: TSystem.cxx:1666
create variable transformations
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:148
Double_t Log(Double_t x)
Returns the natural logarithm of x.
Definition: TMath.h:754