Logo ROOT  
Reference Guide
PyMethodBase.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : PyMethodBase *
8 * *
9 * Description: *
10 * Virtual base class for all MVA method based on python *
11 * *
12 **********************************************************************************/
13
14#include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15#include <TMVA/PyMethodBase.h>
16
17#include "TMVA/DataSet.h"
18#include "TMVA/DataSetInfo.h"
19#include "TMVA/MsgLogger.h"
20#include "TMVA/Results.h"
21#include "TMVA/Timer.h"
22#include "TMVA/Tools.h"
23
24#include "TSystem.h"
25
26#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
27#include <numpy/arrayobject.h>
28
29using namespace TMVA;
30
31namespace TMVA {
32namespace Internal {
33class PyGILRAII {
34 PyGILState_STATE m_GILState;
35
36public:
37 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
38 ~PyGILRAII() { PyGILState_Release(m_GILState); }
39};
40} // namespace Internal
41
42/// get current Python executable used by ROOT
44 TString python_version = gSystem->GetFromPipe("root-config --python-version");
45 if (python_version.IsNull()) {
46 TMVA::gTools().Log() << kFATAL << "Can't find a valid Python version used to build ROOT" << Endl;
47 return nullptr;
48 }
49#ifdef _MSC_VER
50 // on Windows there is a space before the version and the executable is python.exe
51 // for both versions of Python
52 python_version.ReplaceAll(" ", "");
53 if (python_version[0] == '2' || python_version[0] == '3')
54 return "python";
55#endif
56 if (python_version[0] == '2')
57 return "python";
58 else if (python_version[0] == '3')
59 return "python3";
60
61 TMVA::gTools().Log() << kFATAL << "Invalid Python version used to build ROOT : " << python_version << Endl;
62 return nullptr;
63}
64
65} // namespace TMVA
66
68
69// NOTE: Introduce here nothing that breaks if multiple instances
70// of the same method share these objects, e.g., the local namespace.
71PyObject *PyMethodBase::fModuleBuiltin = NULL;
72PyObject *PyMethodBase::fEval = NULL;
73PyObject *PyMethodBase::fOpen = NULL;
74
75PyObject *PyMethodBase::fModulePickle = NULL;
76PyObject *PyMethodBase::fPickleDumps = NULL;
77PyObject *PyMethodBase::fPickleLoads = NULL;
78
79PyObject *PyMethodBase::fMain = NULL;
80PyObject *PyMethodBase::fGlobalNS = NULL;
81
82///////////////////////////////////////////////////////////////////////////////
83
84PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
85 const TString &theOption)
86 : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
87 fClassifier(NULL)
88{
89 if (!PyIsInitialized()) {
91 }
92
93 // Set up private local namespace for each method instance
94 fLocalNS = PyDict_New();
95 if (!fLocalNS) {
96 Log() << kFATAL << "Can't init local namespace" << Endl;
97 }
98}
99
100///////////////////////////////////////////////////////////////////////////////
101
103 DataSetInfo &dsi,
104 const TString &weightFile): MethodBase(methodType, dsi, weightFile),
105 fClassifier(NULL)
106{
107 if (!PyIsInitialized()) {
108 PyInitialize();
109 }
110
111 // Set up private local namespace for each method instance
112 fLocalNS = PyDict_New();
113 if (!fLocalNS) {
114 Log() << kFATAL << "Can't init local namespace" << Endl;
115 }
116}
117
118///////////////////////////////////////////////////////////////////////////////
119
121{
122 // should we delete here fLocalNS ?
123}
124
125///////////////////////////////////////////////////////////////////////////////
126/// Evaluate Python code
127///
128/// \param[in] code Python code as string
129/// \return Python object from evaluation of code line
130///
131/// Take a Python code as input and evaluate it in the local namespace. Then,
132/// return the result as Python object.
133
135{
137 PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
138 PyObject *result = PyObject_CallObject(fEval, pycode);
139 Py_DECREF(pycode);
140 return result;
141}
142
143///////////////////////////////////////////////////////////////////////////////
144/// Initialize Python interpreter
145///
146/// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
147/// a private local namespace `fLocalNS`. This prohibits the interference
148/// of instances of the same method with the same factory, e.g., by overriding
149/// variables in the same local namespace.
150
152{
154
155 bool pyIsInitialized = PyIsInitialized();
156 if (!pyIsInitialized) {
157 Py_Initialize();
158 }
159
161 if (!pyIsInitialized) {
162 _import_array();
163 }
164
165 // note fMain is a borrowed reference
166 fMain = PyImport_AddModule("__main__");
167 if (!fMain) {
168 Log << kFATAL << "Can't import __main__" << Endl;
169 Log << Endl;
170 }
171 Py_INCREF(fMain);
172
173 fGlobalNS = PyModule_GetDict(fMain);
174 if (!fGlobalNS) {
175 Log << kFATAL << "Can't init global namespace" << Endl;
176 Log << Endl;
177 }
178 Py_INCREF(fGlobalNS);
179
180 #if PY_MAJOR_VERSION < 3
181 //preparing objects for eval
182 PyObject *bName = PyUnicode_FromString("__builtin__");
183 // Import the file as a Python module.
184 // returns a new reference
185 fModuleBuiltin = PyImport_Import(bName);
186 if (!fModuleBuiltin) {
187 Log << kFATAL << "Can't import __builtin__" << Endl;
188 Log << Endl;
189 }
190 #else
191 //preparing objects for eval
192 PyObject *bName = PyUnicode_FromString("builtins");
193 // Import the file as a Python module.
194 fModuleBuiltin = PyImport_Import(bName);
195 if (!fModuleBuiltin) {
196 Log << kFATAL << "Can't import builtins" << Endl;
197 Log << Endl;
198 }
199 #endif
200
201 // note mDict is a borrowed reference
202 PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
203 fEval = PyDict_GetItemString(mDict, "eval");
204 fOpen = PyDict_GetItemString(mDict, "open");
205 // fEval and fOpen are borrowed referencers and we need to keep them alive
206 if (fEval) Py_INCREF(fEval);
207 if (fOpen) Py_INCREF(fOpen);
208
209 // bName is a new reference (from PyUnicode_FromString)
210 Py_DECREF(bName);
211
212 //preparing objects for pickle
213 PyObject *pName = PyUnicode_FromString("pickle");
214 // Import the file as a Python module.
215 // return object is a new reference !
216 fModulePickle = PyImport_Import(pName);
217 if (!fModulePickle) {
218 Log << kFATAL << "Can't import pickle" << Endl;
219 Log << Endl;
220 }
221 PyObject *pDict = PyModule_GetDict(fModulePickle);
222 // note the following return objects are borrowed references
223 fPickleDumps = PyDict_GetItemString(pDict, "dump");
224 fPickleLoads = PyDict_GetItemString(pDict, "load");
225 if (fPickleDumps) Py_INCREF(fPickleDumps);
226 if (fPickleLoads) Py_INCREF(fPickleLoads);
227
228 Py_DECREF(pName);
229}
230
231///////////////////////////////////////////////////////////////////////////////
232// Finalize Python interpreter
233
235{
236 if (fEval) Py_DECREF(fEval);
237 if (fOpen) Py_DECREF(fOpen);
238 if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
239 if (fPickleDumps) Py_DECREF(fPickleDumps);
240 if (fPickleLoads) Py_DECREF(fPickleLoads);
241 if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
242 if (fGlobalNS) Py_DECREF(fGlobalNS);
243 Py_Finalize();
244}
245
246///////////////////////////////////////////////////////////////////////////////
247/// Check Python interpreter initialization status
248///
249/// \return Boolean whether interpreter is initialized
250
252{
253 if (!Py_IsInitialized()) return kFALSE;
254 if (!fEval) return kFALSE;
255 if (!fModuleBuiltin) return kFALSE;
256 if (!fPickleDumps) return kFALSE;
257 if (!fPickleLoads) return kFALSE;
258 return kTRUE;
259}
260
261///////////////////////////////////////////////////////////////////////////////
262/// Serialize Python object
263///
264/// \param[in] path Path where object is written to file
265/// \param[in] obj Python object
266///
267/// The input Python object is serialized and written to a file. The Python
268/// module `pickle` is used to do so.
269
271{
273
274 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
275 PyObject *file = PyObject_CallObject(fOpen,file_arg);
276 PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
277 PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
278
279 Py_DECREF(file_arg);
280 Py_DECREF(file);
281 Py_DECREF(model_arg);
282 Py_DECREF(model_data);
283}
284
285///////////////////////////////////////////////////////////////////////////////
286/// Unserialize Python object
287///
288/// \param[in] path Path to serialized Python object
289/// \param[in] obj Python object where the unserialized Python object is loaded
290/// \return Error code
291
293{
294 // Load file
295 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
296 PyObject *file = PyObject_CallObject(fOpen,file_arg);
297 if(!file) return 1;
298
299 // Load object from file using pickle
300 PyObject *model_arg = Py_BuildValue("(O)", file);
301 *obj = PyObject_CallObject(fPickleLoads , model_arg);
302 if(!obj) return 2;
303
304 Py_DECREF(file_arg);
305 Py_DECREF(file);
306 Py_DECREF(model_arg);
307
308 return 0;
309}
310
311///////////////////////////////////////////////////////////////////////////////
312/// Execute Python code from string
313///
314/// \param[in] code Python code as string
315/// \param[in] errorMessage Error message which shall be shown if the execution fails
316/// \param[in] start Start symbol
317///
318/// Helper function to run python code from string in local namespace with
319/// error handling
320/// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
321/// Py_single_input, Py_file_input)
322
323void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
324 fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
325 if (!fPyReturn) {
326 Log() << kWARNING << "Failed to run python code: " << code << Endl;
327 Log() << kWARNING << "Python error message:" << Endl;
328 PyErr_Print();
329 Log() << kFATAL << errorMessage << Endl;
330 }
331}
332
333///////////////////////////////////////////////////////////////////////////////
334/// Execute Python code from string
335///
336/// \param[in] code Python code as string
337/// \param[in] globalNS Global Namespace for Python Session
338/// \param[in] localNS Local Namespace for Python Session
339///
340/// Overloaded static Helper function to run python code
341/// from string and throw runtime error if the Python session
342/// is unable to execute the code
343
344void PyMethodBase::PyRunString(TString code, PyObject *globalNS, PyObject *localNS){
345 PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
346 if (!fPyReturn) {
347 std::cout<<"\nPython error message:\n";
348 PyErr_Print();
349 throw std::runtime_error("\nFailed to run python code: "+code);
350 }
351}
352
353///////////////////////////////////////////////////////////////////////////////
354/// Returns `const char*` from Python string in PyObject
355///
356/// \param[in] string Python String object
357/// \return String representation in `const char*`
358
360 PyObject* encodedString = PyUnicode_AsUTF8String(string);
361 const char* cstring = PyBytes_AsString(encodedString);
362 return cstring;
363}
364
365//////////////////////////////////////////////////////////////////////////////////
366/// \brief Utility function which retrieves and returns the values of the Tuple
367/// object as a vector of size_t
368///
369/// \param[in] tupleObject Python Tuple object
370/// \return vector of tuple members
371
372std::vector<size_t> PyMethodBase::GetDataFromTuple(PyObject* tupleObject){
373 std::vector<size_t>tupleVec;
374 for(Py_ssize_t tupleIter=0;tupleIter<PyTuple_Size(tupleObject);++tupleIter){
375 auto itemObj = PyTuple_GetItem(tupleObject,tupleIter);
376 if (itemObj == Py_None)
377 tupleVec.push_back(0); // case shape is for example (None,2,3)
378 else
379 tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
380 }
381 return tupleVec;
382}
383
384//////////////////////////////////////////////////////////////////////////////////
385/// \brief Utility function which retrieves and returns the values of the List
386/// object as a vector of size_t
387///
388/// \param[in] listObject Python List object
389/// \return vector of list members
390
391std::vector<size_t> PyMethodBase::GetDataFromList(PyObject* listObject){
392 std::vector<size_t>listVec;
393 for(Py_ssize_t listIter=0; listIter<PyList_Size(listObject);++listIter){
394 listVec.push_back((size_t)PyLong_AsLong(PyList_GetItem(listObject,listIter)));
395 }
396 return listVec;
397}
398
399//////////////////////////////////////////////////////////////////////////////////
400/// \brief Utility function which checks if a given key is present in a Python
401/// dictionary object and returns the associated value or throws runtime
402/// error. This is to replace PyDict_GetItemWithError in Python 2.
403///
404/// \param[in] listObject Python Dict object
405/// \return Associated value PyObject
407 #if PY_MAJOR_VERSION >= 3 // using PyDict_GetItemWithError that is available only in Python3
408 return PyDict_GetItemWithError(dict,PyUnicode_FromString(key));
409 #else
410 if(!PyDict_Contains(dict, PyUnicode_FromString(key))){
411 throw std::runtime_error(std::string("Key ")+key+" does not exist in the dictionary.");
412 } else {
413 return PyDict_GetItemString(dict, key);
414 }
415 #endif
416}
417
#define PyBytes_AsString
Definition: CPyCppyy.h:86
_object PyObject
Definition: PyMethodBase.h:43
#define Py_single_input
Definition: PyMethodBase.h:44
const Bool_t kFALSE
Definition: RtypesCore.h:101
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:375
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
R__EXTERN TSystem * gSystem
Definition: TSystem.h:560
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
Virtual base Class for all MVA method.
Definition: MethodBase.h:111
ostringstream derivative to redirect and format output
Definition: MsgLogger.h:57
PyObject * fPyReturn
Definition: PyMethodBase.h:118
static std::vector< size_t > GetDataFromTuple(PyObject *tupleObject)
Utility function which retrieves and returns the values of the Tuple object as a vector of size_t.
static int PyIsInitialized()
Check Python interpreter initialization status.
static std::vector< size_t > GetDataFromList(PyObject *listObject)
Utility function which retrieves and returns the values of the List object as a vector of size_t.
static PyObject * fOpen
Definition: PyMethodBase.h:126
static PyObject * fPickleDumps
Definition: PyMethodBase.h:130
PyObject * Eval(TString code)
Evaluate Python code.
static PyObject * fMain
Definition: PyMethodBase.h:133
static void PyInitialize()
Initialize Python interpreter.
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
static void PyFinalize()
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
static const char * PyStringAsString(PyObject *string)
Returns const char* from Python string in PyObject.
static PyObject * fPickleLoads
Definition: PyMethodBase.h:131
static PyObject * fGlobalNS
Definition: PyMethodBase.h:134
static PyObject * fModulePickle
Definition: PyMethodBase.h:129
static PyObject * fModuleBuiltin
Definition: PyMethodBase.h:124
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
static PyObject * fEval
Definition: PyMethodBase.h:125
static PyObject * GetValueFromDict(PyObject *dict, const char *key)
Utility function which checks if a given key is present in a Python dictionary object and returns the...
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=256)
Execute Python code from string.
PyObject * fLocalNS
Definition: PyMethodBase.h:135
MsgLogger & Log() const
Definition: Tools.h:228
@ kWARNING
Definition: Types.h:59
@ kFATAL
Definition: Types.h:61
Basic string class.
Definition: TString.h:136
const char * Data() const
Definition: TString.h:369
TString & ReplaceAll(const TString &s1, const TString &s2)
Definition: TString.h:693
Bool_t IsNull() const
Definition: TString.h:407
virtual TString GetFromPipe(const char *command)
Execute command and return output in TString.
Definition: TSystem.cxx:683
create variable transformations
TString Python_Executable()
Function to find current Python executable used by ROOT If Python2 is installed return "python" Inste...
Tools & gTools()
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:148
Definition: file.py:1