Logo ROOT  
Reference Guide
PyMethodBase.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : PyMethodBase *
8 * *
9 * Description: *
10 * Virtual base class for all MVA method based on python *
11 * *
12 **********************************************************************************/
13
14#include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15#include <TMVA/PyMethodBase.h>
16
17#include "TMVA/DataSet.h"
18#include "TMVA/DataSetInfo.h"
19#include "TMVA/MsgLogger.h"
20#include "TMVA/Results.h"
21#include "TMVA/Timer.h"
22#include "TMVA/Tools.h"
23
24#include "TSystem.h"
25
26#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
27#include <numpy/arrayobject.h>
28
29#include <cwchar>
30
31using namespace TMVA;
32
33namespace TMVA {
34namespace Internal {
35class PyGILRAII {
36 PyGILState_STATE m_GILState;
37
38public:
39 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
40 ~PyGILRAII() { PyGILState_Release(m_GILState); }
41};
42} // namespace Internal
43
44/// get current Python executable used by ROOT
46 TString python_version = gSystem->GetFromPipe("root-config --python-version");
47 if (python_version.IsNull()) {
48 TMVA::gTools().Log() << kFATAL << "Can't find a valid Python version used to build ROOT" << Endl;
49 return nullptr;
50 }
51#ifdef _MSC_VER
52 // on Windows there is a space before the version and the executable is python.exe
53 // for both versions of Python
54 python_version.ReplaceAll(" ", "");
55 if (python_version[0] == '2' || python_version[0] == '3')
56 return "python";
57#endif
58 if (python_version[0] == '2')
59 return "python";
60 else if (python_version[0] == '3')
61 return "python3";
62
63 TMVA::gTools().Log() << kFATAL << "Invalid Python version used to build ROOT : " << python_version << Endl;
64 return nullptr;
65}
66
67} // namespace TMVA
68
70
71// NOTE: Introduce here nothing that breaks if multiple instances
72// of the same method share these objects, e.g., the local namespace.
73PyObject *PyMethodBase::fModuleBuiltin = NULL;
74PyObject *PyMethodBase::fEval = NULL;
75PyObject *PyMethodBase::fOpen = NULL;
76
77PyObject *PyMethodBase::fModulePickle = NULL;
78PyObject *PyMethodBase::fPickleDumps = NULL;
79PyObject *PyMethodBase::fPickleLoads = NULL;
80
81PyObject *PyMethodBase::fMain = NULL;
82PyObject *PyMethodBase::fGlobalNS = NULL;
83
84///////////////////////////////////////////////////////////////////////////////
85
86PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
87 const TString &theOption)
88 : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
89 fClassifier(NULL)
90{
91 if (!PyIsInitialized()) {
93 }
94
95 // Set up private local namespace for each method instance
96 fLocalNS = PyDict_New();
97 if (!fLocalNS) {
98 Log() << kFATAL << "Can't init local namespace" << Endl;
99 }
100}
101
102///////////////////////////////////////////////////////////////////////////////
103
105 DataSetInfo &dsi,
106 const TString &weightFile): MethodBase(methodType, dsi, weightFile),
107 fClassifier(NULL)
108{
109 if (!PyIsInitialized()) {
110 PyInitialize();
111 }
112
113 // Set up private local namespace for each method instance
114 fLocalNS = PyDict_New();
115 if (!fLocalNS) {
116 Log() << kFATAL << "Can't init local namespace" << Endl;
117 }
118}
119
120///////////////////////////////////////////////////////////////////////////////
121
123{
124 // should we delete here fLocalNS ?
125}
126
127///////////////////////////////////////////////////////////////////////////////
128/// Evaluate Python code
129///
130/// \param[in] code Python code as string
131/// \return Python object from evaluation of code line
132///
133/// Take a Python code as input and evaluate it in the local namespace. Then,
134/// return the result as Python object.
135
137{
139 PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
140 PyObject *result = PyObject_CallObject(fEval, pycode);
141 Py_DECREF(pycode);
142 return result;
143}
144
145///////////////////////////////////////////////////////////////////////////////
146/// Initialize Python interpreter
147///
148/// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
149/// a private local namespace `fLocalNS`. This prohibits the interference
150/// of instances of the same method with the same factory, e.g., by overriding
151/// variables in the same local namespace.
152
154{
156
157 bool pyIsInitialized = PyIsInitialized();
158 if (!pyIsInitialized) {
159 Py_Initialize();
160 }
161
163 if (!pyIsInitialized) {
164 _import_array();
165 }
166
167 // note fMain is a borrowed reference
168 fMain = PyImport_AddModule("__main__");
169 if (!fMain) {
170 Log << kFATAL << "Can't import __main__" << Endl;
171 Log << Endl;
172 }
173 Py_INCREF(fMain);
174
175 fGlobalNS = PyModule_GetDict(fMain);
176 if (!fGlobalNS) {
177 Log << kFATAL << "Can't init global namespace" << Endl;
178 Log << Endl;
179 }
180 Py_INCREF(fGlobalNS);
181
182 #if PY_MAJOR_VERSION < 3
183 //preparing objects for eval
184 PyObject *bName = PyUnicode_FromString("__builtin__");
185 // Import the file as a Python module.
186 // returns a new reference
187 fModuleBuiltin = PyImport_Import(bName);
188 if (!fModuleBuiltin) {
189 Log << kFATAL << "Can't import __builtin__" << Endl;
190 Log << Endl;
191 }
192 #else
193 //preparing objects for eval
194 PyObject *bName = PyUnicode_FromString("builtins");
195 // Import the file as a Python module.
196 fModuleBuiltin = PyImport_Import(bName);
197 if (!fModuleBuiltin) {
198 Log << kFATAL << "Can't import builtins" << Endl;
199 Log << Endl;
200 }
201 #endif
202
203 // note mDict is a borrowed reference
204 PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
205 fEval = PyDict_GetItemString(mDict, "eval");
206 fOpen = PyDict_GetItemString(mDict, "open");
207 // fEval and fOpen are borrowed referencers and we need to keep them alive
208 if (fEval) Py_INCREF(fEval);
209 if (fOpen) Py_INCREF(fOpen);
210
211 // bName is a new reference (from PyUnicode_FromString)
212 Py_DECREF(bName);
213
214 //preparing objects for pickle
215 PyObject *pName = PyUnicode_FromString("pickle");
216 // Import the file as a Python module.
217 // return object is a new reference !
218 fModulePickle = PyImport_Import(pName);
219 if (!fModulePickle) {
220 Log << kFATAL << "Can't import pickle" << Endl;
221 Log << Endl;
222 }
223 PyObject *pDict = PyModule_GetDict(fModulePickle);
224 // note the following return objects are borrowed references
225 fPickleDumps = PyDict_GetItemString(pDict, "dump");
226 fPickleLoads = PyDict_GetItemString(pDict, "load");
227 if (fPickleDumps) Py_INCREF(fPickleDumps);
228 if (fPickleLoads) Py_INCREF(fPickleLoads);
229
230 Py_DECREF(pName);
231}
232
233///////////////////////////////////////////////////////////////////////////////
234// Finalize Python interpreter
235
237{
238 if (fEval) Py_DECREF(fEval);
239 if (fOpen) Py_DECREF(fOpen);
240 if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
241 if (fPickleDumps) Py_DECREF(fPickleDumps);
242 if (fPickleLoads) Py_DECREF(fPickleLoads);
243 if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
244 if (fGlobalNS) Py_DECREF(fGlobalNS);
245 Py_Finalize();
246}
247
248///////////////////////////////////////////////////////////////////////////////
249/// Set program name for Python interpeter
250///
251/// \param[in] name Program name
252
254{
255 #if PY_MAJOR_VERSION < 3
256 Py_SetProgramName(const_cast<char*>(name.Data()));
257 #else
258 Py_SetProgramName((wchar_t *)name.Data());
259 #endif
260}
261
262
263///////////////////////////////////////////////////////////////////////////////
264
265size_t mystrlen(const char* s) { return strlen(s); }
266
267///////////////////////////////////////////////////////////////////////////////
268
269size_t mystrlen(const wchar_t* s) { return wcslen(s); }
270
271///////////////////////////////////////////////////////////////////////////////
272/// Get program name from Python interpreter
273///
274/// \return Program name
275
277{
278 auto progName = ::Py_GetProgramName();
279 return std::string(progName, progName + mystrlen(progName));
280}
281
282///////////////////////////////////////////////////////////////////////////////
283/// Check Python interpreter initialization status
284///
285/// \return Boolean whether interpreter is initialized
286
288{
289 if (!Py_IsInitialized()) return kFALSE;
290 if (!fEval) return kFALSE;
291 if (!fModuleBuiltin) return kFALSE;
292 if (!fPickleDumps) return kFALSE;
293 if (!fPickleLoads) return kFALSE;
294 return kTRUE;
295}
296
297///////////////////////////////////////////////////////////////////////////////
298/// Serialize Python object
299///
300/// \param[in] path Path where object is written to file
301/// \param[in] obj Python object
302///
303/// The input Python object is serialized and written to a file. The Python
304/// module `pickle` is used to do so.
305
307{
309
310 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
311 PyObject *file = PyObject_CallObject(fOpen,file_arg);
312 PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
313 PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
314
315 Py_DECREF(file_arg);
316 Py_DECREF(file);
317 Py_DECREF(model_arg);
318 Py_DECREF(model_data);
319}
320
321///////////////////////////////////////////////////////////////////////////////
322/// Unserialize Python object
323///
324/// \param[in] path Path to serialized Python object
325/// \param[in] obj Python object where the unserialized Python object is loaded
326/// \return Error code
327
329{
330 // Load file
331 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
332 PyObject *file = PyObject_CallObject(fOpen,file_arg);
333 if(!file) return 1;
334
335 // Load object from file using pickle
336 PyObject *model_arg = Py_BuildValue("(O)", file);
337 *obj = PyObject_CallObject(fPickleLoads , model_arg);
338 if(!obj) return 2;
339
340 Py_DECREF(file_arg);
341 Py_DECREF(file);
342 Py_DECREF(model_arg);
343
344 return 0;
345}
346
347///////////////////////////////////////////////////////////////////////////////
348/// Execute Python code from string
349///
350/// \param[in] code Python code as string
351/// \param[in] errorMessage Error message which shall be shown if the execution fails
352/// \param[in] start Start symbol
353///
354/// Helper function to run python code from string in local namespace with
355/// error handling
356/// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
357/// Py_single_input, Py_file_input)
358
359void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
360 fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
361 if (!fPyReturn) {
362 Log() << kWARNING << "Failed to run python code: " << code << Endl;
363 Log() << kWARNING << "Python error message:" << Endl;
364 PyErr_Print();
365 Log() << kFATAL << errorMessage << Endl;
366 }
367}
368
369///////////////////////////////////////////////////////////////////////////////
370/// Execute Python code from string
371///
372/// \param[in] code Python code as string
373/// \param[in] globalNS Global Namespace for Python Session
374/// \param[in] localNS Local Namespace for Python Session
375///
376/// Overloaded static Helper function to run python code
377/// from string and throw runtime error if the Python session
378/// is unable to execute the code
379
380void PyMethodBase::PyRunString(TString code, PyObject *globalNS, PyObject *localNS){
381 PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
382 if (!fPyReturn) {
383 std::cout<<"\nPython error message:\n";
384 PyErr_Print();
385 throw std::runtime_error("\nFailed to run python code: "+code);
386 }
387}
388
389///////////////////////////////////////////////////////////////////////////////
390/// Returns `const char*` from Python string in PyObject
391///
392/// \param[in] string Python String object
393/// \return String representation in `const char*`
394
396 PyObject* encodedString = PyUnicode_AsUTF8String(string);
397 const char* cstring = PyBytes_AsString(encodedString);
398 return cstring;
399}
400
401//////////////////////////////////////////////////////////////////////////////////
402/// \brief Utility function which retrieves and returns the values of the Tuple
403/// object as a vector of size_t
404///
405/// \param[in] tupleObject Python Tuple object
406/// \return vector of tuple members
407
408std::vector<size_t> PyMethodBase::GetDataFromTuple(PyObject* tupleObject){
409 std::vector<size_t>tupleVec;
410 for(Py_ssize_t tupleIter=0;tupleIter<PyTuple_Size(tupleObject);++tupleIter){
411 auto itemObj = PyTuple_GetItem(tupleObject,tupleIter);
412 if (itemObj == Py_None)
413 tupleVec.push_back(0); // case shape is for example (None,2,3)
414 else
415 tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
416 }
417 return tupleVec;
418}
419
420//////////////////////////////////////////////////////////////////////////////////
421/// \brief Utility function which retrieves and returns the values of the List
422/// object as a vector of size_t
423///
424/// \param[in] listObject Python List object
425/// \return vector of list members
426
427std::vector<size_t> PyMethodBase::GetDataFromList(PyObject* listObject){
428 std::vector<size_t>listVec;
429 for(Py_ssize_t listIter=0; listIter<PyList_Size(listObject);++listIter){
430 listVec.push_back((size_t)PyLong_AsLong(PyList_GetItem(listObject,listIter)));
431 }
432 return listVec;
433}
#define PyBytes_AsString
Definition: CPyCppyy.h:86
size_t mystrlen(const char *s)
_object PyObject
Definition: PyMethodBase.h:43
#define Py_single_input
Definition: PyMethodBase.h:44
const Bool_t kFALSE
Definition: RtypesCore.h:101
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:375
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
char name[80]
Definition: TGX11.cxx:110
R__EXTERN TSystem * gSystem
Definition: TSystem.h:559
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
Virtual base Class for all MVA method.
Definition: MethodBase.h:111
ostringstream derivative to redirect and format output
Definition: MsgLogger.h:57
PyObject * fPyReturn
Definition: PyMethodBase.h:120
static std::vector< size_t > GetDataFromTuple(PyObject *tupleObject)
Utility function which retrieves and returns the values of the Tuple object as a vector of size_t.
static int PyIsInitialized()
Check Python interpreter initialization status.
static std::vector< size_t > GetDataFromList(PyObject *listObject)
Utility function which retrieves and returns the values of the List object as a vector of size_t.
static PyObject * fOpen
Definition: PyMethodBase.h:128
static PyObject * fPickleDumps
Definition: PyMethodBase.h:132
PyObject * Eval(TString code)
Evaluate Python code.
static TString Py_GetProgramName()
Get program name from Python interpreter.
static PyObject * fMain
Definition: PyMethodBase.h:135
static void PyInitialize()
Initialize Python interpreter.
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
static void PyFinalize()
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
static const char * PyStringAsString(PyObject *string)
Returns const char* from Python string in PyObject.
static PyObject * fPickleLoads
Definition: PyMethodBase.h:133
static void PySetProgramName(TString name)
Set program name for Python interpeter.
static PyObject * fGlobalNS
Definition: PyMethodBase.h:136
static PyObject * fModulePickle
Definition: PyMethodBase.h:131
static PyObject * fModuleBuiltin
Definition: PyMethodBase.h:126
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
static PyObject * fEval
Definition: PyMethodBase.h:127
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=256)
Execute Python code from string.
PyObject * fLocalNS
Definition: PyMethodBase.h:137
MsgLogger & Log() const
Definition: Tools.h:228
@ kWARNING
Definition: Types.h:59
@ kFATAL
Definition: Types.h:61
Basic string class.
Definition: TString.h:136
const char * Data() const
Definition: TString.h:369
TString & ReplaceAll(const TString &s1, const TString &s2)
Definition: TString.h:692
Bool_t IsNull() const
Definition: TString.h:407
virtual TString GetFromPipe(const char *command)
Execute command and return output in TString.
Definition: TSystem.cxx:683
static constexpr double s
create variable transformations
TString Python_Executable()
Function to find current Python executable used by ROOT If Python2 is installed return "python" Inste...
Tools & gTools()
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:148
Definition: file.py:1