Logo ROOT  
Reference Guide
PyMethodBase.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : PyMethodBase *
8 * *
9 * Description: *
10 * Virtual base class for all MVA method based on python *
11 * *
12 **********************************************************************************/
13
14#include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15#include <TMVA/PyMethodBase.h>
16
17#include "TMVA/DataSet.h"
18#include "TMVA/DataSetInfo.h"
19#include "TMVA/MsgLogger.h"
20#include "TMVA/Results.h"
21#include "TMVA/Timer.h"
22
23#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
24#include <numpy/arrayobject.h>
25
26#include <cwchar>
27
28using namespace TMVA;
29
30namespace TMVA {
31namespace Internal {
32class PyGILRAII {
33 PyGILState_STATE m_GILState;
34
35public:
36 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
37 ~PyGILRAII() { PyGILState_Release(m_GILState); }
38};
39} // namespace Internal
40} // namespace TMVA
41
43
44// NOTE: Introduce here nothing that breaks if multiple instances
45// of the same method share these objects, e.g., the local namespace.
46PyObject *PyMethodBase::fModuleBuiltin = NULL;
47PyObject *PyMethodBase::fEval = NULL;
48PyObject *PyMethodBase::fOpen = NULL;
49
50PyObject *PyMethodBase::fModulePickle = NULL;
51PyObject *PyMethodBase::fPickleDumps = NULL;
52PyObject *PyMethodBase::fPickleLoads = NULL;
53
54PyObject *PyMethodBase::fMain = NULL;
55PyObject *PyMethodBase::fGlobalNS = NULL;
56
57///////////////////////////////////////////////////////////////////////////////
58
59PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
60 const TString &theOption)
61 : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
62 fClassifier(NULL)
63{
64 if (!PyIsInitialized()) {
66 }
67
68 // Set up private local namespace for each method instance
69 fLocalNS = PyDict_New();
70 if (!fLocalNS) {
71 Log() << kFATAL << "Can't init local namespace" << Endl;
72 }
73}
74
75///////////////////////////////////////////////////////////////////////////////
76
78 DataSetInfo &dsi,
79 const TString &weightFile): MethodBase(methodType, dsi, weightFile),
80 fClassifier(NULL)
81{
82 if (!PyIsInitialized()) {
84 }
85
86 // Set up private local namespace for each method instance
87 fLocalNS = PyDict_New();
88 if (!fLocalNS) {
89 Log() << kFATAL << "Can't init local namespace" << Endl;
90 }
91}
92
93///////////////////////////////////////////////////////////////////////////////
94
96{
97 // should we delete here fLocalNS ?
98}
99
100///////////////////////////////////////////////////////////////////////////////
101/// Evaluate Python code
102///
103/// \param[in] code Python code as string
104/// \return Python object from evaluation of code line
105///
106/// Take a Python code as input and evaluate it in the local namespace. Then,
107/// return the result as Python object.
108
110{
112 PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
113 PyObject *result = PyObject_CallObject(fEval, pycode);
114 Py_DECREF(pycode);
115 return result;
116}
117
118///////////////////////////////////////////////////////////////////////////////
119/// Initialize Python interpreter
120///
121/// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
122/// a private local namespace `fLocalNS`. This prohibits the interference
123/// of instances of the same method with the same factory, e.g., by overriding
124/// variables in the same local namespace.
125
127{
129
130 bool pyIsInitialized = PyIsInitialized();
131 if (!pyIsInitialized) {
132 Py_Initialize();
133 }
134
136 if (!pyIsInitialized) {
137 _import_array();
138 }
139
140 // note fMain is a borrowed reference
141 fMain = PyImport_AddModule("__main__");
142 if (!fMain) {
143 Log << kFATAL << "Can't import __main__" << Endl;
144 Log << Endl;
145 }
146 Py_INCREF(fMain);
147
148 fGlobalNS = PyModule_GetDict(fMain);
149 if (!fGlobalNS) {
150 Log << kFATAL << "Can't init global namespace" << Endl;
151 Log << Endl;
152 }
153 Py_INCREF(fGlobalNS);
154
155 #if PY_MAJOR_VERSION < 3
156 //preparing objects for eval
157 PyObject *bName = PyUnicode_FromString("__builtin__");
158 // Import the file as a Python module.
159 // returns a new reference
160 fModuleBuiltin = PyImport_Import(bName);
161 if (!fModuleBuiltin) {
162 Log << kFATAL << "Can't import __builtin__" << Endl;
163 Log << Endl;
164 }
165 #else
166 //preparing objects for eval
167 PyObject *bName = PyUnicode_FromString("builtins");
168 // Import the file as a Python module.
169 fModuleBuiltin = PyImport_Import(bName);
170 if (!fModuleBuiltin) {
171 Log << kFATAL << "Can't import builtins" << Endl;
172 Log << Endl;
173 }
174 #endif
175
176 // note mDict is a borrowed reference
177 PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
178 fEval = PyDict_GetItemString(mDict, "eval");
179 fOpen = PyDict_GetItemString(mDict, "open");
180 // fEval and fOpen are borrowed referencers and we need to keep them alive
181 if (fEval) Py_INCREF(fEval);
182 if (fOpen) Py_INCREF(fOpen);
183
184 // bName is a new reference (from PyUnicode_FromString)
185 Py_DECREF(bName);
186
187 //preparing objects for pickle
188 PyObject *pName = PyUnicode_FromString("pickle");
189 // Import the file as a Python module.
190 // return object is a new reference !
191 fModulePickle = PyImport_Import(pName);
192 if (!fModulePickle) {
193 Log << kFATAL << "Can't import pickle" << Endl;
194 Log << Endl;
195 }
196 PyObject *pDict = PyModule_GetDict(fModulePickle);
197 // note the following return objects are borrowed references
198 fPickleDumps = PyDict_GetItemString(pDict, "dump");
199 fPickleLoads = PyDict_GetItemString(pDict, "load");
200 if (fPickleDumps) Py_INCREF(fPickleDumps);
201 if (fPickleLoads) Py_INCREF(fPickleLoads);
202
203 Py_DECREF(pName);
204}
205
206///////////////////////////////////////////////////////////////////////////////
207// Finalize Python interpreter
208
210{
211 if (fEval) Py_DECREF(fEval);
212 if (fOpen) Py_DECREF(fOpen);
213 if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
214 if (fPickleDumps) Py_DECREF(fPickleDumps);
215 if (fPickleLoads) Py_DECREF(fPickleLoads);
216 if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
217 if (fGlobalNS) Py_DECREF(fGlobalNS);
218 Py_Finalize();
219}
220
221///////////////////////////////////////////////////////////////////////////////
222/// Set program name for Python interpeter
223///
224/// \param[in] name Program name
225
227{
228 #if PY_MAJOR_VERSION < 3
229 Py_SetProgramName(const_cast<char*>(name.Data()));
230 #else
231 Py_SetProgramName((wchar_t *)name.Data());
232 #endif
233}
234
235
236///////////////////////////////////////////////////////////////////////////////
237
238size_t mystrlen(const char* s) { return strlen(s); }
239
240///////////////////////////////////////////////////////////////////////////////
241
242size_t mystrlen(const wchar_t* s) { return wcslen(s); }
243
244///////////////////////////////////////////////////////////////////////////////
245/// Get program name from Python interpreter
246///
247/// \return Program name
248
250{
251 auto progName = ::Py_GetProgramName();
252 return std::string(progName, progName + mystrlen(progName));
253}
254
255///////////////////////////////////////////////////////////////////////////////
256/// Check Python interpreter initialization status
257///
258/// \return Boolean whether interpreter is initialized
259
261{
262 if (!Py_IsInitialized()) return kFALSE;
263 if (!fEval) return kFALSE;
264 if (!fModuleBuiltin) return kFALSE;
265 if (!fPickleDumps) return kFALSE;
266 if (!fPickleLoads) return kFALSE;
267 return kTRUE;
268}
269
270///////////////////////////////////////////////////////////////////////////////
271/// Serialize Python object
272///
273/// \param[in] path Path where object is written to file
274/// \param[in] obj Python object
275///
276/// The input Python object is serialized and written to a file. The Python
277/// module `pickle` is used to do so.
278
280{
282
283 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
284 PyObject *file = PyObject_CallObject(fOpen,file_arg);
285 PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
286 PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
287
288 Py_DECREF(file_arg);
289 Py_DECREF(file);
290 Py_DECREF(model_arg);
291 Py_DECREF(model_data);
292}
293
294///////////////////////////////////////////////////////////////////////////////
295/// Unserialize Python object
296///
297/// \param[in] path Path to serialized Python object
298/// \param[in] obj Python object where the unserialized Python object is loaded
299/// \return Error code
300
302{
303 // Load file
304 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
305 PyObject *file = PyObject_CallObject(fOpen,file_arg);
306 if(!file) return 1;
307
308 // Load object from file using pickle
309 PyObject *model_arg = Py_BuildValue("(O)", file);
310 *obj = PyObject_CallObject(fPickleLoads , model_arg);
311 if(!obj) return 2;
312
313 Py_DECREF(file_arg);
314 Py_DECREF(file);
315 Py_DECREF(model_arg);
316
317 return 0;
318}
319
320///////////////////////////////////////////////////////////////////////////////
321/// Execute Python code from string
322///
323/// \param[in] code Python code as string
324/// \param[in] errorMessage Error message which shall be shown if the execution fails
325/// \param[in] start Start symbol
326///
327/// Helper function to run python code from string in local namespace with
328/// error handling
329/// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
330/// Py_single_input, Py_file_input)
331
332void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
333 fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
334 if (!fPyReturn) {
335 Log() << kWARNING << "Failed to run python code: " << code << Endl;
336 Log() << kWARNING << "Python error message:" << Endl;
337 PyErr_Print();
338 Log() << kFATAL << errorMessage << Endl;
339 }
340}
341
342///////////////////////////////////////////////////////////////////////////////
343/// Execute Python code from string
344///
345/// \param[in] code Python code as string
346/// \param[in] globalNS Global Namespace for Python Session
347/// \param[in] localNS Local Namespace for Python Session
348///
349/// Overloaded static Helper function to run python code
350/// from string and throw runtime error if the Python session
351/// is unable to execute the code
352
353void PyMethodBase::PyRunString(TString code, PyObject *globalNS, PyObject *localNS){
354 PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
355 if (!fPyReturn) {
356 std::cout<<"\nPython error message:\n";
357 PyErr_Print();
358 throw std::runtime_error("\nFailed to run python code: "+code);
359 }
360}
361
362///////////////////////////////////////////////////////////////////////////////
363/// Returns `const char*` from Python string in PyObject
364///
365/// \param[in] string Python String object
366/// \return String representation in `const char*`
367
369 PyObject* encodedString = PyUnicode_AsUTF8String(string);
370 const char* cstring = PyBytes_AsString(encodedString);
371 return cstring;
372}
#define PyBytes_AsString
Definition: CPyCppyy.h:86
size_t mystrlen(const char *s)
_object PyObject
Definition: PyMethodBase.h:42
#define Py_single_input
Definition: PyMethodBase.h:44
int Int_t
Definition: RtypesCore.h:45
const Bool_t kFALSE
Definition: RtypesCore.h:101
const Bool_t kTRUE
Definition: RtypesCore.h:100
#define ClassImp(name)
Definition: Rtypes.h:364
char name[80]
Definition: TGX11.cxx:110
MsgLogger & Log() const
Definition: Configurable.h:122
Class that contains all the data information.
Definition: DataSetInfo.h:62
Virtual base Class for all MVA method.
Definition: MethodBase.h:111
ostringstream derivative to redirect and format output
Definition: MsgLogger.h:57
PyObject * fPyReturn
Definition: PyMethodBase.h:114
static int PyIsInitialized()
Check Python interpreter initialization status.
static PyObject * fOpen
Definition: PyMethodBase.h:122
static PyObject * fPickleDumps
Definition: PyMethodBase.h:126
PyObject * Eval(TString code)
Evaluate Python code.
static TString Py_GetProgramName()
Get program name from Python interpreter.
static PyObject * fMain
Definition: PyMethodBase.h:129
static void PyInitialize()
Initialize Python interpreter.
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
static void PyFinalize()
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
static const char * PyStringAsString(PyObject *string)
Returns const char* from Python string in PyObject.
static PyObject * fPickleLoads
Definition: PyMethodBase.h:127
static void PySetProgramName(TString name)
Set program name for Python interpeter.
virtual ~PyMethodBase()
static PyObject * fGlobalNS
Definition: PyMethodBase.h:130
static PyObject * fModulePickle
Definition: PyMethodBase.h:125
static PyObject * fModuleBuiltin
Definition: PyMethodBase.h:120
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=Py_single_input)
Execute Python code from string.
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
static PyObject * fEval
Definition: PyMethodBase.h:121
PyObject * fLocalNS
Definition: PyMethodBase.h:131
@ kWARNING
Definition: Types.h:59
@ kFATAL
Definition: Types.h:61
Basic string class.
Definition: TString.h:136
const char * Data() const
Definition: TString.h:369
static constexpr double s
create variable transformations
MsgLogger & Endl(MsgLogger &ml)
Definition: MsgLogger.h:148
Definition: file.py:1