Logo ROOT  
Reference Guide
 
Loading...
Searching...
No Matches
PyMethodBase.cxx
Go to the documentation of this file.
1// @(#)root/tmva/pymva $Id$
2// Authors: Omar Zapata, Lorenzo Moneta, Sergei Gleyzer 2015, Stefan Wunsch 2017
3
4/**********************************************************************************
5 * Project: TMVA - a Root-integrated toolkit for multivariate data analysis *
6 * Package: TMVA *
7 * Class : PyMethodBase *
8 * *
9 * Description: *
10 * Virtual base class for all MVA method based on python *
11 * *
12 **********************************************************************************/
13
14#include <Python.h> // Needs to be included first to avoid redefinition of _POSIX_C_SOURCE
15#include <TMVA/PyMethodBase.h>
16
17#include "TMVA/DataSet.h"
18#include "TMVA/DataSetInfo.h"
19#include "TMVA/MsgLogger.h"
20#include "TMVA/Results.h"
21#include "TMVA/Timer.h"
22#include "TMVA/Tools.h"
23
24#include "TSystem.h"
25
26#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
27#include <numpy/arrayobject.h>
28
29using namespace TMVA;
30
31namespace TMVA {
32namespace Internal {
33class PyGILRAII {
34 PyGILState_STATE m_GILState;
35
36public:
37 PyGILRAII() : m_GILState(PyGILState_Ensure()) {}
38 ~PyGILRAII() { PyGILState_Release(m_GILState); }
39};
40} // namespace Internal
41
42/// get current Python executable used by ROOT
44 TString python_version = gSystem->GetFromPipe("root-config --python-version");
45 if (python_version.IsNull()) {
46 TMVA::gTools().Log() << kFATAL << "Can't find a valid Python version used to build ROOT" << Endl;
47 return nullptr;
48 }
49#ifdef _MSC_VER
50 // on Windows there is a space before the version and the executable is python.exe
51 // for both versions of Python
52 python_version.ReplaceAll(" ", "");
53 if (python_version[0] == '2' || python_version[0] == '3')
54 return "python";
55#endif
56 if (python_version[0] == '2')
57 return "python";
58 else if (python_version[0] == '3')
59 return "python3";
60
61 TMVA::gTools().Log() << kFATAL << "Invalid Python version used to build ROOT : " << python_version << Endl;
62 return nullptr;
63}
64
65} // namespace TMVA
66
68
69// NOTE: Introduce here nothing that breaks if multiple instances
70// of the same method share these objects, e.g., the local namespace.
74
78
81
82///////////////////////////////////////////////////////////////////////////////
83
84PyMethodBase::PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi,
85 const TString &theOption)
86 : MethodBase(jobName, methodType, methodTitle, dsi, theOption),
87 fClassifier(NULL)
88{
89 if (!PyIsInitialized()) {
91 }
92
93 // Set up private local namespace for each method instance
94 fLocalNS = PyDict_New();
95 if (!fLocalNS) {
96 Log() << kFATAL << "Can't init local namespace" << Endl;
97 }
98}
99
100///////////////////////////////////////////////////////////////////////////////
101
103 DataSetInfo &dsi,
104 const TString &weightFile): MethodBase(methodType, dsi, weightFile),
105 fClassifier(NULL)
106{
107 if (!PyIsInitialized()) {
108 PyInitialize();
109 }
110
111 // Set up private local namespace for each method instance
112 fLocalNS = PyDict_New();
113 if (!fLocalNS) {
114 Log() << kFATAL << "Can't init local namespace" << Endl;
115 }
116}
117
118///////////////////////////////////////////////////////////////////////////////
119
121{
122 // should we delete here fLocalNS ?
123 //PyFinalize();
124 if (fLocalNS) Py_DECREF(fLocalNS);
125}
126
127///////////////////////////////////////////////////////////////////////////////
128/// Evaluate Python code
129///
130/// \param[in] code Python code as string
131/// \return Python object from evaluation of code line
132///
133/// Take a Python code as input and evaluate it in the local namespace. Then,
134/// return the result as Python object.
135
137{
139 PyObject *pycode = Py_BuildValue("(sOO)", code.Data(), fGlobalNS, fLocalNS);
140 PyObject *result = PyObject_CallObject(fEval, pycode);
141 Py_DECREF(pycode);
142 return result;
143}
144
145///////////////////////////////////////////////////////////////////////////////
146/// Initialize Python interpreter
147///
148/// NOTE: We introduce a shared global namespace `fGlobalNS`, but using
149/// a private local namespace `fLocalNS`. This prohibits the interference
150/// of instances of the same method with the same factory, e.g., by overriding
151/// variables in the same local namespace.
152
154{
156
157 bool pyIsInitialized = PyIsInitialized();
158 if (!pyIsInitialized) {
159 Py_Initialize();
160 }
161
163 if (!pyIsInitialized) {
164 _import_array();
165 }
166
167 // note fMain is a borrowed reference
168 fMain = PyImport_AddModule("__main__");
169 if (!fMain) {
170 Log << kFATAL << "Can't import __main__" << Endl;
171 Log << Endl;
172 }
173 Py_INCREF(fMain);
174
175 fGlobalNS = PyModule_GetDict(fMain);
176 if (!fGlobalNS) {
177 Log << kFATAL << "Can't init global namespace" << Endl;
178 Log << Endl;
179 }
180 Py_INCREF(fGlobalNS);
181
182 #if PY_MAJOR_VERSION < 3
183 //preparing objects for eval
184 PyObject *bName = PyUnicode_FromString("__builtin__");
185 // Import the file as a Python module.
186 // returns a new reference
187 fModuleBuiltin = PyImport_Import(bName);
188 if (!fModuleBuiltin) {
189 Log << kFATAL << "Can't import __builtin__" << Endl;
190 Log << Endl;
191 }
192 #else
193 //preparing objects for eval
194 PyObject *bName = PyUnicode_FromString("builtins");
195 // Import the file as a Python module.
196 fModuleBuiltin = PyImport_Import(bName);
197 if (!fModuleBuiltin) {
198 Log << kFATAL << "Can't import builtins" << Endl;
199 Log << Endl;
200 }
201 #endif
202
203 // note mDict is a borrowed reference
204 PyObject *mDict = PyModule_GetDict(fModuleBuiltin);
205 fEval = PyDict_GetItemString(mDict, "eval");
206 fOpen = PyDict_GetItemString(mDict, "open");
207 // fEval and fOpen are borrowed referencers and we need to keep them alive
208 if (fEval) Py_INCREF(fEval);
209 if (fOpen) Py_INCREF(fOpen);
210
211 // bName is a new reference (from PyUnicode_FromString)
212 Py_DECREF(bName);
213
214 //preparing objects for pickle
215 PyObject *pName = PyUnicode_FromString("pickle");
216 // Import the file as a Python module.
217 // return object is a new reference !
218 fModulePickle = PyImport_Import(pName);
219 if (!fModulePickle) {
220 Log << kFATAL << "Can't import pickle" << Endl;
221 Log << Endl;
222 }
223 PyObject *pDict = PyModule_GetDict(fModulePickle);
224 // note the following return objects are borrowed references
225 fPickleDumps = PyDict_GetItemString(pDict, "dump");
226 fPickleLoads = PyDict_GetItemString(pDict, "load");
227 if (fPickleDumps) Py_INCREF(fPickleDumps);
228 if (fPickleLoads) Py_INCREF(fPickleLoads);
229
230 Py_DECREF(pName);
231}
232
233///////////////////////////////////////////////////////////////////////////////
234// Finalize Python interpreter
235
237{
238 if (fEval) Py_DECREF(fEval);
239 if (fOpen) Py_DECREF(fOpen);
240 if (fModuleBuiltin) Py_DECREF(fModuleBuiltin);
241 if (fPickleDumps) Py_DECREF(fPickleDumps);
242 if (fPickleLoads) Py_DECREF(fPickleLoads);
243 if(fMain) Py_DECREF(fMain);//objects fGlobalNS and fLocalNS will be free here
244 if (fGlobalNS) Py_DECREF(fGlobalNS);
245 Py_Finalize();
246}
247
248///////////////////////////////////////////////////////////////////////////////
249/// Check Python interpreter initialization status
250///
251/// \return Boolean whether interpreter is initialized
252
254{
255 if (!Py_IsInitialized()) return kFALSE;
256 if (!fEval) return kFALSE;
257 if (!fModuleBuiltin) return kFALSE;
258 if (!fPickleDumps) return kFALSE;
259 if (!fPickleLoads) return kFALSE;
260 return kTRUE;
261}
262
263///////////////////////////////////////////////////////////////////////////////
264/// Serialize Python object
265///
266/// \param[in] path Path where object is written to file
267/// \param[in] obj Python object
268///
269/// The input Python object is serialized and written to a file. The Python
270/// module `pickle` is used to do so.
271
273{
275
276 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"wb");
277 PyObject *file = PyObject_CallObject(fOpen,file_arg);
278 PyObject *model_arg = Py_BuildValue("(OO)", obj,file);
279 PyObject *model_data = PyObject_CallObject(fPickleDumps , model_arg);
280
281 Py_DECREF(file_arg);
282 Py_DECREF(file);
283 Py_DECREF(model_arg);
284 Py_DECREF(model_data);
285}
286
287///////////////////////////////////////////////////////////////////////////////
288/// Unserialize Python object
289///
290/// \param[in] path Path to serialized Python object
291/// \param[in] obj Python object where the unserialized Python object is loaded
292/// \return Error code
293
295{
296 // Load file
297 PyObject *file_arg = Py_BuildValue("(ss)", path.Data(),"rb");
298 PyObject *file = PyObject_CallObject(fOpen,file_arg);
299 if(!file) return 1;
300
301 // Load object from file using pickle
302 PyObject *model_arg = Py_BuildValue("(O)", file);
303 *obj = PyObject_CallObject(fPickleLoads , model_arg);
304 if(!obj) return 2;
305
306 Py_DECREF(file_arg);
307 Py_DECREF(file);
308 Py_DECREF(model_arg);
309
310 return 0;
311}
312
313///////////////////////////////////////////////////////////////////////////////
314/// Execute Python code from string
315///
316/// \param[in] code Python code as string
317/// \param[in] errorMessage Error message which shall be shown if the execution fails
318/// \param[in] start Start symbol
319///
320/// Helper function to run python code from string in local namespace with
321/// error handling
322/// `start` defines the start symbol defined in PyRun_String (Py_eval_input,
323/// Py_single_input, Py_file_input)
324
325void PyMethodBase::PyRunString(TString code, TString errorMessage, int start) {
326 //std::cout << "Run: >> " << code << std::endl;
327 fPyReturn = PyRun_String(code, start, fGlobalNS, fLocalNS);
328 if (!fPyReturn) {
329 Log() << kWARNING << "Failed to run python code: " << code << Endl;
330 Log() << kWARNING << "Python error message:" << Endl;
331 PyErr_Print();
332 Log() << kFATAL << errorMessage << Endl;
333 }
334}
335
336///////////////////////////////////////////////////////////////////////////////
337/// Execute Python code from string
338///
339/// \param[in] code Python code as string
340/// \param[in] globalNS Global Namespace for Python Session
341/// \param[in] localNS Local Namespace for Python Session
342///
343/// Overloaded static Helper function to run python code
344/// from string and throw runtime error if the Python session
345/// is unable to execute the code
346
347void PyMethodBase::PyRunString(TString code, PyObject *globalNS, PyObject *localNS){
348 PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
349 if (!fPyReturn) {
350 std::cout<<"\nPython error message:\n";
351 PyErr_Print();
352 throw std::runtime_error("\nFailed to run python code: "+code);
353 }
354}
355
356///////////////////////////////////////////////////////////////////////////////
357/// Returns `const char*` from Python string in PyObject
358///
359/// \param[in] string Python String object
360/// \return String representation in `const char*`
361
363 PyObject* encodedString = PyUnicode_AsUTF8String(string);
364 const char* cstring = PyBytes_AsString(encodedString);
365 return cstring;
366}
367
368//////////////////////////////////////////////////////////////////////////////////
369/// \brief Utility function which retrieves and returns the values of the Tuple
370/// object as a vector of size_t
371///
372/// \param[in] tupleObject Python Tuple object
373/// \return vector of tuple members
374
375std::vector<size_t> PyMethodBase::GetDataFromTuple(PyObject* tupleObject){
376 std::vector<size_t>tupleVec;
377 for(Py_ssize_t tupleIter=0;tupleIter<PyTuple_Size(tupleObject);++tupleIter){
378 auto itemObj = PyTuple_GetItem(tupleObject,tupleIter);
379 if (itemObj == Py_None)
380 tupleVec.push_back(0); // case shape is for example (None,2,3)
381 else
382 tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
383 }
384 return tupleVec;
385}
386
387//////////////////////////////////////////////////////////////////////////////////
388/// \brief Utility function which retrieves and returns the values of the List
389/// object as a vector of size_t
390///
391/// \param[in] listObject Python List object
392/// \return vector of list members
393
394std::vector<size_t> PyMethodBase::GetDataFromList(PyObject* listObject){
395 std::vector<size_t>listVec;
396 for(Py_ssize_t listIter=0; listIter<PyList_Size(listObject);++listIter){
397 listVec.push_back((size_t)PyLong_AsLong(PyList_GetItem(listObject,listIter)));
398 }
399 return listVec;
400}
401
402//////////////////////////////////////////////////////////////////////////////////
403/// \brief Utility function which checks if a given key is present in a Python
404/// dictionary object and returns the associated value or throws runtime
405/// error.
406///
407/// \param[in] listObject Python Dict object
408/// \return Associated value PyObject
410{
411 return PyDict_GetItemWithError(dict, PyUnicode_FromString(key));
412}
#define PyBytes_AsString
Definition CPyCppyy.h:86
_object PyObject
#define Py_single_input
constexpr Bool_t kFALSE
Definition RtypesCore.h:101
constexpr Bool_t kTRUE
Definition RtypesCore.h:100
#define ClassImp(name)
Definition Rtypes.h:377
Option_t Option_t TPoint TPoint const char GetTextMagnitude GetFillStyle GetLineColor GetLineWidth GetMarkerStyle GetTextAlign GetTextColor GetTextSize void char Point_t Rectangle_t WindowAttributes_t Float_t Float_t Float_t Int_t Int_t UInt_t UInt_t Rectangle_t result
R__EXTERN TSystem * gSystem
Definition TSystem.h:560
MsgLogger & Log() const
Class that contains all the data information.
Definition DataSetInfo.h:62
Virtual base Class for all MVA method.
Definition MethodBase.h:111
ostringstream derivative to redirect and format output
Definition MsgLogger.h:57
static std::vector< size_t > GetDataFromTuple(PyObject *tupleObject)
Utility function which retrieves and returns the values of the Tuple object as a vector of size_t.
static int PyIsInitialized()
Check Python interpreter initialization status.
static std::vector< size_t > GetDataFromList(PyObject *listObject)
Utility function which retrieves and returns the values of the List object as a vector of size_t.
static PyObject * fOpen
static PyObject * fPickleDumps
PyObject * Eval(TString code)
Evaluate Python code.
static PyObject * fMain
static void PyInitialize()
Initialize Python interpreter.
static void Serialize(TString file, PyObject *classifier)
Serialize Python object.
static void PyFinalize()
static Int_t UnSerialize(TString file, PyObject **obj)
Unserialize Python object.
static const char * PyStringAsString(PyObject *string)
Returns const char* from Python string in PyObject.
static PyObject * fPickleLoads
static PyObject * fGlobalNS
static PyObject * fModulePickle
static PyObject * fModuleBuiltin
PyMethodBase(const TString &jobName, Types::EMVA methodType, const TString &methodTitle, DataSetInfo &dsi, const TString &theOption="")
static PyObject * fEval
static PyObject * GetValueFromDict(PyObject *dict, const char *key)
Utility function which checks if a given key is present in a Python dictionary object and returns the...
void PyRunString(TString code, TString errorMessage="Failed to run python code", int start=256)
Execute Python code from string.
MsgLogger & Log() const
Definition Tools.h:228
Basic string class.
Definition TString.h:139
const char * Data() const
Definition TString.h:378
TString & ReplaceAll(const TString &s1, const TString &s2)
Definition TString.h:702
Bool_t IsNull() const
Definition TString.h:416
virtual TString GetFromPipe(const char *command)
Execute command and return output in TString.
Definition TSystem.cxx:667
create variable transformations
TString Python_Executable()
Function to find current Python executable used by ROOT If "Python3" is installed,...
Tools & gTools()
MsgLogger & Endl(MsgLogger &ml)
Definition MsgLogger.h:148