ProxyWrappers.cxx

Go to the documentation of this file.

// Bindings
#include "cpyrt.h"
 
using namespace cppjit;
#include "CPPClassMethod.h"
#include "CPPConstructor.h"
#include "CPPDataMember.h"
#include "CPPExcInstance.h"
#include "CPPFunction.h"
#include "CPPGetSetItem.h"
#include "CPPInstance.h"
#include "CPPMethod.h"
#include "CPPOperator.h"
#include "CPPOverload.h"
#include "CPPScope.h"
#include "MemoryRegulator.h"
#include "ProxyWrappers.h"
#include "PyStrings.h"
#include "Pythonize.h"
#include "TemplateProxy.h"
#include "TupleOfInstances.h"
#include "TypeManip.h"
#include "Utility.h"
#include "cppjit_interop.h"
 
// Standard
#include <algorithm>
#include <deque>
#include <map>
#include <string>
#include <unordered_set>
#include <vector>
 
//- data _______________________________________________________________________
namespace cppjit::cpyrt {
extern PyObject* gThisModule;
extern PyObject* gPyTypeMap;
extern std::unordered_set<interop::TCppScope_t> gPinnedTypes;
} // namespace cppjit::cpyrt
 
// to prevent having to walk scopes, track python classes by C++ class
typedef std::unordered_map<interop::TCppScope_t, PyObject*> PyClassMap_t;
static PyClassMap_t gPyClasses;
 
//- helpers --------------------------------------------------------------------
 
namespace cppjit::cpyrt {
 
typedef struct {
  PyObject_HEAD PyObject* dict;
} proxyobject;
 
// helper for creating new C++ proxy python types
static PyObject* CreateNewCppProxyClass(interop::TCppScope_t klass,
                                        PyObject* pybases) {
  // Create a new python shadow class with the required hierarchy and
  // meta-classes.
  PyObject* pymetabases = PyTuple_New(PyTuple_GET_SIZE(pybases));
  for (int i = 0; i < PyTuple_GET_SIZE(pybases); ++i) {
    PyObject* btype = (PyObject*)Py_TYPE(PyTuple_GetItem(pybases, i));
    Py_INCREF(btype);
    PyTuple_SET_ITEM(pymetabases, i, btype);
  }
 
  std::string name = interop::GetFinalName(klass);
 
  // create meta-class, add a dummy __module__ to pre-empt the default setting
  PyObject* args =
      Py_BuildValue((char*)"sO{}", (name + "_meta").c_str(), pymetabases);
  PyDict_SetItem(PyTuple_GET_ITEM(args, 2), PyStrings::gModule, Py_True);
  Py_DECREF(pymetabases);
 
  PyObject* pymeta = (PyObject*)CPPScopeMeta_New(klass, args);
  Py_DECREF(args);
  if (!pymeta)
    return nullptr;
 
  // alright, and now we really badly want to get rid of the dummy ...
  PyObject* dictproxy = PyObject_GetAttr(pymeta, PyStrings::gDict);
  PyDict_DelItem(((proxyobject*)dictproxy)->dict, PyStrings::gModule);
 
  // create actual class
  args = Py_BuildValue((char*)"sO{}", name.c_str(), pybases);
  PyObject* pyclass =
      ((PyTypeObject*)pymeta)->tp_new((PyTypeObject*)pymeta, args, nullptr);
 
  Py_DECREF(args);
  Py_DECREF(pymeta);
 
  return pyclass;
}
 
static inline void AddPropertyToClass(PyObject* pyclass,
                                      interop::TCppScope_t scope,
                                      interop::TCppScope_t data) {
  cpyrt::CPPDataMember* property = cpyrt::CPPDataMember_New(scope, data);
  PyObject* pname = PyUnicode_InternFromString(
      const_cast<char*>(property->GetName().c_str()));
 
  // allow access at the instance level
  PyType_Type.tp_setattro(pyclass, pname, (PyObject*)property);
  if (PyErr_Occurred()) {
    // CPPDataMember.tp_descr_set raises an error
    // when all of the following conditions are met
    // 1. data is static const
    // 2. scope is a derived class
    // 3. data is defined in both parent and base class (using
    // parent::attribute;)
    PyErr_Clear();
  }
 
  // allow access at the class level (always add after setting instance level)
  if (interop::IsStaticDatamember(data) || interop::IsEnumConstant(data))
    PyType_Type.tp_setattro((PyObject*)Py_TYPE(pyclass), pname,
                            (PyObject*)property);
 
  // cleanup
  Py_DECREF(pname);
  Py_DECREF(property);
}
 
static inline void AddScopeToParent(PyObject* parent, const std::string& name,
                                    PyObject* newscope) {
  PyObject* pyname = PyUnicode_InternFromString((char*)name.c_str());
  if (CPPScope_Check(parent))
    PyType_Type.tp_setattro(parent, pyname, newscope);
  else
    PyObject_SetAttr(parent, pyname, newscope);
  Py_DECREF(pyname);
}
 
static inline PyObject* GetAttrDirect(PyObject* pyclass, PyObject* pyname) {
  // get an attribute without causing getattr lookups
  PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict);
  if (dct) {
    PyObject* attr = PyObject_GetItem(dct, pyname);
    Py_DECREF(dct);
    return attr;
  }
  return nullptr;
}
 
} // namespace cppjit::cpyrt
 
//- public functions ---------------------------------------------------------
namespace cppjit::cpyrt {
 
static inline void sync_templates(PyObject* pyclass,
                                  const std::string& mtCppName,
                                  const std::string& mtName) {
  PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict);
  PyObject* pyname = PyUnicode_FromString(const_cast<char*>(mtName.c_str()));
  PyObject* attr = PyObject_GetItem(dct, pyname);
  if (!attr)
    PyErr_Clear();
  Py_DECREF(dct);
  if (!TemplateProxy_Check(attr)) {
    TemplateProxy* pytmpl = TemplateProxy_New(mtCppName, mtName, pyclass);
    if (CPPOverload_Check(attr))
      pytmpl->MergeOverload((CPPOverload*)attr);
    PyType_Type.tp_setattro(pyclass, pyname, (PyObject*)pytmpl);
    Py_DECREF(pytmpl);
  }
  Py_XDECREF(attr);
  Py_DECREF(pyname);
}
 
static int BuildScopeProxyDict(interop::TCppScope_t scope, PyObject* pyclass,
                               const unsigned int flags) {
  // Collect methods and data for the given scope, and add them to the given
  // python proxy object.
 
  // some properties that'll affect building the dictionary
  bool isNamespace = interop::IsNamespace(scope);
  bool isComplete = interop::IsComplete(scope);
  bool hasConstructor = false;
  interop::TCppMethod_t potGetItem = nullptr;
 
  // load all public methods and data members
  typedef std::vector<PyCallable*> Callables_t;
  typedef std::unordered_map<std::string, Callables_t> CallableCache_t;
  CallableCache_t cache;
 
  // bypass custom __getattr__ for efficiency
  getattrofunc oldgetattro = Py_TYPE(pyclass)->tp_getattro;
  Py_TYPE(pyclass)->tp_getattro = PyType_Type.tp_getattro;
 
  // functions in namespaces are properly found through lazy lookup, so do not
  // create them until needed (the same is not true for data members)
  std::vector<interop::TCppMethod_t> methods;
  if (isComplete)
    interop::GetClassMethods(scope, methods);
 
  for (auto& method : methods) {
 
    // do not expose non-public methods as the Cling wrappers as those won't
    // compile; exception: a non-public method whose declaring class is not this
    // scope can only have entered the enumeration through a public
    // using-declaration (the backend drops non-public shadows), so it is
    // publicly callable through this class and its wrapper compiles (e.g.
    // MSVC's std::shared_ptr re-exposes the protected std::_Ptr_base<T>::get
    // with `public: using _Mybase::get;`)
    if (!interop::IsPublicMethod(method) &&
        interop::GetParentScope(interop::TCppScope_t(method.data)) == scope)
      continue;
 
    // process the method based on its name
    std::string mtCppName = interop::GetName(interop::TCppScope_t(method.data));
 
    // special case trackers
    bool setupSetItem = false;
    bool isConstructor = interop::IsConstructor(method);
 
    // Here isTemplate means to ignore templated constructors, that is handled
    // in the next loop.
    bool isTemplate = interop::IsTemplatedMethod(method) && !isConstructor;
    bool isStubbedOperator = false;
 
    // filter empty names (happens for namespaces, is bug?)
    if (mtCppName == "")
      continue;
 
    // filter C++ destructors
    if (mtCppName[0] == '~')
      continue;
 
    // translate operators
    std::string mtName = Utility::MapOperatorName(
        mtCppName, interop::GetMethodNumArgs(method), &isStubbedOperator);
    if (mtName.empty())
      continue;
 
    // operator[]/() returning a reference type will be used for __setitem__
    bool isCall = mtName == "__call__";
    if (isCall || mtName == "__getitem__") {
      const std::string& qual_return =
          interop::GetMethodReturnTypeAsString(method);
      const std::string& cpd = TypeManip::compound(qual_return);
      if (!cpd.empty() && cpd[cpd.size() - 1] == '&' &&
          qual_return.find("const", 0, 5) == std::string::npos) {
        if (isCall && !potGetItem)
          potGetItem = method;
        setupSetItem = true; // will add methods as overloads
      } else if (isCall && 1 < interop::GetMethodNumArgs(method)) {
        // not a non-const by-ref return, thus better __getitem__ candidate; the
        // requirement for multiple arguments is that there is otherwise no
        // benefit over the use of normal __getitem__ (this allows
        // multi-indexing arguments, which is clean in Python, but not allowed
        // in C++)
        potGetItem = method;
      }
    }
 
    // template members; handled by adding a dispatcher to the class
    bool storeOnTemplate = isTemplate;
    if (storeOnTemplate) {
      sync_templates(pyclass, mtCppName, mtName);
      // continue processing to actually add the method so that the proxy can
      // find it on the class when called explicitly
    }
 
    // construct the holder
    PyCallable* pycall = nullptr;
    if (interop::IsStaticMethod(method)) // class method
      pycall = new CPPClassMethod(scope, method);
    else if (isNamespace) // free function
      pycall = new CPPFunction(scope, method);
    else if (isConstructor) { // ctor
      mtName = "__init__";
      hasConstructor = true;
      if (!interop::IsAbstract(scope)) {
        if (flags & CPPScope::kIsMultiCross) {
          pycall = new CPPMultiConstructor(scope, method);
        } else
          pycall = new CPPConstructor(scope, method);
      } else
        pycall = new CPPAbstractClassConstructor(scope, method);
    } else if (isStubbedOperator) {
      pycall = new CPPOperator(scope, method, mtName);
    } else // member function
      pycall = new CPPMethod(scope, method);
 
    if (storeOnTemplate) {
      // template proxy was already created in sync_templates call above, so
      // add only here, not to the cache of collected methods
      PyObject* attr =
          PyObject_GetAttrString(pyclass, const_cast<char*>(mtName.c_str()));
      if (isTemplate)
        ((TemplateProxy*)attr)->AdoptTemplate(pycall);
      else
        ((TemplateProxy*)attr)->AdoptMethod(pycall);
      Py_DECREF(attr);
 
      // for operator[]/() that returns by ref, also add __setitem__
      if (setupSetItem) {
        TemplateProxy* pysi =
            (TemplateProxy*)GetAttrDirect(pyclass, PyStrings::gSetItem);
        if (!TemplateProxy_Check(pysi)) {
          CPPOverload* precursor =
              (CPPOverload_Check(pysi)) ? (CPPOverload*)pysi : nullptr;
          if (pysi && !precursor)
            Py_DECREF(pysi); // something unknown, just drop it
          pysi = TemplateProxy_New(mtCppName, "__setitem__", pyclass);
          if (precursor)
            pysi->MergeOverload(precursor);
          Py_XDECREF(precursor);
          PyObject_SetAttrString(pyclass, const_cast<char*>("__setitem__"),
                                 (PyObject*)pysi);
        }
        if (isTemplate)
          pysi->AdoptTemplate(new CPPSetItem(scope, method));
        else
          pysi->AdoptMethod(new CPPSetItem(scope, method));
        Py_XDECREF(pysi);
      }
 
    } else {
      // lookup method dispatcher and store method
      Callables_t& md =
          (*(cache.insert(std::make_pair(mtName, Callables_t())).first)).second;
      md.push_back(pycall);
 
      // special case for operator[]/() that returns by ref, use for
      // getitem/call and setitem
      if (setupSetItem) {
        Callables_t& setitem =
            (*(cache
                   .insert(std::make_pair(std::string("__setitem__"),
                                          Callables_t()))
                   .first))
                .second;
        setitem.push_back(new CPPSetItem(scope, method));
      }
    }
  }
 
  // add proxies for un-instantiated/non-overloaded templated methods
  std::vector<interop::TCppMethod_t> templ_methods;
  interop::GetTemplatedMethods(scope, templ_methods);
  for (auto& method : templ_methods) {
    const std::string mtCppName =
        interop::GetName(interop::TCppScope_t(method.data));
    // the number of arguments isn't known until instantiation and as far as C++
    // is concerned, all same-named operators are simply overloads; so will
    // pre-emptively add both names if with and without arguments differ,
    // letting the normal overload mechanism resolve on call
    bool isConstructor = interop::IsConstructor(method);
 
    // first add with no arguments
    std::string mtName0 =
        isConstructor ? "__init__" : Utility::MapOperatorName(mtCppName, false);
    sync_templates(pyclass, mtCppName, mtName0);
 
    // then add when taking arguments, if this method is different
    if (!isConstructor) {
      std::string mtName1 = Utility::MapOperatorName(mtCppName, true);
      if (mtName0 != mtName1)
        sync_templates(pyclass, mtCppName, mtName1);
    }
  }
 
  // add a pseudo-default ctor, if none defined
  if (!hasConstructor) {
    PyCallable* defctor = nullptr;
    if (!isComplete) {
      ((CPPScope*)pyclass)->fFlags |= CPPScope::kIsInComplete;
      defctor =
          new CPPIncompleteClassConstructor(scope, interop::TCppMethod_t{});
    } else if (interop::IsAbstract(scope)) {
      defctor = new CPPAbstractClassConstructor(scope, interop::TCppMethod_t{});
    } else if (isNamespace) {
      defctor = new CPPNamespaceConstructor(scope, interop::TCppMethod_t{});
    } else {
      defctor =
          new CPPAllPrivateClassConstructor(scope, interop::TCppMethod_t{});
    }
 
    if (defctor)
      cache["__init__"].push_back(defctor);
  }
 
  // map __call__ to __getitem__ if also mapped to __setitem__
  if (potGetItem) {
    Callables_t& getitem =
        (*(cache
               .insert(
                   std::make_pair(std::string("__getitem__"), Callables_t()))
               .first))
            .second;
    getitem.push_back(new CPPGetItem(scope, potGetItem));
  }
 
  // add the methods to the class dictionary
  PyObject* dct = PyObject_GetAttr(pyclass, PyStrings::gDict);
  for (CallableCache_t::iterator imd = cache.begin(); imd != cache.end();
       ++imd) {
    // in order to prevent removing templated editions of this method (which
    // were set earlier, above, as a different proxy object), we'll check and
    // add this method flagged as a generic one (to be picked up by the
    // templated one as appropriate) if a template exists
    PyObject* pyname =
        PyUnicode_FromString(const_cast<char*>(imd->first.c_str()));
    PyObject* attr = PyObject_GetItem(dct, pyname);
    Py_DECREF(pyname);
    if (TemplateProxy_Check(attr)) {
      // template exists, supply it with the non-templated method overloads
      for (auto cit : imd->second)
        ((TemplateProxy*)attr)->AdoptMethod(cit);
    } else {
      if (!attr)
        PyErr_Clear();
      // normal case, add a new method
      CPPOverload* method = CPPOverload_New(imd->first, imd->second);
      PyObject* pymname = PyUnicode_InternFromString(
          const_cast<char*>(method->GetName().c_str()));
      PyType_Type.tp_setattro(pyclass, pymname, (PyObject*)method);
      Py_DECREF(pymname);
      Py_DECREF(method);
    }
 
    Py_XDECREF(attr); // could have been found in base class or non-existent
  }
  Py_DECREF(dct);
 
  // collect data members (including enums)
  std::vector<interop::TCppScope_t> datamembers;
  interop::GetDatamembers(scope, datamembers);
  for (auto& datamember : datamembers) {
    // allow only public members
    if (!interop::IsPublicData(datamember))
      continue;
 
    // enum datamembers (this in conjunction with previously collected enums
    // above)
    if (interop::IsEnumType(interop::GetDatamemberType(datamember)) &&
        interop::IsEnumConstant(datamember)) {
      // two options: this is a static variable, or it is the enum value, the
      // latter already exists, so check for it and move on if set
      PyObject* eset = PyObject_GetAttrString(
          pyclass,
          const_cast<char*>(interop::GetFinalName(datamember).c_str()));
      if (eset) {
        Py_DECREF(eset);
        continue;
      }
 
      PyErr_Clear();
 
      // it could still be that this is an anonymous enum, which is not in the
      // list provided by the class
      if (strstr(interop::GetDatamemberTypeAsString(datamember).c_str(),
                 "(anonymous)") != 0 ||
          strstr(interop::GetDatamemberTypeAsString(datamember).c_str(),
                 "(unnamed)") != 0) {
        AddPropertyToClass(pyclass, scope, datamember);
        continue;
      }
    }
 
    // properties (aka public (static) data members)
    AddPropertyToClass(pyclass, scope, datamember);
  }
 
  // restore custom __getattr__
  Py_TYPE(pyclass)->tp_getattro = oldgetattro;
 
  // all ok, done
  return 0;
}
 
//----------------------------------------------------------------------------
static void CollectUniqueBases(interop::TCppScope_t klass,
                               std::deque<interop::TCppScope_t>& uqb) {
  // collect bases in acceptable mro order, while removing duplicates (this may
  // break the overload resolution in esoteric cases, but otherwise the class
  // can not be used at all, as CPython will refuse the mro).
  size_t nbases = interop::GetNumBases(klass);
 
  for (size_t ibase = 0; ibase < nbases; ++ibase) {
    interop::TCppScope_t tp = interop::GetBaseScope(klass, ibase);
    int decision = 2;
    if (!tp)
      continue; // means this base with not be available Python-side
    for (size_t ibase2 = 0; ibase2 < uqb.size(); ++ibase2) {
      if (uqb[ibase2] == tp) { // not unique ... skip
        decision = 0;
        break;
      }
 
      if (interop::IsSubclass(tp, uqb[ibase2])) {
        // mro requirement: sub-type has to follow base
        decision = 1;
        break;
      }
    }
 
    if (decision == 1) {
      uqb.push_front(tp);
    } else if (decision == 2) {
      uqb.push_back(tp);
    }
    // skipped if decision == 0 (not unique)
  }
}
 
static PyObject* BuildCppClassBases(interop::TCppScope_t klass) {
  // Build a tuple of python proxy classes of all the bases of the given
  // 'klass'.
  std::deque<interop::TCppScope_t> uqb;
  CollectUniqueBases(klass, uqb);
 
  // allocate a tuple for the base classes, special case for first base
  size_t nbases = uqb.size();
 
  PyObject* pybases = PyTuple_New(nbases ? nbases : 1);
  if (!pybases)
    return nullptr;
 
  // build all the bases
  if (nbases == 0) {
    Py_INCREF((PyObject*)(void*)&CPPInstance_Type);
    PyTuple_SET_ITEM(pybases, 0, (PyObject*)(void*)&CPPInstance_Type);
  } else {
    for (std::deque<interop::TCppScope_t>::size_type ibase = 0; ibase < nbases;
         ++ibase) {
      PyObject* pyclass = CreateScopeProxy(uqb[ibase]);
      if (!pyclass) {
        Py_DECREF(pybases);
        return nullptr;
      }
 
      PyTuple_SET_ITEM(pybases, ibase, pyclass);
    }
 
    // special case, if true python types enter the hierarchy, make sure that
    // the first base seen is still the CPPInstance_Type
    if (!PyObject_IsSubclass(PyTuple_GET_ITEM(pybases, 0),
                             (PyObject*)&CPPInstance_Type)) {
      PyObject* newpybases = PyTuple_New(nbases + 1);
      Py_INCREF((PyObject*)(void*)&CPPInstance_Type);
      PyTuple_SET_ITEM(newpybases, 0, (PyObject*)(void*)&CPPInstance_Type);
      for (int ibase = 0; ibase < (int)nbases; ++ibase) {
        PyObject* pyclass = PyTuple_GET_ITEM(pybases, ibase);
        Py_INCREF(pyclass);
        PyTuple_SET_ITEM(newpybases, ibase + 1, pyclass);
      }
      Py_DECREF(pybases);
      pybases = newpybases;
    }
  }
 
  return pybases;
}
 
} // namespace cppjit::cpyrt
 
//----------------------------------------------------------------------------
PyObject* cpyrt::GetScopeProxy(interop::TCppScope_t scope) {
  // Retrieve scope proxy from the known ones.
  PyClassMap_t::iterator pci = gPyClasses.find(scope);
  if (pci != gPyClasses.end()) {
    PyObject* pyclass = cpyrt_GetWeakRef(pci->second);
    if (pyclass)
      return pyclass;
  }
 
  return nullptr;
}
namespace cppjit::cpyrt {
PyObject* CppType_To_PyObject(interop::TCppType_t type, std::string name,
                              interop::TCppScope_t parent_scope,
                              PyObject* parent) {
  interop::TCppType_t resolved_type = interop::ResolveType(type);
  if (gPyTypeMap) {
    const std::string& resolved = interop::GetTypeAsString(resolved_type);
    PyObject* tc =
        PyDict_GetItemString(gPyTypeMap, resolved.c_str()); // borrowed
    if (tc && PyCallable_Check(tc)) {
      const std::string& scName = interop::GetScopedFinalName(parent_scope);
      PyObject* nt =
          PyObject_CallFunction(tc, (char*)"ss", name.c_str(),
                                scName != "<unnamed>" ? scName.c_str() : "");
      if (nt) {
        if (parent) {
          AddScopeToParent(parent, name, nt);
          Py_DECREF(parent);
        }
        return nt;
      }
      PyErr_Clear();
    }
  }
  return nullptr;
}
} // namespace cppjit::cpyrt
 
//----------------------------------------------------------------------------
PyObject* cpyrt::CreateScopeProxy(PyObject*, PyObject* args) {
  // Build a python shadow class for the named C++ class.
  std::string cname = PyUnicode_AsUTF8(PyTuple_GetItem(args, 0));
  if (PyErr_Occurred())
    return nullptr;
 
  return CreateScopeProxy(cname);
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::CreateScopeProxy(const std::string& name, PyObject* parent,
                                  const unsigned flags) {
  // Build a python shadow class for the named C++ class or namespace.
 
  // determine complete scope name, if a python parent has been given
  interop::TCppScope_t parent_scope;
  if (parent) {
    if (CPPScope_Check(parent))
      parent_scope = ((CPPScope*)parent)->fCppType;
    else {
      PyObject* parname = PyObject_GetAttr(parent, PyStrings::gName);
      if (!parname) {
        PyErr_Format(PyExc_SystemError, "given scope has no name for %s",
                     name.c_str());
        return nullptr;
      }
 
      // should be a string
      std::string scName = PyUnicode_AsUTF8(parname);
      Py_DECREF(parname);
      if (PyErr_Occurred())
        return nullptr;
      parent_scope = interop::GetScope(scName);
    }
 
    Py_INCREF(parent);
  }
 
  // retrieve C++ class (this verifies name, and is therefore done first)
  if (name == "") {
    interop::TCppScope_t klass = interop::GetGlobalScope();
    Py_INCREF(gThisModule);
    parent = gThisModule;
    return CreateScopeProxy(klass, parent, flags);
  } else if (interop::TCppScope_t klass =
                 interop::GetScope(name, parent_scope)) {
    if (interop::IsTypedefed(klass) &&
        interop::IsPointerType(interop::GetTypeFromScope(klass)) &&
        interop::IsClass(interop::GetUnderlyingScope(klass)))
      return nullptr; // this is handled by the caller; typedef to class pointer
    return CreateScopeProxy(klass, parent, flags);
  } else if (interop::IsBuiltin(name)) {
    interop::TCppType_t type = interop::GetType(name);
    PyObject* result = nullptr;
    if (type)
      result = CppType_To_PyObject(type, name, parent_scope, parent);
    if (result)
      return result;
  }
  // all options have been exhausted: it doesn't exist as such
  PyErr_Format(PyExc_TypeError, "\'%s\' is not a known C++ class",
               name.c_str());
  Py_XDECREF(parent);
  return nullptr;
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::CreateScopeProxy(interop::TCppScope_t scope, PyObject* parent,
                                  const unsigned flags) {
  // Convenience function with a lookup first through the known existing
  // proxies.
  PyObject* pyclass = GetScopeProxy(scope);
  if (pyclass)
    return pyclass;
 
  interop::TCppScope_t parent_scope = nullptr;
  if (CPPScope_Check(parent))
    parent_scope = ((CPPScope*)parent)->fCppType;
  if (!parent_scope)
    parent_scope = interop::GetParentScope(scope);
 
  if (!parent) {
    if (parent_scope)
      parent = CreateScopeProxy(parent_scope);
    else {
      Py_INCREF(gThisModule);
      parent = gThisModule;
    }
  }
 
  std::string name = interop::GetFinalName(scope);
  bool is_typedef = false;
  std::string typedefed_name = "";
  if (interop::IsTypedefed(scope)) {
    is_typedef = true;
    typedefed_name = interop::GetFullName(scope);
    interop::TCppScope_t underlying_scope = interop::GetUnderlyingScope(scope);
    if ((underlying_scope) && (underlying_scope != scope)) {
      scope = underlying_scope;
    } else {
      if (PyObject* result = CppType_To_PyObject(
              interop::GetTypeFromScope(scope), name, parent_scope, parent))
        return result;
 
      PyErr_Format(PyExc_TypeError, "\'%s\' is not a known C++ class",
                   interop::GetScopedFinalName(scope).c_str());
      Py_XDECREF(parent);
      return nullptr;
    }
  }
 
  if (interop::IsTemplate(scope)) {
    // a "naked" templated class is requested: return callable proxy for
    // instantiations
    PyObject* pytcl = PyObject_GetAttr(gThisModule, PyStrings::gTemplate);
    PyObject* cppscope = PyLong_FromVoidPtr(scope.data);
    PyObject* pytemplate = PyObject_CallFunction(
        pytcl, const_cast<char*>("sO"),
        const_cast<char*>(interop::GetScopedFinalName(scope).c_str()),
        cppscope);
    Py_DECREF(pytcl);
 
    // cache the result
    AddScopeToParent(parent, name, pytemplate);
 
    // done, next step should be a call into this template
    Py_XDECREF(parent);
    return pytemplate;
  }
 
  if (interop::IsEnumScope(scope))
    return nullptr;
 
  // locate class by ID, if possible, to prevent parsing scopes/templates anew
  PyObject* pyscope = GetScopeProxy(scope);
  if (pyscope) {
    if (parent) {
      AddScopeToParent(parent, name, pyscope);
      Py_DECREF(parent);
    }
    return pyscope;
  }
 
  // if the scope was earlier found as actual, then we're done already,
  // otherwise build a new scope proxy
  if (!pyscope) {
    // construct the base classes
    PyObject* pybases = BuildCppClassBases(scope);
    if (pybases != 0) {
      // create a fresh Python class, given bases, name, and empty dictionary
      pyscope = CreateNewCppProxyClass(scope, pybases);
      Py_DECREF(pybases);
    }
 
    // fill the dictionary, if successful
    if (pyscope) {
      if (BuildScopeProxyDict(scope, pyscope, flags)) {
        // something failed in building the dictionary
        Py_DECREF(pyscope);
        pyscope = nullptr;
      }
    }
 
    // store a ref from cppjit scope id to new python class
    if (pyscope && !(((CPPScope*)pyscope)->fFlags & CPPScope::kIsInComplete)) {
      gPyClasses[scope] = PyWeakref_NewRef(pyscope, nullptr);
 
      if (!(((CPPScope*)pyscope)->fFlags & CPPScope::kIsNamespace)) {
        // add python-style features to classes only
        if (!Pythonize(pyscope, scope)) {
          Py_DECREF(pyscope);
          pyscope = nullptr;
        }
      } else {
        // add to sys.modules to allow importing from this namespace
        PyObject* pyfullname = PyObject_GetAttr(pyscope, PyStrings::gModule);
        PyUnicode_AppendAndDel(&pyfullname, PyUnicode_FromString("."));
        PyUnicode_AppendAndDel(&pyfullname,
                               PyObject_GetAttr(pyscope, PyStrings::gName));
        PyObject* modules = PySys_GetObject(const_cast<char*>("modules"));
        if (modules && PyDict_Check(modules))
          PyDict_SetItem(modules, pyfullname, pyscope);
        Py_DECREF(pyfullname);
      }
    }
  }
 
  // store on parent if found/created and complete
  if (pyscope && !(((CPPScope*)pyscope)->fFlags & CPPScope::kIsInComplete)) {
    // FIXME: This is to mimic original behaviour. Still required?
    if (interop::IsTemplateInstantiation(scope))
      name = interop::GetScopedFinalName(scope);
    if (is_typedef && !typedefed_name.empty())
      AddScopeToParent(parent, typedefed_name, pyscope);
    AddScopeToParent(parent, name, pyscope);
  }
  Py_DECREF(parent);
 
  // all done
  return pyscope;
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::CreateExcScopeProxy(PyObject* pyscope, PyObject* pyname,
                                     PyObject* parent) {
  // To allow use of C++ exceptions in lieue of Python exceptions, they need to
  // derive from BaseException, which can not mix with the normal CPPInstance
  // and use of the meta-class. Instead, encapsulate them in a forwarding class
  // that derives from Pythons Exception class
 
  // start with creation of CPPExcInstance type base classes
  std::deque<interop::TCppScope_t> uqb;
  CollectUniqueBases(((CPPScope*)pyscope)->fCppType, uqb);
  size_t nbases = uqb.size();
 
  // Support for multiple bases actually can not actually work as-is: the reason
  // for deriving from BaseException is to guarantee the layout needed for
  // storing traces. If some other base is std::exception (as e.g.
  // boost::bad_any_cast) or also derives from std::exception, then there are
  // two trace locations. OTOH, if the other class is a non-exception type, then
  // the exception class does not need to derive from it because it can never be
  // caught as that type forwarding to the proxy will work as expected, through,
  // which is good enough).
  //
  // The code below restricts the hierarchy to a single base class, picking the
  // "best" by filtering std::exception and non-exception bases.
 
  PyObject* pybases = PyTuple_New(1);
  if (nbases == 0) {
    Py_INCREF((PyObject*)(void*)&CPPExcInstance_Type);
    PyTuple_SET_ITEM(pybases, 0, (PyObject*)(void*)&CPPExcInstance_Type);
  } else {
    PyObject* best_base = nullptr;
 
    for (std::deque<interop::TCppScope_t>::size_type ibase = 0; ibase < nbases;
         ++ibase) {
      // retrieve bases through their enclosing scope to guarantee treatment as
      // exception classes and proper caching
      interop::TCppScope_t parent_scope = interop::GetParentScope(uqb[ibase]);
      PyObject* base_parent = CreateScopeProxy(parent_scope);
      if (!base_parent) {
        Py_DECREF(pybases);
        return nullptr;
      }
 
      PyObject* excbase = PyObject_GetAttrString(
          base_parent, interop::GetFinalName(uqb[ibase]).c_str());
      Py_DECREF(base_parent);
      if (!excbase) {
        Py_DECREF(pybases);
        return nullptr;
      }
 
      if (PyType_IsSubtype((PyTypeObject*)excbase, &CPPExcInstance_Type)) {
        Py_XDECREF(best_base);
        best_base = excbase;
 
        if (interop::GetScopedFinalName(uqb[ibase]) != "std::exception")
          break;
      } else {
        // just skip: there will be at least one exception derived base class
        Py_DECREF(excbase);
      }
    }
 
    PyTuple_SET_ITEM(pybases, 0, best_base);
  }
 
  PyObject* args = Py_BuildValue((char*)"OO{}", pyname, pybases);
 
  // meta-class attributes (__cpp_name__, etc.) can not be resolved lazily so
  // add them directly instead in case they are needed
  PyObject* dct = PyTuple_GET_ITEM(args, 2);
  PyDict_SetItem(dct, PyStrings::gUnderlying, pyscope);
  PyDict_SetItem(dct, PyStrings::gName,
                 PyObject_GetAttr(pyscope, PyStrings::gName));
  PyDict_SetItem(dct, PyStrings::gCppName,
                 PyObject_GetAttr(pyscope, PyStrings::gCppName));
  PyDict_SetItem(dct, PyStrings::gModule,
                 PyObject_GetAttr(pyscope, PyStrings::gModule));
 
  // create the actual exception class
  PyObject* exc_pyscope = PyType_Type.tp_new(&PyType_Type, args, nullptr);
  Py_DECREF(args);
  Py_DECREF(pybases);
 
  // cache the result for future lookups and return
  PyType_Type.tp_setattro(parent, pyname, exc_pyscope);
  return exc_pyscope;
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::BindCppObjectNoCast(interop::TCppObject_t address,
                                     interop::TCppScope_t klass,
                                     const unsigned flags) {
  // only known or knowable objects will be bound (null object is ok)
  if (!klass) {
    PyErr_SetString(PyExc_TypeError, "attempt to bind C++ object w/o class");
    return nullptr;
  }
 
  // retrieve python class
  PyObject* pyclass = CreateScopeProxy(klass);
  if (!pyclass)
    return nullptr; // error has been set in CreateScopeProxy
 
  bool noReg = flags & (CPPInstance::kNoMemReg | CPPInstance::kNoWrapConv);
  bool isRef = flags & CPPInstance::kIsReference;
  void* r_address =
      isRef ? (address ? *(void**)address.data : nullptr) : address.data;
 
  // check whether the object to be bound is a smart pointer that needs
  // embedding
  PyObject* smart_type = (!(flags & CPPInstance::kNoWrapConv) &&
                          (((CPPClass*)pyclass)->fFlags & CPPScope::kIsSmart))
                             ? pyclass
                             : nullptr;
  if (smart_type) {
    interop::TCppScope_t underlying =
        ((CPPSmartClass*)smart_type)->fUnderlyingType;
 
    // Downcast the underlying object to its actual (most derived) class, as
    // BindCppObject does for raw pointers, but only when the cast is safe:
    //  * the actual class must really derive from the declared one;
    //    GetActualClass() can report a base (e.g. when the actual class has no
    //    dictionary of its own), and such an up-cast must be rejected.
    //  * the cast must need no pointer adjustment: the dereferencer always
    //    returns a pointer to the declared class, so a non-zero offset could
    //    not be applied consistently on later member access (multiple
    //    inheritance).
    if (address && !isRef) {
      if (void* deref =
              interop::CallR(((CPPSmartClass*)smart_type)->fDereferencer,
                             address, 0, nullptr)) {
        interop::TCppScope_t clActual =
            interop::GetActualClass(underlying, deref);
        if (clActual && clActual != underlying &&
            interop::IsSubclass(clActual, underlying) &&
            interop::GetBaseOffset(clActual, underlying, deref,
                                   -1 /* down-cast */) == 0)
          underlying = clActual;
      }
    }
 
    pyclass = CreateScopeProxy(underlying);
    if (!pyclass) {
      // simply restore and expose as the actual smart pointer class
      pyclass = smart_type;
      smart_type = nullptr;
    }
  }
 
  // TODO: make sure that a consistent address is used (may have to be done in
  // BindCppObject)
  if (address && !(flags & CPPInstance::kIsValue) /* always fresh */ &&
      !noReg) {
    PyObject* oldPyObject =
        MemoryRegulator::RetrievePyObject(r_address, pyclass);
 
    // embedded smart pointers are registered with the class of the underlying
    // type, b/c there is no Python proxy for the smart pointer itself in that
    // case; check whether the result found matches the smart-ness and if so,
    // whether the smart pointer found is the same as the one requested (note:
    // ptr-ptr requires old object to be a reference to enable re-use)
    if (oldPyObject) {
      CPPInstance* o_pyobj = ((CPPInstance*)oldPyObject);
 
      if ((bool)smart_type == o_pyobj->IsSmart() &&
          r_address ==
              (smart_type ? o_pyobj->GetObjectRaw() : o_pyobj->GetObject()) &&
          (!(flags & CPPInstance::kIsPtrPtr) ||
           (o_pyobj->fFlags & CPPInstance::kIsReference))) {
        Py_DECREF(pyclass);
        return oldPyObject;
      } else { // reference or naked v.s. smart pointer mismatch
        Py_DECREF(oldPyObject);
      }
    }
  }
 
  // instantiate an object of this class
  PyObject* args = PyTuple_New(0);
  CPPInstance* pyobj = (CPPInstance*)((PyTypeObject*)pyclass)
                           ->tp_new((PyTypeObject*)pyclass, args, nullptr);
  Py_DECREF(args);
 
  // bind, register and return if successful
  if (pyobj != 0) { // fill proxy value?
    unsigned objflags =
        flags & (CPPInstance::kIsReference | CPPInstance::kIsPtrPtr |
                 CPPInstance::kIsValue | CPPInstance::kIsOwner |
                 CPPInstance::kIsActual);
    pyobj->Set(address.data, (CPPInstance::EFlags)objflags);
 
    if (smart_type)
      pyobj->SetSmart(smart_type);
 
    // do not register null pointers, references (regulated objects can be
    // referenced, but referenced objects can not be regulated), or direct usage
    // of smart pointers or iterators
    if (address && !isRef && !noReg)
      MemoryRegulator::RegisterPyObject(
          pyobj, smart_type ? pyobj->GetObjectRaw() : pyobj->GetObject());
  }
 
  // successful completion; wrap exception options to make them raiseable,
  // normal return otherwise
  if (((CPPClass*)pyclass)->fFlags & CPPScope::kIsException) {
    PyObject* exc_obj =
        CPPExcInstance_Type.tp_new(&CPPExcInstance_Type, nullptr, nullptr);
    ((CPPExcInstance*)exc_obj)->fCppInstance = (PyObject*)pyobj;
    Py_DECREF(pyclass);
    return exc_obj;
  }
 
  Py_DECREF(pyclass);
  return (PyObject*)pyobj;
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::BindCppObject(interop::TCppObject_t address,
                               interop::TCppScope_t klass,
                               const unsigned flags) {
  // if the object is a null pointer, return a typed one (as needed for
  // overloading)
  if (!address)
    return BindCppObjectNoCast(address, klass, flags);
 
  // only known or knowable objects will be bound
  if (!klass) {
    PyErr_SetString(PyExc_TypeError, "attempt to bind C++ object w/o class");
    return nullptr;
  }
 
  bool isRef = flags & CPPInstance::kIsReference;
 
  // downcast to real class for object returns, unless pinned
  // TODO: should the memory regulator for klass be searched first, so that if
  // successful, no down-casting is attempted?
  // TODO: optimize for final classes
  unsigned new_flags = flags;
  if (!isRef && (gPinnedTypes.empty() ||
                 gPinnedTypes.find(klass) == gPinnedTypes.end())) {
    interop::TCppScope_t clActual = interop::GetActualClass(klass, address);
 
    if (clActual) {
      if (clActual != klass) {
        intptr_t offset =
            interop::GetBaseOffset(clActual, klass, address, -1 /* down-cast */,
                                   true /* report errors */);
        if (offset != -1) { // may fail if clActual not fully defined
          address = (void*)((intptr_t)address.data + offset);
          klass = clActual;
        }
      }
      new_flags |= CPPInstance::kIsActual;
    }
  }
 
  // actual binding (returned object may be zero w/ a python exception set)
  return BindCppObjectNoCast(address, klass, new_flags);
}
 
//----------------------------------------------------------------------------
PyObject* cpyrt::BindCppObjectArray(interop::TCppObject_t address,
                                    interop::TCppScope_t klass, cdims_t dims) {
  // TODO: this function exists for symmetry; need to figure out if it's useful
  return TupleOfInstances_New(address, klass, dims);
}