Dispatcher.cxx

Go to the documentation of this file.

// Bindings
#include "cpyrt.h"
 
using namespace cppjit;
#include "CPPScope.h"
#include "Dispatcher.h"
#include "ProxyWrappers.h"
#include "PyStrings.h"
#include "TypeManip.h"
#include "Utility.h"
#include "cppjit_interop.h"
 
// Standard
#include <set>
#include <sstream>
 
//----------------------------------------------------------------------------
static inline void InjectMethod(interop::TCppMethod_t method,
                                const std::string& mtCppName,
                                std::ostringstream& code) {
  // inject implementation for an overridden method
  using namespace cppjit::cpyrt;
 
  // method declaration
  std::string retType = interop::GetMethodReturnTypeAsString(method);
  code << "  " << retType << " " << mtCppName << "(";
 
  // build out the signature with predictable formal names
  interop::TCppIndex_t nArgs = interop::GetMethodNumArgs(method);
  std::vector<std::string> argtypes;
  argtypes.reserve(nArgs);
  for (interop::TCppIndex_t i = 0; i < nArgs; ++i) {
    argtypes.push_back(interop::GetMethodArgCanonTypeAsString(method, i));
    if (i != 0)
      code << ", ";
    code << argtypes.back() << " arg" << i;
  }
  code << ") ";
  if (interop::IsConstMethod(method))
    code << "const ";
  code << "{\n";
 
  // on destruction, the Python object may go first, in which case provide a
  // diagnostic warning (raising a PyException may not be possible as this could
  // happen during program shutdown); note that this means that the actual
  // result will be the default and the caller may need to act on that, but
  // that's still an improvement over a possible crash
  code << "    cppjit::cpyrt::PythonGILRAII python_gil_raii;\n" // acquire GIL
          "    PyObject* iself = (PyObject*)_internal_self;\n"
          "    if (!iself || iself == Py_None) {\n"
          "      PyErr_Warn(PyExc_RuntimeWarning, (char*)\"Call attempted on "
          "deleted python-side proxy\");\n"
          "      return";
  if (retType != "void") {
    if (retType.back() != '*')
      code << " (" << cpyrt::TypeManip::remove_const(retType) << "){}";
    else
      code << " nullptr";
  }
  code << ";\n"
          "    }\n"
          "    Py_INCREF(iself);\n";
 
  // start actual function body
  Utility::ConstructCallbackPreamble(retType, argtypes, code);
 
  // perform actual method call
  code << "    PyObject* mtPyName = PyUnicode_FromString(\"" << mtCppName
       << "\");\n"
          "    PyObject* pyresult = PyObject_CallMethodObjArgs(iself, mtPyName";
  for (interop::TCppIndex_t i = 0; i < nArgs; ++i)
    code << ", pyargs[" << i << "]";
  code << ", NULL);\n    Py_DECREF(mtPyName);\n    Py_DECREF(iself);\n";
 
  // close
  Utility::ConstructCallbackReturn(retType, (int)nArgs, code);
}
 
//----------------------------------------------------------------------------
namespace {
struct BaseInfo {
  BaseInfo(interop::TCppScope_t t, std::string&& bn, std::string&& bns)
      : btype(t), bname(bn), bname_scoped(bns) {}
  interop::TCppScope_t btype;
  std::string bname;
  std::string bname_scoped;
};
 
typedef std::vector<BaseInfo> BaseInfos_t;
typedef std::vector<interop::TCppMethod_t> Ctors_t;
typedef std::vector<Ctors_t> AllCtors_t;
typedef std::vector<std::pair<interop::TCppMethod_t, size_t>> CtorInfos_t;
} // unnamed namespace
 
static void build_constructors(const std::string& derivedName,
                               const BaseInfos_t& base_infos,
                               const AllCtors_t& ctors,
                               std::ostringstream& code,
                               const CtorInfos_t& methods = CtorInfos_t{},
                               size_t idx = 0) {
  if (idx < ctors.size()) {
    for (const auto& method : ctors[idx]) {
      size_t argsmin = (size_t)interop::GetMethodReqArgs(method);
      size_t argsmax = (size_t)interop::GetMethodNumArgs(method);
      for (size_t i = argsmin; i <= argsmax; ++i) {
        CtorInfos_t methods1{methods};
        methods1.emplace_back(method, i);
        build_constructors(derivedName, base_infos, ctors, code, methods1,
                           idx + 1);
      }
    }
  } else {
    // this is as deep as we go; start writing
    code << "  " << derivedName << "(";
 
    // declare arguments
    std::vector<size_t> arg_tots;
    arg_tots.reserve(methods.size());
    for (Ctors_t::size_type i = 0; i < methods.size(); ++i) {
      const auto& cinfo = methods[i];
      if (i != 0 && (arg_tots.back() || 1 < arg_tots.size()))
        code << ", ";
      size_t nArgs = cinfo.second;
      arg_tots.push_back(i == 0 ? nArgs : nArgs + arg_tots.back());
 
      if (i != 0)
        code << "__cppjit_internal::Sep*";
      size_t offset = (i != 0 ? arg_tots[i - 1] : 0);
      for (size_t j = 0; j < nArgs; ++j) {
        if (i != 0 || j != 0)
          code << ", ";
        code << interop::GetMethodArgCanonTypeAsString(cinfo.first, j) << " a"
             << (j + offset);
      }
    }
    code << ") : ";
 
    // pass arguments to base constructors
    for (BaseInfos_t::size_type i = 0; i < base_infos.size(); ++i) {
      if (i != 0)
        code << ", ";
      code << base_infos[i].bname << "(";
      size_t first = (i != 0 ? arg_tots[i - 1] : 0);
      for (size_t j = first; j < arg_tots[i]; ++j) {
        if (j != first)
          code << ", ";
        bool isRValue =
            cpyrt::TypeManip::compound(interop::GetMethodArgCanonTypeAsString(
                methods[i].first, j - first)) == "&&";
        if (isRValue)
          code << "std::move(";
        code << "a" << j;
        if (isRValue)
          code << ")";
      }
      code << ")";
    }
    code << " {}\n";
  }
}
 
namespace {
 
using namespace cppjit::interop;
 
static inline std::vector<TCppMethod_t>
FindBaseMethod(TCppScope_t tbase, const std::string mtCppName) {
  // Recursively walk the inheritance tree to find the overloads of the named
  // method
  std::vector<TCppMethod_t> result;
  result = GetMethodsFromName(tbase, mtCppName);
  if (result.empty()) {
    for (TCppIndex_t ibase = 0; ibase < GetNumBases(tbase); ++ibase) {
      TCppScope_t b = interop::GetBaseScope(tbase, ibase);
      result = FindBaseMethod(b, mtCppName);
      if (!result.empty())
        break;
    }
  }
  return result;
}
 
} // unnamed namespace
 
bool cpyrt::InsertDispatcher(CPPScope* klass, PyObject* bases, PyObject* dct,
                             std::ostringstream& err) {
  // Scan all methods in dct and where it overloads base methods in klass,
  // create dispatchers on the C++ side. Then interject the dispatcher class.
 
  if (!PyTuple_Check(bases) || !PyTuple_GET_SIZE(bases) || !dct ||
      !PyDict_Check(dct)) {
    err << "internal error: expected tuple of bases and proper dictionary";
    return false;
  }
 
  if (!Utility::IncludePython()) {
    err << "failed to include Python.h";
    return false;
  }
 
  // collect all bases, error checking the hierarchy along the way
  const Py_ssize_t nBases = PyTuple_GET_SIZE(bases);
  BaseInfos_t base_infos;
  base_infos.reserve(nBases);
  for (Py_ssize_t ibase = 0; ibase < nBases; ++ibase) {
    if (!CPPScope_Check(PyTuple_GET_ITEM(bases, ibase)))
      continue;
 
    interop::TCppScope_t basetype =
        ((CPPScope*)PyTuple_GET_ITEM(bases, ibase))->fCppType;
 
    if (!basetype) {
      err << "base class is incomplete";
      break;
    }
 
    if (interop::IsNamespace(basetype)) {
      err << interop::GetScopedFinalName(basetype) << " is a namespace";
      break;
    }
 
    if (!interop::HasVirtualDestructor(basetype)) {
      const std::string& bname = interop::GetScopedFinalName(basetype);
      if (PyErr_WarnEx(
              PyExc_RuntimeWarning,
              (char*)("class \"" + bname + "\" has no virtual destructor")
                  .c_str(),
              1) < 0)
        return false;
    }
 
    base_infos.emplace_back(
        basetype, TypeManip::template_base(interop::GetFinalName(basetype)),
        interop::GetScopedFinalName(basetype));
  }
 
  // TODO: check deep hierarchy for multiple inheritance
  bool isDeepHierarchy =
      klass->fCppType && base_infos.front().btype != klass->fCppType;
 
  // once classes can be extended, should consider re-use; for now, since
  // derived python classes can differ in what they override, simply use
  // different shims
  static int counter = 0;
  std::ostringstream osname;
  osname << "Dispatcher" << ++counter;
  const std::string& derivedName = osname.str();
 
  // generate proxy class with the relevant method dispatchers
  std::ostringstream code;
 
  // start class declaration
  code << "namespace __cppjit_internal {\n"
       << "class " << derivedName << " : ";
  for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) {
    if (ibase != 0)
      code << ", ";
    code << "public ::" << base_infos[ibase].bname_scoped;
  }
  code << " {\n";
  if (!isDeepHierarchy)
    code << "protected:\n  cppjit::cpyrt::DispatchPtr _internal_self;\n";
  code << "public:\n";
 
  // add a virtual destructor for good measure, which is allowed to be
  // "overridden" by the conventional __destruct__ method (note that __del__ is
  // always called, too, if provided, but only when the Python object goes away;
  // furthermore, if the Python object goes before the C++ one, only __del__ is
  // called)
  if (PyMapping_HasKeyString(dct, (char*)"__destruct__")) {
    code << "  virtual ~" << derivedName
         << "() {\n"
            "    cppjit::cpyrt::PythonGILRAII python_gil_raii;\n"
            "    PyObject* iself = (PyObject*)_internal_self;\n"
            "    if (!iself || iself == Py_None)\n"
            "      return;\n" // safe, as destructor always returns void
            "    Py_INCREF(iself);\n"
            "    PyObject* mtPyName = PyUnicode_FromString(\"__destruct__\");\n"
            "    PyObject* pyresult = PyObject_CallMethodObjArgs(iself, "
            "mtPyName, NULL);\n"
            "    Py_DECREF(mtPyName);\n    Py_DECREF(iself);\n";
 
    // this being a destructor, print on exception rather than propagate using
    // the magic C++ exception ...
    code << "      if (!pyresult) PyErr_Print();\n"
            "      else { Py_DECREF(pyresult); }\n"
            "  }\n";
  } else
    code << "  virtual ~" << derivedName << "() {}\n";
 
  // methods: first collect all callables, then get overrides from base classes,
  // for those that are still missing, search the hierarchy
  PyObject* clbs = PyDict_New();
  PyObject* items = PyDict_Items(dct);
  for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); ++i) {
    PyObject* value = PyTuple_GET_ITEM(PyList_GET_ITEM(items, i), 1);
    if (PyCallable_Check(value))
      PyDict_SetItem(clbs, PyTuple_GET_ITEM(PyList_GET_ITEM(items, i), 0),
                     value);
  }
  Py_DECREF(items);
  if (PyDict_DelItem(clbs, PyStrings::gInit) != 0)
    PyErr_Clear();
 
  // protected methods and data need their access changed in the C++ trampoline
  // and then exposed on the Python side; so, collect their names as we go along
  std::set<std::string> protected_names;
 
  // simple case: methods from current class (collect constructors along the
  // way)
  int has_default = 0, has_cctor = 0, has_ctors = 0, has_tmpl_ctors = 0;
  AllCtors_t ctors{base_infos.size()};
  for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) {
    const auto& binfo = base_infos[ibase];
 
    std::vector<interop::TCppMethod_t> methods;
    interop::GetClassMethods(binfo.btype, methods);
    bool cctor_found = false, default_found = false, any_ctor_found = false;
    for (auto& method : methods) {
      if (interop::IsConstructor(method)) {
        any_ctor_found = true;
        if ((interop::IsPublicMethod(method) ||
             interop::IsProtectedMethod(method)) &&
            !interop::IsDeletedMethod(method)) {
          interop::TCppIndex_t nreq = interop::GetMethodReqArgs(method);
          if (nreq == 0)
            default_found = true;
          else if (!cctor_found && nreq == 1) {
            const std::string& argtype =
                interop::GetMethodArgCanonTypeAsString(method, 0);
            if (TypeManip::compound(argtype) == "&" &&
                TypeManip::clean_type(argtype, false) == binfo.bname_scoped)
              cctor_found = true;
          }
          ctors[ibase].push_back(method);
        }
        continue;
      }
 
      std::string mtCppName =
          interop::GetName(interop::TCppScope_t(method.data));
      PyObject* key = PyUnicode_FromString(mtCppName.c_str());
      int contains = PyDict_Contains(dct, key);
      if (contains == -1)
        PyErr_Clear();
      if (contains != 1) {
        Py_DECREF(key);
 
        // if the method is protected, we expose it through re-declaration and
        // forwarding (using does not work here b/c there may be private
        // overloads)
        if (interop::IsProtectedMethod(method)) {
          protected_names.insert(mtCppName);
 
          code << "  " << interop::GetMethodReturnTypeAsString(method) << " "
               << mtCppName << "(";
          interop::TCppIndex_t nArgs = interop::GetMethodNumArgs(method);
          for (interop::TCppIndex_t i = 0; i < nArgs; ++i) {
            if (i != 0)
              code << ", ";
            code << interop::GetMethodArgCanonTypeAsString(method, i) << " arg"
                 << i;
          }
          code << ") ";
          if (interop::IsConstMethod(method))
            code << "const ";
          code << "{\n    return " << binfo.bname << "::" << mtCppName << "(";
          for (interop::TCppIndex_t i = 0; i < nArgs; ++i) {
            if (i != 0)
              code << ", ";
            if (interop::IsRValueReferenceType(
                    interop::GetMethodArgType(method, i)))
              code << "std::move(arg" << i << ")";
            else
              code << "arg" << i;
          }
          code << ");\n  }\n";
        }
 
        continue;
      }
 
      InjectMethod(method, mtCppName, code);
 
      if (PyDict_DelItem(clbs, key) != 0)
        PyErr_Clear(); // happens for overloads
      Py_DECREF(key);
    }
 
    // support for templated ctors in single inheritance (TODO: also multi
    // possible?)
    if (base_infos.size() == 1) {
      std::vector<interop::TCppMethod_t> templ_methods;
      interop::GetTemplatedMethods(binfo.btype, templ_methods);
      for (auto& method : templ_methods) {
        if (interop::IsConstructor(method)) {
          any_ctor_found = true;
          has_tmpl_ctors += 1;
          break; // one suffices to map as argument packs are used
        }
      }
    }
 
    // count the cctors and default ctors to determine whether each base has one
    if (cctor_found || (!cctor_found && !any_ctor_found))
      has_cctor += 1;
    if (default_found || (!default_found && !any_ctor_found))
      has_default += 1;
    if (any_ctor_found && !has_tmpl_ctors)
      has_ctors += 1;
  }
 
  // try to locate left-overs in base classes
  for (const auto& binfo : base_infos) {
    if (PyDict_Size(clbs)) {
      size_t nbases = interop::GetNumBases(binfo.btype);
      for (size_t ibase = 0; ibase < nbases; ++ibase) {
        interop::TCppScope_t tbase =
            (interop::TCppScope_t)interop::GetBaseScope(binfo.btype, ibase);
 
        PyObject* keys = PyDict_Keys(clbs);
        for (Py_ssize_t i = 0; i < PyList_GET_SIZE(keys); ++i) {
          // TODO: should probably invert this looping; but that makes handling
          // overloads clunky
          PyObject* key = PyList_GET_ITEM(keys, i);
          std::string mtCppName = PyUnicode_AsUTF8(key);
          const auto& methods = FindBaseMethod(tbase, mtCppName);
          for (auto method : methods)
            InjectMethod(method, mtCppName, code);
          if (!methods.empty()) {
            if (PyDict_DelItem(clbs, key) != 0)
              PyErr_Clear();
          }
        }
        Py_DECREF(keys);
      }
    }
  }
  Py_DECREF(clbs);
 
  // constructors: build up from the argument types of the base class, for use
  // by the Python derived class (inheriting with/ "using" does not work b/c
  // base class constructors may have been deleted),
  build_constructors(derivedName, base_infos, ctors, code);
 
  // for working with C++ templates, additional constructors are needed to make
  // sure the python object is properly carried, but they can only be generated
  // if the base class supports them
  if (1 < nBases && (!has_ctors || has_default == nBases))
    code << "  " << derivedName << "() {}\n";
  if (has_cctor == nBases) {
    code << "  " << derivedName << "(const " << derivedName << "& other) : ";
    for (BaseInfos_t::size_type ibase = 0; ibase < base_infos.size(); ++ibase) {
      if (ibase != 0)
        code << ", ";
      code << base_infos[ibase].bname << "(other)";
    }
    if (!isDeepHierarchy)
      code << ", _internal_self(other._internal_self, this)";
    code << " {}\n";
  }
  if (has_tmpl_ctors && base_infos.size() == 1) {
    // support for templated ctors in single inheritance (TODO: also multi
    // possible?)
    code << "  template<typename ...Args>\n  " << derivedName
         << "(Args... args) : " << base_infos[0].bname << "(args...) {}\n";
  }
 
  // destructor: default is fine
 
  // pull in data members that are protected
  bool setPublic = false;
  for (const auto& binfo : base_infos) {
    std::vector<interop::TCppScope_t> datamems;
    interop::GetDatamembers(binfo.btype, datamems);
    for (auto data : datamems) {
      if (interop::IsProtectedData(data)) {
        const std::string dm_name = interop::GetFinalName(data);
        if (dm_name != "_internal_self") {
          const std::string& dname = interop::GetFinalName(data);
          protected_names.insert(dname);
          if (!setPublic) {
            code << "public:\n";
            setPublic = true;
          }
          code << "  using " << binfo.bname << "::" << dname << ";\n";
        }
      }
    }
  }
 
  // initialize the dispatch pointer for all direct bases that have one
  BaseInfos_t::size_type disp_inited = 0;
  code << "public:\n  static void _init_dispatchptr(" << derivedName
       << "* inst, PyObject* self) {\n";
  if (1 < base_infos.size()) {
    for (const auto& binfo : base_infos) {
      if (interop::CheckDatamember(binfo.btype, "_internal_self")) {
        code << "    " << binfo.bname << "::_init_dispatchptr(inst, self);\n";
        disp_inited += 1;
      }
    }
  }
  // The dispatch initializer is only used in constructors, and C++ object start
  // out as owned by C++, with Python ownership explicitly set only later. To
  // match, the dispatch pointer needs to start out with a hard reference, i.e.
  // C++ ownership of the dispatch object. If the constructor has __creates__
  // set to True (default), then a call to PythonOwns() will switch the hard ref
  // to a weak ref, preventing accidental circular references.
  if (disp_inited != base_infos.size())
    code << "    new ((void*)&inst->_internal_self) "
            "cppjit::cpyrt::DispatchPtr{self, true};\n";
  code << "  }";
 
  // provide an accessor to re-initialize after round-tripping from C++
  // (internal)
  code << "\n  static PyObject* _get_dispatch(" << derivedName
       << "* inst) {\n"
          "    cppjit::cpyrt::PythonGILRAII python_gil_raii;\n"
          "    PyObject* res = (PyObject*)inst->_internal_self;\n"
          "    Py_XINCREF(res); return res;\n  }";
 
  // finish class declaration
  code << "};\n}";
 
  // finally, compile the code
  if (!interop::Compile(code.str())) {
    err << "failed to compile the dispatcher code";
    return false;
  }
 
  // keep track internally of the actual C++ type (this is used in
  // CPPConstructor to call the dispatcher's one instead of the base)
  interop::TCppScope_t disp =
      interop::GetFullScope("__cppjit_internal::" + derivedName);
  if (!disp) {
    err << "failed to retrieve the internal dispatcher";
    return false;
  }
  klass->fCppType = disp;
 
  // at this point, the dispatcher only lives in C++, as opposed to regular
  // classes that are part of the hierarchy in Python, so create it, which will
  // cache it for later use by e.g. the MemoryRegulator
  unsigned int flags = (unsigned int)(klass->fFlags & CPPScope::kIsMultiCross);
  PyObject* disp_proxy = cpyrt::CreateScopeProxy(disp, 0, flags);
  if (flags)
    ((CPPScope*)disp_proxy)->fFlags |= CPPScope::kIsMultiCross;
  ((CPPScope*)disp_proxy)->fFlags |= CPPScope::kIsPython;
 
  // finally, to expose protected members, copy them over from the C++
  // dispatcher base to the Python dictionary (the C++ dispatcher's Python proxy
  // is not a base of the Python class to keep the inheritance tree intact)
  for (const auto& name : protected_names) {
    PyObject* disp_dct = PyObject_GetAttr(disp_proxy, PyStrings::gDict);
#if PY_VERSION_HEX < 0x30d00f0
    PyObject* pyf = PyMapping_GetItemString(disp_dct, (char*)name.c_str());
#else
    PyObject* pyf = nullptr;
    PyMapping_GetOptionalItemString(disp_dct, (char*)name.c_str(), &pyf);
#endif
    if (pyf) {
      PyObject_SetAttrString((PyObject*)klass, (char*)name.c_str(), pyf);
      Py_DECREF(pyf);
    }
    Py_DECREF(disp_dct);
  }
 
  Py_XDECREF(disp_proxy);
 
  return true;
}