Make all classes with the same instance size derive from a common base

In order to fully satisfy Python's inheritance type layout requirements,
all types should have a common 'solid' base. A solid base is one which
has the same instance size as the derived type (not counting the space
required for the optional `dict_ptr` and `weakrefs_ptr`). Thus, `object`
does not qualify as a solid base for pybind11 types and this can lead to
issues with multiple inheritance.

To get around this, new base types are created: one per unique instance
size. There is going to be very few of these bases. They ensure Python's
MRO checks will pass when multiple bases are involved.

Author: dean0x7d

docs/advanced/classes.rst

@@ -437,24 +437,15 @@ The section on :ref:`properties` discussed the creation of instance properties
 that are implemented in terms of C++ getters and setters.
 
 Static properties can also be created in a similar way to expose getters and
-setters of static class attributes. Two things are important to note:
-
-1. Static properties are implemented by instrumenting the *metaclass* of the
-   class in question -- however, this requires the class to have a modifiable
-   metaclass in the first place. pybind11 provides a ``py::metaclass()``
-   annotation that must be specified in the ``class_`` constructor, or any
-   later method calls to ``def_{property_,∅}_{readwrite,readonly}_static`` will
-   fail (see the example below).
-
-2. For static properties defined in terms of setter and getter functions, note
-   that the implicit ``self`` argument also exists in this case and is used to
-   pass the Python ``type`` subclass instance. This parameter will often not be
-   needed by the C++ side, and the following example illustrates how to
-   instantiate a lambda getter function that ignores it:
+setters of static class attributes. Note that the implicit ``self`` argument
+also exists in this case and is used to pass the Python ``type`` subclass
+instance. This parameter will often not be needed by the C++ side, and the
+following example illustrates how to instantiate a lambda getter function
+that ignores it:
 
 .. code-block:: cpp
 
-    py::class_<Foo>(m, "Foo", py::metaclass())
+    py::class_<Foo>(m, "Foo")
         .def_property_readonly_static("foo", [](py::object /* self */) { return Foo(); });
 
 Operator overloading

include/pybind11/attr.h

@@ -53,8 +53,11 @@ struct dynamic_attr { };
 /// Annotation which enables the buffer protocol for a type
 struct buffer_protocol { };
 
-/// Annotation which requests that a special metaclass is created for a type
-struct metaclass { };
+/// DEPRECATED: Annotation which requests that a special metaclass is created for a type
+struct metaclass {
+    PYBIND11_DEPRECATED("py::metaclass() is no longer required. It's turned on by default now.")
+    metaclass() = default;
+};
 
 /// Annotation to mark enums as an arithmetic type
 struct arithmetic { };
@@ -149,8 +152,7 @@ struct function_record {
 /// Special data structure which (temporarily) holds metadata about a bound class
 struct type_record {
     PYBIND11_NOINLINE type_record()
-        : multiple_inheritance(false), dynamic_attr(false),
-          buffer_protocol(false), metaclass(false) { }
+        : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false) { }
 
     /// Handle to the parent scope
     handle scope;
@@ -188,9 +190,6 @@ struct type_record {
     /// Does the class implement the buffer protocol?
     bool buffer_protocol : 1;
 
-    /// Does the class require its own metaclass?
-    bool metaclass : 1;
-
     /// Is the default (unique_ptr) holder type used?
     bool default_holder : 1;
 
@@ -354,11 +353,8 @@ struct process_attribute<buffer_protocol> : process_attribute_default<buffer_pro
     static void init(const buffer_protocol &, type_record *r) { r->buffer_protocol = true; }
 };
 
-template <>
-struct process_attribute<metaclass> : process_attribute_default<metaclass> {
-    static void init(const metaclass &, type_record *r) { r->metaclass = true; }
-};
-
+// DEPRECATED
+template <> struct process_attribute<metaclass> : process_attribute_default<metaclass> { };
 
 /// Process an 'arithmetic' attribute for enums (does nothing here)
 template <>

include/pybind11/cast.h

@@ -26,6 +26,7 @@ struct type_info {
     PyTypeObject *type;
     size_t type_size;
     void (*init_holder)(PyObject *, const void *);
+    void (*dealloc)(PyObject *);
     std::vector<PyObject *(*)(PyObject *, PyTypeObject *)> implicit_conversions;
     std::vector<std::pair<const std::type_info *, void *(*)(void *)>> implicit_casts;
     std::vector<bool (*)(PyObject *, void *&)> *direct_conversions;

include/pybind11/class_support.h

@@ -143,5 +143,299 @@ inline PyTypeObject* make_default_metaclass() {
     return type;
 }
 
+extern "C" inline PyObject *pybind11_new(PyTypeObject *type, PyObject *, PyObject *) {
+    PyObject *self = type->tp_alloc(type, 0);
+    auto instance = (instance_essentials<void> *) self;
+    auto tinfo = get_type_info(type);
+    instance->value = ::operator new(tinfo->type_size);
+    instance->owned = true;
+    instance->holder_constructed = false;
+    get_internals().registered_instances.emplace(instance->value, self);
+    return self;
+}
+
+extern "C" inline int pybind11_init(PyObject *self, PyObject *, PyObject *) {
+    PyTypeObject *type = Py_TYPE(self);
+    std::string msg;
+#if defined(PYPY_VERSION)
+    msg += handle((PyObject *) type).attr("__module__").cast<std::string>() + ".";
+#endif
+    msg += type->tp_name;
+    msg += ": No constructor defined!";
+    PyErr_SetString(PyExc_TypeError, msg.c_str());
+    return -1;
+}
+
+extern "C" inline void pybind11_dealloc(PyObject *self) {
+    auto instance = (instance_essentials<void> *) self;
+    if (instance->value) {
+        auto type = Py_TYPE(self);
+        get_type_info(type)->dealloc(self);
+
+        auto &registered_instances = get_internals().registered_instances;
+        auto range = registered_instances.equal_range(instance->value);
+        bool found = false;
+        for (auto it = range.first; it != range.second; ++it) {
+            if (type == Py_TYPE(it->second)) {
+                registered_instances.erase(it);
+                found = true;
+                break;
+            }
+        }
+        if (!found)
+            pybind11_fail("pybind11_dealloc(): Tried to deallocate unregistered instance!");
+
+        if (instance->weakrefs)
+            PyObject_ClearWeakRefs(self);
+
+        PyObject **dict_ptr = _PyObject_GetDictPtr(self);
+        if (dict_ptr)
+            Py_CLEAR(*dict_ptr);
+    }
+    Py_TYPE(self)->tp_free(self);
+}
+
+/** Create a type which can be used as a common base for all classes with the same
+    instance size, i.e. all classes with the same `sizeof(holder_type)`. This is
+    needed in order to satisfy Python's requirements for multiple inheritance.
+    Return value: New reference. */
+inline PyObject *make_object_base_type(size_t instance_size) {
+    auto name = "pybind11_object_" + std::to_string(instance_size);
+    auto name_obj = reinterpret_steal<object>(PYBIND11_FROM_STRING(name.c_str()));
+
+    /* Danger zone: from now (and until PyType_Ready), make sure to
+       issue no Python C API calls which could potentially invoke the
+       garbage collector (the GC will call type_traverse(), which will in
+       turn find the newly constructed type in an invalid state) */
+    auto metaclass = get_internals().default_metaclass;
+    auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0);
+    if (!heap_type)
+        pybind11_fail("make_object_base_type(): error allocating type!");
+
+    heap_type->ht_name = name_obj.inc_ref().ptr();
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
+    heap_type->ht_qualname = name_obj.inc_ref().ptr();
+#endif
+
+    auto type = &heap_type->ht_type;
+    type->tp_name = strdup(name.c_str());
+    type->tp_base = &PyBaseObject_Type;
+    type->tp_basicsize = static_cast<ssize_t>(instance_size);
+    type->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
+
+    type->tp_new = pybind11_new;
+    type->tp_init = pybind11_init;
+    type->tp_dealloc = pybind11_dealloc;
+
+    /* Support weak references (needed for the keep_alive feature) */
+    type->tp_weaklistoffset = offsetof(instance_essentials<void>, weakrefs);
+
+    if (PyType_Ready(type) < 0)
+        pybind11_fail("PyType_Ready failed in make_object_base_type():" + error_string());
+
+    assert(!PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));
+    return (PyObject *) heap_type;
+}
+
+/** Return the appropriate base type for the given instance size. The results are cached
+    in `internals.bases` so that only a single base is ever created for any size value.
+    Return value: Borrowed reference. */
+inline PyObject *internals::get_base(size_t instance_size) {
+    auto it = bases.find(instance_size);
+    if (it != bases.end()) {
+        return it->second;
+    } else {
+        auto base = make_object_base_type(instance_size);
+        bases[instance_size] = base;
+        return base;
+    }
+}
+
+extern "C" inline PyObject *pybind11_get_dict(PyObject *self, void *) {
+    PyObject *&dict = *_PyObject_GetDictPtr(self);
+    if (!dict)
+        dict = PyDict_New();
+    Py_XINCREF(dict);
+    return dict;
+}
+
+extern "C" inline int pybind11_set_dict(PyObject *self, PyObject *new_dict, void *) {
+    if (!PyDict_Check(new_dict)) {
+        PyErr_Format(PyExc_TypeError, "__dict__ must be set to a dictionary, not a '%.200s'",
+                     Py_TYPE(new_dict)->tp_name);
+        return -1;
+    }
+    PyObject *&dict = *_PyObject_GetDictPtr(self);
+    Py_INCREF(new_dict);
+    Py_CLEAR(dict);
+    dict = new_dict;
+    return 0;
+}
+
+static PyGetSetDef pybind11_getset[] = {
+    {const_cast<char*>("__dict__"), pybind11_get_dict, pybind11_set_dict, nullptr, nullptr},
+    {nullptr, nullptr, nullptr, nullptr, nullptr}
+};
+
+extern "C" inline int pybind11_traverse(PyObject *self, visitproc visit, void *arg) {
+    PyObject *&dict = *_PyObject_GetDictPtr(self);
+    Py_VISIT(dict);
+    return 0;
+}
+
+extern "C" inline int pybind11_clear(PyObject *self) {
+    PyObject *&dict = *_PyObject_GetDictPtr(self);
+    Py_CLEAR(dict);
+    return 0;
+}
+
+extern "C" inline int pybind11_getbuffer(PyObject *obj, Py_buffer *view, int flags) {
+    auto tinfo = get_type_info(Py_TYPE(obj));
+    if (view == nullptr || obj == nullptr || !tinfo || !tinfo->get_buffer) {
+        if (view)
+            view->obj = nullptr;
+        PyErr_SetString(PyExc_BufferError, "generic_type::getbuffer(): Internal error");
+        return -1;
+    }
+    memset(view, 0, sizeof(Py_buffer));
+    buffer_info *info = tinfo->get_buffer(obj, tinfo->get_buffer_data);
+    view->obj = obj;
+    view->ndim = 1;
+    view->internal = info;
+    view->buf = info->ptr;
+    view->itemsize = (ssize_t) info->itemsize;
+    view->len = view->itemsize;
+    for (auto s : info->shape)
+        view->len *= s;
+    if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT)
+        view->format = const_cast<char *>(info->format.c_str());
+    if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
+        view->ndim = (int) info->ndim;
+        view->strides = (ssize_t *) &info->strides[0];
+        view->shape = (ssize_t *) &info->shape[0];
+    }
+    Py_INCREF(view->obj);
+    return 0;
+}
+
+extern "C" inline void pybind11_releasebuffer(PyObject *, Py_buffer *view) {
+    delete (buffer_info *) view->internal;
+}
+
+/** Create a brand new Python type according to the `type_record` specification.
+    Return value: New reference. */
+inline PyObject* make_new_python_type(const type_record &rec) {
+    auto name = reinterpret_steal<object>(PYBIND11_FROM_STRING(rec.name));
+
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
+    auto ht_qualname = name;
+    if (rec.scope && hasattr(rec.scope, "__qualname__")) {
+        ht_qualname = reinterpret_steal<object>(
+            PyUnicode_FromFormat("%U.%U", rec.scope.attr("__qualname__").ptr(), name.ptr()));
+    }
+#endif
+
+    object module;
+    if (rec.scope) {
+        if (hasattr(rec.scope, "__module__"))
+            module = rec.scope.attr("__module__");
+        else if (hasattr(rec.scope, "__name__"))
+            module = rec.scope.attr("__name__");
+    }
+
+#if !defined(PYPY_VERSION)
+    const auto full_name = module ? str(module).cast<std::string>() + "." + rec.name
+                                  : std::string(rec.name);
+#else
+    const auto full_name = std::string(rec.name);
+#endif
+
+    char *tp_doc = nullptr;
+    if (rec.doc && options::show_user_defined_docstrings()) {
+        /* Allocate memory for docstring (using PyObject_MALLOC, since
+           Python will free this later on) */
+        size_t size = strlen(rec.doc) + 1;
+        tp_doc = (char *) PyObject_MALLOC(size);
+        memcpy((void *) tp_doc, rec.doc, size);
+    }
+
+    auto &internals = get_internals();
+    auto bases = tuple(rec.bases);
+    auto base = (bases.size() == 0) ? internals.get_base(rec.instance_size)
+                                    : bases[0].ptr();
+
+    /* Danger zone: from now (and until PyType_Ready), make sure to
+       issue no Python C API calls which could potentially invoke the
+       garbage collector (the GC will call type_traverse(), which will in
+       turn find the newly constructed type in an invalid state) */
+    auto metaclass = internals.default_metaclass;
+    auto heap_type = (PyHeapTypeObject *) metaclass->tp_alloc(metaclass, 0);
+    if (!heap_type)
+        pybind11_fail(std::string(rec.name) + ": Unable to create type object!");
+
+    heap_type->ht_name = name.release().ptr();
+#if PY_MAJOR_VERSION >= 3 && PY_MINOR_VERSION >= 3
+    heap_type->ht_qualname = ht_qualname.release().ptr();
+#endif
+
+    auto type = &heap_type->ht_type;
+    type->tp_name = strdup(full_name.c_str());
+    type->tp_doc = tp_doc;
+    type->tp_base = (PyTypeObject *) handle(base).inc_ref().ptr();
+    type->tp_basicsize = static_cast<ssize_t>(rec.instance_size);
+    if (bases.size() > 0)
+        type->tp_bases = bases.release().ptr();
+
+    /* Supported protocols */
+    type->tp_as_number = &heap_type->as_number;
+    type->tp_as_sequence = &heap_type->as_sequence;
+    type->tp_as_mapping = &heap_type->as_mapping;
+
+    /* Flags */
+    type->tp_flags |= Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HEAPTYPE;
+#if PY_MAJOR_VERSION < 3
+    type->tp_flags |= Py_TPFLAGS_CHECKTYPES;
+#endif
+
+    /* Support dynamic attributes */
+    if (rec.dynamic_attr) {
+#if defined(PYPY_VERSION)
+        pybind11_fail(std::string(rec.name) + ": dynamic attributes are "
+                                              "currently not supported in "
+                                              "conjunction with PyPy!");
+#endif
+        type->tp_dictoffset = type->tp_basicsize; // place dict at the end
+        type->tp_basicsize += sizeof(PyObject *); // and allocate enough space for it
+        type->tp_getset = pybind11_getset;
+        type->tp_traverse = pybind11_traverse;
+        type->tp_clear = pybind11_clear;
+        type->tp_flags |= Py_TPFLAGS_HAVE_GC;
+    }
+
+    if (rec.buffer_protocol) {
+        type->tp_as_buffer = &heap_type->as_buffer;
+#if PY_MAJOR_VERSION < 3
+        type->tp_flags |= Py_TPFLAGS_HAVE_NEWBUFFER;
+#endif
+        heap_type->as_buffer.bf_getbuffer = pybind11_getbuffer;
+        heap_type->as_buffer.bf_releasebuffer = pybind11_releasebuffer;
+    }
+
+    if (PyType_Ready(type) < 0)
+        pybind11_fail(std::string(rec.name) + ": PyType_Ready failed (" + error_string() + ")!");
+
+    assert(rec.dynamic_attr ? PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC)
+                            : !PyType_HasFeature(type, Py_TPFLAGS_HAVE_GC));
+
+    /* Register type with the parent scope */
+    if (rec.scope)
+        setattr(rec.scope, rec.name, (PyObject *) type);
+
+    if (module) // Needed by pydoc
+        setattr((PyObject *) type, "__module__", module);
+
+    return (PyObject *) type;
+}
+
 NAMESPACE_END(detail)
 NAMESPACE_END(pybind11)