Add py::local() for module-local type bindings

This commit adds a `py::local` attribute that lets you confine a
registered type to the module (more technically, the shared object) in
which it is defined, by registering it with:

    py::class_<C>(m, "C", py::local())

This will allow the C++ class `C` to be registered in different
modules with independent sets of class definitions.  On the Python side,
two such types will be completely distinct; on the C++ side, the
C++ resolves to a different Python types in each module.

Author: jagerman

include/pybind11/attr.h

@@ -64,6 +64,9 @@ struct metaclass {
     explicit metaclass(handle value) : value(value) { }
 };
 
+/// Annotation that marks a class as local to the module:
+struct local { bool value; local(bool v = true) : value(v) { } };
+
 /// Annotation to mark enums as an arithmetic type
 struct arithmetic { };
 
@@ -196,7 +199,7 @@ struct function_record {
 /// Special data structure which (temporarily) holds metadata about a bound class
 struct type_record {
     PYBIND11_NOINLINE type_record()
-        : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false) { }
+        : multiple_inheritance(false), dynamic_attr(false), buffer_protocol(false), local(false) { }
 
     /// Handle to the parent scope
     handle scope;
@@ -243,6 +246,9 @@ struct type_record {
     /// Is the default (unique_ptr) holder type used?
     bool default_holder : 1;
 
+    /// Is the class definition local to the module shared object?
+    bool local : 1;
+
     PYBIND11_NOINLINE void add_base(const std::type_info &base, void *(*caster)(void *)) {
         auto base_info = detail::get_type_info(base, false);
         if (!base_info) {
@@ -408,6 +414,10 @@ struct process_attribute<metaclass> : process_attribute_default<metaclass> {
     static void init(const metaclass &m, type_record *r) { r->metaclass = m.value; }
 };
 
+template <>
+struct process_attribute<local> : process_attribute_default<local> {
+    static void init(const local &l, type_record *r) { r->local = l.value; }
+};
 
 /// Process an 'arithmetic' attribute for enums (does nothing here)
 template <>

include/pybind11/cast.h

@@ -128,6 +128,14 @@ PYBIND11_NOINLINE inline internals &get_internals() {
     return *internals_ptr;
 }
 
+namespace {
+// Works like internals.registered_types_cpp, but for module-local registered types:
+inline type_map<void *> &registered_local_types_cpp() {
+    static type_map<void *> locals{};
+    return locals;
+}
+}
+
 /// A life support system for temporary objects created by `type_caster::load()`.
 /// Adding a patient will keep it alive up until the enclosing function returns.
 class loader_life_support {
@@ -215,7 +223,7 @@ PYBIND11_NOINLINE inline void all_type_info_populate(PyTypeObject *t, std::vecto
             // registered types
             if (i + 1 == check.size()) {
                 // When we're at the end, we can pop off the current element to avoid growing
-                // `check` when adding just one base (which is typical--.e. when there is no
+                // `check` when adding just one base (which is typical--i.e. when there is no
                 // multiple inheritance)
                 check.pop_back();
                 i--;
@@ -259,13 +267,18 @@ PYBIND11_NOINLINE inline detail::type_info* get_type_info(PyTypeObject *type) {
     return bases.front();
 }
 
-PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_info &tp,
+/// Return the type info for a given C++ type; on lookup failure can either throw or return nullptr.
+PYBIND11_NOINLINE inline detail::type_info *get_type_info(const std::type_index &tp,
                                                           bool throw_if_missing = false) {
+    std::type_index type_idx(tp);
     auto &types = get_internals().registered_types_cpp;
-
-    auto it = types.find(std::type_index(tp));
+    auto it = types.find(type_idx);
     if (it != types.end())
         return (detail::type_info *) it->second;
+    auto &locals = registered_local_types_cpp();
+    it = locals.find(type_idx);
+    if (it != locals.end())
+        return (detail::type_info *) it->second;
     if (throw_if_missing) {
         std::string tname = tp.name();
         detail::clean_type_id(tname);
@@ -723,10 +736,8 @@ class type_caster_generic {
     // with .second = nullptr.  (p.first = nullptr is not an error: it becomes None).
     PYBIND11_NOINLINE static std::pair<const void *, const type_info *> src_and_type(
             const void *src, const std::type_info &cast_type, const std::type_info *rtti_type = nullptr) {
-        auto &internals = get_internals();
-        auto it = internals.registered_types_cpp.find(std::type_index(cast_type));
-        if (it != internals.registered_types_cpp.end())
-            return {src, (const type_info *) it->second};
+        if (auto *tpi = get_type_info(cast_type))
+            return {src, const_cast<const type_info *>(tpi)};
 
         // Not found, set error:
         std::string tname = rtti_type ? rtti_type->name() : cast_type.name();
@@ -804,7 +815,6 @@ template <typename type> class type_caster_base : public type_caster_generic {
     template <typename T = itype, enable_if_t<std::is_polymorphic<T>::value, int> = 0>
     static std::pair<const void *, const type_info *> src_and_type(const itype *src) {
         const void *vsrc = src;
-        auto &internals = get_internals();
         auto &cast_type = typeid(itype);
         const std::type_info *instance_type = nullptr;
         if (vsrc) {
@@ -813,9 +823,8 @@ template <typename type> class type_caster_base : public type_caster_generic {
                 // This is a base pointer to a derived type; if it is a pybind11-registered type, we
                 // can get the correct derived pointer (which may be != base pointer) by a
                 // dynamic_cast to most derived type:
-                auto it = internals.registered_types_cpp.find(std::type_index(*instance_type));
-                if (it != internals.registered_types_cpp.end())
-                    return {dynamic_cast<const void *>(src), (const type_info *) it->second};
+                if (auto *tpi = get_type_info(*instance_type))
+                    return {dynamic_cast<const void *>(src), const_cast<const type_info *>(tpi)};
             }
         }
         // Otherwise we have either a nullptr, an `itype` pointer, or an unknown derived pointer, so

include/pybind11/pybind11.h

@@ -844,7 +844,10 @@ class generic_type : public object {
         auto tindex = std::type_index(*rec.type);
         tinfo->direct_conversions = &internals.direct_conversions[tindex];
         tinfo->default_holder = rec.default_holder;
-        internals.registered_types_cpp[tindex] = tinfo;
+        if (rec.local)
+            registered_local_types_cpp()[tindex] = tinfo;
+        else
+            internals.registered_types_cpp[tindex] = tinfo;
         internals.registered_types_py[(PyTypeObject *) m_ptr] = { tinfo };
 
         if (rec.bases.size() > 1 || rec.multiple_inheritance) {
@@ -978,7 +981,7 @@ class class_ : public detail::generic_type {
         generic_type::initialize(record);
 
         if (has_alias) {
-            auto &instances = get_internals().registered_types_cpp;
+            auto &instances = record.local ? registered_local_types_cpp() : get_internals().registered_types_cpp;
             instances[std::type_index(typeid(type_alias))] = instances[std::type_index(typeid(type))];
         }
     }
@@ -1427,7 +1430,7 @@ iterator make_iterator(Iterator first, Sentinel last, Extra &&... extra) {
     typedef detail::iterator_state<Iterator, Sentinel, false, Policy> state;
 
     if (!detail::get_type_info(typeid(state), false)) {
-        class_<state>(handle(), "iterator")
+        class_<state>(handle(), "iterator", pybind11::local())
             .def("__iter__", [](state &s) -> state& { return s; })
             .def("__next__", [](state &s) -> ValueType {
                 if (!s.first_or_done)
@@ -1456,7 +1459,7 @@ iterator make_key_iterator(Iterator first, Sentinel last, Extra &&... extra) {
     typedef detail::iterator_state<Iterator, Sentinel, true, Policy> state;
 
     if (!detail::get_type_info(typeid(state), false)) {
-        class_<state>(handle(), "iterator")
+        class_<state>(handle(), "iterator", pybind11::local())
             .def("__iter__", [](state &s) -> state& { return s; })
             .def("__next__", [](state &s) -> KeyType {
                 if (!s.first_or_done)

tests/CMakeLists.txt

@@ -43,6 +43,7 @@ set(PYBIND11_TEST_FILES
   test_methods_and_attributes.cpp
   test_modules.cpp
   test_multiple_inheritance.cpp
+  test_local_bindings.cpp
   test_numpy_array.cpp
   test_numpy_dtypes.cpp
   test_numpy_vectorize.cpp
@@ -120,36 +121,49 @@ function(pybind11_enable_warnings target_name)
   endif()
 endfunction()
 
+set(test_targets pybind11_tests)
+# If test_local_bindings is being built in pybind11_tests.so we'll also build
+# a separate module pybind11_local_bindings.so for the test:
+list(FIND PYBIND11_TEST_FILES test_local_bindings.cpp test_local_i)
+if (test_local_i GREATER -1)
+  list(APPEND test_targets pybind11_local_bindings)
+endif()
 
-# Create the binding library
-pybind11_add_module(pybind11_tests THIN_LTO pybind11_tests.cpp
-  ${PYBIND11_TEST_FILES} ${PYBIND11_HEADERS})
+set(testdir ${CMAKE_CURRENT_SOURCE_DIR})
+foreach(tgt ${test_targets})
+  set(test_files ${PYBIND11_TEST_FILES})
+  if(NOT tgt STREQUAL "pybind11_tests")
+    set(test_files "")
+  endif()
 
-pybind11_enable_warnings(pybind11_tests)
+  # Create the binding library
+  pybind11_add_module(${tgt} THIN_LTO ${tgt}.cpp
+    ${test_files} ${PYBIND11_HEADERS})
 
-if(MSVC)
-  target_compile_options(pybind11_tests PRIVATE /utf-8)
-endif()
+  pybind11_enable_warnings(${tgt})
 
-if(EIGEN3_FOUND)
-  if (PYBIND11_EIGEN_VIA_TARGET)
-    target_link_libraries(pybind11_tests PRIVATE Eigen3::Eigen)
-  else()
-    target_include_directories(pybind11_tests PRIVATE ${EIGEN3_INCLUDE_DIR})
+  if(MSVC)
+    target_compile_options(${tgt} PRIVATE /utf-8)
   endif()
-  target_compile_definitions(pybind11_tests PRIVATE -DPYBIND11_TEST_EIGEN)
-endif()
 
-set(testdir ${CMAKE_CURRENT_SOURCE_DIR})
+  if(EIGEN3_FOUND)
+    if (PYBIND11_EIGEN_VIA_TARGET)
+      target_link_libraries(${tgt} PRIVATE Eigen3::Eigen)
+    else()
+      target_include_directories(${tgt} PRIVATE ${EIGEN3_INCLUDE_DIR})
+    endif()
+    target_compile_definitions(${tgt} PRIVATE -DPYBIND11_TEST_EIGEN)
+  endif()
 
-# Always write the output file directly into the 'tests' directory (even on MSVC)
-if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
-  set_target_properties(pybind11_tests PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${testdir})
-  foreach(config ${CMAKE_CONFIGURATION_TYPES})
-    string(TOUPPER ${config} config)
-    set_target_properties(pybind11_tests PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} ${testdir})
-  endforeach()
-endif()
+  # Always write the output file directly into the 'tests' directory (even on MSVC)
+  if(NOT CMAKE_LIBRARY_OUTPUT_DIRECTORY)
+    set_target_properties(${tgt} PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${testdir})
+    foreach(config ${CMAKE_CONFIGURATION_TYPES})
+      string(TOUPPER ${config} config)
+      set_target_properties(${tgt} PROPERTIES LIBRARY_OUTPUT_DIRECTORY_${config} ${testdir})
+    endforeach()
+  endif()
+endforeach()
 
 # Make sure pytest is found or produce a fatal error
 if(NOT PYBIND11_PYTEST_FOUND)
@@ -173,7 +187,7 @@ endif()
 
 # A single command to compile and run the tests
 add_custom_target(pytest COMMAND ${PYTHON_EXECUTABLE} -m pytest ${PYBIND11_PYTEST_FILES}
-                  DEPENDS pybind11_tests WORKING_DIRECTORY ${testdir} ${PYBIND11_USES_TERMINAL})
+                  DEPENDS ${test_targets} WORKING_DIRECTORY ${testdir} ${PYBIND11_USES_TERMINAL})
 
 if(PYBIND11_TEST_OVERRIDE)
   add_custom_command(TARGET pytest POST_BUILD
@@ -189,7 +203,7 @@ if (NOT PROJECT_NAME STREQUAL "pybind11")
   return()
 endif()
 
-# Add a post-build comment to show the .so size and, if a previous size, compare it:
+# Add a post-build comment to show the primary test suite .so size and, if a previous size, compare it:
 add_custom_command(TARGET pybind11_tests POST_BUILD
   COMMAND ${PYTHON_EXECUTABLE} ${PROJECT_SOURCE_DIR}/tools/libsize.py
   $<TARGET_FILE:pybind11_tests> ${CMAKE_CURRENT_BINARY_DIR}/sosize-$<TARGET_FILE_NAME:pybind11_tests>.txt)

tests/local_bindings.h

@@ -0,0 +1,26 @@
+#pragma once
+#include "pybind11_tests.h"
+
+/// Simple class used to test py::local:
+template <int> class LocalBase {
+public:
+    LocalBase(int i) : i(i) { }
+    int i = -1;
+};
+
+/// Registered with py::local in both main and secondary modules:
+using LocalType = LocalBase<0>;
+/// Registered without py::local in both modules:
+using NonLocalType = LocalBase<1>;
+/// A second non-local type (for stl_bind tests):
+using NonLocal2 = LocalBase<2>;
+/// Tests within-module, different-compilation-unit local definition conflict:
+using LocalExternal = LocalBase<3>;
+
+// Simple bindings (used with the above):
+template <typename T, int Adjust, typename... Args>
+py::class_<T> bind_local(Args && ...args) {
+    return py::class_<T>(std::forward<Args>(args)...)
+        .def(py::init<int>())
+        .def("get", [](T &i) { return i.i + Adjust; });
+};

tests/pybind11_local_bindings.cpp

@@ -0,0 +1,35 @@
+/*
+    tests/pybind11_local_bindings.cpp -- counterpart to test_local_bindings.cpp
+
+    Copyright (c) 2017 Jason Rhinelander <[email protected]>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "pybind11_tests.h"
+#include "local_bindings.h"
+#include <pybind11/stl_bind.h>
+
+PYBIND11_MODULE(pybind11_local_bindings, m) {
+    m.doc() = "pybind11 local bindings test module";
+
+    // Definitions here are tested by importing both this module and the
+    // pybind11_tests.local_bindings submodule from test_local_bindings.py
+
+    // test_local_bindings
+    // Local to both:
+    bind_local<LocalType, 1>(m, "LocalType", py::local())
+        .def("get2", [](LocalType &t) { return t.i + 2; })
+        ;
+
+    // Can only be called with our python type:
+    m.def("local_value", [](LocalType &l) { return l.i; });
+
+    // test_nonlocal_failure
+    // This registration will fail (global registration when LocalFail is already registered
+    // globally in the main test module):
+    m.def("register_nonlocal", [m]() {
+        bind_local<NonLocalType, 0>(m, "NonLocalType");
+    });
+}

tests/test_class.cpp

@@ -9,6 +9,7 @@
 
 #include "pybind11_tests.h"
 #include "constructor_stats.h"
+#include "local_bindings.h"
 
 TEST_SUBMODULE(class_, m) {
     // test_instance
@@ -184,6 +185,10 @@ TEST_SUBMODULE(class_, m) {
         auto def = new PyMethodDef{"f", f, METH_VARARGS, nullptr};
         return py::reinterpret_steal<py::object>(PyCFunction_NewEx(def, nullptr, m.ptr()));
     }());
+
+    // This test is actually part of test_local_bindings (test_duplicate_local), but we need a
+    // definition in a different compilation unit within the same module:
+    bind_local<LocalExternal, 17>(m, "LocalExternal", py::local());
 }
 
 template <int N> class BreaksBase {};

tests/test_local_bindings.cpp

@@ -0,0 +1,45 @@
+/*
+    tests/test_local_bindings.cpp -- tests the py::local class feature which makes a class binding
+                                     local to the module in which it is defined.
+
+    Copyright (c) 2017 Jason Rhinelander <[email protected]>
+
+    All rights reserved. Use of this source code is governed by a
+    BSD-style license that can be found in the LICENSE file.
+*/
+
+#include "pybind11_tests.h"
+#include "local_bindings.h"
+#include <pybind11/stl_bind.h>
+
+TEST_SUBMODULE(local_bindings, m) {
+
+    // test_local_bindings
+    // Register a class with py::local:
+    bind_local<LocalType, -1>(m, "LocalType", py::local())
+        .def("get3", [](LocalType &t) { return t.i + 3; })
+        ;
+
+    m.def("local_value", [](LocalType &l) { return l.i; });
+
+    // test_nonlocal_failure
+    // The main pybind11 test module is loaded first, so this registration will succeed (the second
+    // one, in pybind11_local_bindings.cpp, is designed to fail):
+    bind_local<NonLocalType, 0>(m, "NonLocalType")
+        .def(py::init<int>())
+        .def("get", [](LocalType &i) { return i.i; })
+        ;
+
+    // test_duplicate_local
+    // py::local declarations should be visible across compilation units that get linked together;
+    // this tries to register a duplicate local.  It depends on a definition in test_class.cpp and
+    // should raise a runtime error from the duplicate definition attempt.  If test_class isn't
+    // available it *also* throws a runtime error (with "test_class not enabled" as value).
+    m.def("register_local_external", [m]() {
+        auto main = py::module::import("pybind11_tests");
+        if (py::hasattr(main, "class_")) {
+            bind_local<LocalExternal, 7>(m, "LocalExternal", py::local());
+        }
+        else throw std::runtime_error("test_class not enabled");
+    });
+}