Modified version of PR #4293 by @wjakob

Modifications are:

* Backward compatibility (no ABI break), as originally under PR #4307.
* Naming: `get_python_state_dict()`, `has_pybind11_internals_capsule()`
* Report error retrieving `internals**` from capsule instead of clearing it.

Locally tested with ASAN, MSAN, TSAN, UBSAN (Google-internal toolchain).

Author: rwgk

include/pybind11/detail/internals.h

@@ -424,9 +424,30 @@ inline void translate_local_exception(std::exception_ptr p) {
 }
 #endif
 
+inline object get_python_state_dict() {
+    object state_dict;
+#if (PYBIND11_INTERNALS_VERSION <= 4 && PY_VERSION_HEX < 0x030C0000)                              \
+    || PY_VERSION_HEX < 0x03080000 || defined(PYPY_VERSION)
+    state_dict = reinterpret_borrow<object>(PyEval_GetBuiltins());
+#else
+#    if PY_VERSION_HEX < 0x03090000
+    PyInterpreterState *istate = _PyInterpreterState_Get();
+#    else
+    PyInterpreterState *istate = PyInterpreterState_Get();
+#    endif
+    if (istate) {
+        state_dict = reinterpret_borrow<object>(PyInterpreterState_GetDict(istate));
+    }
+#endif
+    if (!state_dict) {
+        raise_from(PyExc_SystemError, "pybind11::detail::get_python_state_dict() FAILED");
+    }
+    return state_dict;
+}
+
 /// Return a reference to the current `internals` data
 PYBIND11_NOINLINE internals &get_internals() {
-    auto **&internals_pp = get_internals_pp();
+    internals **&internals_pp = get_internals_pp();
     if (internals_pp && *internals_pp) {
         return **internals_pp;
     }
@@ -448,11 +469,22 @@ PYBIND11_NOINLINE internals &get_internals() {
 #endif
     error_scope err_scope;
 
-    PYBIND11_STR_TYPE id(PYBIND11_INTERNALS_ID);
-    auto builtins = handle(PyEval_GetBuiltins());
-    if (builtins.contains(id) && isinstance<capsule>(builtins[id])) {
-        internals_pp = static_cast<internals **>(capsule(builtins[id]));
+    constexpr const char *id_cstr = PYBIND11_INTERNALS_ID;
+    str id(id_cstr);
 
+    dict state_dict = get_python_state_dict();
+
+    if (state_dict.contains(id_cstr)) {
+        void *raw_ptr = PyCapsule_GetPointer(state_dict[id].ptr(), id_cstr);
+        if (raw_ptr == nullptr) {
+            raise_from(
+                PyExc_SystemError,
+                "pybind11::detail::get_internals(): Retrieve internals** from capsule FAILED");
+        }
+        internals_pp = static_cast<internals **>(raw_ptr);
+    }
+
+    if (internals_pp && *internals_pp) {
         // We loaded builtins through python's builtins, which means that our `error_already_set`
         // and `builtin_exception` may be different local classes than the ones set up in the
         // initial exception translator, below, so add another for our local exception classes.
@@ -488,7 +520,7 @@ PYBIND11_NOINLINE internals &get_internals() {
 #    endif
         internals_ptr->istate = tstate->interp;
 #endif
-        builtins[id] = capsule(internals_pp);
+        state_dict[id] = capsule(internals_pp, id_cstr);
         internals_ptr->registered_exception_translators.push_front(&translate_exception);
         internals_ptr->static_property_type = make_static_property_type();
         internals_ptr->default_metaclass = make_default_metaclass();

tests/test_embed/test_interpreter.cpp

@@ -168,9 +168,8 @@ TEST_CASE("There can be only one interpreter") {
     py::initialize_interpreter();
 }
 
-bool has_pybind11_internals_builtin() {
-    auto builtins = py::handle(PyEval_GetBuiltins());
-    return builtins.contains(PYBIND11_INTERNALS_ID);
+bool has_pybind11_internals_capsule() {
+    return py::detail::get_python_state_dict().contains(PYBIND11_INTERNALS_ID);
 };
 
 bool has_pybind11_internals_static() {
@@ -181,7 +180,7 @@ bool has_pybind11_internals_static() {
 TEST_CASE("Restart the interpreter") {
     // Verify pre-restart state.
     REQUIRE(py::module_::import("widget_module").attr("add")(1, 2).cast<int>() == 3);
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_pybind11_internals_capsule());
     REQUIRE(has_pybind11_internals_static());
     REQUIRE(py::module_::import("external_module").attr("A")(123).attr("value").cast<int>()
             == 123);
@@ -198,10 +197,10 @@ TEST_CASE("Restart the interpreter") {
     REQUIRE(Py_IsInitialized() == 1);
 
     // Internals are deleted after a restart.
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_pybind11_internals_capsule());
     REQUIRE_FALSE(has_pybind11_internals_static());
     pybind11::detail::get_internals();
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_pybind11_internals_capsule());
     REQUIRE(has_pybind11_internals_static());
     REQUIRE(reinterpret_cast<uintptr_t>(*py::detail::get_internals_pp())
             == py::module_::import("external_module").attr("internals_at")().cast<uintptr_t>());
@@ -216,13 +215,13 @@ TEST_CASE("Restart the interpreter") {
               py::detail::get_internals();
               *static_cast<bool *>(ran) = true;
           });
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_pybind11_internals_capsule());
     REQUIRE_FALSE(has_pybind11_internals_static());
     REQUIRE_FALSE(ran);
     py::finalize_interpreter();
     REQUIRE(ran);
     py::initialize_interpreter();
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_pybind11_internals_capsule());
     REQUIRE_FALSE(has_pybind11_internals_static());
 
     // C++ modules can be reloaded.
@@ -244,7 +243,7 @@ TEST_CASE("Subinterpreter") {
 
         REQUIRE(m.attr("add")(1, 2).cast<int>() == 3);
     }
-    REQUIRE(has_pybind11_internals_builtin());
+    REQUIRE(has_pybind11_internals_capsule());
     REQUIRE(has_pybind11_internals_static());
 
     /// Create and switch to a subinterpreter.
@@ -254,7 +253,7 @@ TEST_CASE("Subinterpreter") {
     // Subinterpreters get their own copy of builtins. detail::get_internals() still
     // works by returning from the static variable, i.e. all interpreters share a single
     // global pybind11::internals;
-    REQUIRE_FALSE(has_pybind11_internals_builtin());
+    REQUIRE_FALSE(has_pybind11_internals_capsule());
     REQUIRE(has_pybind11_internals_static());
 
     // Modules tags should be gone.