Factor out pybind11/conduit/pybind11_platform_abi_id.h (#5375)

* Factor out pybind11/compat/wrap_include_python_h.h

* Fixes to resolve tests_packaging failures.

* Factor out pybind11/compat/pybind11_platform_abi_id.h

* Add pybind11/compat/README.txt and a couple source code comments.

* Minor changes to comments.

* Factor out pybind11/compat/pybind11_conduit_v1.h

* Add long comment to pybind11/compat/pybind11_conduit_v1.h

* Add pybind11/compat/README.txt to wheels.

* Add `-fno-exceptions` to compiler options for exo_planet_c_api

* 1. Move `target_compile_options()` into loop over test targets, in case the `"exo_planet_c_api"` target does not exist.  2. Add `-fno-exceptions` option also for `NVHPC`.  3. Also check for `__cpp_exceptions` in exo_planet_c_api.cpp.

* 1. Fix accident (forgot to undo temporary change).  2. Special-case __EMSCRIPTEN__ in exo_planet_c_api.cpp

* Give up on compiling exo_planet_c_api.cpp with MSVC `/EHs-c-`:

There was one trouble maker (all other jobs worked):

Visual Studio 15 2017:

```
cl : Command line warning D9025: overriding '/EHc' with '/EHc-' [C:\projects\pybind11\tests\exo_planet_c_api.vcxproj]
...
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include\xlocale(319): error C2220: warning treated as error - no 'object' file generated [C:\projects\pybind11\tests\exo_planet_c_api.vcxproj]
C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Tools\MSVC\14.16.27023\include\xlocale(319): warning C4530: C++ exception handler used, but unwind semantics are not enabled. Specify /EHsc
```

* Move pybind11/compat to pybind11/conduit as suggested by @henryiii:

#5375 (review)

Author: rwgk

CMakeLists.txt

@@ -143,6 +143,9 @@ set(PYBIND11_HEADERS
     include/pybind11/chrono.h
     include/pybind11/common.h
     include/pybind11/complex.h
+    include/pybind11/conduit/pybind11_conduit_v1.h
+    include/pybind11/conduit/pybind11_platform_abi_id.h
+    include/pybind11/conduit/wrap_include_python_h.h
     include/pybind11/options.h
     include/pybind11/eigen.h
     include/pybind11/eigen/common.h

include/pybind11/conduit/README.txt

@@ -0,0 +1,15 @@
+NOTE
+----
+
+The C++ code here
+
+** only depends on <Python.h> **
+
+and nothing else.
+
+DO NOT ADD CODE WITH OTHER EXTERNAL DEPENDENCIES TO THIS DIRECTORY.
+
+Read on:
+
+pybind11_conduit_v1.h — Type-safe interoperability between different
+                        independent Python/C++ bindings systems.

include/pybind11/conduit/pybind11_conduit_v1.h

@@ -0,0 +1,111 @@
+// Copyright (c) 2024 The pybind Community.
+
+/* The pybind11_conduit_v1 feature enables type-safe interoperability between
+
+* different independent Python/C++ bindings systems,
+
+* including pybind11 versions with different PYBIND11_INTERNALS_VERSION's.
+
+The naming of the feature is a bit misleading:
+
+* The feature is in no way tied to pybind11 internals.
+
+* It just happens to originate from pybind11 and currently still lives there.
+
+* The only external dependency is <Python.h>.
+
+The implementation is a VERY light-weight dependency. It is designed to be
+compatible with any ISO C++11 (or higher) compiler, and does NOT require
+C++ Exception Handling to be enabled.
+
+Please see https://github.com/pybind/pybind11/pull/5296 for more background.
+
+The implementation involves a
+
+def _pybind11_conduit_v1_(
+    self,
+    pybind11_platform_abi_id: bytes,
+    cpp_type_info_capsule: capsule,
+    pointer_kind: bytes) -> capsule
+
+method that is meant to be added to Python objects wrapping C++ objects
+(e.g. pybind11::class_-wrapped types).
+
+The design of the _pybind11_conduit_v1_ feature provides two layers of
+protection against C++ ABI mismatches:
+
+* The first and most important layer is that the pybind11_platform_abi_id's
+  must match between extensions. — This will never be perfect, but is the same
+  pragmatic approach used in pybind11 since 2017
+  (https://github.com/pybind/pybind11/commit/96997a4b9d4ec3d389a570604394af5d5eee2557,
+  PYBIND11_INTERNALS_ID).
+
+* The second layer is that the typeid(std::type_info).name()'s must match
+  between extensions.
+
+The implementation below (which is shorter than this comment!), serves as a
+battle-tested specification. The main API is this one function:
+
+auto *cpp_pointer = pybind11_conduit_v1::get_type_pointer_ephemeral<YourType>(py_obj);
+
+It is meant to be a minimalistic reference implementation, intentionally
+without comprehensive error reporting. It is expected that major bindings
+systems will roll their own, compatible implementations, potentially with
+system-specific error reporting. The essential specifications all bindings
+systems need to agree on are merely:
+
+* PYBIND11_PLATFORM_ABI_ID (const char* literal).
+
+* The cpp_type_info capsule (see below: a void *ptr and a const char *name).
+
+* The cpp_conduit capsule (see below: a void *ptr and a const char *name).
+
+* "raw_pointer_ephemeral" means: the lifetime of the pointer is the lifetime
+  of the py_obj.
+
+*/
+
+// THIS MUST STAY AT THE TOP!
+#include "pybind11_platform_abi_id.h"
+
+#include <Python.h>
+#include <typeinfo>
+
+namespace pybind11_conduit_v1 {
+
+inline void *get_raw_pointer_ephemeral(PyObject *py_obj, const std::type_info *cpp_type_info) {
+    PyObject *cpp_type_info_capsule
+        = PyCapsule_New(const_cast<void *>(static_cast<const void *>(cpp_type_info)),
+                        typeid(std::type_info).name(),
+                        nullptr);
+    if (cpp_type_info_capsule == nullptr) {
+        return nullptr;
+    }
+    PyObject *cpp_conduit = PyObject_CallMethod(py_obj,
+                                                "_pybind11_conduit_v1_",
+                                                "yOy",
+                                                PYBIND11_PLATFORM_ABI_ID,
+                                                cpp_type_info_capsule,
+                                                "raw_pointer_ephemeral");
+    Py_DECREF(cpp_type_info_capsule);
+    if (cpp_conduit == nullptr) {
+        return nullptr;
+    }
+    void *raw_ptr = PyCapsule_GetPointer(cpp_conduit, cpp_type_info->name());
+    Py_DECREF(cpp_conduit);
+    if (PyErr_Occurred()) {
+        return nullptr;
+    }
+    return raw_ptr;
+}
+
+template <typename T>
+T *get_type_pointer_ephemeral(PyObject *py_obj) {
+    void *raw_ptr = get_raw_pointer_ephemeral(py_obj, &typeid(T));
+    if (raw_ptr == nullptr) {
+        return nullptr;
+    }
+    return static_cast<T *>(raw_ptr);
+}
+
+} // namespace pybind11_conduit_v1

include/pybind11/conduit/pybind11_platform_abi_id.h

@@ -0,0 +1,88 @@
+#pragma once
+
+// Copyright (c) 2024 The pybind Community.
+
+// To maximize reusability:
+// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING.
+
+#include "wrap_include_python_h.h"
+
+// Implementation details. DO NOT USE ELSEWHERE. (Unfortunately we cannot #undef them.)
+// This is duplicated here to maximize portability.
+#define PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x) #x
+#define PYBIND11_PLATFORM_ABI_ID_TOSTRING(x) PYBIND11_PLATFORM_ABI_ID_STRINGIFY(x)
+
+// On MSVC, debug and release builds are not ABI-compatible!
+#if defined(_MSC_VER) && defined(_DEBUG)
+#    define PYBIND11_BUILD_TYPE "_debug"
+#else
+#    define PYBIND11_BUILD_TYPE ""
+#endif
+
+// Let's assume that different compilers are ABI-incompatible.
+// A user can manually set this string if they know their
+// compiler is compatible.
+#ifndef PYBIND11_COMPILER_TYPE
+#    if defined(_MSC_VER)
+#        define PYBIND11_COMPILER_TYPE "_msvc"
+#    elif defined(__INTEL_COMPILER)
+#        define PYBIND11_COMPILER_TYPE "_icc"
+#    elif defined(__clang__)
+#        define PYBIND11_COMPILER_TYPE "_clang"
+#    elif defined(__PGI)
+#        define PYBIND11_COMPILER_TYPE "_pgi"
+#    elif defined(__MINGW32__)
+#        define PYBIND11_COMPILER_TYPE "_mingw"
+#    elif defined(__CYGWIN__)
+#        define PYBIND11_COMPILER_TYPE "_gcc_cygwin"
+#    elif defined(__GNUC__)
+#        define PYBIND11_COMPILER_TYPE "_gcc"
+#    else
+#        define PYBIND11_COMPILER_TYPE "_unknown"
+#    endif
+#endif
+
+// Also standard libs
+#ifndef PYBIND11_STDLIB
+#    if defined(_LIBCPP_VERSION)
+#        define PYBIND11_STDLIB "_libcpp"
+#    elif defined(__GLIBCXX__) || defined(__GLIBCPP__)
+#        define PYBIND11_STDLIB "_libstdcpp"
+#    else
+#        define PYBIND11_STDLIB ""
+#    endif
+#endif
+
+#ifndef PYBIND11_BUILD_ABI
+#    if defined(__GXX_ABI_VERSION) // Linux/OSX.
+#        define PYBIND11_BUILD_ABI "_cxxabi" PYBIND11_PLATFORM_ABI_ID_TOSTRING(__GXX_ABI_VERSION)
+#    elif defined(_MSC_VER)               // See PR #4953.
+#        if defined(_MT) && defined(_DLL) // Corresponding to CL command line options /MD or /MDd.
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_md_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        elif defined(_MT) // Corresponding to CL command line options /MT or /MTd.
+#            define PYBIND11_BUILD_ABI "_mt_mscver" PYBIND11_PLATFORM_ABI_ID_TOSTRING(_MSC_VER)
+#        else
+#            if (_MSC_VER) / 100 == 19
+#                define PYBIND11_BUILD_ABI "_none_mscver19"
+#            else
+#                error "Unknown major version for MSC_VER: PLEASE REVISE THIS CODE."
+#            endif
+#        endif
+#    elif defined(__NVCOMPILER)       // NVHPC (PGI-based).
+#        define PYBIND11_BUILD_ABI "" // TODO: What should be here, to prevent UB?
+#    else
+#        error "Unknown platform or compiler: PLEASE REVISE THIS CODE."
+#    endif
+#endif
+
+#ifndef PYBIND11_INTERNALS_KIND
+#    define PYBIND11_INTERNALS_KIND ""
+#endif
+
+#define PYBIND11_PLATFORM_ABI_ID                                                                  \
+    PYBIND11_INTERNALS_KIND PYBIND11_COMPILER_TYPE PYBIND11_STDLIB PYBIND11_BUILD_ABI             \
+        PYBIND11_BUILD_TYPE

include/pybind11/conduit/wrap_include_python_h.h

@@ -0,0 +1,72 @@
+#pragma once
+
+// Copyright (c) 2024 The pybind Community.
+
+// STRONG REQUIREMENT:
+//   This header is a wrapper around `#include <Python.h>`, therefore it
+//   MUST BE INCLUDED BEFORE ANY STANDARD HEADERS are included.
+// See also:
+//   https://docs.python.org/3/c-api/intro.html#include-files
+// Quoting from there:
+//   Note: Since Python may define some pre-processor definitions which affect
+//   the standard headers on some systems, you must include Python.h before
+//   any standard headers are included.
+
+// To maximize reusability:
+// DO NOT ADD CODE THAT REQUIRES C++ EXCEPTION HANDLING.
+
+// Disable linking to pythonX_d.lib on Windows in debug mode.
+#if defined(_MSC_VER) && defined(_DEBUG) && !defined(Py_DEBUG)
+// Workaround for a VS 2022 issue.
+// See https://github.com/pybind/pybind11/pull/3497 for full context.
+// NOTE: This workaround knowingly violates the Python.h include order
+//       requirement (see above).
+#    include <yvals.h>
+#    if _MSVC_STL_VERSION >= 143
+#        include <crtdefs.h>
+#    endif
+#    define PYBIND11_DEBUG_MARKER
+#    undef _DEBUG
+#endif
+
+// Don't let Python.h #define (v)snprintf as macro because they are implemented
+// properly in Visual Studio since 2015.
+#if defined(_MSC_VER)
+#    define HAVE_SNPRINTF 1
+#endif
+
+#if defined(_MSC_VER)
+#    pragma warning(push)
+#    pragma warning(disable : 4505)
+// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed
+#endif
+
+#include <Python.h>
+#include <frameobject.h>
+#include <pythread.h>
+
+#if defined(_MSC_VER)
+#    pragma warning(pop)
+#endif
+
+#if defined(PYBIND11_DEBUG_MARKER)
+#    define _DEBUG
+#    undef PYBIND11_DEBUG_MARKER
+#endif
+
+// Python #defines overrides on all sorts of core functions, which
+// tends to wreak havok in C++ codebases that expect these to work
+// like regular functions (potentially with several overloads).
+#if defined(isalnum)
+#    undef isalnum
+#    undef isalpha
+#    undef islower
+#    undef isspace
+#    undef isupper
+#    undef tolower
+#    undef toupper
+#endif
+
+#if defined(copysign)
+#    undef copysign
+#endif

include/pybind11/detail/common.h

@@ -9,6 +9,11 @@
 
 #pragma once
 
+#include <pybind11/conduit/wrap_include_python_h.h>
+#if PY_VERSION_HEX < 0x03080000
+#    error "PYTHON < 3.8 IS UNSUPPORTED. pybind11 v2.13 was the last to support Python 3.7."
+#endif
+
 #define PYBIND11_VERSION_MAJOR 2
 #define PYBIND11_VERSION_MINOR 14
 #define PYBIND11_VERSION_PATCH 0.dev1
@@ -204,31 +209,6 @@
 #    define PYBIND11_MAYBE_UNUSED __attribute__((__unused__))
 #endif
 
-/* Don't let Python.h #define (v)snprintf as macro because they are implemented
-   properly in Visual Studio since 2015. */
-#if defined(_MSC_VER)
-#    define HAVE_SNPRINTF 1
-#endif
-
-/// Include Python header, disable linking to pythonX_d.lib on Windows in debug mode
-#if defined(_MSC_VER)
-PYBIND11_WARNING_PUSH
-PYBIND11_WARNING_DISABLE_MSVC(4505)
-// C4505: 'PySlice_GetIndicesEx': unreferenced local function has been removed (PyPy only)
-#    if defined(_DEBUG) && !defined(Py_DEBUG)
-// Workaround for a VS 2022 issue.
-// NOTE: This workaround knowingly violates the Python.h include order requirement:
-// https://docs.python.org/3/c-api/intro.html#include-files
-// See https://github.com/pybind/pybind11/pull/3497 for full context.
-#        include <yvals.h>
-#        if _MSVC_STL_VERSION >= 143
-#            include <crtdefs.h>
-#        endif
-#        define PYBIND11_DEBUG_MARKER
-#        undef _DEBUG
-#    endif
-#endif
-
 // https://en.cppreference.com/w/c/chrono/localtime
 #if defined(__STDC_LIB_EXT1__) && !defined(__STDC_WANT_LIB_EXT1__)
 #    define __STDC_WANT_LIB_EXT1__
@@ -263,30 +243,6 @@ PYBIND11_WARNING_DISABLE_MSVC(4505)
 #    endif
 #endif
 
-#include <Python.h>
-#if PY_VERSION_HEX < 0x03080000
-#    error "PYTHON < 3.8 IS UNSUPPORTED. pybind11 v2.13 was the last to support Python 3.7."
-#endif
-#include <frameobject.h>
-#include <pythread.h>
-
-/* Python #defines overrides on all sorts of core functions, which
-   tends to weak havok in C++ codebases that expect these to work
-   like regular functions (potentially with several overloads) */
-#if defined(isalnum)
-#    undef isalnum
-#    undef isalpha
-#    undef islower
-#    undef isspace
-#    undef isupper
-#    undef tolower
-#    undef toupper
-#endif
-
-#if defined(copysign)
-#    undef copysign
-#endif
-
 #if defined(PYBIND11_NUMPY_1_ONLY)
 #    define PYBIND11_INTERNAL_NUMPY_1_ONLY_DETECTED
 #endif
@@ -295,14 +251,6 @@ PYBIND11_WARNING_DISABLE_MSVC(4505)
 #    define PYBIND11_SIMPLE_GIL_MANAGEMENT
 #endif
 
-#if defined(_MSC_VER)
-#    if defined(PYBIND11_DEBUG_MARKER)
-#        define _DEBUG
-#        undef PYBIND11_DEBUG_MARKER
-#    endif
-PYBIND11_WARNING_POP
-#endif
-
 #include <cstddef>
 #include <cstring>
 #include <exception>