From f777165dcb8e772473c82dfc9e461c4fe710ba90 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 4 May 2022 14:35:57 -0400 Subject: [PATCH 01/13] Placeholder commit for 3.11 testing --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 4fd1e7a6fa..36fbfed4f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython From 056965c4d1ee8cf054959507d6199236ea1afaa6 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 5 May 2022 16:29:33 -0400 Subject: [PATCH 02/13] Does this fix it? --- tests/test_multiple_inheritance.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index 5916ae9010..d57470b5ce 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -230,8 +230,10 @@ TEST_SUBMODULE(multiple_inheritance, m) { struct VanillaDictMix1 : Vanilla, WithDict {}; struct VanillaDictMix2 : WithDict, Vanilla {}; py::class_(m, "WithDict", py::dynamic_attr()).def(py::init<>()); - py::class_(m, "VanillaDictMix1").def(py::init<>()); - py::class_(m, "VanillaDictMix2").def(py::init<>()); + py::class_(m, "VanillaDictMix1", py::dynamic_attr()) + .def(py::init<>()); + py::class_(m, "VanillaDictMix2", py::dynamic_attr()) + .def(py::init<>()); // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance From 574bd957a50284c63555706b26a4efe59925feee Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 17 May 2022 12:02:58 -0400 Subject: [PATCH 03/13] Try suggestion --- include/pybind11/detail/class.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index e52ec1d3e6..2bd91734cf 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -543,8 +543,12 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; +#if PY_VERSION_HEX < 0x030B0000 type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it +#else + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; +#endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; From 60a2ab8d34449dc77b268ad730f8e9eb50f176c4 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 4 May 2022 14:35:57 -0400 Subject: [PATCH 04/13] Placeholder commit for 3.11 testing --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 4fd1e7a6fa..36fbfed4f3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,6 +19,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 License :: OSI Approved :: BSD License Programming Language :: Python :: Implementation :: PyPy Programming Language :: Python :: Implementation :: CPython From 5045e2ea7cd34845fb5172c39ceb9dd84036e818 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Thu, 5 May 2022 16:29:33 -0400 Subject: [PATCH 05/13] Does this fix it? --- tests/test_multiple_inheritance.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index 5916ae9010..d57470b5ce 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -230,8 +230,10 @@ TEST_SUBMODULE(multiple_inheritance, m) { struct VanillaDictMix1 : Vanilla, WithDict {}; struct VanillaDictMix2 : WithDict, Vanilla {}; py::class_(m, "WithDict", py::dynamic_attr()).def(py::init<>()); - py::class_(m, "VanillaDictMix1").def(py::init<>()); - py::class_(m, "VanillaDictMix2").def(py::init<>()); + py::class_(m, "VanillaDictMix1", py::dynamic_attr()) + .def(py::init<>()); + py::class_(m, "VanillaDictMix2", py::dynamic_attr()) + .def(py::init<>()); // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance From 78ff6401a68dd2113db35bc45b99e5bd59482d5a Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Tue, 17 May 2022 12:02:58 -0400 Subject: [PATCH 06/13] Try suggestion --- include/pybind11/detail/class.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/pybind11/detail/class.h b/include/pybind11/detail/class.h index e52ec1d3e6..2bd91734cf 100644 --- a/include/pybind11/detail/class.h +++ b/include/pybind11/detail/class.h @@ -543,8 +543,12 @@ extern "C" inline int pybind11_clear(PyObject *self) { inline void enable_dynamic_attributes(PyHeapTypeObject *heap_type) { auto *type = &heap_type->ht_type; type->tp_flags |= Py_TPFLAGS_HAVE_GC; +#if PY_VERSION_HEX < 0x030B0000 type->tp_dictoffset = type->tp_basicsize; // place dict at the end type->tp_basicsize += (ssize_t) sizeof(PyObject *); // and allocate enough space for it +#else + type->tp_flags |= Py_TPFLAGS_MANAGED_DICT; +#endif type->tp_traverse = pybind11_traverse; type->tp_clear = pybind11_clear; From 0d7c4b7cb3bdba42bbe24890129c30ea7cdfafdd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 25 May 2022 09:49:20 -0400 Subject: [PATCH 07/13] fix: try using modern init for embedded interp Signed-off-by: Henry Schreiner --- include/pybind11/embed.h | 78 +++++++++++++++++++++++----------------- tests/requirements.txt | 2 +- 2 files changed, 47 insertions(+), 33 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 98cc3e4664..760d89dbec 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -86,37 +86,6 @@ inline wchar_t *widen_chars(const char *safe_arg) { return widened_arg; } -/// Python 2.x/3.x-compatible version of `PySys_SetArgv` -inline void set_interpreter_argv(int argc, const char *const *argv, bool add_program_dir_to_path) { - // Before it was special-cased in python 3.8, passing an empty or null argv - // caused a segfault, so we have to reimplement the special case ourselves. - bool special_case = (argv == nullptr || argc <= 0); - - const char *const empty_argv[]{"\0"}; - const char *const *safe_argv = special_case ? empty_argv : argv; - if (special_case) { - argc = 1; - } - - auto argv_size = static_cast(argc); - // SetArgv* on python 3 takes wchar_t, so we have to convert. - std::unique_ptr widened_argv(new wchar_t *[argv_size]); - std::vector> widened_argv_entries; - widened_argv_entries.reserve(argv_size); - for (size_t ii = 0; ii < argv_size; ++ii) { - widened_argv_entries.emplace_back(widen_chars(safe_argv[ii])); - if (!widened_argv_entries.back()) { - // A null here indicates a character-encoding failure or the python - // interpreter out of memory. Give up. - return; - } - widened_argv[ii] = widened_argv_entries.back().get(); - } - - auto *pysys_argv = widened_argv.get(); - PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); -} - PYBIND11_NAMESPACE_END(detail) /** \rst @@ -146,9 +115,54 @@ inline void initialize_interpreter(bool init_signal_handlers = true, pybind11_fail("The interpreter is already running"); } +#if PY_VERSION_HEX < 0x030B0000 + Py_InitializeEx(init_signal_handlers ? 1 : 0); - detail::set_interpreter_argv(argc, argv, add_program_dir_to_path); + // Before it was special-cased in python 3.8, passing an empty or null argv + // caused a segfault, so we have to reimplement the special case ourselves. + bool special_case = (argv == nullptr || argc <= 0); + + const char *const empty_argv[]{"\0"}; + const char *const *safe_argv = special_case ? empty_argv : argv; + if (special_case) { + argc = 1; + } + + auto argv_size = static_cast(argc); + // SetArgv* on python 3 takes wchar_t, so we have to convert. + std::unique_ptr widened_argv(new wchar_t *[argv_size]); + std::vector> widened_argv_entries; + widened_argv_entries.reserve(argv_size); + for (size_t ii = 0; ii < argv_size; ++ii) { + widened_argv_entries.emplace_back(detail::widen_chars(safe_argv[ii])); + if (!widened_argv_entries.back()) { + // A null here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + return; + } + widened_argv[ii] = widened_argv_entries.back().get(); + } + + auto *pysys_argv = widened_argv.get(); + + PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); +#else + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.safe_path = add_program_dir_to_path ? 0 : 1; + config.install_signal_handlers = init_signal_handlers ? 1 : 0; + + PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); + if (PyStatus_Exception(status)) { + // A failure here indicates a character-encoding failure or the python + // interpreter out of memory. Give up. + PyConfig_Clear(&config); + return; + } + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); +#endif } /** \rst diff --git a/tests/requirements.txt b/tests/requirements.txt index 1159287fa9..04aafa8cf9 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,4 @@ -build==0.7.0 +build==0.8.0 numpy==1.21.5; platform_python_implementation=="PyPy" and sys_platform=="linux" and python_version=="3.7" numpy==1.19.3; platform_python_implementation!="PyPy" and python_version=="3.6" numpy==1.21.5; platform_python_implementation!="PyPy" and python_version>="3.7" and python_version<"3.10" From 9887cdc8cc0eb5109ed8221ff72eea931405691b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 25 May 2022 10:02:21 -0400 Subject: [PATCH 08/13] fix: error message changed in 3.11 --- tests/test_methods_and_attributes.py | 30 +++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/test_methods_and_attributes.py b/tests/test_methods_and_attributes.py index 54597fa788..0a2ae1239a 100644 --- a/tests/test_methods_and_attributes.py +++ b/tests/test_methods_and_attributes.py @@ -1,9 +1,21 @@ +import sys + import pytest import env # noqa: F401 from pybind11_tests import ConstructorStats from pybind11_tests import methods_and_attributes as m +NO_GETTER_MSG = ( + "unreadable attribute" if sys.version_info < (3, 11) else "object has no getter" +) +NO_SETTER_MSG = ( + "can't set attribute" if sys.version_info < (3, 11) else "object has no setter" +) +NO_DELETER_MSG = ( + "can't delete attribute" if sys.version_info < (3, 11) else "object has no deleter" +) + def test_methods_and_attributes(): instance1 = m.ExampleMandA() @@ -102,32 +114,32 @@ def test_properties(): with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_writeonly # unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly = 4 assert instance.def_property_readonly == 4 with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_impossible # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) with pytest.raises(AttributeError) as excinfo: instance.def_property_impossible = 5 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) def test_static_properties(): assert m.TestProperties.def_readonly_static == 1 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_readonly_static = 2 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_readwrite_static = 2 assert m.TestProperties.def_readwrite_static == 2 with pytest.raises(AttributeError) as excinfo: dummy = m.TestProperties.def_writeonly_static # unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_writeonly_static = 3 assert m.TestProperties.def_readonly_static == 3 @@ -135,14 +147,14 @@ def test_static_properties(): assert m.TestProperties.def_property_readonly_static == 3 with pytest.raises(AttributeError) as excinfo: m.TestProperties.def_property_readonly_static = 99 - assert "can't set attribute" in str(excinfo.value) + assert NO_SETTER_MSG in str(excinfo.value) m.TestProperties.def_property_static = 4 assert m.TestProperties.def_property_static == 4 with pytest.raises(AttributeError) as excinfo: dummy = m.TestProperties.def_property_writeonly_static - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) m.TestProperties.def_property_writeonly_static = 5 assert m.TestProperties.def_property_static == 5 @@ -160,7 +172,7 @@ def test_static_properties(): with pytest.raises(AttributeError) as excinfo: dummy = instance.def_property_writeonly_static # noqa: F841 unused var - assert "unreadable attribute" in str(excinfo.value) + assert NO_GETTER_MSG in str(excinfo.value) instance.def_property_writeonly_static = 4 assert instance.def_property_static == 4 @@ -180,7 +192,7 @@ def test_static_properties(): properties_override = m.TestPropertiesOverride() with pytest.raises(AttributeError) as excinfo: del properties_override.def_readonly - assert "can't delete attribute" in str(excinfo.value) + assert NO_DELETER_MSG in str(excinfo.value) def test_static_cls(): From 1cd2513a3c68d807543375dce2fca3b6a71e9b15 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 4 Jul 2022 11:01:03 -0400 Subject: [PATCH 09/13] fix: apply logic in Python manually Signed-off-by: Henry Schreiner --- include/pybind11/embed.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index 760d89dbec..f6c8d9063f 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -149,8 +149,7 @@ inline void initialize_interpreter(bool init_signal_handlers = true, PySys_SetArgvEx(argc, pysys_argv, static_cast(add_program_dir_to_path)); #else PyConfig config; - PyConfig_InitPythonConfig(&config); - config.safe_path = add_program_dir_to_path ? 0 : 1; + PyConfig_InitIsolatedConfig(&config); config.install_signal_handlers = init_signal_handlers ? 1 : 0; PyStatus status = PyConfig_SetBytesArgv(&config, argc, const_cast(argv)); @@ -158,10 +157,19 @@ inline void initialize_interpreter(bool init_signal_handlers = true, // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. PyConfig_Clear(&config); - return; + throw std::runtime_error("Failed to prepare CPython"); } - Py_InitializeFromConfig(&config); + status = Py_InitializeFromConfig(&config); PyConfig_Clear(&config); + if (PyStatus_Exception(status)) { + throw std::runtime_error("Failed to init CPython"); + } + if (add_program_dir_to_path) { + PyRun_SimpleString("import sys, os.path; " + "sys.path.insert(0, " + "os.path.abspath(os.path.dirname(sys.argv[0])) " + "if sys.argv and os.path.exists(sys.argv[0]) else '')"); + } #endif } From 9b0dad5a06245669c2e02c7b26e4114a5ecc90bd Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Mon, 4 Jul 2022 13:32:09 -0400 Subject: [PATCH 10/13] fix autodetect dynamic attrs in 3.11 --- include/pybind11/attr.h | 7 ++++--- tests/test_multiple_inheritance.cpp | 6 ++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 14600e9bb0..1fd17178c9 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -345,9 +345,10 @@ struct type_record { bases.append((PyObject *) base_info->type); - if (base_info->type->tp_dictoffset != 0) { - dynamic_attr = true; - } + dynamic_attr |= base_info->type->tp_dictoffset != 0; +#if PY_VERSION_HEX >= 0x030B0000 + dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; +#endif if (caster) { base_info->implicit_casts.emplace_back(type, caster); diff --git a/tests/test_multiple_inheritance.cpp b/tests/test_multiple_inheritance.cpp index d57470b5ce..5916ae9010 100644 --- a/tests/test_multiple_inheritance.cpp +++ b/tests/test_multiple_inheritance.cpp @@ -230,10 +230,8 @@ TEST_SUBMODULE(multiple_inheritance, m) { struct VanillaDictMix1 : Vanilla, WithDict {}; struct VanillaDictMix2 : WithDict, Vanilla {}; py::class_(m, "WithDict", py::dynamic_attr()).def(py::init<>()); - py::class_(m, "VanillaDictMix1", py::dynamic_attr()) - .def(py::init<>()); - py::class_(m, "VanillaDictMix2", py::dynamic_attr()) - .def(py::init<>()); + py::class_(m, "VanillaDictMix1").def(py::init<>()); + py::class_(m, "VanillaDictMix2").def(py::init<>()); // test_diamond_inheritance // Issue #959: segfault when constructing diamond inheritance instance From e0e567336c01ba0e583cdbed404c5fc83734b80b Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 5 Jul 2022 09:08:15 -0400 Subject: [PATCH 11/13] fix: include error message if possible in error Signed-off-by: Henry Schreiner --- include/pybind11/embed.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/pybind11/embed.h b/include/pybind11/embed.h index f6c8d9063f..d6999cd779 100644 --- a/include/pybind11/embed.h +++ b/include/pybind11/embed.h @@ -157,12 +157,14 @@ inline void initialize_interpreter(bool init_signal_handlers = true, // A failure here indicates a character-encoding failure or the python // interpreter out of memory. Give up. PyConfig_Clear(&config); - throw std::runtime_error("Failed to prepare CPython"); + throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg + : "Failed to prepare CPython"); } status = Py_InitializeFromConfig(&config); PyConfig_Clear(&config); if (PyStatus_Exception(status)) { - throw std::runtime_error("Failed to init CPython"); + throw std::runtime_error(PyStatus_IsError(status) ? status.err_msg + : "Failed to init CPython"); } if (add_program_dir_to_path) { PyRun_SimpleString("import sys, os.path; " From 04d60f1d029f73d849ac2be1209c72c4796a781d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 5 Jul 2022 15:19:28 -0400 Subject: [PATCH 12/13] ci: enable standard Python 3.11 testing Signed-off-by: Henry Schreiner --- .github/workflows/ci.yml | 5 +++-- .github/workflows/upstream.yml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b537e6e1e..1f7fd87e37 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: - '3.6' - '3.9' - '3.10' + - '3.11-dev' - 'pypy-3.7' - 'pypy-3.8' - 'pypy-3.9' @@ -185,8 +186,8 @@ jobs: - python-version: "3.9" python-debug: true valgrind: true - # - python-version: "3.11-dev" - # python-debug: false + - python-version: "3.11-dev" + python-debug: false name: "🐍 ${{ matrix.python-version }}${{ matrix.python-debug && '-dbg' || '' }} (deadsnakes)${{ matrix.valgrind && ' • Valgrind' || '' }} • x64" runs-on: ubuntu-latest diff --git a/.github/workflows/upstream.yml b/.github/workflows/upstream.yml index 174dc24965..ee6cde29b2 100644 --- a/.github/workflows/upstream.yml +++ b/.github/workflows/upstream.yml @@ -14,7 +14,7 @@ env: jobs: standard: - name: "🐍 3.11 dev • ubuntu-latest • x64" + name: "🐍 3.11 latest internals • ubuntu-latest • x64" runs-on: ubuntu-latest if: "contains(github.event.pull_request.labels.*.name, 'python dev')" From 6f43cc4966f42d7fa42f86576476ce0f86f73298 Mon Sep 17 00:00:00 2001 From: Aaron Gokaslan Date: Wed, 6 Jul 2022 12:10:33 -0400 Subject: [PATCH 13/13] Make dynamic attrs condtiion exclusive to ver. --- include/pybind11/attr.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/pybind11/attr.h b/include/pybind11/attr.h index 1fd17178c9..db7cd8efff 100644 --- a/include/pybind11/attr.h +++ b/include/pybind11/attr.h @@ -345,8 +345,9 @@ struct type_record { bases.append((PyObject *) base_info->type); +#if PY_VERSION_HEX < 0x030B0000 dynamic_attr |= base_info->type->tp_dictoffset != 0; -#if PY_VERSION_HEX >= 0x030B0000 +#else dynamic_attr |= (base_info->type->tp_flags & Py_TPFLAGS_MANAGED_DICT) != 0; #endif