diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a3..19417df698c8e7 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -45,19 +45,19 @@ # endif #endif -// gh-111506: The free-threaded build is not compatible with the limited API -// or the stable ABI. -#if defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED) -# error "The limited API is not currently supported in the free-threaded build" -#endif +#if defined(Py_GIL_DISABLED) +# if defined(Py_LIMITED_API) && !defined(_Py_OPAQUE_PYOBJECT) +# error "Py_LIMITED_API is not currently supported in the free-threaded build" +# endif -#if defined(Py_GIL_DISABLED) && defined(_MSC_VER) -# include // __readgsqword() -#endif +# if defined(_MSC_VER) +# include // __readgsqword() +# endif -#if defined(Py_GIL_DISABLED) && defined(__MINGW32__) -# include // __readgsqword() -#endif +# if defined(__MINGW32__) +# include // __readgsqword() +# endif +#endif // Py_GIL_DISABLED // Include Python header files #include "pyport.h" diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 2a17c891dda811..17634a93f8fa6f 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -36,6 +36,7 @@ PyAPI_FUNC(PyObject *) PyModuleDef_Init(PyModuleDef*); PyAPI_DATA(PyTypeObject) PyModuleDef_Type; #endif +#ifndef _Py_OPAQUE_PYOBJECT typedef struct PyModuleDef_Base { PyObject_HEAD /* The function used to re-initialize the module. @@ -63,6 +64,7 @@ typedef struct PyModuleDef_Base { 0, /* m_index */ \ _Py_NULL, /* m_copy */ \ } +#endif // _Py_OPAQUE_PYOBJECT #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03050000 /* New in 3.5 */ @@ -104,6 +106,8 @@ struct PyModuleDef_Slot { PyAPI_FUNC(int) PyUnstable_Module_SetGIL(PyObject *module, void *gil); #endif + +#ifndef _Py_OPAQUE_PYOBJECT struct PyModuleDef { PyModuleDef_Base m_base; const char* m_name; @@ -115,6 +119,7 @@ struct PyModuleDef { inquiry m_clear; freefunc m_free; }; +#endif // _Py_OPAQUE_PYOBJECT #ifdef __cplusplus } diff --git a/Include/object.h b/Include/object.h index c75e9db0cbd935..b1bcc9481871b4 100644 --- a/Include/object.h +++ b/Include/object.h @@ -56,6 +56,11 @@ whose size is determined when the object is allocated. # define Py_REF_DEBUG #endif +#if defined(_Py_OPAQUE_PYOBJECT) && !defined(Py_LIMITED_API) +# error "_Py_OPAQUE_PYOBJECT only makes sense with Py_LIMITED_API" +#endif + +#ifndef _Py_OPAQUE_PYOBJECT /* PyObject_HEAD defines the initial segment of every PyObject. */ #define PyObject_HEAD PyObject ob_base; @@ -99,6 +104,8 @@ whose size is determined when the object is allocated. * not necessarily a byte count. */ #define PyObject_VAR_HEAD PyVarObject ob_base; +#endif // !defined(_Py_OPAQUE_PYOBJECT) + #define Py_INVALID_SIZE (Py_ssize_t)-1 /* PyObjects are given a minimum alignment so that the least significant bits @@ -112,7 +119,9 @@ whose size is determined when the object is allocated. * by hand. Similarly every pointer to a variable-size Python object can, * in addition, be cast to PyVarObject*. */ -#ifndef Py_GIL_DISABLED +#ifdef _Py_OPAQUE_PYOBJECT + /* PyObject is opaque */ +#elif !defined(Py_GIL_DISABLED) struct _object { #if (defined(__GNUC__) || defined(__clang__)) \ && !(defined __STDC_VERSION__ && __STDC_VERSION__ >= 201112L) @@ -168,15 +177,18 @@ struct _object { Py_ssize_t ob_ref_shared; // shared (atomic) reference count PyTypeObject *ob_type; }; -#endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* Cast argument to PyObject* type. */ #define _PyObject_CAST(op) _Py_CAST(PyObject*, (op)) -typedef struct { +#ifndef _Py_OPAQUE_PYOBJECT +struct PyVarObject { PyObject ob_base; Py_ssize_t ob_size; /* Number of items in variable part */ -} PyVarObject; +}; +#endif +typedef struct PyVarObject PyVarObject; /* Cast argument to PyVarObject* type. */ #define _PyVarObject_CAST(op) _Py_CAST(PyVarObject*, (op)) @@ -286,6 +298,7 @@ PyAPI_FUNC(PyTypeObject*) Py_TYPE(PyObject *ob); PyAPI_DATA(PyTypeObject) PyLong_Type; PyAPI_DATA(PyTypeObject) PyBool_Type; +#ifndef _Py_OPAQUE_PYOBJECT // bpo-39573: The Py_SET_SIZE() function must be used to set an object size. static inline Py_ssize_t Py_SIZE(PyObject *ob) { assert(Py_TYPE(ob) != &PyLong_Type); @@ -295,6 +308,7 @@ static inline Py_ssize_t Py_SIZE(PyObject *ob) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SIZE(ob) Py_SIZE(_PyObject_CAST(ob)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { return Py_TYPE(ob) == type; @@ -304,6 +318,7 @@ static inline int Py_IS_TYPE(PyObject *ob, PyTypeObject *type) { #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline void Py_SET_TYPE(PyObject *ob, PyTypeObject *type) { ob->ob_type = type; } @@ -323,6 +338,7 @@ static inline void Py_SET_SIZE(PyVarObject *ob, Py_ssize_t size) { #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_SET_SIZE(ob, size) Py_SET_SIZE(_PyVarObject_CAST(ob), (size)) #endif +#endif // !defined(_Py_OPAQUE_PYOBJECT) /* diff --git a/Include/refcount.h b/Include/refcount.h index 457972b6dcfd9f..ba34461fefcbb0 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -117,6 +117,7 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); #endif #endif +#ifndef _Py_OPAQUE_PYOBJECT static inline Py_ALWAYS_INLINE int _Py_IsImmortal(PyObject *op) { #if defined(Py_GIL_DISABLED) @@ -140,6 +141,7 @@ static inline Py_ALWAYS_INLINE int _Py_IsStaticImmortal(PyObject *op) #endif } #define _Py_IsStaticImmortal(op) _Py_IsStaticImmortal(_PyObject_CAST(op)) +#endif // !defined(_Py_OPAQUE_PYOBJECT) // Py_SET_REFCNT() implementation for stable ABI PyAPI_FUNC(void) _Py_SetRefcnt(PyObject *ob, Py_ssize_t refcnt); diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 93e7b2043d397a..fb93c6ccbb637d 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -12,7 +12,10 @@ from test import support -SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SOURCES = [ + os.path.join(os.path.dirname(__file__), 'extension.c'), + os.path.join(os.path.dirname(__file__), 'create_moduledef.c'), +] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -35,17 +38,22 @@ class BaseTests: def test_build(self): self.check_build('_test_cext') - def check_build(self, extension_name, std=None, limited=False): + def check_build(self, extension_name, std=None, limited=False, + opaque_pyobject=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, - std=std, limited=limited) + std=std, limited=limited, + opaque_pyobject=opaque_pyobject) - def _check_build(self, extension_name, python_exe, std, limited): + def _check_build(self, extension_name, python_exe, std, limited, + opaque_pyobject): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + for source in SOURCES: + dest = os.path.join(pkg_dir, os.path.basename(source)) + shutil.copy(source, dest) def run_cmd(operation, cmd): env = os.environ.copy() @@ -53,6 +61,8 @@ def run_cmd(operation, cmd): env['CPYTHON_TEST_STD'] = std if limited: env['CPYTHON_TEST_LIMITED'] = '1' + if opaque_pyobject: + env['CPYTHON_TEST_OPAQUE_PYOBJECT'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: @@ -107,6 +117,11 @@ def test_build_limited_c11(self): def test_build_c11(self): self.check_build('_test_c11_cext', std='c11') + def test_build_opaque_pyobject(self): + # Test with _Py_OPAQUE_PYOBJECT + self.check_build('_test_limited_opaque_cext', limited=True, + opaque_pyobject=True) + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") def test_build_c99(self): # In public docs, we say C API is compatible with C11. However, diff --git a/Lib/test/test_cext/create_moduledef.c b/Lib/test/test_cext/create_moduledef.c new file mode 100644 index 00000000000000..249c3163552c03 --- /dev/null +++ b/Lib/test/test_cext/create_moduledef.c @@ -0,0 +1,29 @@ +// Workaround for testing _Py_OPAQUE_PYOBJECT. +// See end of 'extension.c' + + +#undef _Py_OPAQUE_PYOBJECT +#undef Py_LIMITED_API +#include "Python.h" + + +// (repeated definition to avoid creating a header) +extern PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots); + +PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots) { + + static struct PyModuleDef _testcext_module = { + PyModuleDef_HEAD_INIT, + }; + if (!_testcext_module.m_name) { + _testcext_module.m_name = name; + _testcext_module.m_doc = doc; + _testcext_module.m_methods = methods; + _testcext_module.m_slots = slots; + } + return PyModuleDef_Init(&_testcext_module); +} diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 4be2f24c60d44b..73fc67ae59d18f 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -78,6 +78,9 @@ _testcext_exec( return 0; } +#define _FUNC_NAME(NAME) PyInit_ ## NAME +#define FUNC_NAME(NAME) _FUNC_NAME(NAME) + // Converting from function pointer to void* has undefined behavior, but // works on all known platforms, and CPython's module and type slots currently // need it. @@ -96,9 +99,10 @@ static PyModuleDef_Slot _testcext_slots[] = { _Py_COMP_DIAG_POP - PyDoc_STRVAR(_testcext_doc, "C test extension."); +#ifndef _Py_OPAQUE_PYOBJECT + static struct PyModuleDef _testcext_module = { PyModuleDef_HEAD_INIT, // m_base STR(MODULE_NAME), // m_name @@ -112,11 +116,30 @@ static struct PyModuleDef _testcext_module = { }; -#define _FUNC_NAME(NAME) PyInit_ ## NAME -#define FUNC_NAME(NAME) _FUNC_NAME(NAME) - PyMODINIT_FUNC FUNC_NAME(MODULE_NAME)(void) { return PyModuleDef_Init(&_testcext_module); } + +#else // _Py_OPAQUE_PYOBJECT + +// Opaque PyObject means that PyModuleDef is also opaque and cannot be +// declared statically. See PEP 793. +// So, this part of module creation is split into a separate source file +// which uses non-limited API. + +// (repeated definition to avoid creating a header) +extern PyObject *testcext_create_moduledef( + const char *name, const char *doc, + PyMethodDef *methods, PyModuleDef_Slot *slots); + + +PyMODINIT_FUNC +FUNC_NAME(MODULE_NAME)(void) +{ + return testcext_create_moduledef( + STR(MODULE_NAME), _testcext_doc, _testcext_methods, _testcext_slots); +} + +#endif // _Py_OPAQUE_PYOBJECT diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 587585e8086e92..4d71e4751f7afd 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -59,8 +59,11 @@ def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + opaque_pyobject = bool(os.environ.get("CPYTHON_TEST_OPAQUE_PYOBJECT", "")) internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) + sources = [SOURCE] + if not internal: cflags = list(PUBLIC_CFLAGS) else: @@ -93,6 +96,11 @@ def main(): version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + # Define _Py_OPAQUE_PYOBJECT macro + if opaque_pyobject: + cflags.append(f'-D_Py_OPAQUE_PYOBJECT') + sources.append('create_moduledef.c') + if internal: cflags.append('-DTEST_INTERNAL_C_API=1') @@ -120,7 +128,7 @@ def main(): ext = Extension( module_name, - sources=[SOURCE], + sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, library_dirs=library_dirs)