gh-116322: Add Py_mod_gil module slot (#116882)

This PR adds the ability to enable the GIL if it was disabled at
interpreter startup, and modifies the multi-phase module initialization
path to enable the GIL when loading a module, unless that module's spec
includes a slot indicating it can run safely without the GIL.

PEP 703 called the constant for the slot `Py_mod_gil_not_used`; I went
with `Py_MOD_GIL_NOT_USED` for consistency with gh-104148.

A warning will be issued up to once per interpreter for the first
GIL-using module that is loaded. If `-v` is given, a shorter message
will be printed to stderr every time a GIL-using module is loaded
(including the first one that issues a warning).

Author: swtaarrs

Doc/c-api/module.rst

@@ -411,6 +411,31 @@ The available slot types are:
 
    .. versionadded:: 3.12
 
+.. c:macro:: Py_mod_gil
+
+   Specifies one of the following values:
+
+   .. c:macro:: Py_MOD_GIL_USED
+
+      The module depends on the presence of the global interpreter lock (GIL),
+      and may access global state without synchronization.
+
+   .. c:macro:: Py_MOD_GIL_NOT_USED
+
+      The module is safe to run without an active GIL.
+
+   This slot is ignored by Python builds not configured with
+   :option:`--disable-gil`.  Otherwise, it determines whether or not importing
+   this module will cause the GIL to be automatically enabled. See
+   :envvar:`PYTHON_GIL` and :option:`-X gil <-X>` for more detail.
+
+   Multiple ``Py_mod_gil`` slots may not be specified in one module definition.
+
+   If ``Py_mod_gil`` is not specified, the import machinery defaults to
+   ``Py_MOD_GIL_USED``.
+
+   .. versionadded: 3.13
+
 See :PEP:`489` for more details on multi-phase initialization.
 
 Low-level module creation functions
@@ -609,6 +634,19 @@ state:
 
    .. versionadded:: 3.9
 
+.. c:function:: int PyModule_ExperimentalSetGIL(PyObject *module, void *gil)
+
+   Indicate that *module* does or does not support running without the global
+   interpreter lock (GIL), using one of the values from
+   :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization
+   function. If this function is not called during module initialization, the
+   import machinery assumes the module does not support running without the
+   GIL. This function is only available in Python builds configured with
+   :option:`--disable-gil`.
+   Return ``-1`` on error, ``0`` on success.
+
+   .. versionadded:: 3.13
+
 
 Module lookup
 ^^^^^^^^^^^^^

Include/internal/pycore_moduleobject.h

@@ -22,6 +22,9 @@ typedef struct {
     PyObject *md_weaklist;
     // for logging purposes after md_dict is cleared
     PyObject *md_name;
+#ifdef Py_GIL_DISABLED
+    void *md_gil;
+#endif
 } PyModuleObject;
 
 static inline PyModuleDef* _PyModule_GetDef(PyObject *mod) {

Include/moduleobject.h

@@ -76,9 +76,13 @@ struct PyModuleDef_Slot {
 #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030c0000
 #  define Py_mod_multiple_interpreters 3
 #endif
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000
+#  define Py_mod_gil 4
+#endif
+
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 3
+#define _Py_mod_LAST_SLOT 4
 #endif
 
 #endif /* New in 3.5 */
@@ -90,6 +94,16 @@ struct PyModuleDef_Slot {
 #  define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2)
 #endif
 
+/* for Py_mod_gil: */
+#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030d0000
+#  define Py_MOD_GIL_USED ((void *)0)
+#  define Py_MOD_GIL_NOT_USED ((void *)1)
+#endif
+
+#if !defined(Py_LIMITED_API) && defined(Py_GIL_DISABLED)
+PyAPI_FUNC(int) PyModule_ExperimentalSetGIL(PyObject *module, void *gil);
+#endif
+
 struct PyModuleDef {
   PyModuleDef_Base m_base;
   const char* m_name;

Lib/test/test_importlib/extension/_test_nonmodule_cases.py

@@ -0,0 +1,44 @@
+import types
+import unittest
+from test.test_importlib import util
+
+machinery = util.import_importlib('importlib.machinery')
+
+from test.test_importlib.extension.test_loader import MultiPhaseExtensionModuleTests
+
+
+class NonModuleExtensionTests:
+    setUp = MultiPhaseExtensionModuleTests.setUp
+    load_module_by_name = MultiPhaseExtensionModuleTests.load_module_by_name
+
+    def _test_nonmodule(self):
+        # Test returning a non-module object from create works.
+        name = self.name + '_nonmodule'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+
+    # issue 27782
+    def test_nonmodule_with_methods(self):
+        # Test creating a non-module object with methods defined.
+        name = self.name + '_nonmodule_with_methods'
+        mod = self.load_module_by_name(name)
+        self.assertNotEqual(type(mod), type(unittest))
+        self.assertEqual(mod.three, 3)
+        self.assertEqual(mod.bar(10, 1), 9)
+
+    def test_null_slots(self):
+        # Test that NULL slots aren't a problem.
+        name = self.name + '_null_slots'
+        module = self.load_module_by_name(name)
+        self.assertIsInstance(module, types.ModuleType)
+        self.assertEqual(module.__name__, name)
+
+
+(Frozen_NonModuleExtensionTests,
+ Source_NonModuleExtensionTests
+ ) = util.test_both(NonModuleExtensionTests, machinery=machinery)
+
+
+if __name__ == '__main__':
+    unittest.main()

Lib/test/test_importlib/extension/test_loader.py

@@ -10,7 +10,8 @@
 import warnings
 import importlib.util
 import importlib
-from test.support import MISSING_C_DOCSTRINGS
+from test import support
+from test.support import MISSING_C_DOCSTRINGS, script_helper
 
 
 class LoaderTests:
@@ -325,29 +326,6 @@ def test_unloadable_nonascii(self):
             self.load_module_by_name(name)
         self.assertEqual(cm.exception.name, name)
 
-    def test_nonmodule(self):
-        # Test returning a non-module object from create works.
-        name = self.name + '_nonmodule'
-        mod = self.load_module_by_name(name)
-        self.assertNotEqual(type(mod), type(unittest))
-        self.assertEqual(mod.three, 3)
-
-    # issue 27782
-    def test_nonmodule_with_methods(self):
-        # Test creating a non-module object with methods defined.
-        name = self.name + '_nonmodule_with_methods'
-        mod = self.load_module_by_name(name)
-        self.assertNotEqual(type(mod), type(unittest))
-        self.assertEqual(mod.three, 3)
-        self.assertEqual(mod.bar(10, 1), 9)
-
-    def test_null_slots(self):
-        # Test that NULL slots aren't a problem.
-        name = self.name + '_null_slots'
-        module = self.load_module_by_name(name)
-        self.assertIsInstance(module, types.ModuleType)
-        self.assertEqual(module.__name__, name)
-
     def test_bad_modules(self):
         # Test SystemError is raised for misbehaving extensions.
         for name_base in [
@@ -401,5 +379,14 @@ def test_nonascii(self):
  ) = util.test_both(MultiPhaseExtensionModuleTests, machinery=machinery)
 
 
+class NonModuleExtensionTests(unittest.TestCase):
+    def test_nonmodule_cases(self):
+        # The test cases in this file cause the GIL to be enabled permanently
+        # in free-threaded builds, so they are run in a subprocess to isolate
+        # this effect.
+        script = support.findfile("test_importlib/extension/_test_nonmodule_cases.py")
+        script_helper.run_test_script(script)
+
+
 if __name__ == '__main__':
     unittest.main()

Lib/test/test_sys.py

@@ -1606,7 +1606,10 @@ def get_gen(): yield 1
         check(int(PyLong_BASE**2-1), vsize('') + 2*self.longdigit)
         check(int(PyLong_BASE**2), vsize('') + 3*self.longdigit)
         # module
-        check(unittest, size('PnPPP'))
+        if support.Py_GIL_DISABLED:
+            check(unittest, size('PPPPPP'))
+        else:
+            check(unittest, size('PPPPP'))
         # None
         check(None, size(''))
         # NotImplementedType

Misc/NEWS.d/next/Core and Builtins/2024-03-12-13-51-09.gh-issue-116322.q8TcDQ.rst

@@ -0,0 +1,5 @@
+Extension modules may indicate to the runtime that they can run without the
+GIL. Multi-phase init modules do so by calling providing
+``Py_MOD_GIL_NOT_USED`` for the ``Py_mod_gil`` slot, while single-phase init
+modules call ``PyModule_ExperimentalSetGIL(mod, Py_MOD_GIL_NOT_USED)`` from
+their init function.

Modules/_abc.c

@@ -970,6 +970,7 @@ _abcmodule_free(void *module)
 static PyModuleDef_Slot _abcmodule_slots[] = {
     {Py_mod_exec, _abcmodule_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };
 

Modules/_asynciomodule.c

@@ -3795,6 +3795,7 @@ module_exec(PyObject *mod)
 static struct PyModuleDef_Slot module_slots[] = {
     {Py_mod_exec, module_exec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL},
 };
 

Modules/_bisectmodule.c

@@ -462,6 +462,7 @@ bisect_modexec(PyObject *m)
 static PyModuleDef_Slot bisect_slots[] = {
     {Py_mod_exec, bisect_modexec},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
+    {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}
 };