Skip to content

Commit 0d8fb0b

Browse files
authored
[3.14] gh-137109: refactor warning about threads when forking (GH-141438) (GH-141614)
This splits the OS API specific functionality to get the number of threads out from the fallback Python method and warning raising code itself. This way the OS APIs can be queried before we've run `os.register_at_fork(after_in_parent=...)` registered functions which themselves may (re)start threads that would otherwise be detected. This is best effort. If the OS APIs are either unavailable or fail, the warning generating code still falls back to looking at the Python threading state after the CPython interpreter world has been restarted and the after_in_parent calls have been made. The common case for most Linux and macOS environments should work today. This also lines up with the existing TODO refactoring, we may choose to expose this API to get the number of OS threads in the `os` module in the future. Note: This is a simplified backport that maintains the void return type for warn_about_fork_with_threads() and keeps PyErr_Clear() in the warning path, as the error handling changes from fd8f42d are not needed in 3.14.
1 parent 58c9d25 commit 0d8fb0b

File tree

2 files changed

+65
-43
lines changed

2 files changed

+65
-43
lines changed
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
The :mod:`os.fork` and related forking APIs will no longer warn in the
2+
common case where Linux or macOS platform APIs return the number of threads
3+
in a process and find the answer to be 1 even when a
4+
:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a
5+
thread.

Modules/posixmodule.c

Lines changed: 60 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -7975,53 +7975,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before,
79757975
// running in the process. Best effort, silent if unable to count threads.
79767976
// Constraint: Quick. Never overcounts. Never leaves an error set.
79777977
//
7978-
// This should only be called from the parent process after
7978+
// This MUST only be called from the parent process after
79797979
// PyOS_AfterFork_Parent().
79807980
static void
7981-
warn_about_fork_with_threads(const char* name)
7981+
warn_about_fork_with_threads(
7982+
const char* name, // Name of the API to use in the warning message.
7983+
const Py_ssize_t num_os_threads // Only trusted when >= 1.
7984+
)
79827985
{
79837986
// It's not safe to issue the warning while the world is stopped, because
79847987
// other threads might be holding locks that we need, which would deadlock.
79857988
assert(!_PyRuntime.stoptheworld.world_stopped);
79867989

7987-
// TODO: Consider making an `os` module API to return the current number
7988-
// of threads in the process. That'd presumably use this platform code but
7989-
// raise an error rather than using the inaccurate fallback.
7990-
Py_ssize_t num_python_threads = 0;
7991-
#if defined(__APPLE__) && defined(HAVE_GETPID)
7992-
mach_port_t macos_self = mach_task_self();
7993-
mach_port_t macos_task;
7994-
if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) {
7995-
thread_array_t macos_threads;
7996-
mach_msg_type_number_t macos_n_threads;
7997-
if (task_threads(macos_task, &macos_threads,
7998-
&macos_n_threads) == KERN_SUCCESS) {
7999-
num_python_threads = macos_n_threads;
8000-
}
8001-
}
8002-
#elif defined(__linux__)
8003-
// Linux /proc/self/stat 20th field is the number of threads.
8004-
FILE* proc_stat = fopen("/proc/self/stat", "r");
8005-
if (proc_stat) {
8006-
size_t n;
8007-
// Size chosen arbitrarily. ~60% more bytes than a 20th column index
8008-
// observed on the author's workstation.
8009-
char stat_line[160];
8010-
n = fread(&stat_line, 1, 159, proc_stat);
8011-
stat_line[n] = '\0';
8012-
fclose(proc_stat);
8013-
8014-
char *saveptr = NULL;
8015-
char *field = strtok_r(stat_line, " ", &saveptr);
8016-
unsigned int idx;
8017-
for (idx = 19; idx && field; --idx) {
8018-
field = strtok_r(NULL, " ", &saveptr);
8019-
}
8020-
if (idx == 0 && field) { // found the 20th field
8021-
num_python_threads = atoi(field); // 0 on error
8022-
}
8023-
}
8024-
#endif
7990+
Py_ssize_t num_python_threads = num_os_threads;
80257991
if (num_python_threads <= 0) {
80267992
// Fall back to just the number our threading module knows about.
80277993
// An incomplete view of the world, but better than nothing.
@@ -8074,6 +8040,51 @@ warn_about_fork_with_threads(const char* name)
80748040
PyErr_Clear();
80758041
}
80768042
}
8043+
8044+
// If this returns <= 0, we were unable to successfully use any OS APIs.
8045+
// Returns a positive number of threads otherwise.
8046+
static Py_ssize_t get_number_of_os_threads(void)
8047+
{
8048+
// TODO: Consider making an `os` module API to return the current number
8049+
// of threads in the process. That'd presumably use this platform code but
8050+
// raise an error rather than using the inaccurate fallback.
8051+
Py_ssize_t num_python_threads = 0;
8052+
#if defined(__APPLE__) && defined(HAVE_GETPID)
8053+
mach_port_t macos_self = mach_task_self();
8054+
mach_port_t macos_task;
8055+
if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) {
8056+
thread_array_t macos_threads;
8057+
mach_msg_type_number_t macos_n_threads;
8058+
if (task_threads(macos_task, &macos_threads,
8059+
&macos_n_threads) == KERN_SUCCESS) {
8060+
num_python_threads = macos_n_threads;
8061+
}
8062+
}
8063+
#elif defined(__linux__)
8064+
// Linux /proc/self/stat 20th field is the number of threads.
8065+
FILE* proc_stat = fopen("/proc/self/stat", "r");
8066+
if (proc_stat) {
8067+
size_t n;
8068+
// Size chosen arbitrarily. ~60% more bytes than a 20th column index
8069+
// observed on the author's workstation.
8070+
char stat_line[160];
8071+
n = fread(&stat_line, 1, 159, proc_stat);
8072+
stat_line[n] = '\0';
8073+
fclose(proc_stat);
8074+
8075+
char *saveptr = NULL;
8076+
char *field = strtok_r(stat_line, " ", &saveptr);
8077+
unsigned int idx;
8078+
for (idx = 19; idx && field; --idx) {
8079+
field = strtok_r(NULL, " ", &saveptr);
8080+
}
8081+
if (idx == 0 && field) { // found the 20th field
8082+
num_python_threads = atoi(field); // 0 on error
8083+
}
8084+
}
8085+
#endif
8086+
return num_python_threads;
8087+
}
80778088
#endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK
80788089

80798090
#ifdef HAVE_FORK1
@@ -8108,10 +8119,12 @@ os_fork1_impl(PyObject *module)
81088119
/* child: this clobbers and resets the import lock. */
81098120
PyOS_AfterFork_Child();
81108121
} else {
8122+
// Called before AfterFork_Parent in case those hooks start threads.
8123+
Py_ssize_t num_os_threads = get_number_of_os_threads();
81118124
/* parent: release the import lock. */
81128125
PyOS_AfterFork_Parent();
81138126
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
8114-
warn_about_fork_with_threads("fork1");
8127+
warn_about_fork_with_threads("fork1", num_os_threads);
81158128
}
81168129
if (pid == -1) {
81178130
errno = saved_errno;
@@ -8157,10 +8170,12 @@ os_fork_impl(PyObject *module)
81578170
/* child: this clobbers and resets the import lock. */
81588171
PyOS_AfterFork_Child();
81598172
} else {
8173+
// Called before AfterFork_Parent in case those hooks start threads.
8174+
Py_ssize_t num_os_threads = get_number_of_os_threads();
81608175
/* parent: release the import lock. */
81618176
PyOS_AfterFork_Parent();
81628177
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
8163-
warn_about_fork_with_threads("fork");
8178+
warn_about_fork_with_threads("fork", num_os_threads);
81648179
}
81658180
if (pid == -1) {
81668181
errno = saved_errno;
@@ -9014,10 +9029,12 @@ os_forkpty_impl(PyObject *module)
90149029
/* child: this clobbers and resets the import lock. */
90159030
PyOS_AfterFork_Child();
90169031
} else {
9032+
// Called before AfterFork_Parent in case those hooks start threads.
9033+
Py_ssize_t num_os_threads = get_number_of_os_threads();
90179034
/* parent: release the import lock. */
90189035
PyOS_AfterFork_Parent();
90199036
// After PyOS_AfterFork_Parent() starts the world to avoid deadlock.
9020-
warn_about_fork_with_threads("forkpty");
9037+
warn_about_fork_with_threads("forkpty", num_os_threads);
90219038
}
90229039
if (pid == -1) {
90239040
return posix_error();

0 commit comments

Comments
 (0)