From 82816cb0c6bbd07ac355d2150f7320d9aceb79c5 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 14:46:48 +0800 Subject: [PATCH 1/6] Add test skip for Unix Datagram tests. --- Apple/testbed/__main__.py | 9 +++++++++ Lib/test/support/__init__.py | 3 ++- Lib/test/test_socketserver.py | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py index f3407ecdf7e734..42eb60a4c8dc02 100644 --- a/Apple/testbed/__main__.py +++ b/Apple/testbed/__main__.py @@ -1,5 +1,6 @@ import argparse import json +import os import re import shutil import subprocess @@ -78,6 +79,13 @@ def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): check=True, ) + # Any environment variable prefixed with TEST_RUNNER_ is exposed into the + # test runner environment. There are some variables (like those identifying + # CI platforms) that can be useful to have access to. + test_env = os.environ.copy() + if "GITHUB_ACTIONS" in os.environ: + test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"] + print("Running test project...") # Test execution *can't* be run -quiet; verbose mode # is how we see the output of the test output. @@ -85,6 +93,7 @@ def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): ["xcodebuild", "test-without-building"] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env=test_env, ) while line := (process.stdout.readline()).decode(*DECODE_ARGS): # Strip the timestamp/process prefix from each log line diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 098bdcc0542b90..0a50912ff0ea8c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -68,7 +68,7 @@ "BrokenIter", "in_systemd_nspawn_sync_suppressed", "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", - "reset_code", + "reset_code", "on_github_actions" ] @@ -1369,6 +1369,7 @@ def reset_code(f: types.FunctionType) -> types.FunctionType: f.__code__ = f.__code__.replace() return f +on_github_actions = "GITHUB_ACTIONS" in os.environ #======================================================================= # Check for the presence of docstrings. diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index a5e03020e5bd57..33e43b54ad8f57 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -204,11 +204,15 @@ def test_ForkingUnixStreamServer(self): socketserver.StreamRequestHandler, self.stream_examine) + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_UDPServer(self): self.run_server(socketserver.UDPServer, socketserver.DatagramRequestHandler, self.dgram_examine) + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_ThreadingUDPServer(self): self.run_server(socketserver.ThreadingUDPServer, socketserver.DatagramRequestHandler, From 4624828b4342a02dde29317f338c6dd34f9623db Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 15:26:25 +0800 Subject: [PATCH 2/6] Document testbed behavior, and add changenote. --- Apple/iOS/README.md | 11 +++++++++++ .../2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst | 2 ++ 2 files changed, 13 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md index 124a05657aae09..7ee257b5d648f4 100644 --- a/Apple/iOS/README.md +++ b/Apple/iOS/README.md @@ -224,6 +224,17 @@ Once you have a built an XCframework, you can test that framework by running: $ python Apple test iOS +This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or +iPhone 16e, or similar), and run the test suite on the most recent version of +iOS that is available. You can specify a simulator using the `--simulator` +command line argument, providing the name of the simulator (e.g., `--simulator +'iPhone 16 Pro'`). You can also use this argument to control the OS version used +for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the +tests on an iPhone 16 Pro running iOS 18.2. + +If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS` +environment variable will be exposed to the iOS process at runtime. + ### Testing a single-architecture framework The `Apple/testbed` folder that contains an Xcode project that is able to run diff --git a/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst b/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst new file mode 100644 index 00000000000000..9efbf0162dd1c1 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2025-10-29-15-20-19.gh-issue-140702.ZXtW8h.rst @@ -0,0 +1,2 @@ +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. From 3d6ff98e81d5e0922999bc8f710b1c0db59d8eb4 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 15:30:21 +0800 Subject: [PATCH 3/6] Add visibility to running on GitHub Actions. --- Apple/testbed/TestbedTests/TestbedTests.m | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Apple/testbed/TestbedTests/TestbedTests.m b/Apple/testbed/TestbedTests/TestbedTests.m index 80741097e4c80d..f7788c47f2c229 100644 --- a/Apple/testbed/TestbedTests/TestbedTests.m +++ b/Apple/testbed/TestbedTests/TestbedTests.m @@ -35,6 +35,9 @@ - (void)testPython { setenv("NO_COLOR", "1", true); setenv("PYTHON_COLORS", "0", true); + if (getenv("GITHUB_ACTIONS")) { + NSLog(@"Running in a GitHub Actions environment"); + } // Arguments to pass into the test suite runner. // argv[0] must identify the process; any subsequent arg // will be handled as if it were an argument to `python -m test` From fb461aa8da436f8b798614138c7400eb3114f544 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 15:30:36 +0800 Subject: [PATCH 4/6] Helps to skip the right tests... --- Lib/test/test_socketserver.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 33e43b54ad8f57..d0874bf9e0d6a0 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -204,15 +204,11 @@ def test_ForkingUnixStreamServer(self): socketserver.StreamRequestHandler, self.stream_examine) - @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_UDPServer(self): self.run_server(socketserver.UDPServer, socketserver.DatagramRequestHandler, self.dgram_examine) - @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_ThreadingUDPServer(self): self.run_server(socketserver.ThreadingUDPServer, socketserver.DatagramRequestHandler, @@ -227,12 +223,16 @@ def test_ForkingUDPServer(self): self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_UnixDatagramServer(self): self.run_server(socketserver.UnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "Test fails regularly on iOS simulator on Github Actions - See #140702") def test_ThreadingUnixDatagramServer(self): self.run_server(socketserver.ThreadingUnixDatagramServer, socketserver.DatagramRequestHandler, From 5276271b1bc751b38c01cb974626692a5bf8e243 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 15:31:25 +0800 Subject: [PATCH 5/6] Modify skip message as suggested. --- Lib/test/test_socketserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index d0874bf9e0d6a0..07e8cc11256fc3 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -224,7 +224,7 @@ def test_ForkingUDPServer(self): @requires_unix_sockets @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "Test fails regularly on iOS simulator on Github Actions - See #140702") + "gh-140702: Test fails regularly on iOS simulator on Github Actions") def test_UnixDatagramServer(self): self.run_server(socketserver.UnixDatagramServer, socketserver.DatagramRequestHandler, @@ -232,7 +232,7 @@ def test_UnixDatagramServer(self): @requires_unix_sockets @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "Test fails regularly on iOS simulator on Github Actions - See #140702") + "gh-140702: Test fails regularly on iOS simulator on Github Actions") def test_ThreadingUnixDatagramServer(self): self.run_server(socketserver.ThreadingUnixDatagramServer, socketserver.DatagramRequestHandler, From e94c31587f0e05374df3cf92fecb1281c70d3182 Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Wed, 29 Oct 2025 16:07:34 +0800 Subject: [PATCH 6/6] Correct capitalisation of GitHub Co-authored-by: Hugo van Kemenade <1324225+hugovk@users.noreply.github.com> --- Lib/test/test_socketserver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 07e8cc11256fc3..ca33a9a6dac8d3 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -224,7 +224,7 @@ def test_ForkingUDPServer(self): @requires_unix_sockets @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "gh-140702: Test fails regularly on iOS simulator on Github Actions") + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_UnixDatagramServer(self): self.run_server(socketserver.UnixDatagramServer, socketserver.DatagramRequestHandler, @@ -232,7 +232,7 @@ def test_UnixDatagramServer(self): @requires_unix_sockets @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, - "gh-140702: Test fails regularly on iOS simulator on Github Actions") + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_ThreadingUnixDatagramServer(self): self.run_server(socketserver.ThreadingUnixDatagramServer, socketserver.DatagramRequestHandler,