From 662845b7fc9ab6d99078083042ef962314197d86 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 11 Sep 2023 19:06:56 +0200 Subject: [PATCH] gh-109276: libregrtest calls random.seed() before each test libregrtest now calls random.seed() before running each test file when -r/--randomize command line option is used. Moreover, it's also called in worker processes. It should help to make tests more deterministic. Previously, it was only called once in the main process before running all test files and it was not called in worker processes. * Convert some f-strings to regular strings in test_regrtest when f-string is not needed. * Remove unused all_methods variable from test_regrtest. * Add RunTests members are now mandatory. --- Lib/test/libregrtest/main.py | 13 +++- Lib/test/libregrtest/runtests.py | 44 ++++++------- Lib/test/libregrtest/setup.py | 4 ++ Lib/test/test_regrtest.py | 61 +++++++++++++++---- ...-09-11-19-11-57.gh-issue-109276.qxI4OG.rst | 6 ++ 5 files changed, 94 insertions(+), 34 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 2c0a6c204373cc..f52deac329dc84 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -112,8 +112,11 @@ def __init__(self, ns: Namespace): self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold - self.use_resources: list[str] = ns.use_resources - self.python_cmd: list[str] | None = ns.python + self.use_resources: tuple[str] = tuple(ns.use_resources) + if ns.python: + self.python_cmd: tuple[str] = tuple(ns.python) + else: + self.python_cmd = None self.coverage: bool = ns.trace self.coverage_dir: StrPath | None = ns.coverdir self.tmp_dir: StrPath | None = ns.tempdir @@ -377,8 +380,11 @@ def create_run_tests(self, tests: TestTuple): return RunTests( tests, fail_fast=self.fail_fast, + fail_env_changed=self.fail_env_changed, match_tests=self.match_tests, ignore_tests=self.ignore_tests, + match_tests_dict=None, + rerun=None, forever=self.forever, pgo=self.pgo, pgo_extended=self.pgo_extended, @@ -393,6 +399,9 @@ def create_run_tests(self, tests: TestTuple): gc_threshold=self.gc_threshold, use_resources=self.use_resources, python_cmd=self.python_cmd, + randomize=self.randomize, + random_seed=self.random_seed, + json_fd=None, ) def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 64f8f6ab0ff305..656958fa71312f 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -16,29 +16,31 @@ class HuntRefleak: @dataclasses.dataclass(slots=True, frozen=True) class RunTests: tests: TestTuple - fail_fast: bool = False - fail_env_changed: bool = False - match_tests: FilterTuple | None = None - ignore_tests: FilterTuple | None = None - match_tests_dict: FilterDict | None = None - rerun: bool = False - forever: bool = False - pgo: bool = False - pgo_extended: bool = False - output_on_failure: bool = False - timeout: float | None = None - verbose: int = 0 - quiet: bool = False - hunt_refleak: HuntRefleak | None = None - test_dir: StrPath | None = None - use_junit: bool = False - memory_limit: str | None = None - gc_threshold: int | None = None - use_resources: list[str] = dataclasses.field(default_factory=list) - python_cmd: list[str] | None = None + fail_fast: bool + fail_env_changed: bool + match_tests: FilterTuple | None + ignore_tests: FilterTuple | None + match_tests_dict: FilterDict | None + rerun: bool + forever: bool + pgo: bool + pgo_extended: bool + output_on_failure: bool + timeout: float | None + verbose: int + quiet: bool + hunt_refleak: HuntRefleak | None + test_dir: StrPath | None + use_junit: bool + memory_limit: str | None + gc_threshold: int | None + use_resources: tuple[str] + python_cmd: tuple[str] | None + randomize: bool + random_seed: int | None # On Unix, it's a file descriptor. # On Windows, it's a handle. - json_fd: int | None = None + json_fd: int | None def copy(self, **override): state = dataclasses.asdict(self) diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index 353a0f70b94ab2..1c40b7c7b3bbfd 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,5 +1,6 @@ import faulthandler import os +import random import signal import sys import unittest @@ -127,3 +128,6 @@ def setup_tests(runtests: RunTests): if runtests.gc_threshold is not None: gc.set_threshold(runtests.gc_threshold) + + if runtests.randomize: + random.seed(runtests.random_seed) diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 466b6f66797b7a..7cf3d05a6e6d70 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -11,6 +11,7 @@ import locale import os.path import platform +import random import re import subprocess import sys @@ -504,7 +505,7 @@ def list_regex(line_format, tests): if rerun is not None: regex = list_regex('%s re-run test%s', [rerun.name]) self.check_line(output, regex) - regex = LOG_PREFIX + fr"Re-running 1 failed tests in verbose mode" + regex = LOG_PREFIX + r"Re-running 1 failed tests in verbose mode" self.check_line(output, regex) regex = fr"Re-running {rerun.name} in verbose mode" if rerun.match: @@ -1019,13 +1020,13 @@ def test_run(self): forever=True) @without_optimizer - def check_leak(self, code, what, *, multiprocessing=False): + def check_leak(self, code, what, *, run_workers=False): test = self.create_test('huntrleaks', code=code) filename = 'reflog.txt' self.addCleanup(os_helper.unlink, filename) cmd = ['--huntrleaks', '3:3:'] - if multiprocessing: + if run_workers: cmd.append('-j1') cmd.append(test) output = self.run_tests(*cmd, @@ -1044,7 +1045,7 @@ def check_leak(self, code, what, *, multiprocessing=False): self.assertIn(line2, reflog) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') - def check_huntrleaks(self, *, multiprocessing: bool): + def check_huntrleaks(self, *, run_workers: bool): # test --huntrleaks code = textwrap.dedent(""" import unittest @@ -1055,13 +1056,13 @@ class RefLeakTest(unittest.TestCase): def test_leak(self): GLOBAL_LIST.append(object()) """) - self.check_leak(code, 'references', multiprocessing=multiprocessing) + self.check_leak(code, 'references', run_workers=run_workers) def test_huntrleaks(self): - self.check_huntrleaks(multiprocessing=False) + self.check_huntrleaks(run_workers=False) def test_huntrleaks_mp(self): - self.check_huntrleaks(multiprocessing=True) + self.check_huntrleaks(run_workers=True) @unittest.skipUnless(support.Py_DEBUG, 'need a debug build') def test_huntrleaks_fd_leak(self): @@ -1139,8 +1140,6 @@ def test_method3(self): def test_method4(self): pass """) - all_methods = ['test_method1', 'test_method2', - 'test_method3', 'test_method4'] testname = self.create_test(code=code) # only run a subset @@ -1762,7 +1761,7 @@ def test_mp_decode_error(self): if encoding is None: encoding = sys.__stdout__.encoding if encoding is None: - self.skipTest(f"cannot get regrtest worker encoding") + self.skipTest("cannot get regrtest worker encoding") nonascii = b"byte:\xa0\xa9\xff\n" try: @@ -1789,7 +1788,7 @@ def test_mp_decode_error(self): stats=0) def test_doctest(self): - code = textwrap.dedent(fr''' + code = textwrap.dedent(r''' import doctest import sys from test import support @@ -1827,6 +1826,46 @@ def load_tests(loader, tests, pattern): randomize=True, stats=TestStats(1, 1, 0)) + def _check_random_seed(self, run_workers: bool): + # gh-109276: When -r/--randomize is used, random.seed() is called + # with the same random seed before running each test file. + code = textwrap.dedent(r''' + import random + import unittest + + class RandomSeedTest(unittest.TestCase): + def test_randint(self): + numbers = [random.randint(0, 1000) for _ in range(10)] + print(f"Random numbers: {numbers}") + ''') + tests = [self.create_test(name=f'test_random{i}', code=code) + for i in range(1, 3+1)] + + random_seed = 856_656_202 + cmd = ["--randomize", f"--randseed={random_seed}"] + if run_workers: + # run as many worker processes than the number of tests + cmd.append(f'-j{len(tests)}') + cmd.extend(tests) + output = self.run_tests(*cmd) + + random.seed(random_seed) + # Make the assumption that nothing consume entropy between libregrest + # setup_tests() which calls random.seed() and RandomSeedTest calling + # random.randint(). + numbers = [random.randint(0, 1000) for _ in range(10)] + expected = f"Random numbers: {numbers}" + + regex = r'^Random numbers: .*$' + matches = re.findall(regex, output, flags=re.MULTILINE) + self.assertEqual(matches, [expected] * len(tests)) + + def test_random_seed(self): + self._check_random_seed(run_workers=False) + + def test_random_seed_workers(self): + self._check_random_seed(run_workers=True) + class TestUtils(unittest.TestCase): def test_format_duration(self): diff --git a/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst new file mode 100644 index 00000000000000..cf4074b2fe23cc --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-09-11-19-11-57.gh-issue-109276.qxI4OG.rst @@ -0,0 +1,6 @@ +libregrtest now calls :func:`random.seed()` before running each test file +when ``-r/--randomize`` command line option is used. Moreover, it's also +called in worker processes. It should help to make tests more +deterministic. Previously, it was only called once in the main process before +running all test files and it was not called in worker processes. Patch by +Victor Stinner.