From 669f95acf6ddcdc639e946163f8536d8fd351997 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Tue, 17 Dec 2019 23:00:53 +0100 Subject: [PATCH 01/13] bpo-13601: always use line-buffering for sys.stderr --- Doc/library/sys.rst | 7 ++++--- Misc/ACKS | 1 + .../2019-12-17-22-32-11.bpo-13601.vNP4LC.rst | 2 ++ Python/pylifecycle.c | 2 +- 4 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a824fb95e8ecfc..ecfdddf4658e47 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1446,9 +1446,10 @@ always available. for the Windows console, this only applies when :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also set. - * When interactive, ``stdout`` and ``stderr`` streams are line-buffered. - Otherwise, they are block-buffered like regular text files. You can - override this value with the :option:`-u` command-line option. + * When interactive, the ``stdout`` stream is line-buffered. Otherwise, + it is block-buffered like regular text files. The ``stderr`` stream + is line-buffered in both cases. You can override this behaviour with + the :option:`-u` command-line option. .. note:: diff --git a/Misc/ACKS b/Misc/ACKS index 253e2f6133d587..27f076d61fd768 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -1500,6 +1500,7 @@ Steven Scott Nick Seidenman Michael Seifert Žiga Seilnacht +Jendrik Seipp Michael Selik Yury Selivanov Fred Sells diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst new file mode 100644 index 00000000000000..830edf72e176ee --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst @@ -0,0 +1,2 @@ +``sys.stderr`` is always line-buffered now, even if ``stderr`` is +redirected to a file. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 823d96e86a4388..d6d692cfdb83a3 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1812,7 +1812,7 @@ create_stdio(const PyConfig *config, PyObject* io, write_through = Py_True; else write_through = Py_False; - if (isatty && buffered_stdio) + if (buffered_stdio && (isatty || fd == fileno(stderr))) line_buffering = Py_True; else line_buffering = Py_False; From 70c374801c30f64b476dc33a0aea83f6788b4889 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Tue, 17 Dec 2019 23:24:57 +0100 Subject: [PATCH 02/13] Add versionchanged note to docs. --- Doc/library/sys.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index ecfdddf4658e47..a6e3963392f03c 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1451,6 +1451,9 @@ always available. is line-buffered in both cases. You can override this behaviour with the :option:`-u` command-line option. + .. versionchanged:: 3.9 + Use line-buffering for non-interactive ``stderr``. + .. note:: To write or read binary data from/to the standard streams, use the From 76203d3e32e0443322b014468fc428caf3503a7d Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 19 Dec 2019 15:10:14 +0100 Subject: [PATCH 03/13] Add tests. --- Lib/test/test_cmd_line.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 47810020dd353c..eca66d128f5be3 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -219,6 +219,24 @@ def check_output(text): ) check_output(text) + def test_non_interactive_output_buffering(self): + # Test buffering for stdout and stderr. + cases = [ + # Binary stdout is unbuffered. + ('sys.stdout.buffer', b'x\n', b'y', b'', b''), + # Binary stderr is unbuffered (can't be line-buffered). + ('sys.stderr.buffer', b'x\n', b'y', b'', b''), + # Text stdout is unbuffered. + ('sys.stdout', 'x\n', 'y', b'', b''), + # Text stderr is line-buffered. + ('sys.stderr', 'x\n', 'y', b'', b'x\n'), + ] + for buf, txt1, txt2, out_ok, err_ok in cases: + code = f'import os, sys; {buf}.write({txt1!r}); {buf}.write({txt2!r}); os._exit(0)' + rc, out, err = assert_python_ok('-c', code) + self.assertEqual(out, out_ok) + self.assertEqual(err, err_ok) + def test_unbuffered_output(self): # Test expected operation of the '-u' switch for stream in ('stdout', 'stderr'): From 2e0b5e47066e2ee6080bde4be6ef2daadcf52df2 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 19 Dec 2019 23:04:46 +0100 Subject: [PATCH 04/13] Describe effects of -u option. --- Doc/library/sys.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index a6e3963392f03c..939be43cdcc550 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1448,8 +1448,9 @@ always available. * When interactive, the ``stdout`` stream is line-buffered. Otherwise, it is block-buffered like regular text files. The ``stderr`` stream - is line-buffered in both cases. You can override this behaviour with - the :option:`-u` command-line option. + is line-buffered in both cases. You can make both streams unbuffered + by passing the :option:`-u` command-line option or setting the + :envvar:`PYTHONUNBUFFERED` environment variable. .. versionchanged:: 3.9 Use line-buffering for non-interactive ``stderr``. From fa90013f29e9f80718a30dbd03f262ae509414d4 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Thu, 19 Dec 2019 23:13:37 +0100 Subject: [PATCH 05/13] Describe previous stderr buffering. --- Doc/library/sys.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 939be43cdcc550..2ad77d82fe3422 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1453,7 +1453,7 @@ always available. :envvar:`PYTHONUNBUFFERED` environment variable. .. versionchanged:: 3.9 - Use line-buffering for non-interactive ``stderr``. + Make non-interactive ``stderr`` line-buffered instead of fully buffered. .. note:: From 20dbd8070bc5a43ec7f99531fca6cbdb7f929913 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 00:10:10 +0100 Subject: [PATCH 06/13] Replace \r\n by \n before checking results. --- Lib/test/test_cmd_line.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index eca66d128f5be3..1912764fffa9f4 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -234,8 +234,8 @@ def test_non_interactive_output_buffering(self): for buf, txt1, txt2, out_ok, err_ok in cases: code = f'import os, sys; {buf}.write({txt1!r}); {buf}.write({txt2!r}); os._exit(0)' rc, out, err = assert_python_ok('-c', code) - self.assertEqual(out, out_ok) - self.assertEqual(err, err_ok) + self.assertEqual(out.replace(b'\r\n', b'\n'), out_ok) + self.assertEqual(err.replace(b'\r\n', b'\n'), err_ok) def test_unbuffered_output(self): # Test expected operation of the '-u' switch From ea9df735dab388fde5822da24bca8c48a73e08b0 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 12:35:55 +0100 Subject: [PATCH 07/13] Add non-functional tests. --- Lib/test/test_cmd_line.py | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1912764fffa9f4..966d1aa5e6ac3f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,6 +2,7 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution +import ast import os import subprocess import sys @@ -219,8 +220,36 @@ def check_output(text): ) check_output(text) + def test_interactive_output_buffering(self): + cases = [ + ('sys.stdout.isatty()', True), + ('sys.stdout.write_through', False), + ('sys.stdout.line_buffering', True), + ('sys.stderr.isatty()', True), + ('sys.stderr.write_through', False), + ('sys.stderr.line_buffering', True), + ] + for attr, value in cases: + self.assertEqual(eval(attr), value, f'{attr} is not {value}') + def test_non_interactive_output_buffering(self): - # Test buffering for stdout and stderr. + def get_value(attr): + code = f'import sys; print({attr})' + rc, out, err = assert_python_ok('-c', code) + return ast.literal_eval(out.rstrip().decode()) + + cases = [ + ('sys.stdout.isatty()', False), + ('sys.stdout.write_through', False), + ('sys.stdout.line_buffering', False), + ('sys.stderr.isatty()', False), + ('sys.stderr.write_through', False), + ('sys.stderr.line_buffering', True), + ] + for attr, value in cases: + self.assertEqual(get_value(attr), value, f'{attr} is not {value}') + + def test_non_interactive_output_buffering_functional(self): cases = [ # Binary stdout is unbuffered. ('sys.stdout.buffer', b'x\n', b'y', b'', b''), From d58e8be128d5652e168a05d45a0d3ffd3a9a406c Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 12:38:28 +0100 Subject: [PATCH 08/13] Remove functional tests. --- Lib/test/test_cmd_line.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 966d1aa5e6ac3f..3b3105ba312663 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -249,23 +249,6 @@ def get_value(attr): for attr, value in cases: self.assertEqual(get_value(attr), value, f'{attr} is not {value}') - def test_non_interactive_output_buffering_functional(self): - cases = [ - # Binary stdout is unbuffered. - ('sys.stdout.buffer', b'x\n', b'y', b'', b''), - # Binary stderr is unbuffered (can't be line-buffered). - ('sys.stderr.buffer', b'x\n', b'y', b'', b''), - # Text stdout is unbuffered. - ('sys.stdout', 'x\n', 'y', b'', b''), - # Text stderr is line-buffered. - ('sys.stderr', 'x\n', 'y', b'', b'x\n'), - ] - for buf, txt1, txt2, out_ok, err_ok in cases: - code = f'import os, sys; {buf}.write({txt1!r}); {buf}.write({txt2!r}); os._exit(0)' - rc, out, err = assert_python_ok('-c', code) - self.assertEqual(out.replace(b'\r\n', b'\n'), out_ok) - self.assertEqual(err.replace(b'\r\n', b'\n'), err_ok) - def test_unbuffered_output(self): # Test expected operation of the '-u' switch for stream in ('stdout', 'stderr'): From d46d8ca3b4e8387280a4ffa9cbcb2bb437de8c5a Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 13:12:18 +0100 Subject: [PATCH 09/13] Remove failing tests. --- Lib/test/test_cmd_line.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 3b3105ba312663..556ec360488177 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -222,10 +222,8 @@ def check_output(text): def test_interactive_output_buffering(self): cases = [ - ('sys.stdout.isatty()', True), ('sys.stdout.write_through', False), ('sys.stdout.line_buffering', True), - ('sys.stderr.isatty()', True), ('sys.stderr.write_through', False), ('sys.stderr.line_buffering', True), ] From 7799476734a105de837bc8750950f8ec53051e2b Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 14:19:16 +0100 Subject: [PATCH 10/13] Remove failing test. --- Lib/test/test_cmd_line.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 556ec360488177..b0a836f3e8447f 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -220,16 +220,6 @@ def check_output(text): ) check_output(text) - def test_interactive_output_buffering(self): - cases = [ - ('sys.stdout.write_through', False), - ('sys.stdout.line_buffering', True), - ('sys.stderr.write_through', False), - ('sys.stderr.line_buffering', True), - ] - for attr, value in cases: - self.assertEqual(eval(attr), value, f'{attr} is not {value}') - def test_non_interactive_output_buffering(self): def get_value(attr): code = f'import sys; print({attr})' From 4258402dcf8d2116283467054b1db1c14debecf2 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Fri, 20 Dec 2019 23:00:13 +0100 Subject: [PATCH 11/13] Invoke Python only once for buffer test. --- Lib/test/test_cmd_line.py | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index b0a836f3e8447f..ee96473322dba0 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -2,11 +2,11 @@ # Most tests are executed with environment variables ignored # See test_cmd_line_script.py for testing of script execution -import ast import os import subprocess import sys import tempfile +import textwrap import unittest from test import support from test.support.script_helper import ( @@ -221,21 +221,19 @@ def check_output(text): check_output(text) def test_non_interactive_output_buffering(self): - def get_value(attr): - code = f'import sys; print({attr})' - rc, out, err = assert_python_ok('-c', code) - return ast.literal_eval(out.rstrip().decode()) - - cases = [ - ('sys.stdout.isatty()', False), - ('sys.stdout.write_through', False), - ('sys.stdout.line_buffering', False), - ('sys.stderr.isatty()', False), - ('sys.stderr.write_through', False), - ('sys.stderr.line_buffering', True), - ] - for attr, value in cases: - self.assertEqual(get_value(attr), value, f'{attr} is not {value}') + code = textwrap.dedent(""" + import sys + out = sys.stdout + print(out.isatty(), out.write_through, out.line_buffering) + err = sys.stderr + print(err.isatty(), err.write_through, err.line_buffering) + """) + args = [sys.executable, '-c', code] + proc = subprocess.run(args, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, check=True) + self.assertEqual(proc.stdout, + 'False False False\n' + 'False False True\n') def test_unbuffered_output(self): # Test expected operation of the '-u' switch From e5530eff189e8ad19550c49d9a79d091c7fd68a3 Mon Sep 17 00:00:00 2001 From: Jendrik Seipp Date: Sat, 21 Dec 2019 20:37:58 +0100 Subject: [PATCH 12/13] Exlain Python's unbuffered mode in changelog. --- .../2019-12-17-22-32-11.bpo-13601.vNP4LC.rst | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst b/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst index 830edf72e176ee..f2c9495a59afb1 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2019-12-17-22-32-11.bpo-13601.vNP4LC.rst @@ -1,2 +1,6 @@ -``sys.stderr`` is always line-buffered now, even if ``stderr`` is -redirected to a file. +By default, ``sys.stderr`` is line-buffered now, even if ``stderr`` is +redirected to a file. You can still make ``sys.stderr`` unbuffered by +passing the :option:`-u` command-line option or setting the +:envvar:`PYTHONUNBUFFERED` environment variable. + +(Contributed by Jendrik Seipp in bpo-13601.) From 83c53051d8517cf7b5a6c59cb2f86686eecaa7b9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 1 Jan 2020 22:41:13 +0100 Subject: [PATCH 13/13] Improve wording of `versionchanged` clause. --- Doc/library/sys.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 2ad77d82fe3422..0aae263ff5f4c1 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -1453,7 +1453,8 @@ always available. :envvar:`PYTHONUNBUFFERED` environment variable. .. versionchanged:: 3.9 - Make non-interactive ``stderr`` line-buffered instead of fully buffered. + Non-interactive ``stderr`` is now line-buffered instead of fully + buffered. .. note::