[3.9] bpo-41069: Make TESTFN and the CWD for tests containing non-ascii characters. (GH-21035). (GH-21156)

serhiy-storchaka · web-flow · commit f925407a19ee · 2020-06-25T20:39:12.000+03:00
(cherry picked from commit 700cfa8)
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -596,6 +596,7 @@ def create_temp_dir(self):
             test_cwd = 'test_python_worker_{}'.format(pid)
         else:
             test_cwd = 'test_python_{}'.format(pid)
+        test_cwd += support.FS_NONASCII
         test_cwd = os.path.join(self.tmp_dir, test_cwd)
         return test_cwd
 
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
@@ -720,21 +720,21 @@ def requires_lzma(reason='requires lzma'):
 # Filename used for testing
 if os.name == 'java':
     # Jython disallows @ in module names
-    TESTFN = '$test'
+    TESTFN_ASCII = '$test'
 else:
-    TESTFN = '@test'
+    TESTFN_ASCII = '@test'
 
 # Disambiguate TESTFN for parallel testing, while letting it remain a valid
 # module name.
-TESTFN = "{}_{}_tmp".format(TESTFN, os.getpid())
+TESTFN_ASCII = "{}_{}_tmp".format(TESTFN_ASCII, os.getpid())
 
 # Define the URL of a dedicated HTTP server for the network tests.
 # The URL must use clear-text HTTP: no redirection to encrypted HTTPS.
 TEST_HTTP_URL = "http://www.pythontest.net"
 
 # FS_NONASCII: non-ASCII character encodable by os.fsencode(),
-# or None if there is no such character.
-FS_NONASCII = None
+# or an empty string if there is no such character.
+FS_NONASCII = ''
 for character in (
     # First try printable and common characters to have a readable filename.
     # For each character, the encoding list are just example of encodings able
@@ -781,7 +781,7 @@ def requires_lzma(reason='requires lzma'):
         break
 
 # TESTFN_UNICODE is a non-ascii filename
-TESTFN_UNICODE = TESTFN + "-\xe0\xf2\u0258\u0141\u011f"
+TESTFN_UNICODE = TESTFN_ASCII + "-\xe0\xf2\u0258\u0141\u011f"
 if sys.platform == 'darwin':
     # In Mac OS X's VFS API file names are, by definition, canonically
     # decomposed Unicode, encoded using UTF-8. See QA1173:
@@ -799,7 +799,7 @@ def requires_lzma(reason='requires lzma'):
     if sys.getwindowsversion().platform >= 2:
         # Different kinds of characters from various languages to minimize the
         # probability that the whole name is encodable to MBCS (issue #9819)
-        TESTFN_UNENCODABLE = TESTFN + "-\u5171\u0141\u2661\u0363\uDC80"
+        TESTFN_UNENCODABLE = TESTFN_ASCII + "-\u5171\u0141\u2661\u0363\uDC80"
         try:
             TESTFN_UNENCODABLE.encode(TESTFN_ENCODING)
         except UnicodeEncodeError:
@@ -816,7 +816,7 @@ def requires_lzma(reason='requires lzma'):
         b'\xff'.decode(TESTFN_ENCODING)
     except UnicodeDecodeError:
         # 0xff will be encoded using the surrogate character u+DCFF
-        TESTFN_UNENCODABLE = TESTFN \
+        TESTFN_UNENCODABLE = TESTFN_ASCII \
             + b'-\xff'.decode(TESTFN_ENCODING, 'surrogateescape')
     else:
         # File system encoding (eg. ISO-8859-* encodings) can encode
@@ -850,13 +850,14 @@ def requires_lzma(reason='requires lzma'):
     try:
         name.decode(TESTFN_ENCODING)
     except UnicodeDecodeError:
-        TESTFN_UNDECODABLE = os.fsencode(TESTFN) + name
+        TESTFN_UNDECODABLE = os.fsencode(TESTFN_ASCII) + name
         break
 
 if FS_NONASCII:
-    TESTFN_NONASCII = TESTFN + '-' + FS_NONASCII
+    TESTFN_NONASCII = TESTFN_ASCII + FS_NONASCII
 else:
     TESTFN_NONASCII = None
+TESTFN = TESTFN_NONASCII or TESTFN_ASCII
 
 # Save the initial cwd
 SAVEDCWD = os.getcwd()
diff --git a/Lib/test/test_binhex.py b/Lib/test/test_binhex.py
@@ -13,9 +13,10 @@
 class BinHexTestCase(unittest.TestCase):
 
     def setUp(self):
-        self.fname1 = support.TESTFN + "1"
-        self.fname2 = support.TESTFN + "2"
-        self.fname3 = support.TESTFN + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
+        # binhex supports only file names encodable to Latin1
+        self.fname1 = support.TESTFN_ASCII + "1"
+        self.fname2 = support.TESTFN_ASCII + "2"
+        self.fname3 = support.TESTFN_ASCII + "very_long_filename__very_long_filename__very_long_filename__very_long_filename__"
 
     def tearDown(self):
         support.unlink(self.fname1)
diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py
@@ -41,8 +41,9 @@ def test_syshook_no_logdir_default_format(self):
             rc, out, err = assert_python_failure(
                   '-c',
                   ('import cgitb; cgitb.enable(logdir=%s); '
-                   'raise ValueError("Hello World")') % repr(tracedir))
-        out = out.decode(sys.getfilesystemencoding())
+                   'raise ValueError("Hello World")') % repr(tracedir),
+                  PYTHONIOENCODING='utf-8')
+        out = out.decode()
         self.assertIn("ValueError", out)
         self.assertIn("Hello World", out)
         self.assertIn("<strong>&lt;module&gt;</strong>", out)
@@ -56,8 +57,9 @@ def test_syshook_no_logdir_text_format(self):
             rc, out, err = assert_python_failure(
                   '-c',
                   ('import cgitb; cgitb.enable(format="text", logdir=%s); '
-                   'raise ValueError("Hello World")') % repr(tracedir))
-        out = out.decode(sys.getfilesystemencoding())
+                   'raise ValueError("Hello World")') % repr(tracedir),
+                  PYTHONIOENCODING='utf-8')
+        out = out.decode()
         self.assertIn("ValueError", out)
         self.assertIn("Hello World", out)
         self.assertNotIn('<p>', out)
diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py
@@ -456,13 +456,15 @@ def _get_run_args(self, args):
 
     def assertRunOK(self, *args, **env_vars):
         rc, out, err = script_helper.assert_python_ok(
-                         *self._get_run_args(args), **env_vars)
+                         *self._get_run_args(args), **env_vars,
+                         PYTHONIOENCODING='utf-8')
         self.assertEqual(b'', err)
         return out
 
     def assertRunNotOK(self, *args, **env_vars):
         rc, out, err = script_helper.assert_python_failure(
-                        *self._get_run_args(args), **env_vars)
+                        *self._get_run_args(args), **env_vars,
+                        PYTHONIOENCODING='utf-8')
         return rc, out, err
 
     def assertCompiled(self, fn):
diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py
@@ -1353,7 +1353,7 @@ def test_audit_run_file(self):
                                       returncode=1)
 
     def test_audit_run_interactivehook(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("import sys", file=f)
             print("sys.__interactivehook__ = lambda: None", file=f)
@@ -1366,7 +1366,7 @@ def test_audit_run_interactivehook(self):
             os.unlink(startup)
 
     def test_audit_run_startup(self):
-        startup = os.path.join(self.oldcwd, support.TESTFN) + (support.FS_NONASCII or '') + ".py"
+        startup = os.path.join(self.oldcwd, support.TESTFN) + ".py"
         with open(startup, "w", encoding="utf-8") as f:
             print("pass", file=f)
         try:
diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py
@@ -1061,8 +1061,9 @@ def test_filename_in_syntaxerror(self):
             file_path = os.path.join(cwd, 't.py')
             with open(file_path, 'w') as f:
                 f.write('f"{a b}"') # This generates a SyntaxError
-            _, _, stderr = assert_python_failure(file_path)
-        self.assertIn(file_path, stderr.decode('utf-8'))
+            _, _, stderr = assert_python_failure(file_path,
+                                                 PYTHONIOENCODING='ascii')
+        self.assertIn(file_path.encode('ascii', 'backslashreplace'), stderr)
 
     def test_loop(self):
         for i in range(1000):
diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py
@@ -534,7 +534,7 @@ def test_import(self):
 class PathLikeTests(unittest.TestCase):
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         create_file(self.file_name, b"test_genericpath.PathLikeTests")
diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py
@@ -328,8 +328,15 @@ def test_metadata(self):
             cmByte = fRead.read(1)
             self.assertEqual(cmByte, b'\x08') # deflate
 
+            try:
+                expectedname = self.filename.encode('Latin-1') + b'\x00'
+                expectedflags = b'\x08' # only the FNAME flag is set
+            except UnicodeEncodeError:
+                expectedname = b''
+                expectedflags = b'\x00'
+
             flagsByte = fRead.read(1)
-            self.assertEqual(flagsByte, b'\x08') # only the FNAME flag is set
+            self.assertEqual(flagsByte, expectedflags)
 
             mtimeBytes = fRead.read(4)
             self.assertEqual(mtimeBytes, struct.pack('<i', mtime)) # little-endian
@@ -344,9 +351,8 @@ def test_metadata(self):
             # RFC 1952 specifies that this is the name of the input file, if any.
             # However, the gzip module defaults to storing the name of the output
             # file in this field.
-            expected = self.filename.encode('Latin-1') + b'\x00'
-            nameBytes = fRead.read(len(expected))
-            self.assertEqual(nameBytes, expected)
+            nameBytes = fRead.read(len(expectedname))
+            self.assertEqual(nameBytes, expectedname)
 
             # Since no other flags were set, the header ends here.
             # Rather than process the compressed data, let's seek to the trailer.
@@ -358,6 +364,10 @@ def test_metadata(self):
             isizeBytes = fRead.read(4)
             self.assertEqual(isizeBytes, struct.pack('<i', len(data1)))
 
+    def test_metadata_ascii_name(self):
+        self.filename = support.TESTFN_ASCII
+        self.test_metadata()
+
     def test_compresslevel_metadata(self):
         # see RFC 1952: http://www.faqs.org/rfcs/rfc1952.html
         # specifically, discussion of XFL in section 2.3.1
diff --git a/Lib/test/test_msilib.py b/Lib/test/test_msilib.py
@@ -1,13 +1,13 @@
 """ Test suite for the code in msilib """
 import os
 import unittest
-from test.support import TESTFN, FS_NONASCII, import_module, unlink
+from test.support import TESTFN, import_module, unlink
 msilib = import_module('msilib')
 import msilib.schema
 
 
 def init_database():
-    path = TESTFN + (FS_NONASCII or '') + '.msi'
+    path = TESTFN + '.msi'
     db = msilib.init_database(
         path,
         msilib.schema,
diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py
@@ -725,7 +725,7 @@ class PathLikeTests(NtpathTestCase):
     path = ntpath
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         with open(self.file_name, 'xb', 0) as file:
diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py
@@ -1173,7 +1173,7 @@ def setUp(self):
         os.makedirs(t2_path)
 
         for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path:
-            with open(path, "x") as f:
+            with open(path, "x", encoding='utf-8') as f:
                 f.write("I'm " + path + " and proud of it.  Blame test_os.\n")
 
         if support.can_symlink():
@@ -2359,7 +2359,7 @@ def setUp(self):
             file_name = 'FILE%d' % i
             file_path = os.path.join(support.TESTFN, file_name)
             os.makedirs(dir_path)
-            with open(file_path, 'w') as f:
+            with open(file_path, 'w', encoding='utf-8') as f:
                 f.write("I'm %s and proud of it. Blame test_os.\n" % file_path)
             self.created_paths.extend([dir_name, file_name])
         self.created_paths.sort()
@@ -3737,7 +3737,7 @@ def test_path_t_converter(self):
         if os.name == 'nt':
             bytes_fspath = bytes_filename = None
         else:
-            bytes_filename = support.TESTFN.encode('ascii')
+            bytes_filename = os.fsencode(support.TESTFN)
             bytes_fspath = FakePath(bytes_filename)
         fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT)
         self.addCleanup(support.unlink, support.TESTFN)
diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py
@@ -1198,6 +1198,7 @@ def _run_pdb(self, pdb_args, commands):
                 stdout=subprocess.PIPE,
                 stdin=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
+                env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
         ) as proc:
             stdout, stderr = proc.communicate(str.encode(commands))
         stdout = stdout and bytes.decode(stdout)
@@ -1353,10 +1354,11 @@ def start_pdb():
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE,
             stderr=subprocess.STDOUT,
+            env={**os.environ, 'PYTHONIOENCODING': 'utf-8'}
             )
         self.addCleanup(proc.stdout.close)
         stdout, stderr = proc.communicate(b'cont\n')
-        self.assertNotIn('Error', stdout.decode(),
+        self.assertNotIn(b'Error', stdout,
                          "Got an error running test script under PDB")
 
     def test_issue36250(self):
@@ -1382,10 +1384,11 @@ def start_pdb():
             stdout=subprocess.PIPE,
             stdin=subprocess.PIPE,
             stderr=subprocess.STDOUT,
+            env = {**os.environ, 'PYTHONIOENCODING': 'utf-8'}
             )
         self.addCleanup(proc.stdout.close)
         stdout, stderr = proc.communicate(b'cont\ncont\n')
-        self.assertNotIn('Error', stdout.decode(),
+        self.assertNotIn(b'Error', stdout,
                          "Got an error running test script under PDB")
 
     def test_issue16180(self):
@@ -1425,8 +1428,8 @@ def test_readrc_kwarg(self):
                 )
                 with proc:
                     stdout, stderr = proc.communicate(b'q\n')
-                    self.assertNotIn("NameError: name 'invalid' is not defined",
-                                  stdout.decode())
+                    self.assertNotIn(b"NameError: name 'invalid' is not defined",
+                                  stdout)
 
         finally:
             if save_home is not None:
diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py
@@ -627,7 +627,7 @@ class PathLikeTests(unittest.TestCase):
     path = posixpath
 
     def setUp(self):
-        self.file_name = support.TESTFN.lower()
+        self.file_name = support.TESTFN
         self.file_path = FakePath(support.TESTFN)
         self.addCleanup(support.unlink, self.file_name)
         with open(self.file_name, 'xb', 0) as file:
diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py
@@ -2305,7 +2305,8 @@ def test_test_command(self):
     def test_test_command_verbose(self):
         for tar_name in testtarnames:
             for opt in '-v', '--verbose':
-                out = self.tarfilecmd(opt, '-t', tar_name)
+                out = self.tarfilecmd(opt, '-t', tar_name,
+                                      PYTHONIOENCODING='utf-8')
                 self.assertIn(b'is a tar archive.\n', out)
 
     def test_test_command_invalid_file(self):
@@ -2376,7 +2377,8 @@ def test_create_command_verbose(self):
                                   'and-utf8-bom-sig-only.txt')]
         for opt in '-v', '--verbose':
             try:
-                out = self.tarfilecmd(opt, '-c', tmpname, *files)
+                out = self.tarfilecmd(opt, '-c', tmpname, *files,
+                                      PYTHONIOENCODING='utf-8')
                 self.assertIn(b' file created.', out)
                 with tarfile.open(tmpname) as tar:
                     tar.getmembers()
@@ -2434,7 +2436,8 @@ def test_extract_command_verbose(self):
         for opt in '-v', '--verbose':
             try:
                 with support.temp_cwd(tarextdir):
-                    out = self.tarfilecmd(opt, '-e', tmpname)
+                    out = self.tarfilecmd(opt, '-e', tmpname,
+                                          PYTHONIOENCODING='utf-8')
                 self.assertIn(b' file is extracted.', out)
             finally:
                 support.rmtree(tarextdir)
diff --git a/Lib/test/test_tools/test_pathfix.py b/Lib/test/test_tools/test_pathfix.py
@@ -30,16 +30,18 @@ def pathfix(self, shebang, pathfix_flags, exitcode=0, stdout='', stderr='',
         with open(filename, 'w', encoding='utf8') as f:
             f.write(f'{shebang}\n' + 'print("Hello world")\n')
 
+        encoding = sys.getfilesystemencoding()
         proc = subprocess.run(
             [sys.executable, self.script,
              *pathfix_flags, '-n', pathfix_arg],
-            capture_output=True, text=1)
+            env={**os.environ, 'PYTHONIOENCODING': encoding},
+            capture_output=True)
 
         if stdout == '' and proc.returncode == 0:
             stdout = f'{filename}: updating\n'
         self.assertEqual(proc.returncode, exitcode, proc)
-        self.assertEqual(proc.stdout, stdout, proc)
-        self.assertEqual(proc.stderr, stderr, proc)
+        self.assertEqual(proc.stdout.decode(encoding), stdout.replace('\n', os.linesep), proc)
+        self.assertEqual(proc.stderr.decode(encoding), stderr.replace('\n', os.linesep), proc)
 
         with open(filename, 'r', encoding='utf8') as f:
             output = f.read()
diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py
diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py
diff --git a/Lib/test/test_uu.py b/Lib/test/test_uu.py
diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py
diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py
diff --git a/Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst b/Misc/NEWS.d/next/Tests/2020-06-22-00-21-12.bpo-41069.bLZkX-.rst
diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c
