bpo-36763: Fix _PyPreConfig_InitCompatConfig() utf8_mode (GH-13518)

* _PyPreConfig_InitCompatConfig() sets utf8_mode to 0.
* Change Py_UTF8Mode default value to 0.
* Fix _PyPreConfig_Copy(): copy also _config_init attrbibute.
* _PyPreConfig_AsDict() exports _config_init
* Fix _PyPreConfig_GetGlobalConfig(): use Py_UTF8Mode if it's greater
  than 0, even if utf8_mode >= 0.
* Add unit tests on environment variables using Python API.

Author: vstinner

Lib/test/test_embed.py

@@ -287,6 +287,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     IGNORE_CONFIG = object()
 
     PRE_CONFIG_COMPAT = {
+        '_config_init': API_COMPAT,
         'allocator': PYMEM_ALLOCATOR_NOT_SET,
         'parse_argv': 0,
         'configure_locale': 1,
@@ -299,11 +300,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'legacy_windows_fs_encoding': 0,
         })
     PRE_CONFIG_PYTHON = dict(PRE_CONFIG_COMPAT,
+        _config_init=API_PYTHON,
         parse_argv=1,
         coerce_c_locale=GET_DEFAULT_CONFIG,
         utf8_mode=GET_DEFAULT_CONFIG,
     )
     PRE_CONFIG_ISOLATED = dict(PRE_CONFIG_COMPAT,
+        _config_init=API_ISOLATED,
         configure_locale=0,
         isolated=1,
         use_environment=0,
@@ -388,10 +391,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         })
 
     CORE_CONFIG_PYTHON = dict(CORE_CONFIG_COMPAT,
+        _config_init=API_PYTHON,
         configure_c_stdio=1,
         parse_argv=1,
     )
     CORE_CONFIG_ISOLATED = dict(CORE_CONFIG_COMPAT,
+        _config_init=API_ISOLATED,
         isolated=1,
         use_environment=0,
         user_site_directory=0,
@@ -611,7 +616,6 @@ def check_config(self, testname, expected_config=None,
         else:
             default_config = self.CORE_CONFIG_COMPAT
         expected_config = dict(default_config, **expected_config)
-        expected_config['_config_init'] = api
 
         self.get_expected_config(expected_preconfig,
                                  expected_config, env,
@@ -708,10 +712,9 @@ def test_init_from_config(self):
         self.check_config("test_init_from_config", config, preconfig,
                           api=API_COMPAT)
 
-    def test_init_env(self):
+    def test_init_compat_env(self):
         preconfig = {
             'allocator': PYMEM_ALLOCATOR_MALLOC,
-            'utf8_mode': 1,
         }
         config = {
             'use_hash_seed': 1,
@@ -732,9 +735,36 @@ def test_init_env(self):
             'faulthandler': 1,
             'warnoptions': ['EnvVar'],
         }
-        self.check_config("test_init_env", config, preconfig,
+        self.check_config("test_init_compat_env", config, preconfig,
                           api=API_COMPAT)
 
+    def test_init_python_env(self):
+        preconfig = {
+            'allocator': PYMEM_ALLOCATOR_MALLOC,
+            'utf8_mode': 1,
+        }
+        config = {
+            'use_hash_seed': 1,
+            'hash_seed': 42,
+            'tracemalloc': 2,
+            'import_time': 1,
+            'malloc_stats': 1,
+            'inspect': 1,
+            'optimization_level': 2,
+            'module_search_path_env': '/my/path',
+            'pycache_prefix': 'env_pycache_prefix',
+            'write_bytecode': 0,
+            'verbose': 1,
+            'buffered_stdio': 0,
+            'stdio_encoding': 'iso8859-1',
+            'stdio_errors': 'replace',
+            'user_site_directory': 0,
+            'faulthandler': 1,
+            'warnoptions': ['EnvVar'],
+        }
+        self.check_config("test_init_python_env", config, preconfig,
+                          api=API_PYTHON)
+
     def test_init_env_dev_mode(self):
         preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
         config = dict(dev_mode=1,

Programs/_testembed.c

@@ -638,7 +638,7 @@ static void set_all_env_vars(void)
 }
 
 
-static int test_init_env(void)
+static int test_init_compat_env(void)
 {
     /* Test initialization from environment variables */
     Py_IgnoreEnvironmentFlag = 0;
@@ -650,6 +650,29 @@ static int test_init_env(void)
 }
 
 
+static int test_init_python_env(void)
+{
+    _PyInitError err;
+
+    set_all_env_vars();
+
+    _PyCoreConfig config;
+    err = _PyCoreConfig_InitPythonConfig(&config);
+    if (_PyInitError_Failed(err)) {
+        _Py_ExitInitError(err);
+    }
+    config.program_name = L"./_testembed";
+
+    err = _Py_InitializeFromConfig(&config);
+    if (_PyInitError_Failed(err)) {
+        _Py_ExitInitError(err);
+    }
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static void set_all_env_vars_dev_mode(void)
 {
     putenv("PYTHONMALLOC=");
@@ -1257,7 +1280,8 @@ static struct TestCase TestCases[] = {
     {"test_init_from_config", test_init_from_config},
     {"test_init_parse_argv", test_init_parse_argv},
     {"test_init_dont_parse_argv", test_init_dont_parse_argv},
-    {"test_init_env", test_init_env},
+    {"test_init_compat_env", test_init_compat_env},
+    {"test_init_python_env", test_init_python_env},
     {"test_init_env_dev_mode", test_init_env_dev_mode},
     {"test_init_env_dev_mode_alloc", test_init_env_dev_mode_alloc},
     {"test_init_dont_configure_locale", test_init_dont_configure_locale},

Python/coreconfig.c

@@ -107,9 +107,8 @@ static const char usage_6[] =
 /* --- Global configuration variables ----------------------------- */
 
 /* UTF-8 mode (PEP 540): if equals to 1, use the UTF-8 encoding, and change
-   stdin and stdout error handler to "surrogateescape". It is equal to
-   -1 by default: unknown, will be set by Py_Main() */
-int Py_UTF8Mode = -1;
+   stdin and stdout error handler to "surrogateescape". */
+int Py_UTF8Mode = 0;
 int Py_DebugFlag = 0; /* Needed by parser.c */
 int Py_VerboseFlag = 0; /* Needed by import.c */
 int Py_QuietFlag = 0; /* Needed by sysmodule.c */

Python/preconfig.c

@@ -272,7 +272,16 @@ _PyPreConfig_InitCompatConfig(_PyPreConfig *config)
     config->isolated = -1;
     config->use_environment = -1;
     config->configure_locale = 1;
-    config->utf8_mode = -1;
+
+    /* bpo-36443: C locale coercion (PEP 538) and UTF-8 Mode (PEP 540)
+       are disabled by default using the Compat configuration.
+
+       Py_UTF8Mode=1 enables the UTF-8 mode. PYTHONUTF8 environment variable
+       is ignored (even if use_environment=1). */
+    config->utf8_mode = 0;
+    config->coerce_c_locale = 0;
+    config->coerce_c_locale_warn = 0;
+
     config->dev_mode = -1;
     config->allocator = PYMEM_ALLOCATOR_NOT_SET;
 #ifdef MS_WINDOWS
@@ -353,6 +362,7 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
 {
 #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR
 
+    COPY_ATTR(_config_init);
     COPY_ATTR(parse_argv);
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
@@ -393,6 +403,7 @@ _PyPreConfig_AsDict(const _PyPreConfig *config)
             } \
         } while (0)
 
+    SET_ITEM_INT(_config_init);
     SET_ITEM_INT(parse_argv);
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
@@ -452,7 +463,9 @@ _PyPreConfig_GetGlobalConfig(_PyPreConfig *config)
 
     COPY_FLAG(isolated, Py_IsolatedFlag);
     COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag);
-    COPY_FLAG(utf8_mode, Py_UTF8Mode);
+    if (Py_UTF8Mode > 0) {
+        config->utf8_mode = Py_UTF8Mode;
+    }
 #ifdef MS_WINDOWS
     COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag);
 #endif