BPO-41100: macOS branch

Lib/_osx_support.py

@@ -481,6 +481,8 @@ def get_platform_osx(_config_vars, osname, release, machine):
 
             if len(archs) == 1:
                 machine = archs[0]
+            elif archs == ('arm64', 'x86_64'):
+                machine = 'universal2'
             elif archs == ('i386', 'ppc'):
                 machine = 'fat'
             elif archs == ('i386', 'x86_64'):

Lib/ctypes/macholib/dyld.py

@@ -6,6 +6,11 @@
 from ctypes.macholib.framework import framework_info
 from ctypes.macholib.dylib import dylib_info
 from itertools import *
+try:
+     from _ctypes import _dyld_shared_cache_contains_path
+except ImportError:
+     def _dyld_shared_cache_contains_path(*args):
+         raise NotImplementedError
 
 __all__ = [
     'dyld_find', 'framework_find',
@@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None):
                 dyld_executable_path_search(name, executable_path),
                 dyld_default_search(name, env),
             ), env):
+
         if os.path.isfile(path):
             return path
+        try:
+            if _dyld_shared_cache_contains_path(path):
+                return path
+        except NotImplementedError:
+             pass
+
     raise ValueError("dylib %s could not be found" % (name,))
 
 def framework_find(fn, executable_path=None, env=None):

Lib/ctypes/test/test_macholib.py

@@ -45,19 +45,22 @@ def find_lib(name):
 class MachOTest(unittest.TestCase):
     @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test')
     def test_find(self):
-
-        self.assertEqual(find_lib('pthread'),
-                             '/usr/lib/libSystem.B.dylib')
+        # On Mac OS 11, system dylibs are only present in the shared cache,
+        # so symlinks like libpthread.dylib -> libSystem.B.dylib will not
+        # be resolved by dyld_find
+        self.assertIn(find_lib('pthread'),
+                              ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib'))
 
         result = find_lib('z')
         # Issue #21093: dyld default search path includes $HOME/lib and
         # /usr/local/lib before /usr/lib, which caused test failures if
         # a local copy of libz exists in one of them. Now ignore the head
         # of the path.
-        self.assertRegex(result, r".*/lib/libz\..*.*\.dylib")
+        self.assertRegex(result, r".*/lib/libz.*\.dylib")
 
-        self.assertEqual(find_lib('IOKit'),
-                             '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit')
+        self.assertIn(find_lib('IOKit'),
+                              ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit',
+                              '/System/Library/Frameworks/IOKit.framework/IOKit'))
 
 if __name__ == "__main__":
     unittest.main()

Lib/distutils/tests/test_build_ext.py

@@ -493,7 +493,7 @@ def _try_compile_deployment_target(self, operator, target):
         # format the target value as defined in the Apple
         # Availability Macros.  We can't use the macro names since
         # at least one value we test with will not exist yet.
-        if target[1] < 10:
+        if target[:2] < (10, 10):
             # for 10.1 through 10.9.x -> "10n0"
             target = '%02d%01d0' % target
         else:

Lib/test/test_bytes.py

@@ -1035,6 +1035,7 @@ def test_from_format(self):
             c_char_p)
 
         PyBytes_FromFormat = pythonapi.PyBytes_FromFormat
+        PyBytes_FromFormat.argtypes = (c_char_p,)
         PyBytes_FromFormat.restype = py_object
 
         # basic tests

Lib/test/test_platform.py

@@ -246,7 +246,7 @@ def test_mac_ver(self):
             self.assertEqual(res[1], ('', '', ''))
 
             if sys.byteorder == 'little':
-                self.assertIn(res[2], ('i386', 'x86_64'))
+                self.assertIn(res[2], ('i386', 'x86_64', 'arm64'))
             else:
                 self.assertEqual(res[2], 'PowerPC')
 

Lib/test/test_unicode.py

@@ -2513,11 +2513,13 @@ class CAPITest(unittest.TestCase):
     def test_from_format(self):
         import_helper.import_module('ctypes')
         from ctypes import (
+            c_char_p,
             pythonapi, py_object, sizeof,
             c_int, c_long, c_longlong, c_ssize_t,
             c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p)
         name = "PyUnicode_FromFormat"
         _PyUnicode_FromFormat = getattr(pythonapi, name)
+        _PyUnicode_FromFormat.argtypes = (c_char_p,)
         _PyUnicode_FromFormat.restype = py_object
 
         def PyUnicode_FromFormat(format, *args):

Mac/BuildScript/build-installer.py

@@ -116,14 +116,16 @@ def getFullVersion():
 DEPSRC = os.path.join(WORKDIR, 'third-party')
 DEPSRC = os.path.expanduser('~/Universal/other-sources')
 
-universal_opts_map = { '32-bit': ('i386', 'ppc',),
+universal_opts_map = { 'universal2': ('arm64', 'x86_64'),
+                       '32-bit': ('i386', 'ppc',),
                        '64-bit': ('x86_64', 'ppc64',),
                        'intel':  ('i386', 'x86_64'),
                        'intel-32':  ('i386',),
                        'intel-64':  ('x86_64',),
                        '3-way':  ('ppc', 'i386', 'x86_64'),
                        'all':    ('i386', 'ppc', 'x86_64', 'ppc64',) }
 default_target_map = {
+        'universal2': '10.9',
         '64-bit': '10.5',
         '3-way': '10.5',
         'intel': '10.5',
@@ -148,6 +150,10 @@ def getFullVersion():
 # $MACOSX_DEPLOYMENT_TARGET -> minimum OS X level
 DEPTARGET = '10.5'
 
+# If true only builds the 3th-party dependencies
+# in $WORKDIR
+DEPS_ONLY=False
+
 def getDeptargetTuple():
     return tuple([int(n) for n in DEPTARGET.split('.')[0:2]])
 
@@ -190,6 +196,27 @@ def getTargetCompilers():
 def internalTk():
     return getDeptargetTuple() >= (10, 6)
 
+
+def tweak_tcl_build(basedir, archList):
+    with open("Makefile", "r") as fp:
+        contents = fp.readlines()
+
+    # For reasons I don't understand the tcl configure script 
+    # decides that some stdlib symbols aren't present, before 
+    # deciding that strtod is broken. 
+    new_contents = []
+    for line in contents:
+        if line.startswith("COMPAT_OBJS"):
+            # note: the space before strtod.o is intentional,
+            # the detection of a broken strtod results in
+            # "fixstrod.o" on this line.
+            for nm in ("strstr.o", "strtoul.o", " strtod.o"):
+                line = line.replace(nm, "")
+        new_contents.append(line)
+
+    with open("Makefile", "w") as fp:
+        fp.writelines(new_contents)
+
 # List of names of third party software built with this installer.
 # The names will be inserted into the rtf version of the License.
 THIRD_PARTY_LIBS = []
@@ -215,6 +242,9 @@ def library_recipes():
               buildrecipe=build_universal_openssl,
               configure=None,
               install=None,
+              patches=[
+                  "openssl-mac-arm64.patch",
+                   ],
           ),
     ])
 
@@ -231,6 +261,7 @@ def library_recipes():
                     '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),),
               ],
               useLDFlags=False,
+              buildrecipe=tweak_tcl_build,
               install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{
                   "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')),
                   "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())),
@@ -596,7 +627,7 @@ def checkEnvironment():
                                                     ev, os.environ[ev]))
                 del os.environ[ev]
 
-    base_path = '/bin:/sbin:/usr/bin:/usr/sbin'
+    base_path = '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin'
     if 'SDK_TOOLS_BIN' in os.environ:
         base_path = os.environ['SDK_TOOLS_BIN'] + ':' + base_path
     # Xcode 2.5 on OS X 10.4 does not include SetFile in its usr/bin;
@@ -618,14 +649,15 @@ def parseOptions(args=None):
     global UNIVERSALOPTS, UNIVERSALARCHS, ARCHLIST, CC, CXX
     global FW_VERSION_PREFIX
     global FW_SSL_DIRECTORY
+    global DEPS_ONLY
 
     if args is None:
         args = sys.argv[1:]
 
     try:
         options, args = getopt.getopt(args, '?hb',
                 [ 'build-dir=', 'third-party=', 'sdk-path=' , 'src-dir=',
-                  'dep-target=', 'universal-archs=', 'help' ])
+                  'dep-target=', 'universal-archs=', 'deps-only', 'help' ])
     except getopt.GetoptError:
         print(sys.exc_info()[1])
         sys.exit(1)
@@ -646,6 +678,9 @@ def parseOptions(args=None):
         elif k in ('--third-party',):
             DEPSRC=v
 
+        elif k in ('--deps-only',):
+            DEPS_ONLY=True
+
         elif k in ('--sdk-path',):
             print(" WARNING: --sdk-path is no longer supported")
 
@@ -691,6 +726,8 @@ def parseOptions(args=None):
     print(" -- Building a Python %s framework at patch level %s"
                 % (getVersion(), getFullVersion()))
     print("")
+    if DEPS_ONLY:
+        print("Stopping after building third-party libraries")
 
 def extractArchive(builddir, archiveName):
     """
@@ -801,6 +838,7 @@ def build_openssl_arch(archbase, arch):
         arch_opts = {
             "i386": ["darwin-i386-cc"],
             "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"],
+            "arm64": ["darwin64-arm64-cc" ],
             "ppc": ["darwin-ppc-cc"],
             "ppc64": ["darwin64-ppc-cc"],
         }
@@ -1656,6 +1694,9 @@ def main():
     # Then build third-party libraries such as sleepycat DB4.
     buildLibraries()
 
+    if DEPS_ONLY:
+        sys.exit(1)
+
     # Now build python itself
     buildPython()
 

Mac/BuildScript/openssl-mac-arm64.patch

@@ -0,0 +1,41 @@
+diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf
+--- openssl-1.1.1g-orig/Configurations/10-main.conf	2020-04-21 14:22:39.000000000 +0200
++++ openssl-1.1.1g/Configurations/10-main.conf	2020-07-26 12:21:32.000000000 +0200
+@@ -1557,6 +1557,14 @@
+         bn_ops           => "SIXTY_FOUR_BIT_LONG",
+         perlasm_scheme   => "macosx",
+     },
++    "darwin64-arm64-cc" => {
++        inherit_from     => [ "darwin-common", asm("aarch64_asm") ],
++        CFLAGS           => add("-Wall"),
++        cflags           => add("-arch arm64"),
++        lib_cppflags     => add("-DL_ENDIAN"),
++        bn_ops           => "SIXTY_FOUR_BIT_LONG",
++        perlasm_scheme   => "ios64",
++    },
+
+ ##### GNU Hurd
+     "hurd-x86" => {
+diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config
+--- openssl-1.1.1g-orig/config	2020-04-21 14:22:39.000000000 +0200
++++ openssl-1.1.1g/config	2020-07-26 12:21:59.000000000 +0200
+@@ -255,6 +255,9 @@
+ 		;;
+ 	    x86_64)
+ 		echo "x86_64-apple-darwin${VERSION}"
++                ;;
++	    arm64)
++		echo "arm64-apple-darwin${VERSION}"
+ 		;;
+ 	    *)
+ 		echo "i686-apple-darwin${VERSION}"
+@@ -497,6 +500,9 @@
+ 	else
+ 	    OUT="darwin64-x86_64-cc"
+ 	fi ;;
++  x86_64-apple-darwin*)
++	OUT="darwin64-arm64-cc"
++        ;;
+   armv6+7-*-iphoneos)
+ 	__CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7"
+ 	__CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7"

Mac/Tools/pythonw.c

@@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
     size_t count;
     cpu_type_t cpu_types[1];
     short flags = 0;
-#ifdef __LP64__
-    int   ch;
-#endif
 
     if ((errno = posix_spawnattr_init(spawnattr)) != 0) {
         err(2, "posix_spawnattr_int");
@@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr)
 
 #elif defined(__ppc__)
     cpu_types[0] = CPU_TYPE_POWERPC;
+
 #elif defined(__i386__)
     cpu_types[0] = CPU_TYPE_X86;
+
+#elif defined(__arm64__)
+    cpu_types[0] = CPU_TYPE_ARM64;
+
 #else
 #       error "Unknown CPU"
+
 #endif
 
     if (posix_spawnattr_setbinpref_np(spawnattr, count,
@@ -220,7 +223,8 @@ main(int argc, char **argv) {
     /* We're weak-linking to posix-spawnv to ensure that
      * an executable build on 10.5 can work on 10.4.
      */
-    if (posix_spawn != NULL) {
+
+    if (&posix_spawn != NULL) {
         posix_spawnattr_t spawnattr = NULL;
 
         setup_spawnattr(&spawnattr);

Modules/_ctypes/callbacks.c

@@ -1,6 +1,8 @@
 #include "Python.h"
 #include "frameobject.h"
 
+#include <stdbool.h>
+
 #include <ffi.h>
 #ifdef MS_WIN32
 #include <windows.h>
@@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself)
     Py_XDECREF(self->callable);
     Py_XDECREF(self->restype);
     if (self->pcl_write)
-        ffi_closure_free(self->pcl_write);
+        Py_ffi_closure_free(self->pcl_write);
     PyObject_GC_Del(self);
 }
 
@@ -361,8 +363,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable,
 
     assert(CThunk_CheckExact((PyObject *)p));
 
-    p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure),
-                                                                         &p->pcl_exec);
+    p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec);
     if (p->pcl_write == NULL) {
         PyErr_NoMemory();
         goto error;
@@ -408,13 +409,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable,
                      "ffi_prep_cif failed with %d", result);
         goto error;
     }
-#if defined(X86_DARWIN) || defined(POWERPC_DARWIN)
-    result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p);
+#if HAVE_FFI_PREP_CLOSURE_LOC
+#   if USING_APPLE_OS_LIBFFI
+#      define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)
+#   else
+#      define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME true
+#   endif
+    if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) {
+        result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn,
+                                    p,
+                                    p->pcl_exec);
+    } else
+#endif
+    {
+#if USING_APPLE_OS_LIBFFI && defined(__arm64__)
+        PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing");
+        goto error;
 #else
-    result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn,
-                                  p,
-                                  p->pcl_exec);
+#ifdef MACOSX
+        #pragma clang diagnostic push
+        #pragma clang diagnostic ignored "-Wdeprecated-declarations"
 #endif
+        result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p);
+
+#ifdef MACOSX
+        #pragma clang diagnostic pop
+#endif
+
+#endif
+    }
     if (result != FFI_OK) {
         PyErr_Format(PyExc_RuntimeError,
                      "ffi_prep_closure failed with %d", result);