Build self-hosted trunk from last tag, after bringing back upstreamed patches

.github/workflows/pull_request.yml

@@ -28,10 +28,10 @@ jobs:
        fail-fast: false
        matrix:
          include:
-           #- swift-version: 'swift-6.2-branch'
-           #  build-type: 'docker'
-           #  build-compiler: '1'
-           #  runner: 'self-hosted'
+           - swift-version: 'swift-6.2-branch'
+             build-type: 'docker'
+             build-compiler: '1'
+             runner: 'self-hosted'
            #- swift-version: 'development'
            #  build-type: 'docker'
            #  build-compiler: '1'

swift-ci/sdks/android/patches/apply-patches.sh

@@ -5,8 +5,8 @@ cd ${1}
 
 case "${BUILD_SCHEME}" in
     swift-*-branch)
-        git apply -v -C1 ${patches_dir}/swift-android.patch
         git apply -v -C1 ${patches_dir}/swift-android-devel.patch
+        git apply -v -C1 ${patches_dir}/swift-android.patch
         ;;
     development)
         git apply -v -C1 ${patches_dir}/swift-android.patch

swift-ci/sdks/android/patches/swift-android-devel.patch

@@ -1,17 +1,25 @@
-diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py
-index bfb69965890..b5e9f5349c2 100644
---- a/swift/utils/swift_build_support/swift_build_support/cmake.py
-+++ b/swift/utils/swift_build_support/swift_build_support/cmake.py
-@@ -153,10 +153,8 @@ class CMake(object):
-             toolchain_path = product.native_toolchain_path(args.host_target)
-             cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER',
-                                           os.path.join(toolchain_path, 'bin', 'swiftc'))
--            define("CMAKE_C_COMPILER:PATH", os.path.join(toolchain_path,
--                                                         'bin', 'clang'))
--            define("CMAKE_CXX_COMPILER:PATH", os.path.join(toolchain_path,
--                                                           'bin', 'clang++'))
-+            define("CMAKE_C_COMPILER:PATH", toolchain.cc)
-+            define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx)
-             define("CMAKE_Swift_COMPILER:PATH", cmake_swiftc_path)
-         else:
-             cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER', toolchain.swiftc)
+diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py
+index 47e7ab79905..d88c3c242ad 100644
+--- a/swift/utils/swift_build_support/swift_build_support/products/product.py
++++ b/swift/utils/swift_build_support/swift_build_support/products/product.py
+@@ -389,7 +395,7 @@ class Product(object):
+         sysroot_arch, vendor, abi = self.get_linux_target_components(arch)
+         return '{}-{}-linux-{}'.format(sysroot_arch, vendor, abi)
+
+-    def generate_linux_toolchain_file(self, platform, arch):
++    def generate_linux_toolchain_file(self, platform, arch, crosscompiling=True):
+         """
+         Generates a new CMake tolchain file that specifies Linux as a target
+         platform.
+@@ -402,8 +408,9 @@ class Product(object):
+
+         toolchain_args = {}
+
+-        toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
+-        toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
++        if crosscompiling:
++            toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
++            toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
+
+         # We only set the actual sysroot if we are actually cross
+         # compiling. This is important since otherwise cmake seems to change the

swift-ci/sdks/android/patches/swift-android-trunk.patch

@@ -10,21 +10,3 @@ index 3f36fc979cb..8713ae8fea8 100755
              "grep",
              "--extended-regexp",
              "--recursive",
-diff --git a/swift/utils/swift_build_support/swift_build_support/cmake.py b/swift/utils/swift_build_support/swift_build_support/cmake.py
-index 9a0f5b43e2d..5c7d1320a2d 100644
---- a/swift/utils/swift_build_support/swift_build_support/cmake.py
-+++ b/swift/utils/swift_build_support/swift_build_support/cmake.py
-@@ -150,11 +150,8 @@ class CMake(object):
-             define("CMAKE_CXX_COMPILER_LAUNCHER:PATH", args.cmake_cxx_launcher)
-
-         if self.prefer_native_toolchain and product:
--            clang_tools_path = product.native_clang_tools_path(args.host_target)
--            define("CMAKE_C_COMPILER:PATH", os.path.join(clang_tools_path,
--                                                         'bin', 'clang'))
--            define("CMAKE_CXX_COMPILER:PATH", os.path.join(clang_tools_path,
--                                                           'bin', 'clang++'))
-+            define("CMAKE_C_COMPILER:PATH", toolchain.cc)
-+            define("CMAKE_CXX_COMPILER:PATH", toolchain.cxx)
-
-             toolchain_path = product.native_toolchain_path(args.host_target)
-             cmake_swiftc_path = os.getenv('CMAKE_Swift_COMPILER',

swift-ci/sdks/android/patches/swift-android.patch

@@ -22,43 +22,228 @@ index 16e05052609..7ab8cebfab8 100755
 
                    cmake_options=(
                      -DENABLE_SWIFT=YES
+diff --git a/swift/utils/swift_build_support/swift_build_support/products/product.py b/swift/utils/swift_build_support/swift_build_support/products/product.py
+index d88c3c242ad..dad4a539075 100644
+--- a/swift/utils/swift_build_support/swift_build_support/products/product.py
++++ b/swift/utils/swift_build_support/swift_build_support/products/product.py
+@@ -409,18 +409,33 @@ class Product(object):
+         toolchain_args = {}
+
+         if crosscompiling:
+-            toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
+-            toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
++            if platform == "linux":
++                toolchain_args['CMAKE_SYSTEM_NAME'] = 'Linux'
++                toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch
++            elif platform == "android":
++                toolchain_args['CMAKE_SYSTEM_NAME'] = 'Android'
++                toolchain_args['CMAKE_SYSTEM_VERSION'] = self.args.android_api_level
++                toolchain_args['CMAKE_SYSTEM_PROCESSOR'] = arch if not arch == 'armv7' \
++                                                           else 'armv7-a'
++                toolchain_args['CMAKE_ANDROID_NDK'] = self.args.android_ndk
++                toolchain_args['CMAKE_FIND_ROOT_PATH'] = self.args.cross_compile_deps_path
++                # This is a workaround for a CMake 3.30+ bug,
++                # https://gitlab.kitware.com/cmake/cmake/-/issues/26154, and can
++                # be removed once that is fixed.
++                toolchain_args['CMAKE_SHARED_LINKER_FLAGS'] = '\"\"'
+
+         # We only set the actual sysroot if we are actually cross
+         # compiling. This is important since otherwise cmake seems to change the
+         # RUNPATH to be a relative rather than an absolute path, breaking
+         # certain cmark tests (and maybe others).
+-        maybe_sysroot = self.get_linux_sysroot(platform, arch)
+-        if maybe_sysroot is not None:
+-            toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot
+-
+-        target = self.get_linux_target(platform, arch)
++        if platform == "linux":
++            maybe_sysroot = self.get_linux_sysroot(platform, arch)
++            if maybe_sysroot is not None:
++                toolchain_args['CMAKE_SYSROOT'] = maybe_sysroot
++
++            target = self.get_linux_target(platform, arch)
++        elif platform == "android":
++            target = '%s-unknown-linux-android%s' % (arch, self.args.android_api_level)
+         if self.toolchain.cc.endswith('clang'):
+             toolchain_args['CMAKE_C_COMPILER_TARGET'] = target
+         if self.toolchain.cxx.endswith('clang++'):
+@@ -466,10 +481,30 @@ class Product(object):
+                 platform, arch,
+                 macos_deployment_version=override_macos_deployment_version)
+             self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file)
+-        elif platform == "linux":
+-            toolchain_file = self.generate_linux_toolchain_file(platform, arch)
++        elif platform == "linux" or platform == "android":
++            # Always cross-compile for linux, but not on Android, as a native
++            # compile on Android does not use the NDK and its CMake config.
++            cross_compile = platform == "linux" or \
++                self.is_cross_compile_target(host_target)
++            toolchain_file = self.generate_linux_toolchain_file(platform, arch,
++                                                                cross_compile)
+             self.cmake_options.define('CMAKE_TOOLCHAIN_FILE:PATH', toolchain_file)
+
++            if cross_compile and platform == "android":
++                resource_dir = None
++                # build-script-impl products build before the install and use
++                # the Swift stdlib from the compiler build directory instead,
++                # while products built even before that currently do not support
++                # cross-compiling Swift.
++                if not self.is_before_build_script_impl_product() and \
++                   not self.is_build_script_impl_product():
++                    install_path = self.host_install_destdir(host_target) + \
++                        self.args.install_prefix
++                    resource_dir = '%s/lib/swift' % install_path
++                flags = targets.StdlibDeploymentTarget.get_target_for_name(
++                    host_target).platform.swift_flags(self.args, resource_dir)
++                self.cmake_options.define('CMAKE_Swift_FLAGS', flags)
++
+         return toolchain_file
+
+     def get_openbsd_toolchain_file(self):
 diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py
-index 324d1a77eea..e88601a8701 100644
+index 417056efdd0..177ea9f0623 100644
 --- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py
 +++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing.py
-@@ -13,6 +13,9 @@
- import os
-
- from build_swift.build_swift.versions import Version
-+from ..host_specific_configuration \
-+    import HostSpecificConfiguration
-+from ..targets import StdlibDeploymentTarget
-
- from . import cmake_product
- from . import product
-@@ -115,6 +117,25 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct):
-         # FIXME: If we build macros for the builder, specify the path.
-         self.cmake_options.define('SwiftTesting_MACRO', 'NO')
-
-+        if host_target.startswith('android') and self.is_cross_compile_target(host_target):
-+            host_config = HostSpecificConfiguration(host_target, self.args)
-+            self.cmake_options.extend(host_config.cmake_options)
-+            triple = '%s-unknown-linux-android%s' % (self.args.android_arch,
-+                                                     self.args.android_api_level)
-+            flags = '-target %s ' % (triple)
+@@ -127,3 +127,11 @@ class SwiftTestingCMakeShim(cmake_product.CMakeProduct):
+         install_prefix = install_destdir + self.args.install_prefix
+
+         self.install_with_cmake(['install'], install_prefix)
 +
-+            flags += '-resource-dir %s/lib/swift ' % (
-+                     self.host_install_destdir(host_target) + self.args.install_prefix)
++    @classmethod
++    def is_build_script_impl_product(cls):
++        return False
 +
-+            ndk_path = StdlibDeploymentTarget.get_target_for_name(host_target).platform.ndk_toolchain_path(self.args)
-+            flags += '-sdk %s/sysroot ' % (ndk_path)
-+            flags += '-tools-directory %s/bin' % (ndk_path)
-+            self.cmake_options.define('CMAKE_Swift_FLAGS', flags)
-+            self.cmake_options.define('CMAKE_Swift_COMPILER_TARGET', triple)
-+            self.cmake_options.define('CMAKE_CXX_COMPILER_WORKS', 'True')
-+            self.cmake_options.define('CMAKE_SHARED_LINKER_FLAGS', '')
-+            self.cmake_options.define('CMAKE_FIND_ROOT_PATH', self.args.cross_compile_deps_path)
++    @classmethod
++    def is_before_build_script_impl_product(cls):
++        return False
+diff --git a/swift/utils/swift_build_support/swift_build_support/targets.py b/swift/utils/swift_build_support/swift_build_support/targets.py
+index fba09416ddb..67b81daba12 100644
+--- a/swift/utils/swift_build_support/swift_build_support/targets.py
++++ b/swift/utils/swift_build_support/swift_build_support/targets.py
+@@ -72,7 +72,7 @@ class Platform(object):
+                 return True
+         return False
+
+-    def swift_flags(self, args):
++    def swift_flags(self, args, resource_path=None):
+         """
+         Swift compiler flags for a platform, useful for cross-compiling
+         """
+@@ -154,12 +154,15 @@ class AndroidPlatform(Platform):
+         """
+         return True
+
+-    def swift_flags(self, args):
++    def swift_flags(self, args, resource_path=None):
+         flags = '-target %s-unknown-linux-android%s ' % (args.android_arch,
+                                                          args.android_api_level)
+
+-        flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % (
+-                 args.build_root, self.name, args.android_arch)
++        if resource_path is not None:
++            flags += '-resource-dir %s ' % (resource_path)
++        else:
++            flags += '-resource-dir %s/swift-%s-%s/lib/swift ' % (
++                     args.build_root, self.name, args.android_arch)
+
+         android_toolchain_path = self.ndk_toolchain_path(args)
+
+diff --git a/swift/utils/build_swift/build_swift/driver_arguments.py b/swift/utils/build_swift/build_swift/driver_arguments.py
+index 2b7d6d07799..e0d04d22825 100644
+--- a/swift/utils/build_swift/build_swift/driver_arguments.py
++++ b/swift/utils/build_swift/build_swift/driver_arguments.py
+@@ -677,6 +677,12 @@ def create_argument_parser():
+                 "for each cross-compiled toolchain's destdir, useful when building "
+                 "multiple toolchains and can be disabled if only cross-compiling one.")
+
++    option('--cross-compile-build-swift-tools', toggle_true,
++           default=True,
++           help="Cross-compile the Swift compiler, other host tools from the "
++                "compiler repository, and various macros for each listed "
++                "--cross-compile-hosts platform.")
 +
-         self.generate_toolchain_file_for_darwin_or_linux(
-             host_target, override_macos_deployment_version=override_deployment_version)
-         self.build_with_cmake([], self.args.build_variant, [],
+     option('--stdlib-deployment-targets', store,
+            type=argparse.ShellSplitType(),
+            default=None,
+diff --git a/swift/utils/build_swift/tests/expected_options.py b/swift/utils/build_swift/tests/expected_options.py
+index dddb505174e..6052e1f3da6 100644
+--- a/swift/utils/build_swift/tests/expected_options.py
++++ b/swift/utils/build_swift/tests/expected_options.py
+@@ -156,6 +156,7 @@ EXPECTED_DEFAULTS = {
+     'compiler_vendor': defaults.COMPILER_VENDOR,
+     'coverage_db': None,
+     'cross_compile_append_host_target_to_destdir': True,
++    'cross_compile_build_swift_tools': True,
+     'cross_compile_deps_path': None,
+     'cross_compile_hosts': [],
+     'infer_cross_compile_hosts_on_darwin': False,
+@@ -628,6 +629,7 @@ EXPECTED_OPTIONS = [
+     EnableOption('--build-swift-clang-overlays'),
+     EnableOption('--build-swift-remote-mirror'),
+     EnableOption('--cross-compile-append-host-target-to-destdir'),
++    EnableOption('--cross-compile-build-swift-tools'),
+     EnableOption('--color-in-tests'),
+     EnableOption('--distcc'),
+     EnableOption('--sccache'),
+diff --git a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py
+index a0289515fd0..355d72fe21f 100644
+--- a/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py
++++ b/swift/utils/swift_build_support/swift_build_support/build_script_invocation.py
+@@ -119,6 +119,8 @@ class BuildScriptInvocation(object):
+             "--cmake-generator", args.cmake_generator,
+             "--cross-compile-append-host-target-to-destdir", str(
+                 args.cross_compile_append_host_target_to_destdir).lower(),
++            "--cross-compile-build-swift-tools", str(
++                args.cross_compile_build_swift_tools).lower(),
+             "--build-jobs", str(args.build_jobs),
+             "--lit-jobs", str(args.lit_jobs),
+             "--common-cmake-options=%s" % ' '.join(
+diff --git a/swift/utils/swift_build_support/swift_build_support/products/llvm.py b/swift/utils/swift_build_support/swift_build_support/products/llvm.py
+index ffae1d66702..bc6e00e7ea3 100644
+--- a/swift/utils/swift_build_support/swift_build_support/products/llvm.py
++++ b/swift/utils/swift_build_support/swift_build_support/products/llvm.py
+@@ -249,7 +249,8 @@ class LLVM(cmake_product.CMakeProduct):
+                 # space/time efficient than -g on that platform.
+                 llvm_cmake_options.define('LLVM_USE_SPLIT_DWARF:BOOL', 'YES')
+
+-        if not self.args._build_llvm:
++        if not self.args._build_llvm or (not self.args.cross_compile_build_swift_tools and self.is_cross_compile_target(host_target)):
++            self.args._build_llvm = False
+             # Indicating we don't want to build LLVM at all should
+             # override everything.
+             build_targets = []
+@@ -484,7 +484,7 @@ class LLVM(cmake_product.CMakeProduct):
+         Whether or not this product should be installed with the given
+         arguments.
+         """
+-        return self.args.install_llvm
++        return self.args.install_llvm and (self.args.cross_compile_build_swift_tools or not self.is_cross_compile_target(host_target))
+
+     def install(self, host_target):
+         """
+diff --git a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py
+index ddb3b553de7..7195d921da2 100644
+--- a/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py
++++ b/swift/utils/swift_build_support/swift_build_support/products/swift_testing_macros.py
+@@ -42,13 +42,19 @@ class SwiftTestingMacros(product.Product):
+         return True
+
+     def should_build(self, host_target):
+-        return True
++        build_macros = not self.is_cross_compile_target(host_target) or self.args.cross_compile_build_swift_tools
++        if not build_macros:
++            print("Skipping building Testing Macros for %s, because the host tools are not being built", host_target)
++        return build_macros
+
+     def should_test(self, host_target):
+         return False
+
+     def should_install(self, host_target):
+-        return self.args.install_swift_testing_macros
++        install_macros = self.args.install_swift_testing_macros and (not self.is_cross_compile_target(host_target) or self.args.cross_compile_build_swift_tools)
++        if self.args.install_swift_testing_macros and not install_macros:
++            print("Skipping installing Testing Macros for %s, because the host tools are not being built", host_target)
++        return install_macros
+
+     def _cmake_product(self, host_target):
+         build_root = os.path.dirname(self.build_dir)