From 04e78e057882547d0cae37d84f145637f336794b Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Thu, 26 Sep 2024 14:12:19 -0700 Subject: [PATCH] [build] Build sourcekit-lsp for multiple arches and lipo them Instead of building SourceKit-LSP using SwiftPM's multi-arch xcbuild backend, build it for only one arch at a time and then run `lipo` to merge the two resulting binaries. This should allow us to share build products between building installing and testing and also eliminates other quirks resulting from the xcbuild backend. --- .../products/indexstoredb.py | 96 ++++++------------- .../products/sourcekitlsp.py | 83 ++++++++++++++-- 2 files changed, 106 insertions(+), 73 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/indexstoredb.py b/utils/swift_build_support/swift_build_support/products/indexstoredb.py index 95c05f72720f9..b0902c0ab0197 100644 --- a/utils/swift_build_support/swift_build_support/products/indexstoredb.py +++ b/utils/swift_build_support/swift_build_support/products/indexstoredb.py @@ -26,7 +26,6 @@ from . import swiftsyntax from . import xctest from .. import shell -from .. import targets class IndexStoreDB(product.Product): @@ -50,14 +49,13 @@ def should_build(self, host_target): return True def build(self, host_target): - run_build_script_helper('build', host_target, self, self.args) + self.run_build_script_helper('build', host_target) def should_test(self, host_target): return self.args.test_indexstoredb def test(self, host_target): - run_build_script_helper('test', host_target, self, self.args, - self.args.test_indexstoredb_sanitize_all) + self.run_build_script_helper('test', host_target) def should_install(self, host_target): return False @@ -81,64 +79,32 @@ def get_dependencies(cls): swiftpm.SwiftPM, swiftsyntax.SwiftSyntax] - -def run_build_script_helper(action, host_target, product, args, - sanitize_all=False, clean=False): - script_path = os.path.join( - product.source_dir, 'Utilities', 'build-script-helper.py') - - install_destdir = product.host_install_destdir(host_target) - toolchain_path = product.native_toolchain_path(host_target) - is_release = product.is_release() - configuration = 'release' if is_release else 'debug' - helper_cmd = [ - script_path, - action, - '--package-path', product.source_dir, - '--build-path', product.build_dir, - '--configuration', configuration, - '--toolchain', toolchain_path, - '--ninja-bin', product.toolchain.ninja, - '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH, - ] - if args.verbose_build: - helper_cmd.append('--verbose') - - if sanitize_all: - helper_cmd.append('--sanitize-all') - elif args.enable_asan: - helper_cmd.extend(['--sanitize', 'address']) - elif args.enable_ubsan: - helper_cmd.extend(['--sanitize', 'undefined']) - elif args.enable_tsan: - helper_cmd.extend(['--sanitize', 'thread']) - - if clean: - helper_cmd.append('--clean') - - # Pass Cross compile host info unless we're testing. - # It doesn't make sense to run tests of the cross compile host. - if product.has_cross_compile_hosts() and action != 'test': - if product.is_darwin_host(host_target): - if len(args.cross_compile_hosts) != 1: - raise RuntimeError("Cross-Compiling indexstoredb to multiple " + - "targets is not supported") - helper_cmd += ['--cross-compile-host', args.cross_compile_hosts[0]] - elif product.is_cross_compile_target(host_target): - helper_cmd.extend(['--cross-compile-host', host_target]) - build_toolchain_path = install_destdir + args.install_prefix - resource_dir = '%s/lib/swift' % build_toolchain_path - helper_cmd += [ - '--cross-compile-config', - targets.StdlibDeploymentTarget.get_target_for_name(host_target).platform - .swiftpm_config(args, output_dir=build_toolchain_path, - swift_toolchain=toolchain_path, - resource_path=resource_dir) - ] - - if action == 'install' and product.product_name() == "sourcekitlsp": - helper_cmd.extend([ - '--prefix', install_destdir + args.install_prefix - ]) - - shell.call(helper_cmd) + def run_build_script_helper(self, action, host_target): + script_path = os.path.join( + self.source_dir, 'Utilities', 'build-script-helper.py') + + toolchain_path = self.native_toolchain_path(host_target) + configuration = 'release' if self.is_release() else 'debug' + helper_cmd = [ + script_path, + action, + '--package-path', self.source_dir, + '--build-path', self.build_dir, + '--configuration', configuration, + '--toolchain', toolchain_path, + '--ninja-bin', self.toolchain.ninja, + '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH, + ] + if self.args.verbose_build: + helper_cmd.append('--verbose') + + if self.args.test_indexstoredb_sanitize_all: + helper_cmd.append('--sanitize-all') + elif self.args.enable_asan: + helper_cmd.extend(['--sanitize', 'address']) + elif self.args.enable_ubsan: + helper_cmd.extend(['--sanitize', 'undefined']) + elif self.args.enable_tsan: + helper_cmd.extend(['--sanitize', 'thread']) + + shell.call(helper_cmd) diff --git a/utils/swift_build_support/swift_build_support/products/sourcekitlsp.py b/utils/swift_build_support/swift_build_support/products/sourcekitlsp.py index 9f1772eec0267..6b3adf7fabecc 100644 --- a/utils/swift_build_support/swift_build_support/products/sourcekitlsp.py +++ b/utils/swift_build_support/swift_build_support/products/sourcekitlsp.py @@ -10,9 +10,12 @@ # # ---------------------------------------------------------------------------- +import os + +from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH + from . import cmark from . import foundation -from . import indexstoredb from . import libcxx from . import libdispatch from . import llbuild @@ -21,6 +24,8 @@ from . import swift from . import swiftpm from . import xctest +from .. import shell +from .. import targets class SourceKitLSP(product.Product): @@ -43,24 +48,35 @@ def is_swiftpm_unified_build_product(cls): def should_build(self, host_target): return True + def _for_each_host_target(self, base_target, body): + body(base_target) + + # For Darwin host, 'build' is only called for the builder. + # Manually iterate the cross compile hosts. + if self.has_cross_compile_hosts() and self.is_darwin_host(base_target): + for target in self.args.cross_compile_hosts: + body(target) + def build(self, host_target): - indexstoredb.run_build_script_helper( - 'build', host_target, self, self.args) + self._for_each_host_target( + host_target, + lambda target: self.run_build_script_helper('build', host_target, target) + ) def should_test(self, host_target): return self.args.test_sourcekitlsp def test(self, host_target): - indexstoredb.run_build_script_helper( - 'test', host_target, self, self.args, - self.args.test_sourcekitlsp_sanitize_all) + self.run_build_script_helper('test', host_target, host_target) def should_install(self, host_target): return self.args.install_sourcekitlsp def install(self, host_target): - indexstoredb.run_build_script_helper( - 'install', host_target, self, self.args) + self._for_each_host_target( + host_target, + lambda target: self.run_build_script_helper('install', host_target, target) + ) @classmethod def get_dependencies(cls): @@ -73,3 +89,54 @@ def get_dependencies(cls): xctest.XCTest, llbuild.LLBuild, swiftpm.SwiftPM] + + def run_build_script_helper(self, action, base_target, host_target): + # base_target is the machine that's driving the build. + # host_target is the target we are bulding for. + script_path = os.path.join( + self.source_dir, 'Utilities', 'build-script-helper.py') + + install_destdir = self.host_install_destdir(host_target) + toolchain_path = self.native_toolchain_path(base_target) + configuration = 'release' if self.is_release() else 'debug' + helper_cmd = [ + script_path, + action, + '--package-path', self.source_dir, + '--build-path', self.build_dir, + '--configuration', configuration, + '--toolchain', toolchain_path, + '--ninja-bin', self.toolchain.ninja, + '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH, + ] + if self.args.verbose_build: + helper_cmd.append('--verbose') + + if self.args.test_sourcekitlsp_sanitize_all: + helper_cmd.append('--sanitize-all') + elif self.args.enable_asan: + helper_cmd += ['--sanitize', 'address'] + elif self.args.enable_ubsan: + helper_cmd += ['--sanitize', 'undefined'] + elif self.args.enable_tsan: + helper_cmd += ['--sanitize', 'thread'] + + if self.has_cross_compile_hosts(): + helper_cmd += ['--cross-compile-host', host_target] + if self.is_cross_compile_target(host_target) and \ + not self.is_darwin_host(host_target): + build_toolchain_path = install_destdir + self.args.install_prefix + resource_dir = '%s/lib/swift' % build_toolchain_path + helper_cmd += [ + '--cross-compile-config', + targets.StdlibDeploymentTarget.get_target_for_name(host_target) + .platform + .swiftpm_config(self.args, output_dir=build_toolchain_path, + swift_toolchain=toolchain_path, + resource_path=resource_dir) + ] + + if action == 'install': + helper_cmd += ['--prefix', install_destdir + self.args.install_prefix] + + shell.call(helper_cmd)