From 620735bc817e9e5605c945545157f2cf07239df9 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Mon, 5 Jun 2023 16:30:49 +0100 Subject: [PATCH] [Linux] Add compression libraries to the build. Add zlib, zstd and liblzma to the build. rdar://110263358 --- utils/build-script-impl | 89 +++++++++++++- .../build_swift/driver_arguments.py | 23 +++- utils/build_swift/tests/expected_options.py | 6 + .../build_script_invocation.py | 21 +++- .../swift_build_support/products/__init__.py | 6 +- .../swift_build_support/products/liblzma.py | 114 ++++++++++++++++++ .../swift_build_support/products/zlib.py | 9 ++ .../swift_build_support/products/zstd.py | 95 +++++++++++++++ .../update-checkout-config.json | 19 ++- 9 files changed, 375 insertions(+), 7 deletions(-) create mode 100644 utils/swift_build_support/swift_build_support/products/liblzma.py create mode 100644 utils/swift_build_support/swift_build_support/products/zstd.py diff --git a/utils/build-script-impl b/utils/build-script-impl index 99302d005d42d..b1f1b201b6c1a 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -276,7 +276,9 @@ components=( libdispatch libicu libxml2 + liblzma zlib + zstd curl llbuild lldb @@ -1205,12 +1207,18 @@ LIBDISPATCH_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch" LIBDISPATCH_STATIC_SOURCE_DIR="${WORKSPACE}/swift-corelibs-libdispatch" LIBICU_SOURCE_DIR="${WORKSPACE}/icu" LIBCXX_SOURCE_DIR="${WORKSPACE}/llvm-project/runtimes" +LIBLZMA_SOURCE_DIR="${WORKSPACE}/liblzma" +ZLIB_SOURCE_DIR="${WORKSPACE}/zlib" +ZSTD_SOURCE_DIR="${WORKSPACE}/zstd/build/cmake" SWIFT_PATH_TO_STRING_PROCESSING_SOURCE="${WORKSPACE}/swift-experimental-string-processing" SWIFTSYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax" SWIFT_SYNTAX_SOURCE_DIR="${WORKSPACE}/swift-syntax" [[ "${SKIP_BUILD_LIBCXX}" ]] || PRODUCTS+=(libcxx) [[ "${SKIP_BUILD_LIBICU}" ]] || PRODUCTS+=(libicu) +[[ "${SKIP_BUILD_LIBLZMA}" ]] || PRODUCTS+=(liblzma) +[[ "${SKIP_BUILD_ZLIB}" ]] || PRODUCTS+=(zlib) +[[ "${SKIP_BUILD_ZSTD}" ]] || PRODUCTS+=(zstd) [[ "${SKIP_BUILD_SWIFT}" ]] || PRODUCTS+=(swift) [[ "${SKIP_BUILD_LLDB}" ]] || PRODUCTS+=(lldb) [[ "${SKIP_BUILD_LIBDISPATCH}" ]] || PRODUCTS+=(libdispatch) @@ -1300,6 +1308,15 @@ function build_directory_bin() { ;; libicu) ;; + liblzma) + echo "${root}/${LIBLZMA_BUILD_TYPE}/bin" + ;; + zlib) + echo "${root}/${ZLIB_BUILD_TYPE}/bin" + ;; + zstd) + echo "${root}/${ZSTD_BUILD_TYPE}/bin" + ;; *) echo "error: unknown product: ${product}" exit 1 @@ -1416,6 +1433,15 @@ function cmake_config_opt() { ;; libicu) ;; + liblzma) + echo "--config ${LIBLZMA_BUILD_TYPE}" + ;; + zlib) + echo "--config ${ZLIB_BUILD_TYPE}" + ;; + zstd) + echo "--config ${ZSTD_BUILD_TYPE}" + ;; *) echo "error: unknown product: ${product}" exit 1 @@ -1518,7 +1544,6 @@ function copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain() { # # Start with native deployment targets because the resulting tools are used during cross-compilation. - for host in "${ALL_HOSTS[@]}"; do # Skip this pass when the only action to execute can't match. if ! [[ $(should_execute_host_actions_for_phase ${host} build) ]]; then @@ -1532,14 +1557,20 @@ for host in "${ALL_HOSTS[@]}"; do # Don't echo anything if only executing an individual action. if [[ "${ONLY_EXECUTE}" = "all" ]]; then echo "Building the standard library for: ${SWIFT_STDLIB_TARGETS[@]}" + if [[ "${SWIFT_TEST_TARGETS[@]}" ]] && ! [[ "${SKIP_TEST_SWIFT}" ]]; then + echo "1" echo "Running Swift tests for: ${SWIFT_TEST_TARGETS[@]}" fi + if ! [[ "${SKIP_TEST_BENCHMARKS}" ]] && - [[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]]; then + [[ "${SWIFT_RUN_BENCHMARK_TARGETS[@]}" ]]; then + echo "3" echo "Running Swift benchmarks for: ${SWIFT_RUN_BENCHMARK_TARGETS[@]}" fi + if [ $(true_false "${BUILD_SWIFT_LIBEXEC}") == "TRUE" ]; then + echo "5" echo "Building the auxiliary executables for: ${SWIFT_LIBEXEC_TARGETS[@]}" fi fi @@ -2625,6 +2656,48 @@ for host in "${ALL_HOSTS[@]}"; do # libicu builds itself and doesn't use cmake continue ;; + liblzma) + if [[ "${host}" != "linux-"* ]]; then + break + fi + + LIBLZMA_BUILD_DIR=$(build_directory ${host} ${product}) + + cmake_options=( + -DCMAKE_BUILD_TYPE:STRING="${LIBLZMA_BUILD_TYPE}" + -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang" + -DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++" + -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})" + ) + ;; + zlib) + if [[ "${host}" != "linux-"* ]]; then + break + fi + + ZLIB_BUILD_DIR=$(build_directory ${host} ${product}) + + cmake_options=( + -DCMAKE_BUILD_TYPE:STRING="${ZLIB_BUILD_TYPE}" + -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang" + -DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++" + -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})" + ) + ;; + zstd) + if [[ "${host}" != "linux-"* ]]; then + break + fi + + ZSTD_BUILD_DIR=$(build_directory ${host} ${product}) + + cmake_options=( + -DCMAKE_BUILD_TYPE:STRING="${ZSTD_BUILD_TYPE}" + -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang" + -DCMAKE_CXX_COMPILER:PATHJ="${CLANG_BIN}/clang++" + -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})" + ) + ;; *) echo "error: unknown product: ${product}" exit 1 @@ -2952,6 +3025,12 @@ for host in "${ALL_HOSTS[@]}"; do echo "--- Finished tests for ${product} ---" continue ;; + liblzma) + ;; + zlib) + ;; + zstd) + ;; *) echo "error: unknown product: ${product}" exit 1 @@ -3149,6 +3228,12 @@ for host in "${ALL_HOSTS[@]}"; do call cp -a "${ICU_TMP_INSTALL_DIR}/share/icuswift" "${ICU_INSTALL_DIR}share" continue ;; + liblzma) + ;; + zlib) + ;; + zstd) + ;; *) echo "error: unknown product: ${product}" exit 1 diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 3fc18b3c281bc..5bbb075a4bb3a 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -104,6 +104,12 @@ def _apply_default_arguments(args): if args.zlib_build_variant is None: args.zlib_build_variant = args.build_variant + if args.zstd_build_variant is None: + args.zstd_build_variant = args.build_variant + + if args.liblzma_build_variant is None: + args.liblzma_build_variant = args.build_variant + if args.curl_build_variant is None: args.curl_build_variant = args.build_variant @@ -771,9 +777,12 @@ def create_argument_parser(): option('--static-zlib', toggle_true('build_zlib'), default=False, help='build static zlib') - option('--static-curl', toggle_true('build_curl'), default=False, help='build static curl libraries') + option('--static-liblzma', toggle_true('build_liblzma'), default=False, + help='build static liblzma') + option('--static-zstd', toggle_true('build_zstd'), default=False, + help='build static zstd') option('--playgroundsupport', toggle_true('build_playgroundsupport'), help='build PlaygroundSupport') @@ -903,6 +912,14 @@ def create_argument_parser(): const='Debug', help='build the Debug variant of libcurl') + option('--debug-zstd', store('zstd_build_variant'), + const='Debug', + help='build the Debug variant of zstd') + + option('--debug-liblzma', store('liblzma_build_variant'), + const='Debug', + help='build the Debug variant of liblzma') + # ------------------------------------------------------------------------- # Assertions group @@ -1374,6 +1391,10 @@ def create_argument_parser(): help='skip building zlib') option('--skip-build-curl', toggle_false('build_curl'), help='skip building curl') + option('--skip-build-liblzma', toggle_false('build_lzma'), + help='skip building liblzma') + option('--skip-build-zstd', toggle_false('build_zstd'), + help='skip building zstd') # We need to list --skip-test-swift explicitly because otherwise argparse # will auto-expand arguments like --skip-test-swift to the only known diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index f69e99b448b2c..b51fd3d8ca195 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -457,6 +457,8 @@ class BuildScriptImplOption(_BaseOption): SetOption('--debug-libicu', dest='libicu_build_variant', value='Debug'), SetOption('--debug-libxml2', dest='libxml2_build_variant', value='Debug'), SetOption('--debug-zlib', dest='zlib_build_variant', value='Debug'), + SetOption('--debug-zstd', dest='zstd_build_variant', value='Debug'), + SetOption('--debug-liblzma', dest='liblzma_build_variant', value='Debug'), SetOption('--debug-curl', dest='curl_build_variant', value='Debug'), SetOption('--debug-lldb', dest='lldb_build_variant', value='Debug'), SetOption('--lldb-build-with-xcode', dest='lldb_build_with_xcode', @@ -601,6 +603,8 @@ class BuildScriptImplOption(_BaseOption): EnableOption('--libicu', dest='build_libicu'), EnableOption('--static-libxml2', dest='build_libxml2'), EnableOption('--static-zlib', dest='build_zlib'), + EnableOption('--static-zstd', dest='build_zstd'), + EnableOption('--static-liblzma', dest='build_liblzma'), EnableOption('--static-curl', dest='build_curl'), EnableOption('--indexstore-db', dest='build_indexstoredb'), EnableOption('--test-indexstore-db-sanitize-all', @@ -717,6 +721,8 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-build-libxml2', dest='build_libxml2'), DisableOption('--skip-build-zlib', dest='build_zlib'), DisableOption('--skip-build-curl', dest='build_curl'), + DisableOption('--skip-build-zstd', dest='build_zstd'), + DisableOption('--skip-build-liblzma', dest='build_liblzma'), DisableOption('--skip-build-compiler-rt', dest='build_compiler_rt'), ChoicesOption('--compiler-vendor', diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 4d70897286f6a..760a76d7ba14b 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -259,6 +259,17 @@ def convert_to_impl_arguments(self): if args.build_early_swiftsyntax: impl_args += ["--swift-earlyswiftsyntax"] + # Add the paths of the compression libraries + zlib_src = os.path.join(self.workspace.source_root, "zlib") + zstd_src = os.path.join(self.workspace.source_root, "zstd") + liblzma_src = os.path.join(self.workspace.source_root, "liblzma") + + args.extra_cmake_options += [ + '-DSWIFT_PATH_TO_ZLIB_SOURCE:PATH={}'.format(zlib_src), + '-DSWIFT_PATH_TO_ZSTD_SOURCE:PATH={}'.format(zstd_src), + '-DSWIFT_PATH_TO_LIBLZMA_SOURCE:PATH={}'.format(liblzma_src), + ] + # Then add subproject install flags that either skip building them /or/ # if we are going to build them and install_all is set, we also install # them. @@ -274,7 +285,9 @@ def convert_to_impl_arguments(self): (args.build_libicu, "libicu"), (args.build_libxml2, 'libxml2'), (args.build_zlib, 'zlib'), - (args.build_curl, 'curl') + (args.build_curl, 'curl'), + (args.build_zstd, 'zstd'), + (args.build_liblzma, 'liblzma'), ] for (should_build, string_name) in conditional_subproject_configs: if not should_build and not self.args.infer_dependencies: @@ -600,6 +613,12 @@ def compute_product_pipelines(self): builder.add_product(products.curl.LibCurl, is_enabled=self.args.build_curl) + builder.add_product(products.Zstd, + is_enabled=self.args.build_zstd) + + builder.add_product(products.Liblzma, + is_enabled=self.args.build_liblzma) + # Begin a build-script-impl pipeline for handling the compiler toolchain # and a subset of the tools that we build. We build these in this manner # to preserve current build-script-impl run behavior as we transition diff --git a/utils/swift_build_support/swift_build_support/products/__init__.py b/utils/swift_build_support/swift_build_support/products/__init__.py index 40fd43960af40..1e2c94c2791bd 100644 --- a/utils/swift_build_support/swift_build_support/products/__init__.py +++ b/utils/swift_build_support/swift_build_support/products/__init__.py @@ -42,6 +42,8 @@ from .tsan_libdispatch import TSanLibDispatch from .xctest import XCTest from .zlib import Zlib +from .zstd import Zstd +from .liblzma import Liblzma __all__ = [ 'BackDeployConcurrency', @@ -75,5 +77,7 @@ 'Benchmarks', 'TSanLibDispatch', 'SwiftDocC', - 'SwiftDocCRender' + 'SwiftDocCRender', + 'Zstd', + 'Liblzma', ] diff --git a/utils/swift_build_support/swift_build_support/products/liblzma.py b/utils/swift_build_support/swift_build_support/products/liblzma.py new file mode 100644 index 0000000000000..a76b989ee1cc3 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/liblzma.py @@ -0,0 +1,114 @@ +# swift_build_support/products/liblzma.py ------------------------------------ +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from . import cmake_product +from . import earlyswiftdriver + + +class Liblzma(cmake_product.CMakeProduct): + @classmethod + def is_build_script_impl_product(cls): + """is_build_script_impl_product -> bool + + Whether this product is produced by build-script-impl + """ + return False + + @classmethod + def is_before_build_script_impl_product(cls): + """is_before_build_script_impl_product -> bool + + Whether this product is built before any build-script-impl products + """ + return True + + @classmethod + def is_nondarwin_only_build_product(cls): + return True + + @classmethod + def get_dependencies(cls): + return [earlyswiftdriver.EarlySwiftDriver] + + def should_build(self, host_target): + """should_build() -> Bool + + Return True if liblzma should be built + """ + return self.args.build_liblzma + + def should_test(self, host_target): + """should_test() -> Bool + + Returns True if liblzma should be tested. + Currently is set to false + """ + return False + + def should_install(self, host_target): + """should_install() -> Bool + + Returns True + If we're building liblzma, you're going to need it + """ + return self.args.build_liblzma + + def install(self, host_target): + """ + Install liblzma to the target location + """ + path = self.host_install_destdir(host_target) + self.install_with_cmake(['install'], path) + + # Remove the binaries (we don't want them and we can't turn them off) + try: + os.unlink(os.path.join(path, 'usr', 'bin', 'xz')) + os.unlink(os.path.join(path, 'usr', 'bin', 'xzdec')) + except FileNotFoundError: + pass + + try: + os.removedirs(os.path.join(path, 'usr', 'bin')) + except (FileNotFoundError, OSError): + pass + + # Also remove the man pages + try: + os.unlink(os.path.join(path, 'usr', 'share', 'man1', 'xz.1')) + os.unlink(os.path.join(path, 'usr', 'share', 'man1', 'xzdec.1')) + except FileNotFoundError: + pass + + try: + os.removedirs(os.path.join(path, 'usr', 'share', 'man1')) + except (FileNotFoundError, OSError): + pass + + def build(self, host_target): + self.cmake_options.define('BUILD_SHARED_LIBS', 'NO') + self.cmake_options.define('CREATE_XZ_SYMLINKS', 'NO') + self.cmake_options.define('CREATE_LZMA_SYMLINKS', 'NO') + self.cmake_options.define('CMAKE_POSITION_INDEPENDENT_CODE', 'YES') + + if self.args.liblzma_build_variant is None: + self.args.liblzma_build_variant = "Release" + self.cmake_options.define('CMAKE_BUILD_TYPE:STRING', + self.args.liblzma_build_variant) + self.cmake_options.define('CMAKE_BUILD_TYPE', 'RELEASE') + self.cmake_options.define('SKIP_INSTALL_FILES', 'YES') + self.cmake_options.define('CMAKE_INSTALL_PREFIX', '/usr') + self.cmake_options.define('BUILD_EXECUTABLES', 'NO') + + self.generate_toolchain_file_for_darwin_or_linux(host_target) + self.build_with_cmake(["liblzma"], self.args.liblzma_build_variant, []) diff --git a/utils/swift_build_support/swift_build_support/products/zlib.py b/utils/swift_build_support/swift_build_support/products/zlib.py index c19189456f622..5fd8f6abd0247 100644 --- a/utils/swift_build_support/swift_build_support/products/zlib.py +++ b/utils/swift_build_support/swift_build_support/products/zlib.py @@ -10,6 +10,8 @@ # # ---------------------------------------------------------------------------- +import os + from . import cmake_product from . import earlyswiftdriver @@ -69,6 +71,13 @@ def install(self, host_target): path = self.host_install_destdir(host_target) self.install_with_cmake(['install'], path) + # Remove the unwanted shared libraries + for (root, dirs, files) in os.walk(os.path.join(path, 'usr', 'lib')): + for file in files: + if file.startswith('libz.so') or (file.startswith('zlib') + and file.endswith('.dll')): + os.unlink(os.path.join(root, file)) + def build(self, host_target): self.cmake_options.define('BUILD_SHARED_LIBS', 'NO') self.cmake_options.define('CMAKE_POSITION_INDEPENDENT_CODE', 'YES') diff --git a/utils/swift_build_support/swift_build_support/products/zstd.py b/utils/swift_build_support/swift_build_support/products/zstd.py new file mode 100644 index 0000000000000..048daf6f09e71 --- /dev/null +++ b/utils/swift_build_support/swift_build_support/products/zstd.py @@ -0,0 +1,95 @@ +# swift_build_support/products/zstd.py ------------------------------------ +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See https://swift.org/LICENSE.txt for license information +# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +# +# ---------------------------------------------------------------------------- + +import os + +from . import cmake_product +from . import earlyswiftdriver + + +class Zstd(cmake_product.CMakeProduct): + def __init__(self, args, toolchain, source_dir, build_dir): + # We need to modify the source dir because the CMakeLists.txt is in + # a subdirectory + super().__init__(args, toolchain, + os.path.join(source_dir, 'build', 'cmake'), + build_dir) + + @classmethod + def is_build_script_impl_product(cls): + """is_build_script_impl_product -> bool + + Whether this product is produced by build-script-impl + """ + return False + + @classmethod + def is_before_build_script_impl_product(cls): + """is_before_build_script_impl_product -> bool + + Whether this product is built before any build-script-impl products + """ + return True + + @classmethod + def is_nondarwin_only_build_product(cls): + return True + + @classmethod + def get_dependencies(cls): + return [earlyswiftdriver.EarlySwiftDriver] + + def should_build(self, host_target): + """should_build() -> Bool + + Return True if zstd should be built + """ + return self.args.build_zstd + + def should_test(self, host_target): + """should_test() -> Bool + + Returns True if zstd should be tested. + Currently is set to false + """ + return False + + def should_install(self, host_target): + """should_install() -> Bool + + Returns True + If we're building zstd, you're going to need it + """ + return self.args.build_zstd + + def install(self, host_target): + """ + Install zstd to the target location + """ + path = self.host_install_destdir(host_target) + self.install_with_cmake(['install'], path) + + def build(self, host_target): + self.cmake_options.define('ZSTD_BUILD_SHARED', 'NO') + self.cmake_options.define('ZSTD_BUILD_STATIC', 'YES') + self.cmake_options.define('ZSTD_BUILD_PROGRAMS', 'NO') + self.cmake_options.define('CMAKE_POSITION_INDEPENDENT_CODE', 'YES') + + if self.args.zstd_build_variant is None: + self.args.zstd_build_variant = "Release" + self.cmake_options.define('CMAKE_BUILD_TYPE:STRING', + self.args.zstd_build_variant) + self.cmake_options.define('CMAKE_BUILD_TYPE', 'RELEASE') + self.cmake_options.define('CMAKE_INSTALL_PREFIX', '/usr') + + self.generate_toolchain_file_for_darwin_or_linux(host_target) + self.build_with_cmake(["all"], self.args.zstd_build_variant, []) diff --git a/utils/update_checkout/update-checkout-config.json b/utils/update_checkout/update-checkout-config.json index 9e8df6cf6abbb..ec5ed24b39263 100644 --- a/utils/update_checkout/update-checkout-config.json +++ b/utils/update_checkout/update-checkout-config.json @@ -86,7 +86,19 @@ "swift-llvm-bindings": { "remote": { "id": "apple/swift-llvm-bindings" } }, "llvm-project": { - "remote": { "id": "apple/llvm-project" } } + "remote": { "id": "apple/llvm-project" } }, + "liblzma": { + "remote": { "id": "tukaani-project/xz" }, + "platforms": [ "Linux" ] + }, + "zlib": { + "remote": { "id": "madler/zlib" }, + "platforms": [ "Linux" ] + }, + "zstd": { + "remote": { "id": "facebook/zstd" }, + "platforms": [ "Linux" ] + } }, "default-branch-scheme": "main", "branch-schemes": { @@ -131,7 +143,10 @@ "swift-markdown": "main", "swift-nio": "2.31.2", "swift-nio-ssl": "2.15.0", - "swift-experimental-string-processing": "swift/main" + "swift-experimental-string-processing": "swift/main", + "liblzma": "v5.4.2", + "zlib": "v1.2.13", + "zstd": "v1.5.4" } }, "rebranch": {