From 758665a3d751eb20b53c10fb98b79377a907a97a Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Mon, 6 May 2024 17:34:22 +0100 Subject: [PATCH 1/3] Merge pull request #71839 from al45tair/eng/PR-123503470 [Build] Add the new fully-static Linux SDK. --- CMakeLists.txt | 41 ++++++++- cmake/modules/SwiftConfigureSDK.cmake | 13 ++- stdlib/cmake/modules/AddSwiftStdlib.cmake | 88 +++++++++++++++---- .../linux-static/static-executable-args.lnk | 15 ++++ .../linux-static/static-stdlib-args.lnk | 9 ++ test/CMakeLists.txt | 10 +++ test/TypeRoundTrip/round-trip.swift | 3 +- test/lit.cfg | 4 + utils/build-script | 12 ++- utils/build-script-impl | 49 ++++++++++- .../build_swift/driver_arguments.py | 33 +++++++ utils/build_swift/tests/expected_options.py | 11 +++ .../build_script_invocation.py | 18 ++++ .../host_specific_configuration.py | 4 + .../products/cmake_product.py | 9 ++ .../swift_build_support/products/llvm.py | 47 +++++++++- .../swift_build_support/targets.py | 5 ++ 17 files changed, 341 insertions(+), 30 deletions(-) create mode 100644 stdlib/public/Resources/linux-static/static-executable-args.lnk create mode 100644 stdlib/public/Resources/linux-static/static-stdlib-args.lnk diff --git a/CMakeLists.txt b/CMakeLists.txt index e173b3a18df27..0895a8a778148 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -477,6 +477,21 @@ set(SWIFT_MIN_RUNTIME_VERSION "${DEFAULT_SWIFT_MIN_RUNTIME_VERSION}" CACHE STRIN the compiler itself. This is used on non-Darwin platforms to ensure \ that it's possible to build the compiler using host tools.") +# +# User-configurable Linux specific options. +# + +set(SWIFT_MUSL_PATH "/usr/local/musl" CACHE STRING + "Path to the directory that contains the Musl headers and libraries. \ +This is only required if we have been asked to build the Musl SDK, and \ +defaults to the default install location for Musl.") + +set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "" CACHE STRING + "The architectures to configure when using the static Linux SDK.") + +set(SWIFT_SDK_LINUX_ARCHITECTURES "" CACHE STRING + "The architectures to configure when using the Linux SDK.") + # # User-configurable Android specific options. # @@ -1078,12 +1093,26 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") set(SWIFT_HOST_VARIANT "linux" CACHE STRING "Deployment OS for Swift host tools (the compiler) [linux].") - # Should we build the standard library for the host? is_sdk_requested(LINUX swift_build_linux) if(swift_build_linux) - configure_sdk_unix("Linux" "${SWIFT_HOST_VARIANT_ARCH}") - set(SWIFT_PRIMARY_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") - set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") + if("${SWIFT_SDK_LINUX_ARCHITECTURES}" STREQUAL "") + set(SWIFT_SDK_LINUX_ARCHITECTURES "${SWIFT_HOST_VARIANT_ARCH}") + endif() + + configure_sdk_unix("Linux" "${SWIFT_SDK_LINUX_ARCHITECTURES}") + endif() + + is_sdk_requested(LINUX_STATIC swift_build_linux_static) + if(swift_build_linux_static) + if("${SWIFT_MUSL_PATH}" STREQUAL "") + message(FATAL_ERROR "You must set SWIFT_MUSL_PATH to point to the Musl libraries and headers. Specifically, we expect to find Musl at / for each requested architecture.") + endif() + + if("${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}" STREQUAL "") + set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "aarch64;x86_64") + endif() + + configure_sdk_unix("Linux_Static" "${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}") endif() is_sdk_requested(FREESTANDING swift_build_freestanding) @@ -1092,6 +1121,10 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") # configure_sdk_unix("FREESTANDING" "${SWIFT_HOST_VARIANT_ARCH}") endif() + # Default is Linux SDK for host + set(SWIFT_PRIMARY_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") + set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") + elseif("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "FREEBSD") set(SWIFT_HOST_VARIANT "freebsd" CACHE STRING diff --git a/cmake/modules/SwiftConfigureSDK.cmake b/cmake/modules/SwiftConfigureSDK.cmake index 56fa7d4b001ce..104c6f3c02cda 100644 --- a/cmake/modules/SwiftConfigureSDK.cmake +++ b/cmake/modules/SwiftConfigureSDK.cmake @@ -321,12 +321,20 @@ macro(configure_sdk_unix name architectures) # Static linking is suported on Linux and WASI if("${prefix}" STREQUAL "LINUX" + OR "${prefix}" STREQUAL "LINUX_STATIC" OR "${prefix}" STREQUAL "WASI") set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED TRUE) else() set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED FALSE) endif() + # For LINUX_STATIC, build static only + if("${prefix}" STREQUAL "LINUX_STATIC") + set(SWIFT_SDK_${prefix}_STATIC_ONLY TRUE) + else() + set(SWIFT_SDK_${prefix}_STATIC_ONLY FALSE) + endif() + # GCC on Linux is usually located under `/usr`. # However, Ubuntu 20.04 ships with another GCC installation under `/`, which # does not include libstdc++. Swift build scripts pass `--sysroot=/` to @@ -343,7 +351,7 @@ macro(configure_sdk_unix name architectures) CACHE STRING "Extra flags for compiling the C++ overlay") set(_default_threading_package "pthreads") - if("${prefix}" STREQUAL "LINUX") + if("${prefix}" STREQUAL "LINUX" OR "${prefix}" STREQUAL "LINUX_STATIC") set(_default_threading_package "linux") elseif("${prefix}" STREQUAL "WASI") if(SWIFT_ENABLE_WASI_THREADS) @@ -444,6 +452,9 @@ macro(configure_sdk_unix name architectures) else() set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi") endif() + elseif("${prefix}" STREQUAL "LINUX_STATIC") + set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_TRIPLE "${arch}-swift-linux-musl") + set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_PATH "${SWIFT_MUSL_PATH}/${arch}") else() message(FATAL_ERROR "unknown Unix OS: ${prefix}") endif() diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index d95ce397e7726..c3ad95c05c6a0 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -119,6 +119,12 @@ function(_add_target_variant_c_compile_link_flags) list(APPEND result "--sysroot=${_sysroot}") endif() + if("${CFLAGS_SDK}" STREQUAL "LINUX_STATIC") + list(APPEND result "-isystem" "${SWIFT_MUSL_PATH}/${CFLAGS_ARCH}/usr/include/c++/v1") + list(APPEND result "-DSWIFT_LIBC_IS_MUSL") + endif() + + if("${CFLAGS_SDK}" STREQUAL "ANDROID") # Make sure the Android NDK lld is used. swift_android_tools_path(${CFLAGS_ARCH} tools_path) @@ -512,6 +518,8 @@ function(_add_target_variant_link_flags) if("${LFLAGS_ARCH}" MATCHES "armv5|armv6|armv7|i686") list(APPEND link_libraries "atomic") endif() + elseif("${LFLAGS_SDK}" STREQUAL "LINUX_STATIC") + list(APPEND link_libraries "pthread" "dl") elseif("${LFLAGS_SDK}" STREQUAL "FREEBSD") list(APPEND link_libraries "pthread") elseif("${LFLAGS_SDK}" STREQUAL "OPENBSD") @@ -1352,10 +1360,14 @@ function(add_swift_target_library_single target name) endif() if(target_static) - _list_add_string_suffix( - "${SWIFTLIB_SINGLE_LINK_LIBRARIES}" - "-static" - target_static_depends) + set(target_static_depends) + foreach(dep ${SWIFTLIB_SINGLE_LINK_LIBRARIES}) + if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") + list(APPEND target_static_depends "${dep}-static") + endif() + endforeach() + + # FIXME: should this be target_link_libraries? add_dependencies_multiple_targets( TARGETS "${target_static}" @@ -1500,6 +1512,7 @@ function(add_swift_target_library_single target name) "SHELL:-Xclang --dependent-lib=msvcrt$<$:d>") endif() endif() + target_compile_options(${target} PRIVATE ${c_compile_flags}) target_link_options(${target} PRIVATE @@ -1869,6 +1882,7 @@ function(add_swift_target_library name) SWIFT_MODULE_DEPENDS_HAIKU SWIFT_MODULE_DEPENDS_IOS SWIFT_MODULE_DEPENDS_LINUX + SWIFT_MODULE_DEPENDS_LINUX_STATIC SWIFT_MODULE_DEPENDS_OSX SWIFT_MODULE_DEPENDS_TVOS SWIFT_MODULE_DEPENDS_WASI @@ -1915,6 +1929,17 @@ function(add_swift_target_library name) endif() list_replace(SWIFTLIB_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}") + # Support adding a "NOT" on the front to mean all SDKs except the following + list(GET SWIFTLIB_TARGET_SDKS 0 first_sdk) + if("${first_sdk}" STREQUAL "NOT") + list(REMOVE_AT SWIFTLIB_TARGET_SDKS 0) + list_subtract("${SWIFT_SDKS}" "${SWIFTLIB_TARGET_SDKS}" + "SWIFTLIB_TARGET_SDKS") + endif() + + list_intersect( + "${SWIFTLIB_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTLIB_TARGET_SDKS) + # All Swift code depends on the standard library, except for the standard # library itself. if(SWIFTLIB_HAS_SWIFT_CONTENT AND NOT SWIFTLIB_IS_STDLIB_CORE) @@ -2064,6 +2089,9 @@ function(add_swift_target_library name) elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX}) + elseif(sdk STREQUAL "LINUX_STATIC") + list(APPEND swiftlib_module_depends_flattened + ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX_STATIC}) elseif(sdk STREQUAL "CYGWIN") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_CYGWIN}) @@ -2319,12 +2347,21 @@ function(add_swift_target_library name) set(back_deployment_library_option) endif() + # If the SDK is static only, always build static instead of dynamic + if(SWIFT_SDK_${sdk}_STATIC_ONLY AND SWIFTLIB_SHARED) + set(shared_keyword) + set(static_keyword STATIC) + else() + set(shared_keyword ${SWIFTLIB_SHARED_keyword}) + set(static_keyword ${SWIFTLIB_STATIC_keyword}) + endif() + # Add this library variant. add_swift_target_library_single( ${variant_name} ${name} - ${SWIFTLIB_SHARED_keyword} - ${SWIFTLIB_STATIC_keyword} + ${shared_keyword} + ${static_keyword} ${SWIFTLIB_NO_LINK_NAME_keyword} ${SWIFTLIB_OBJECT_LIBRARY_keyword} ${SWIFTLIB_INSTALL_WITH_SHARED_keyword} @@ -2384,9 +2421,9 @@ function(add_swift_target_library name) if(NOT SWIFTLIB_OBJECT_LIBRARY) # Add dependencies on the (not-yet-created) custom lipo target. foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${DEP}" STREQUAL "icucore" AND - NOT "${DEP}" STREQUAL "dispatch" AND - NOT "${DEP}" STREQUAL "BlocksRuntime") + if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND + NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND + NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") add_dependencies(${VARIANT_NAME} "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}") endif() @@ -2395,9 +2432,9 @@ function(add_swift_target_library name) if (SWIFTLIB_IS_STDLIB AND SWIFTLIB_STATIC) # Add dependencies on the (not-yet-created) custom lipo target. foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${DEP}" STREQUAL "icucore" AND - NOT "${DEP}" STREQUAL "dispatch" AND - NOT "${DEP}" STREQUAL "BlocksRuntime") + if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND + NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND + NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") add_dependencies("${VARIANT_NAME}-static" "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") endif() @@ -2443,7 +2480,7 @@ function(add_swift_target_library name) if(NOT SWIFTLIB_OBJECT_LIBRARY) # Determine the name of the universal library. - if(SWIFTLIB_SHARED) + if(SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY) if("${sdk}" STREQUAL "WINDOWS") set(UNIVERSAL_LIBRARY_NAME "${SWIFTLIB_DIR}/${library_subdir}/${name}.dll") @@ -2455,12 +2492,18 @@ function(add_swift_target_library name) "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_SHARED_LIBRARY_PREFIX}${name}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() else() + if(SWIFTLIB_INSTALL_WITH_SHARED) + set(lib_dir "${SWIFTLIB_DIR}") + else() + set(lib_dir "${SWIFTSTATICLIB_DIR}") + endif() + if("${sdk}" STREQUAL "WINDOWS") set(UNIVERSAL_LIBRARY_NAME - "${SWIFTLIB_DIR}/${library_subdir}/${name}.lib") + "${lib_dir}/${library_subdir}/${name}.lib") else() set(UNIVERSAL_LIBRARY_NAME - "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}") + "${lib_dir}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() endif() @@ -2492,7 +2535,8 @@ function(add_swift_target_library name) precondition(resource_dir_sdk_subdir) - if(SWIFTLIB_SHARED OR SWIFTLIB_INSTALL_WITH_SHARED) + if((SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY) + OR SWIFTLIB_INSTALL_WITH_SHARED) set(resource_dir "swift") set(file_permissions OWNER_READ OWNER_WRITE OWNER_EXECUTE @@ -2868,6 +2912,7 @@ function(add_swift_target_executable name) SWIFT_MODULE_DEPENDS_HAIKU SWIFT_MODULE_DEPENDS_IOS SWIFT_MODULE_DEPENDS_LINUX + SWIFT_MODULE_DEPENDS_LINUX_STATIC SWIFT_MODULE_DEPENDS_OSX SWIFT_MODULE_DEPENDS_TVOS SWIFT_MODULE_DEPENDS_WASI @@ -2924,6 +2969,14 @@ function(add_swift_target_executable name) endif() list_replace(SWIFTEXE_TARGET_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}") + # Support adding a "NOT" on the front to mean all SDKs except the following + list(GET SWIFTEXE_TARGET_TARGET_SDKS 0 first_sdk) + if("${first_sdk}" STREQUAL "NOT") + list(REMOVE_AT SWIFTEXE_TARGET_TARGET_SDKS 0) + list_subtract("${SWIFT_SDKS}" "${SWIFTEXE_TARGET_TARGET_SDKS}" + "SWIFTEXE_TARGET_TARGET_SDKS") + endif() + list_intersect( "${SWIFTEXE_TARGET_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTEXE_TARGET_TARGET_SDKS) @@ -2965,6 +3018,9 @@ function(add_swift_target_executable name) elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID") list(APPEND swiftexe_module_depends_flattened ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX}) + elseif(sdk STREQUAL "LINUX_STATIC") + list(APPEND swiftexe_module_depends_flattened + ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX_STATIC}) elseif(sdk STREQUAL "CYGWIN") list(APPEND swiftexe_module_depends_flattened ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_CYGWIN}) diff --git a/stdlib/public/Resources/linux-static/static-executable-args.lnk b/stdlib/public/Resources/linux-static/static-executable-args.lnk new file mode 100644 index 0000000000000..5b8f1e9ca508c --- /dev/null +++ b/stdlib/public/Resources/linux-static/static-executable-args.lnk @@ -0,0 +1,15 @@ +-static +-lswiftCore +-lswift_RegexParser +-Xlinker +-undefined=pthread_self +-Xlinker +-undefined=pthread_once +-Xlinker +-undefined=pthread_key_create +-ldispatch +-lBlocksRuntime +-lpthread +-ldl +-lc++ +-lm diff --git a/stdlib/public/Resources/linux-static/static-stdlib-args.lnk b/stdlib/public/Resources/linux-static/static-stdlib-args.lnk new file mode 100644 index 0000000000000..d34341a999eed --- /dev/null +++ b/stdlib/public/Resources/linux-static/static-stdlib-args.lnk @@ -0,0 +1,9 @@ +-ldl +-lpthread +-lswiftCore +-ldispatch -lBlocksRuntime +-lc++ +-lm +-Xlinker -export-dynamic +-Xlinker --exclude-libs +-Xlinker ALL diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cf7886047cf4e..017c574f55183 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,6 +47,10 @@ function(get_test_dependencies SDK result_var_name) list(APPEND deps sdk-overlay) endif() + if(SWIFT_BUILD_REMOTE_MIRROR) + list(APPEND deps swiftRemoteInspection) + endif() + set(deps_binaries) if (SWIFT_INCLUDE_TOOLS) @@ -109,6 +113,7 @@ function(get_test_dependencies SDK result_var_name) ("${SDK}" STREQUAL "XROS_SIMULATOR") OR ("${SDK}" STREQUAL "FREESTANDING") OR ("${SDK}" STREQUAL "LINUX") OR + ("${SDK}" STREQUAL "LINUX_STATIC") OR ("${SDK}" STREQUAL "CYGWIN") OR ("${SDK}" STREQUAL "FREEBSD") OR ("${SDK}" STREQUAL "OPENBSD") OR @@ -214,6 +219,7 @@ normalize_boolean_spelling(SWIFT_STDLIB_ENABLE_OBJC_INTEROP) normalize_boolean_spelling(SWIFT_ENABLE_BACKTRACING) normalize_boolean_spelling(SWIFT_BUILD_SWIFT_SYNTAX) normalize_boolean_spelling(SWIFT_ENABLE_SYNCHRONIZATION) +normalize_boolean_spelling(SWIFT_BUILD_REMOTE_MIRROR) is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED) # Get 'SWIFT_HOST_SDKROOT' for lit.site.cfg.in @@ -423,6 +429,10 @@ foreach(SDK ${SWIFT_SDKS}) list(APPEND LIT_ARGS "--param" "synchronization") endif() + if(SWIFT_BUILD_REMOTE_MIRROR) + list(APPEND LIT_ARGS "--param" "remote_mirror") + endif() + list(APPEND LIT_ARGS "--param" "threading=${SWIFT_SDK_${SDK}_THREADING_PACKAGE}") # Enable on-crash backtracing if supported diff --git a/test/TypeRoundTrip/round-trip.swift b/test/TypeRoundTrip/round-trip.swift index 1cc6e568edd5b..bbea5c9881db0 100644 --- a/test/TypeRoundTrip/round-trip.swift +++ b/test/TypeRoundTrip/round-trip.swift @@ -6,12 +6,13 @@ // RUN: echo "func runAllTests() throws {" >> %t/all-tests.swift // RUN: for module in %S/Inputs/testcases/*.swift; do modname=$(basename $module .swift); %target-build-swift -g -static -emit-module-path %t/$modname.swiftmodule -emit-module -emit-library -module-name $modname -o %t/%target-static-library-name($modname) -I%t -L%t $module -lRoundTrip; echo " print(\"--- $modname\")" >> %t/all-tests.swift; echo " $modname.test()" >> %t/all-tests.swift; echo " print(\"\")" >> %t/all-tests.swift; echo "-l$modname" >> %t/link.txt; done // RUN: echo "}" >> %t/all-tests.swift -// RUN: %target-build-swift -g -I%t -o %t/round-trip %s %t/all-tests.swift -L%t %target-cxx-lib $(cat %t/link.txt) -lm -lRoundTrip -lswiftRemoteInspection +// RUN: %target-build-swift -g -I%t -o %t/round-trip %s %t/all-tests.swift -L%t %target-cxx-lib $(cat %t/link.txt) -lm -lRoundTrip -L%swift-lib-dir/swift/%target-sdk-name/%target-arch -lswiftRemoteInspection // RUN: %target-codesign %t/round-trip // RUN: %target-run %t/round-trip | %FileCheck %s // REQUIRES: executable_test // REQUIRES: shell +// REQUIRES: remote_mirror // UNSUPPORTED: use_os_stdlib // UNSUPPORTED: back_deployment_runtime diff --git a/test/lit.cfg b/test/lit.cfg index e8e91c1582eba..c327efdb9f7bd 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -532,6 +532,10 @@ synchronization = lit_config.params.get('synchronization', None) if synchronization is not None: config.available_features.add('synchronization') +remote_mirror = lit_config.params.get('remote_mirror', None) +if remote_mirror is not None: + config.available_features.add('remote_mirror') + test_options = os.environ.get('SWIFT_TEST_OPTIONS') if test_options: config.swift_test_options += ' ' diff --git a/utils/build-script b/utils/build-script index 529865512f780..4d39db779e210 100755 --- a/utils/build-script +++ b/utils/build-script @@ -278,12 +278,13 @@ def default_stdlib_deployment_targets(args): if host_target is None: return None + targets = [host_target] + # OS X build machines configure all Darwin platforms by default. # Put iOS native targets last so that we test them last # (it takes a long time). if host_target == StdlibDeploymentTarget.OSX.x86_64 or \ host_target == StdlibDeploymentTarget.OSX.arm64: - targets = [host_target] if args.build_ios and args.build_ios_simulator: targets.extend(StdlibDeploymentTarget.iOSSimulator.targets) if args.build_ios and args.build_ios_device: @@ -304,9 +305,12 @@ def default_stdlib_deployment_targets(args): args.build_xros and args.build_xros_device: targets.extend(StdlibDeploymentTarget.XROS.targets) return targets - else: - # All other machines only configure their host stdlib by default. - return [host_target] + elif host_target.platform.name == 'linux': + if args.build_linux_static: + targets.append(getattr(StdlibDeploymentTarget.Musl, + host_target.arch)) + + return targets def apply_default_arguments(toolchain, args): diff --git a/utils/build-script-impl b/utils/build-script-impl index 55954d26ab32d..75373fb674cfc 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -93,6 +93,12 @@ KNOWN_SETTINGS=( android-deploy-device-path "" "Path on an Android device to which built Swift stdlib products will be deployed" android-ndk "" "An absolute path to the NDK that will be used as a libc implementation for Android builds" + ## Linux Options + musl-path "/usr/local/musl" "The path to the Musl headers and libraries" + linux-archs "" "The Linux target architecture(s)" + linux-static-archs "" "The fully static Linux target architecture(s)" + build-linux-static "" "set to build Swift stdlibs for statically linked Linux" + ## Darwin Options darwin-crash-reporter-client "" "whether to enable CrashReporter integration, default is 1 on Darwin platforms, 0 otherwise" darwin-deployment-version-ios "11.0" "minimum deployment target version for iOS" @@ -130,6 +136,7 @@ KNOWN_SETTINGS=( ## Skip Build ... skip-build "" "set to configure as usual while skipping the build step" skip-build-android "" "set to skip building Swift stdlibs for Android" + skip-build-linux "" "set to skip building Swift stdlibs for Linux" skip-build-wasm "" "set to skip building Swift stdlibs for WebAssembly" skip-build-benchmarks "" "set to skip building Swift Benchmark Suite" skip-build-clang-tools-extra "" "set to skip building clang-tools-extra as part of llvm" @@ -486,6 +493,8 @@ function verify_host_is_supported() { | linux-powerpc64le \ | linux-riscv64 \ | linux-s390x \ + | linux-static-aarch64 \ + | linux-static-x86_64 \ | macosx-x86_64 \ | macosx-arm64 \ | macosx-arm64e \ @@ -1120,7 +1129,7 @@ function false_true() { CROSS_COMPILE_HOSTS=($CROSS_COMPILE_HOSTS) for t in "${CROSS_COMPILE_HOSTS[@]}"; do case ${t} in - macosx* | iphone* | appletv* | watch* | linux-armv5 | linux-armv6 | linux-armv7 | android-* | openbsd-* ) + macosx* | iphone* | appletv* | watch* | linux-armv5 | linux-armv6 | linux-armv7 | android-* | openbsd-* | linux-static-* ) ;; *) echo "Unknown host to cross-compile for: ${t}" @@ -1428,6 +1437,24 @@ function swift_c_flags() { if [[ $(is_cmake_release_build_type "${SWIFT_BUILD_TYPE}") ]] ; then echo -n " -fno-stack-protector" fi + + local host=$1 + + case $host in + linux-static-*) + echo -n " -D_GNU_SOURCE -DHAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME" + ;; + esac +} + +function maybe_lfts() { + local host=$1 + + case $host in + linux-static-*) + echo -n "-lfts" + ;; + esac } function common_swift_flags() { @@ -1710,6 +1737,21 @@ for host in "${ALL_HOSTS[@]}"; do ) fi + if [[ ! "${SKIP_BUILD_LINUX}" ]]; then + cmake_options=( + "${cmake_options[@]}" + -DSWIFT_SDK_LINUX_ARCHITECTURES:STRING="${LINUX_ARCHS}" + ) + fi + + if [[ "${BUILD_LINUX_STATIC}" ]]; then + cmake_options=( + "${cmake_options[@]}" + -DSWIFT_MUSL_PATH:STRING="${MUSL_PATH}" + -DSWIFT_SDK_LINUX_STATIC_ARCHITECTURES:STRING="${LINUX_STATIC_ARCHS}" + ) + fi + if [[ "${DARWIN_OVERLAY_TARGET}" != "" ]]; then # Split LOCAL_HOST into a pair ``arch-sdk`` # Example LOCAL_HOST: macosx-x86_64 @@ -2494,6 +2536,7 @@ for host in "${ALL_HOSTS[@]}"; do -DENABLE_TESTING:BOOL=NO -DBUILD_SHARED_LIBS=$([[ ${product} == foundation_static ]] && echo "NO" || echo "YES") + -DCMAKE_C_FLAGS="$(swift_c_flags ${host}) $(maybe_lfts ${host})" ) if [[ $(is_cross_tools_host ${host}) ]] ; then @@ -2537,9 +2580,11 @@ for host in "${ALL_HOSTS[@]}"; do cmake_options=( -DENABLE_SWIFT=YES ${cmake_options[@]} + -DCMAKE_BUILD_TYPE:STRING="${LIBDISPATCH_BUILD_TYPE}" -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang" -DCMAKE_CXX_COMPILER:PATH="${CLANG_BIN}/clang++" + -DCMAKE_C_FLAGS="$(swift_c_flags ${host})" -DCMAKE_SWIFT_COMPILER:PATH="${SWIFTC_BIN}" -DCMAKE_Swift_COMPILER:PATH="${SWIFTC_BIN}" -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)" @@ -3080,7 +3125,7 @@ for host in "${ALL_HOSTS[@]}"; do fi case ${host} in - linux-*|freebsd-*|openbsd-*|cygwin-*|haiku-*|android-*) ;; + linux-*|freebsd-*|openbsd-*|cygwin-*|haiku-*|android-*|linux-static-*) ;; *) echo "error: --install-xctest is not supported on this platform" exit 1 diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 37d82d8a48a6d..beb7d41a71464 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -56,6 +56,10 @@ def _apply_default_arguments(args): args.lldb_build_with_xcode is not None: args.build_lldb = True + # Build libc++ if fully static Linux was specified. + if args.build_linux_static and args.build_libcxx is None: + args.build_libcxx = True + # Set the default CMake generator. if args.cmake_generator is None: args.cmake_generator = 'Ninja' @@ -201,6 +205,7 @@ def _apply_default_arguments(args): # If none of tests specified skip swift stdlib test on all platforms if not args.test and not args.validation_test and not args.long_test: args.test_linux = False + args.test_linux_static = False args.test_freebsd = False args.test_cygwin = False args.test_osx = False @@ -1099,6 +1104,8 @@ def create_argument_parser(): help='skip testing Swift stdlibs for Mac OS X') option('--skip-test-linux', toggle_false('test_linux'), help='skip testing Swift stdlibs for Linux') + option('--skip-test-linux-static', toggle_false('test_linux_static'), + help='skip testing Swift stdlibs for fully static Linux') option('--skip-test-freebsd', toggle_false('test_freebsd'), help='skip testing Swift stdlibs for FreeBSD') option('--skip-test-cygwin', toggle_false('test_cygwin'), @@ -1151,8 +1158,12 @@ def create_argument_parser(): option(['-S', '--skip-build'], store_true, help='generate build directory only without building') + option('--build-linux-static', toggle_true, + help='build Swift stdlibs for fully static Linux') + option('--skip-build-linux', toggle_false('build_linux'), help='skip building Swift stdlibs for Linux') + option('--skip-build-freebsd', toggle_false('build_freebsd'), help='skip building Swift stdlibs for FreeBSD') option('--skip-build-cygwin', toggle_false('build_cygwin'), @@ -1385,6 +1396,28 @@ def create_argument_parser(): 'Currently, only armv7, aarch64, and x86_64 are supported. ' '%(default)s is the default.') + # ------------------------------------------------------------------------- + in_group('Build settings for Linux') + + option('--linux-archs', store, + type=argparse.ShellSplitType(), + default=None, + help='Comma-separated list of architectures to use when ' + 'building for Linux.') + + # ------------------------------------------------------------------------- + in_group('Build settings for fully static Linux') + + option('--musl-path', store_path, + default='/usr/local/musl', + help='The path to the Musl headers and libraries.') + + option('--linux-static-archs', store, + type=argparse.ShellSplitType(), + default=['x86_64', 'aarch64'], + help='Comma-separated list of architectures to use when ' + 'building for fully static Linux.') + # ------------------------------------------------------------------------- in_group('Experimental language features') diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index b29f52c2b77e0..8f074319cf739 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -72,6 +72,7 @@ 'build_llbuild': False, 'build_lldb': False, 'build_libcxx': False, + 'build_linux_static': False, 'build_ninja': False, 'build_lld': False, 'build_osx': True, @@ -212,6 +213,7 @@ 'libdispatch_build_variant': 'Debug', 'libicu_build_variant': 'Debug', 'libxml2_build_variant': 'Debug', + 'linux_archs': None, 'lit_jobs': multiprocessing.cpu_count(), 'zlib_build_variant': 'Debug', 'curl_build_variant': 'Debug', @@ -236,6 +238,8 @@ 'lto_type': None, 'maccatalyst': False, 'maccatalyst_ios_tests': False, + 'musl_path': '/usr/local/musl', + 'linux_static_archs': ['x86_64', 'aarch64'], 'native_clang_tools_path': None, 'native_llvm_tools_path': None, 'native_swift_tools_path': None, @@ -282,6 +286,7 @@ 'test_ios_host': False, 'test_ios_simulator': False, 'test_linux': False, + 'test_linux_static': False, 'test_optimize_for_size': None, 'test_optimize_none_with_implicit_dynamic': None, 'test_optimized': None, @@ -685,6 +690,7 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-build-ios-simulator', dest='build_ios_simulator'), DisableOption('--skip-build-linux', dest='build_linux'), + EnableOption('--build-linux-static'), DisableOption('--skip-build-osx', dest='build_osx'), DisableOption('--skip-build-tvos', dest='build_tvos'), DisableOption('--skip-build-tvos-device', dest='build_tvos_device'), @@ -716,6 +722,7 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-test-ios-host', dest='test_ios_host'), DisableOption('--skip-test-ios-simulator', dest='test_ios_simulator'), DisableOption('--skip-test-linux', dest='test_linux'), + DisableOption('--skip-test-linux-static', dest='test_linux_static'), DisableOption('--skip-test-osx', dest='test_osx'), DisableOption('--skip-test-tvos', dest='test_tvos'), DisableOption('--skip-test-tvos-host', dest='test_tvos_host'), @@ -776,6 +783,9 @@ class BuildScriptImplOption(_BaseOption): StrOption('--swift-darwin-supported-archs'), SetTrueOption('--swift-freestanding-is-darwin'), + StrOption('--linux-archs'), + StrOption('--linux-static-archs'), + PathOption('--android-deploy-device-path'), PathOption('--android-ndk'), PathOption('--build-subdir'), @@ -800,6 +810,7 @@ class BuildScriptImplOption(_BaseOption): PathOption('--cmake-cxx-launcher'), PathOption('--swift-profile-instr-use'), PathOption('--swift-runtime-fixed-backtracer-path'), + PathOption('--musl-path'), IntOption('--benchmark-num-o-iterations'), IntOption('--benchmark-num-onone-iterations'), diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index d61731aa4a281..96efa576c943b 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -522,6 +522,24 @@ def convert_to_impl_arguments(self): shlex.quote(opt) for opt in args.extra_dsymutil_args) ] + if args.musl_path: + impl_args += [ + "--musl-path=%s" % (args.musl_path, ) + ] + if args.linux_static_archs: + impl_args += [ + "--linux-static-archs=%s" % ';'.join(args.linux_static_archs) + ] + if args.linux_archs: + impl_args += [ + "--linux-archs=%s" % ';'.join(args.linux_archs) + ] + + if not args.build_linux: + impl_args += ["--skip-build-linux"] + if args.build_linux_static: + impl_args += ["--build-linux-static"] + # Compute the set of host-specific variables, which we pass through to # the build script via environment variables. host_specific_variables = self.compute_host_specific_variables() diff --git a/utils/swift_build_support/swift_build_support/host_specific_configuration.py b/utils/swift_build_support/swift_build_support/host_specific_configuration.py index cbb5b22e52b0c..ede1c81d74c26 100644 --- a/utils/swift_build_support/swift_build_support/host_specific_configuration.py +++ b/utils/swift_build_support/swift_build_support/host_specific_configuration.py @@ -230,6 +230,8 @@ def __platforms_to_skip_build(self, args, stage_dependent_args): platforms_to_skip_build = set() if not stage_dependent_args.build_linux: platforms_to_skip_build.add(StdlibDeploymentTarget.Linux) + if not stage_dependent_args.build_linux_static: + platforms_to_skip_build.add(StdlibDeploymentTarget.LinuxStatic) if not stage_dependent_args.build_freebsd: platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD) if not stage_dependent_args.build_cygwin: @@ -267,6 +269,8 @@ def __platforms_to_skip_test(self, args, stage_dependent_args): platforms_to_skip_test = set() if not stage_dependent_args.test_linux: platforms_to_skip_test.add(StdlibDeploymentTarget.Linux) + if not stage_dependent_args.test_linux_static: + platforms_to_skip_test.add(StdlibDeploymentTarget.LinuxStatic) if not stage_dependent_args.test_freebsd: platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD) if not stage_dependent_args.test_cygwin: diff --git a/utils/swift_build_support/swift_build_support/products/cmake_product.py b/utils/swift_build_support/swift_build_support/products/cmake_product.py index de148ec8c76e6..6b4eaca106265 100644 --- a/utils/swift_build_support/swift_build_support/products/cmake_product.py +++ b/utils/swift_build_support/swift_build_support/products/cmake_product.py @@ -198,6 +198,15 @@ def host_cmake_options(self, host_target): swift_host_triple = 'armv7-unknown-linux-gnueabihf' llvm_target_arch = 'ARM' + elif host_target.startswith('linux-static'): + + if host_target == 'linux-static-aarch64': + swift_host_triple = 'aarch64-swift-linux-musl' + llvm_target_arch = 'AArch64' + elif host_target == 'linux-static-x86_64': + swift_host_triple = 'x86_64-swift-linux-musl' + llvm_target_arch = 'X86' + elif host_target.startswith('macosx') or \ host_target.startswith('iphone') or \ host_target.startswith('appletv') or \ diff --git a/utils/swift_build_support/swift_build_support/products/llvm.py b/utils/swift_build_support/swift_build_support/products/llvm.py index 70a4bae3ffcaf..f9f51d802fd4f 100644 --- a/utils/swift_build_support/swift_build_support/products/llvm.py +++ b/utils/swift_build_support/swift_build_support/products/llvm.py @@ -199,6 +199,35 @@ def copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain( elif self.args.verbose_build: print('no file exists at {}'.format(host_lib_path)) + def install_static_linux_config(self, arch, bin_dir): + """Install the .cfg files to set the relevant Clang options for the + fully static Linux SDK's -swift-linux-musl triple. + + Doing it this way means it's easier to modify the defaults without + having to change the compiler driver.""" + + try: + os.makedirs(bin_dir) + except FileExistsError: + pass + + musl_cfg = os.path.join(bin_dir, f'{arch}-swift-linux-musl-clang.cfg') + with open(musl_cfg, "wt") as f: + f.write(f""" +-target {arch}-swift-linux-musl +-rtlib=compiler-rt +-stdlib=libc++ +-fuse-ld=lld +-unwindlib=libunwind +-lc++abi +-static + """) + for name in (f'{arch}-swift-linux-musl-clang++.cfg', ): + try: + os.symlink(musl_cfg, os.path.join(bin_dir, name)) + except FileExistsError: + pass + def should_build(self, host_target): """should_build() -> Bool @@ -351,6 +380,15 @@ def build(self, host_target): llvm_cmake_options.define('LLVM_INCLUDE_TESTS', 'NO') llvm_cmake_options.define('CLANG_INCLUDE_TESTS', 'NO') + build_root = os.path.dirname(self.build_dir) + host_machine_target = targets.StdlibDeploymentTarget.host_target().name + host_build_dir = os.path.join(build_root, 'llvm-{}'.format( + host_machine_target)) + + # Install config files for linux-static + bin_dir = os.path.join(host_build_dir, 'bin') + self.install_static_linux_config(arch, bin_dir) + if self.is_cross_compile_target(host_target): build_root = os.path.dirname(self.build_dir) host_machine_target = targets.StdlibDeploymentTarget.host_target().name @@ -485,8 +523,13 @@ def install(self, host_target): self.install_with_cmake(install_targets, host_install_destdir) + clang_dest_dir = '{}{}'.format(host_install_destdir, + self.args.install_prefix) + + bin_dir = os.path.join(clang_dest_dir, 'bin') + for arch in self.args.linux_static_archs: + self.install_static_linux_config(arch, bin_dir) + if self.args.llvm_install_components and system() == 'Darwin': - clang_dest_dir = '{}{}'.format(host_install_destdir, - self.args.install_prefix) self.copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain( clang_dest_dir) diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index 4124989e9bc48..9932b854cb692 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -288,6 +288,10 @@ class StdlibDeploymentTarget(object): FreeBSD = Platform("freebsd", archs=["x86_64", "arm64"]) + LinuxStatic = Platform('linux-static', sdk_name='LINUX_STATIC', archs=[ + 'x86_64', + 'aarch64']) + OpenBSD = OpenBSDPlatform("openbsd", archs=["amd64"]) Cygwin = Platform("cygwin", archs=["x86_64"]) @@ -309,6 +313,7 @@ class StdlibDeploymentTarget(object): XROS, XROSSimulator, Freestanding, Linux, + LinuxStatic, FreeBSD, OpenBSD, Cygwin, From 23e40a722bdb930d7dae33b678ecd3cfbaca2300 Mon Sep 17 00:00:00 2001 From: Yuta Saito Date: Tue, 7 May 2024 07:31:12 +0000 Subject: [PATCH 2/3] [Build] Repair the build for non-static-only static stdlib builds When SWIFT_BUILD_STATIC_STDLIB=ON, SWIFT_BUILD_DYNAMIC_STDLIB=OFF, and the sdk being built is not a static-only (e.g. WASI), the build fails due to duplicate custom command rules against the same output path. In the case of WASI, the static archive should be copied into lib/swift by the first lipo target, and then the second lipo target should copy it into lib/swift_static. ``` CMake Error at cmake/modules/SwiftAddCustomCommandTarget.cmake:144 (add_custom_command): Attempt to add a custom rule to output /home/build-user/build/buildbot_linux/wasmstdlib-linux-x86_64/lib/swift_static/wasi/libswiftCore.a.rule which already has a custom rule. Call Stack (most recent call first): stdlib/cmake/modules/AddSwiftStdlib.cmake:673 (add_custom_command_target) stdlib/cmake/modules/AddSwiftStdlib.cmake:2657 (_add_swift_lipo_target) stdlib/public/core/CMakeLists.txt:401 (add_swift_target_library) ``` --- stdlib/cmake/modules/AddSwiftStdlib.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index c3ad95c05c6a0..8bef94ac1529a 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -2492,10 +2492,10 @@ function(add_swift_target_library name) "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_SHARED_LIBRARY_PREFIX}${name}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() else() - if(SWIFTLIB_INSTALL_WITH_SHARED) - set(lib_dir "${SWIFTLIB_DIR}") - else() + if(SWIFT_SDK_${sdk}_STATIC_ONLY) set(lib_dir "${SWIFTSTATICLIB_DIR}") + else() + set(lib_dir "${SWIFTLIB_DIR}") endif() if("${sdk}" STREQUAL "WINDOWS") From f9cac0232285d75dae467c58ddde27e5822990c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Rodri=CC=81guez=20Troitin=CC=83o?= Date: Mon, 6 May 2024 17:04:10 -0700 Subject: [PATCH 3/3] [test][Linux] Improve Gold/LLD detection for function_sections.swift In #72061 the test was modified because LLD and Gold linker map formats are different, so they need different CHECKs. The original method in PR #72061 presents problems when the output is not exactly only one line. These changes use the first line of the linker map file to decide if the Gold or the LLD linker map checks should be used, since the format differs in both for the first line. This way of checking works in my Linux machine when using the default linker, when forcing Gold with `-use-ld=gold` and when forcing LLD with `-use-ld=lld`. --- test/LinkerSections/function_sections.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/LinkerSections/function_sections.swift b/test/LinkerSections/function_sections.swift index f941adec11a39..f7e9df726c05a 100644 --- a/test/LinkerSections/function_sections.swift +++ b/test/LinkerSections/function_sections.swift @@ -1,8 +1,9 @@ // REQUIRES: OS=linux-gnu || OS=freebsd // RUN: %empty-directory(%t) -// RUN: %target-build-swift -Xfrontend -function-sections -emit-module -emit-library -static -parse-stdlib %S/Inputs/FunctionSections.swift -// RUN: %target-build-swift -Xlinker -v -Xlinker --gc-sections -Xlinker -Map=%t/../../FunctionSections.map -I%t/../.. -L%t/../.. -lFunctionSections %S/Inputs/FunctionSectionsUse.swift 2>&1 | sed 's/.*\(gold\|LLD\).*/\1/g' | tr "[:lower:]" "[:upper:]" > %t/../../Linker.txt -// RUN: %FileCheck --check-prefix $(cat %t/../../Linker.txt) %s < %t/../../FunctionSections.map +// RUN: %target-build-swift -Xfrontend -function-sections -emit-module -emit-library -static -parse-stdlib %S/Inputs/FunctionSections.swift -o %t/libFunctionSections.a +// RUN: %target-build-swift -Xlinker --gc-sections -Xlinker -Map=%t/FunctionSections.map -I%t -L%t -lFunctionSections %S/Inputs/FunctionSectionsUse.swift +// RUN: if head -1 %t/FunctionSections.map | grep "Archive members"; then %FileCheck --check-prefix GOLD %s < %t/FunctionSections.map ; fi +// RUN: if head -1 %t/FunctionSections.map | grep "VMA"; then %FileCheck --check-prefix LLD %s < %t/FunctionSections.map ; fi // GOLD: Discarded input sections // GOLD: .text.$s16FunctionSections5func2yyF