From ae1c0455f38c78f14bedee443b6ff8620697fcd2 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Fri, 23 Feb 2024 12:56:08 +0000 Subject: [PATCH 1/9] [Build] Add the new fully-static Linux SDK. Declare a new `LINUX_STATIC` SDK and configure it. Add options to set the build architectures for the `LINUX` and `LINUX_STATIC` SDKs, similar to what we have for Darwin, because we'll be cross-compiling. Also add an option to point the build system at the sources for the musl C library, which we're using for `LINUX_STATIC`. rdar://123503470 --- CMakeLists.txt | 41 +++++++- cmake/modules/SwiftConfigureSDK.cmake | 13 ++- stdlib/cmake/modules/AddSwiftStdlib.cmake | 94 ++++++++++++++----- .../linux-static/static-executable-args.lnk | 15 +++ .../linux-static/static-stdlib-args.lnk | 9 ++ test/CMakeLists.txt | 1 + utils/build-script | 6 ++ utils/build-script-impl | 39 +++++++- .../build_swift/driver_arguments.py | 32 +++++++ utils/build_swift/tests/expected_options.py | 6 ++ .../build_script_invocation.py | 12 +++ .../host_specific_configuration.py | 4 + .../products/cmake_product.py | 9 ++ .../swift_build_support/products/llvm.py | 31 ++++++ .../swift_build_support/targets.py | 8 ++ 15 files changed, 293 insertions(+), 27 deletions(-) create mode 100644 stdlib/public/Resources/linux-static/static-executable-args.lnk create mode 100644 stdlib/public/Resources/linux-static/static-stdlib-args.lnk diff --git a/CMakeLists.txt b/CMakeLists.txt index 7876c3989077e..df92f43770d5e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -479,6 +479,21 @@ set(SWIFT_MIN_RUNTIME_VERSION "${DEFAULT_SWIFT_MIN_RUNTIME_VERSION}" CACHE STRIN the compiler itself. This is used on non-Darwin platforms to ensure \ that it's possible to build the compiler using host tools.") +# +# User-configurable Linux specific options. +# + +set(SWIFT_MUSL_PATH "/usr/local/musl" CACHE STRING + "Path to the directory that contains the Musl headers and libraries. \ +This is only required if we have been asked to build the Musl SDK, and \ +defaults to the default install location for Musl.") + +set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "" CACHE STRING + "The architectures to configure when using the static Linux SDK.") + +set(SWIFT_SDK_LINUX_ARCHITECTURES "" CACHE STRING + "The architectures to configure when using the Linux SDK.") + # # User-configurable Android specific options. # @@ -1095,9 +1110,25 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") # Should we build the standard library for the host? is_sdk_requested(LINUX swift_build_linux) if(swift_build_linux) - configure_sdk_unix("Linux" "${SWIFT_HOST_VARIANT_ARCH}") - set(SWIFT_PRIMARY_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") - set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") + if("${SWIFT_SDK_LINUX_ARCHITECTURES}" STREQUAL "") + set(SWIFT_SDK_LINUX_ARCHITECTURES "${SWIFT_HOST_VARIANT_ARCH}") + endif() + + configure_sdk_unix("Linux" "${SWIFT_SDK_LINUX_ARCHITECTURES}") + endif() + + # Should we build it for LINUX_STATIC? + is_sdk_requested(LINUX_STATIC swift_build_linux_static) + if(swift_build_linux_static) + if("${SWIFT_MUSL_PATH}" STREQUAL "") + message(FATAL_ERROR "You must set SWIFT_MUSL_PATH to point to the Musl libraries and headers. Specifically, we expect to find Musl at / for each requested architecture.") + endif() + + if("${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}" STREQUAL "") + set(SWIFT_SDK_LINUX_STATIC_ARCHITECTURES "aarch64;x86_64") + endif() + + configure_sdk_unix("Linux_Static" "${SWIFT_SDK_LINUX_STATIC_ARCHITECTURES}") endif() is_sdk_requested(FREESTANDING swift_build_freestanding) @@ -1106,6 +1137,10 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") # configure_sdk_unix("FREESTANDING" "${SWIFT_HOST_VARIANT_ARCH}") endif() + # Default is Linux SDK for host + set(SWIFT_PRIMARY_VARIANT_SDK_default "${SWIFT_HOST_VARIANT_SDK}") + set(SWIFT_PRIMARY_VARIANT_ARCH_default "${SWIFT_HOST_VARIANT_ARCH}") + elseif("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "FREEBSD") set(SWIFT_HOST_VARIANT "freebsd" CACHE STRING diff --git a/cmake/modules/SwiftConfigureSDK.cmake b/cmake/modules/SwiftConfigureSDK.cmake index d226ec8bf3574..69147e3ca449e 100644 --- a/cmake/modules/SwiftConfigureSDK.cmake +++ b/cmake/modules/SwiftConfigureSDK.cmake @@ -321,12 +321,20 @@ macro(configure_sdk_unix name architectures) # Static linking is suported on Linux and WASI if("${prefix}" STREQUAL "LINUX" + OR "${prefix}" STREQUAL "LINUX_STATIC" OR "${prefix}" STREQUAL "WASI") set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED TRUE) else() set(SWIFT_SDK_${prefix}_STATIC_LINKING_SUPPORTED FALSE) endif() + # For LINUX_STATIC, build static only + if("${prefix}" STREQUAL "LINUX_STATIC") + set(SWIFT_SDK_${prefix}_STATIC_ONLY TRUE) + else() + set(SWIFT_SDK_${prefix}_STATIC_ONLY FALSE) + endif() + # GCC on Linux is usually located under `/usr`. # However, Ubuntu 20.04 ships with another GCC installation under `/`, which # does not include libstdc++. Swift build scripts pass `--sysroot=/` to @@ -343,7 +351,7 @@ macro(configure_sdk_unix name architectures) CACHE STRING "Extra flags for compiling the C++ overlay") set(_default_threading_package "pthreads") - if("${prefix}" STREQUAL "LINUX") + if("${prefix}" STREQUAL "LINUX" OR "${prefix}" STREQUAL "LINUX_STATIC") set(_default_threading_package "linux") elseif("${prefix}" STREQUAL "WASI") if(SWIFT_ENABLE_WASI_THREADS) @@ -444,6 +452,9 @@ macro(configure_sdk_unix name architectures) else() set(SWIFT_SDK_WASI_ARCH_wasm32_TRIPLE "wasm32-unknown-wasi") endif() + elseif("${prefix}" STREQUAL "LINUX_STATIC") + set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_TRIPLE "${arch}-swift-linux-musl") + set(SWIFT_SDK_LINUX_STATIC_ARCH_${arch}_PATH "${SWIFT_MUSL_PATH}/${arch}") else() message(FATAL_ERROR "unknown Unix OS: ${prefix}") endif() diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index d95ce397e7726..19a737724665b 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -119,6 +119,12 @@ function(_add_target_variant_c_compile_link_flags) list(APPEND result "--sysroot=${_sysroot}") endif() + if("${CFLAGS_SDK}" STREQUAL "LINUX_STATIC") + list(APPEND result "-isystem" "${SWIFT_MUSL_PATH}/${CFLAGS_ARCH}/usr/include/c++/v1") + list(APPEND result "-DSWIFT_LIBC_IS_MUSL") + endif() + + if("${CFLAGS_SDK}" STREQUAL "ANDROID") # Make sure the Android NDK lld is used. swift_android_tools_path(${CFLAGS_ARCH} tools_path) @@ -512,6 +518,8 @@ function(_add_target_variant_link_flags) if("${LFLAGS_ARCH}" MATCHES "armv5|armv6|armv7|i686") list(APPEND link_libraries "atomic") endif() + elseif("${LFLAGS_SDK}" STREQUAL "LINUX_STATIC") + list(APPEND link_libraries "pthread" "dl") elseif("${LFLAGS_SDK}" STREQUAL "FREEBSD") list(APPEND link_libraries "pthread") elseif("${LFLAGS_SDK}" STREQUAL "OPENBSD") @@ -1352,10 +1360,14 @@ function(add_swift_target_library_single target name) endif() if(target_static) - _list_add_string_suffix( - "${SWIFTLIB_SINGLE_LINK_LIBRARIES}" - "-static" - target_static_depends) + set(target_static_depends) + foreach(dep ${SWIFTLIB_SINGLE_LINK_LIBRARIES}) + if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") + list(APPEND target_static_depends "${dep}-static") + endif() + endforeach() + + # FIXME: should this be target_link_libraries? add_dependencies_multiple_targets( TARGETS "${target_static}" @@ -1500,6 +1512,7 @@ function(add_swift_target_library_single target name) "SHELL:-Xclang --dependent-lib=msvcrt$<$:d>") endif() endif() + target_compile_options(${target} PRIVATE ${c_compile_flags}) target_link_options(${target} PRIVATE @@ -1869,6 +1882,7 @@ function(add_swift_target_library name) SWIFT_MODULE_DEPENDS_HAIKU SWIFT_MODULE_DEPENDS_IOS SWIFT_MODULE_DEPENDS_LINUX + SWIFT_MODULE_DEPENDS_LINUX_STATIC SWIFT_MODULE_DEPENDS_OSX SWIFT_MODULE_DEPENDS_TVOS SWIFT_MODULE_DEPENDS_WASI @@ -1915,6 +1929,17 @@ function(add_swift_target_library name) endif() list_replace(SWIFTLIB_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}") + # Support adding a "NOT" on the front to mean all SDKs except the following + list(GET SWIFTLIB_TARGET_SDKS 0 first_sdk) + if("${first_sdk}" STREQUAL "NOT") + list(REMOVE_AT SWIFTLIB_TARGET_SDKS 0) + list_subtract("${SWIFT_SDKS}" "${SWIFTLIB_TARGET_SDKS}" + "SWIFTLIB_TARGET_SDKS") + endif() + + list_intersect( + "${SWIFTLIB_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTLIB_TARGET_SDKS) + # All Swift code depends on the standard library, except for the standard # library itself. if(SWIFTLIB_HAS_SWIFT_CONTENT AND NOT SWIFTLIB_IS_STDLIB_CORE) @@ -2064,7 +2089,10 @@ function(add_swift_target_library name) elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX}) - elseif(sdk STREQUAL "CYGWIN") + elseif(${sdk} STREQUAL "LINUX_STATIC") + list(APPEND swiftlib_module_depends_flattened + ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX_STATIC}) + elseif(${sdk} STREQUAL "CYGWIN") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_CYGWIN}) elseif(sdk STREQUAL "HAIKU") @@ -2319,12 +2347,21 @@ function(add_swift_target_library name) set(back_deployment_library_option) endif() + # If the SDK is static only, always build static instead of dynamic + if(SWIFT_SDK_${sdk}_STATIC_ONLY AND SWIFTLIB_SHARED) + set(shared) + set(static STATIC) + else() + set(shared ${SWIFTLIB_SHARED_keyword}) + set(static ${SWIFTLIB_STATIC_keyword}) + endif() + # Add this library variant. add_swift_target_library_single( ${variant_name} ${name} - ${SWIFTLIB_SHARED_keyword} - ${SWIFTLIB_STATIC_keyword} + ${shared} + ${static} ${SWIFTLIB_NO_LINK_NAME_keyword} ${SWIFTLIB_OBJECT_LIBRARY_keyword} ${SWIFTLIB_INSTALL_WITH_SHARED_keyword} @@ -2383,23 +2420,19 @@ function(add_swift_target_library name) if(NOT SWIFTLIB_OBJECT_LIBRARY) # Add dependencies on the (not-yet-created) custom lipo target. - foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${DEP}" STREQUAL "icucore" AND - NOT "${DEP}" STREQUAL "dispatch" AND - NOT "${DEP}" STREQUAL "BlocksRuntime") + foreach(dep ${SWIFTLIB_LINK_LIBRARIES}) + if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") add_dependencies(${VARIANT_NAME} - "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}") + "${dep}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}") endif() endforeach() if (SWIFTLIB_IS_STDLIB AND SWIFTLIB_STATIC) # Add dependencies on the (not-yet-created) custom lipo target. - foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${DEP}" STREQUAL "icucore" AND - NOT "${DEP}" STREQUAL "dispatch" AND - NOT "${DEP}" STREQUAL "BlocksRuntime") + foreach(dep ${SWIFTLIB_LINK_LIBRARIES}) + if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") add_dependencies("${VARIANT_NAME}-static" - "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") + "${dep}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") endif() endforeach() endif() @@ -2443,7 +2476,7 @@ function(add_swift_target_library name) if(NOT SWIFTLIB_OBJECT_LIBRARY) # Determine the name of the universal library. - if(SWIFTLIB_SHARED) + if(SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY) if("${sdk}" STREQUAL "WINDOWS") set(UNIVERSAL_LIBRARY_NAME "${SWIFTLIB_DIR}/${library_subdir}/${name}.dll") @@ -2455,12 +2488,18 @@ function(add_swift_target_library name) "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_SHARED_LIBRARY_PREFIX}${name}${CMAKE_SHARED_LIBRARY_SUFFIX}") endif() else() + if(SWIFTLIB_INSTALL_WITH_SHARED) + set(lib_dir "${SWIFTLIB_DIR}") + else() + set(lib_dir "${SWIFTSTATICLIB_DIR}") + endif() + if("${sdk}" STREQUAL "WINDOWS") set(UNIVERSAL_LIBRARY_NAME - "${SWIFTLIB_DIR}/${library_subdir}/${name}.lib") + "${lib_dir}/${library_subdir}/${name}.lib") else() set(UNIVERSAL_LIBRARY_NAME - "${SWIFTLIB_DIR}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}") + "${lib_dir}/${library_subdir}/${CMAKE_STATIC_LIBRARY_PREFIX}${name}${CMAKE_STATIC_LIBRARY_SUFFIX}") endif() endif() @@ -2492,7 +2531,8 @@ function(add_swift_target_library name) precondition(resource_dir_sdk_subdir) - if(SWIFTLIB_SHARED OR SWIFTLIB_INSTALL_WITH_SHARED) + if((SWIFTLIB_SHARED AND NOT SWIFT_SDK_${sdk}_STATIC_ONLY) + OR SWIFTLIB_INSTALL_WITH_SHARED) set(resource_dir "swift") set(file_permissions OWNER_READ OWNER_WRITE OWNER_EXECUTE @@ -2868,6 +2908,7 @@ function(add_swift_target_executable name) SWIFT_MODULE_DEPENDS_HAIKU SWIFT_MODULE_DEPENDS_IOS SWIFT_MODULE_DEPENDS_LINUX + SWIFT_MODULE_DEPENDS_LINUX_STATIC SWIFT_MODULE_DEPENDS_OSX SWIFT_MODULE_DEPENDS_TVOS SWIFT_MODULE_DEPENDS_WASI @@ -2924,6 +2965,14 @@ function(add_swift_target_executable name) endif() list_replace(SWIFTEXE_TARGET_TARGET_SDKS ALL_APPLE_PLATFORMS "${SWIFT_DARWIN_PLATFORMS}") + # Support adding a "NOT" on the front to mean all SDKs except the following + list(GET SWIFTEXE_TARGET_TARGET_SDKS 0 first_sdk) + if("${first_sdk}" STREQUAL "NOT") + list(REMOVE_AT SWIFTEXE_TARGET_TARGET_SDKS 0) + list_subtract("${SWIFT_SDKS}" "${SWIFTEXE_TARGET_TARGET_SDKS}" + "SWIFTEXE_TARGET_TARGET_SDKS") + endif() + list_intersect( "${SWIFTEXE_TARGET_TARGET_SDKS}" "${SWIFT_SDKS}" SWIFTEXE_TARGET_TARGET_SDKS) @@ -2965,6 +3014,9 @@ function(add_swift_target_executable name) elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID") list(APPEND swiftexe_module_depends_flattened ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX}) + elseif(sdk STREQUAL "LINUX_STATIC") + list(APPEND swiftexe_module_depends_flattened + ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_LINUX_STATIC}) elseif(sdk STREQUAL "CYGWIN") list(APPEND swiftexe_module_depends_flattened ${SWIFTEXE_TARGET_SWIFT_MODULE_DEPENDS_CYGWIN}) diff --git a/stdlib/public/Resources/linux-static/static-executable-args.lnk b/stdlib/public/Resources/linux-static/static-executable-args.lnk new file mode 100644 index 0000000000000..5b8f1e9ca508c --- /dev/null +++ b/stdlib/public/Resources/linux-static/static-executable-args.lnk @@ -0,0 +1,15 @@ +-static +-lswiftCore +-lswift_RegexParser +-Xlinker +-undefined=pthread_self +-Xlinker +-undefined=pthread_once +-Xlinker +-undefined=pthread_key_create +-ldispatch +-lBlocksRuntime +-lpthread +-ldl +-lc++ +-lm diff --git a/stdlib/public/Resources/linux-static/static-stdlib-args.lnk b/stdlib/public/Resources/linux-static/static-stdlib-args.lnk new file mode 100644 index 0000000000000..d34341a999eed --- /dev/null +++ b/stdlib/public/Resources/linux-static/static-stdlib-args.lnk @@ -0,0 +1,9 @@ +-ldl +-lpthread +-lswiftCore +-ldispatch -lBlocksRuntime +-lc++ +-lm +-Xlinker -export-dynamic +-Xlinker --exclude-libs +-Xlinker ALL diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6f7726b23eec8..cd8fa93451cd5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -109,6 +109,7 @@ function(get_test_dependencies SDK result_var_name) ("${SDK}" STREQUAL "XROS_SIMULATOR") OR ("${SDK}" STREQUAL "FREESTANDING") OR ("${SDK}" STREQUAL "LINUX") OR + ("${SDK}" STREQUAL "LINUX_STATIC") OR ("${SDK}" STREQUAL "CYGWIN") OR ("${SDK}" STREQUAL "FREEBSD") OR ("${SDK}" STREQUAL "OPENBSD") OR diff --git a/utils/build-script b/utils/build-script index 529865512f780..bfe6c4c97257e 100755 --- a/utils/build-script +++ b/utils/build-script @@ -304,6 +304,12 @@ def default_stdlib_deployment_targets(args): args.build_xros and args.build_xros_device: targets.extend(StdlibDeploymentTarget.XROS.targets) return targets + elif host_target.platform.name == 'linux': + targets = [host_target] + if args.build_linux_static: + targets.append(getattr(StdlibDeploymentTarget.Musl, + host_target.arch)) + return targets else: # All other machines only configure their host stdlib by default. return [host_target] diff --git a/utils/build-script-impl b/utils/build-script-impl index 2d37e5738ccb9..f2fb5715aaa6d 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -93,6 +93,11 @@ KNOWN_SETTINGS=( android-deploy-device-path "" "Path on an Android device to which built Swift stdlib products will be deployed" android-ndk "" "An absolute path to the NDK that will be used as a libc implementation for Android builds" + ## Linux Options + musl-path "/usr/local/musl" "The path to the Musl headers and libraries" + linux-arch "" "The Linux target architecture(s)" + linux-static-arch "" "The fully static Linux target architecture(s)" + ## Darwin Options darwin-crash-reporter-client "" "whether to enable CrashReporter integration, default is 1 on Darwin platforms, 0 otherwise" darwin-deployment-version-ios "11.0" "minimum deployment target version for iOS" @@ -130,6 +135,8 @@ KNOWN_SETTINGS=( ## Skip Build ... skip-build "" "set to configure as usual while skipping the build step" skip-build-android "" "set to skip building Swift stdlibs for Android" + skip-build-linux "" "set to skip building Swift stdlibs for Linux" + skip-build-linux-static "" "set to skip building Swift stdlibs for statically linked Linux" skip-build-wasm "" "set to skip building Swift stdlibs for WebAssembly" skip-build-benchmarks "" "set to skip building Swift Benchmark Suite" skip-build-clang-tools-extra "" "set to skip building clang-tools-extra as part of llvm" @@ -485,6 +492,8 @@ function verify_host_is_supported() { | linux-powerpc64le \ | linux-riscv64 \ | linux-s390x \ + | linux-static-aarch64 \ + | linux-static-x86_64 \ | macosx-x86_64 \ | macosx-arm64 \ | macosx-arm64e \ @@ -1119,7 +1128,7 @@ function false_true() { CROSS_COMPILE_HOSTS=($CROSS_COMPILE_HOSTS) for t in "${CROSS_COMPILE_HOSTS[@]}"; do case ${t} in - macosx* | iphone* | appletv* | watch* | linux-armv5 | linux-armv6 | linux-armv7 | android-* | openbsd-* ) + macosx* | iphone* | appletv* | watch* | linux-armv5 | linux-armv6 | linux-armv7 | android-* | openbsd-* | linux-static-* ) ;; *) echo "Unknown host to cross-compile for: ${t}" @@ -1427,6 +1436,14 @@ function swift_c_flags() { if [[ $(is_cmake_release_build_type "${SWIFT_BUILD_TYPE}") ]] ; then echo -n " -fno-stack-protector" fi + + local host=$1 + + case $host in + linux-static-*) + echo -n " -D_GNU_SOURCE -DHAVE_DECL_PROGRAM_INVOCATION_SHORT_NAME" + ;; + esac } function common_swift_flags() { @@ -1709,6 +1726,21 @@ for host in "${ALL_HOSTS[@]}"; do ) fi + if [[ ! "${SKIP_BUILD_LINUX}" ]]; then + cmake_options=( + "${cmake_options[@]}" + -DSWIFT_SDK_LINUX_ARCHITECTURES:STRING="${LINUX_ARCH}" + ) + fi + + if [[ ! "${SKIP_BUILD_LINUX_STATIC}" ]]; then + cmake_options=( + "${cmake_options[@]}" + -DSWIFT_MUSL_PATH:STRING="${MUSL_PATH}" + -DSWIFT_SDK_LINUX_STATIC_ARCHITECTURES:STRING="${LINUX_STATIC_ARCH}" + ) + fi + if [[ "${DARWIN_OVERLAY_TARGET}" != "" ]]; then # Split LOCAL_HOST into a pair ``arch-sdk`` # Example LOCAL_HOST: macosx-x86_64 @@ -2492,6 +2524,7 @@ for host in "${ALL_HOSTS[@]}"; do -DENABLE_TESTING:BOOL=NO -DBUILD_SHARED_LIBS=$([[ ${product} == foundation_static ]] && echo "NO" || echo "YES") + -DCMAKE_C_FLAGS="$(swift_c_flags ${host}) -lfts" ) if [[ $(is_cross_tools_host ${host}) ]] ; then @@ -2535,9 +2568,11 @@ for host in "${ALL_HOSTS[@]}"; do cmake_options=( -DENABLE_SWIFT=YES ${cmake_options[@]} + -DCMAKE_BUILD_TYPE:STRING="${LIBDISPATCH_BUILD_TYPE}" -DCMAKE_C_COMPILER:PATH="${CLANG_BIN}/clang" -DCMAKE_CXX_COMPILER:PATH="${CLANG_BIN}/clang++" + -DCMAKE_C_FLAGS="$(swift_c_flags ${host})" -DCMAKE_SWIFT_COMPILER:PATH="${SWIFTC_BIN}" -DCMAKE_Swift_COMPILER:PATH="${SWIFTC_BIN}" -DCMAKE_Swift_FLAGS:STRING="$(common_swift_flags)" @@ -3078,7 +3113,7 @@ for host in "${ALL_HOSTS[@]}"; do fi case ${host} in - linux-*|freebsd-*|openbsd-*|cygwin-*|haiku-*|android-*) ;; + linux-*|freebsd-*|openbsd-*|cygwin-*|haiku-*|android-*|linux-static-*) ;; *) echo "error: --install-xctest is not supported on this platform" exit 1 diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index e0383d4214125..df4e85747a869 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -56,6 +56,10 @@ def _apply_default_arguments(args): args.lldb_build_with_xcode is not None: args.build_lldb = True + # Build libc++ if fully static Linux was specified. + if args.build_linux_static and args.build_libcxx is None: + args.build_libcxx = True + # Set the default CMake generator. if args.cmake_generator is None: args.cmake_generator = 'Ninja' @@ -1099,6 +1103,8 @@ def create_argument_parser(): help='skip testing Swift stdlibs for Mac OS X') option('--skip-test-linux', toggle_false('test_linux'), help='skip testing Swift stdlibs for Linux') + option('--skip-test-linux-static', toggle_false('test_linux_static'), + help='skip testing Swift stdlibs for fully static Linux') option('--skip-test-freebsd', toggle_false('test_freebsd'), help='skip testing Swift stdlibs for FreeBSD') option('--skip-test-cygwin', toggle_false('test_cygwin'), @@ -1147,8 +1153,12 @@ def create_argument_parser(): option(['-S', '--skip-build'], store_true, help='generate build directory only without building') + option('--build-linux-static', toggle_true, + help='build Swift stdlibs for fully static Linux') + option('--skip-build-linux', toggle_false('build_linux'), help='skip building Swift stdlibs for Linux') + option('--skip-build-freebsd', toggle_false('build_freebsd'), help='skip building Swift stdlibs for FreeBSD') option('--skip-build-cygwin', toggle_false('build_cygwin'), @@ -1381,6 +1391,28 @@ def create_argument_parser(): 'Currently, only armv7, aarch64, and x86_64 are supported. ' '%(default)s is the default.') + # ------------------------------------------------------------------------- + in_group('Build settings for Linux') + + option('--linux-arch', store, + type=argparse.ShellSplitType(), + default=None, + help='Semicolon-separated list of architectures to use when ' + 'building for Linux.') + + # ------------------------------------------------------------------------- + in_group('Build settings for fully static Linux') + + option('--musl-path', store_path, + default='/usr/local/musl', + help='The path to the Musl headers and libraries.') + + option('--linux-static-arch', store, + type=argparse.ShellSplitType(), + default=None, + help='Semicolon-separated list of architectures to use when ' + 'building for fully static Linux.') + # ------------------------------------------------------------------------- in_group('Experimental language features') diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 87af890bfd5b4..4200c2b2b8651 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -72,6 +72,7 @@ 'build_llbuild': False, 'build_lldb': False, 'build_libcxx': False, + 'build_linux_static': False, 'build_ninja': False, 'build_lld': False, 'build_osx': True, @@ -211,6 +212,7 @@ 'libdispatch_build_variant': 'Debug', 'libicu_build_variant': 'Debug', 'libxml2_build_variant': 'Debug', + 'linux_arch': None, 'lit_jobs': multiprocessing.cpu_count(), 'zlib_build_variant': 'Debug', 'curl_build_variant': 'Debug', @@ -235,6 +237,8 @@ 'lto_type': None, 'maccatalyst': False, 'maccatalyst_ios_tests': False, + 'musl_path': '/usr/local/musl', + 'linux_static_arch': None, 'native_clang_tools_path': None, 'native_llvm_tools_path': None, 'native_swift_tools_path': None, @@ -683,6 +687,7 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-build-ios-simulator', dest='build_ios_simulator'), DisableOption('--skip-build-linux', dest='build_linux'), + EnableOption('--build-linux-static'), DisableOption('--skip-build-osx', dest='build_osx'), DisableOption('--skip-build-tvos', dest='build_tvos'), DisableOption('--skip-build-tvos-device', dest='build_tvos_device'), @@ -714,6 +719,7 @@ class BuildScriptImplOption(_BaseOption): DisableOption('--skip-test-ios-host', dest='test_ios_host'), DisableOption('--skip-test-ios-simulator', dest='test_ios_simulator'), DisableOption('--skip-test-linux', dest='test_linux'), + DisableOption('--skip-test-linux-static', dest='test_linux_static'), DisableOption('--skip-test-osx', dest='test_osx'), DisableOption('--skip-test-tvos', dest='test_tvos'), DisableOption('--skip-test-tvos-host', dest='test_tvos_host'), diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 4b591c6806bec..6310f8ec96b70 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -518,6 +518,18 @@ def convert_to_impl_arguments(self): impl_args += [ "--extra-dsymutil-args=%s" % ' '.join( shlex.quote(opt) for opt in args.extra_dsymutil_args) + + if args.musl_path: + impl_args += [ + "--musl-path=%s" % (args.musl_path, ) + ] + if args.linux_static_arch: + impl_args += [ + "--linux-static-arch=%s" % ';'.join(args.linux_static_arch) + ] + if args.linux_arch: + impl_args += [ + "--linux-arch=%s" % ';'.join(args.linux_arch) ] # Compute the set of host-specific variables, which we pass through to diff --git a/utils/swift_build_support/swift_build_support/host_specific_configuration.py b/utils/swift_build_support/swift_build_support/host_specific_configuration.py index cbb5b22e52b0c..ede1c81d74c26 100644 --- a/utils/swift_build_support/swift_build_support/host_specific_configuration.py +++ b/utils/swift_build_support/swift_build_support/host_specific_configuration.py @@ -230,6 +230,8 @@ def __platforms_to_skip_build(self, args, stage_dependent_args): platforms_to_skip_build = set() if not stage_dependent_args.build_linux: platforms_to_skip_build.add(StdlibDeploymentTarget.Linux) + if not stage_dependent_args.build_linux_static: + platforms_to_skip_build.add(StdlibDeploymentTarget.LinuxStatic) if not stage_dependent_args.build_freebsd: platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD) if not stage_dependent_args.build_cygwin: @@ -267,6 +269,8 @@ def __platforms_to_skip_test(self, args, stage_dependent_args): platforms_to_skip_test = set() if not stage_dependent_args.test_linux: platforms_to_skip_test.add(StdlibDeploymentTarget.Linux) + if not stage_dependent_args.test_linux_static: + platforms_to_skip_test.add(StdlibDeploymentTarget.LinuxStatic) if not stage_dependent_args.test_freebsd: platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD) if not stage_dependent_args.test_cygwin: diff --git a/utils/swift_build_support/swift_build_support/products/cmake_product.py b/utils/swift_build_support/swift_build_support/products/cmake_product.py index de148ec8c76e6..6b4eaca106265 100644 --- a/utils/swift_build_support/swift_build_support/products/cmake_product.py +++ b/utils/swift_build_support/swift_build_support/products/cmake_product.py @@ -198,6 +198,15 @@ def host_cmake_options(self, host_target): swift_host_triple = 'armv7-unknown-linux-gnueabihf' llvm_target_arch = 'ARM' + elif host_target.startswith('linux-static'): + + if host_target == 'linux-static-aarch64': + swift_host_triple = 'aarch64-swift-linux-musl' + llvm_target_arch = 'AArch64' + elif host_target == 'linux-static-x86_64': + swift_host_triple = 'x86_64-swift-linux-musl' + llvm_target_arch = 'X86' + elif host_target.startswith('macosx') or \ host_target.startswith('iphone') or \ host_target.startswith('appletv') or \ diff --git a/utils/swift_build_support/swift_build_support/products/llvm.py b/utils/swift_build_support/swift_build_support/products/llvm.py index b961e427c99fc..1aab6a5dd5cb8 100644 --- a/utils/swift_build_support/swift_build_support/products/llvm.py +++ b/utils/swift_build_support/swift_build_support/products/llvm.py @@ -11,6 +11,7 @@ # ---------------------------------------------------------------------------- import os +import errno import shutil from platform import system @@ -347,6 +348,36 @@ def build(self, host_target): llvm_cmake_options.define('LLVM_INCLUDE_TESTS', 'NO') llvm_cmake_options.define('CLANG_INCLUDE_TESTS', 'NO') + build_root = os.path.dirname(self.build_dir) + host_machine_target = targets.StdlibDeploymentTarget.host_target().name + host_build_dir = os.path.join(build_root, 'llvm-{}'.format( + host_machine_target)) + + # Install config files for linux-static + bin_dir = os.path.join(host_build_dir, 'bin') + try: + os.makedirs(bin_dir) + except FileExistsError: + pass + + musl_cfg = os.path.join(bin_dir, f'{arch}-swift-linux-musl-clang.cfg') + with open(musl_cfg, "wt") as f: + f.write(f""" +-target {arch}-swift-linux-musl +-rtlib=compiler-rt +-stdlib=libc++ +-fuse-ld=lld +-unwindlib=libunwind +-lc++abi +-funwind-tables +-fasynchronous-unwind-tables + """) + for name in (f'{arch}-swift-linux-musl-clang++.cfg', ): + try: + os.symlink(musl_cfg, os.path.join(host_build_dir, 'bin', name)) + except FileExistsError: + pass + if self.is_cross_compile_target(host_target): build_root = os.path.dirname(self.build_dir) host_machine_target = targets.StdlibDeploymentTarget.host_target().name diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index 4124989e9bc48..4e233b0d467a9 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -8,8 +8,11 @@ # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +import atexit +import contextlib import os import platform +import importlib.resources from . import cmake from . import shell @@ -288,6 +291,10 @@ class StdlibDeploymentTarget(object): FreeBSD = Platform("freebsd", archs=["x86_64", "arm64"]) + LinuxStatic = Platform('linux-static', sdk_name='LINUX_STATIC', archs=[ + 'x86_64', + 'aarch64']) + OpenBSD = OpenBSDPlatform("openbsd", archs=["amd64"]) Cygwin = Platform("cygwin", archs=["x86_64"]) @@ -309,6 +316,7 @@ class StdlibDeploymentTarget(object): XROS, XROSSimulator, Freestanding, Linux, + LinuxStatic, FreeBSD, OpenBSD, Cygwin, From fdc8ffd36fa0a761252779bb1c14919f1255fdeb Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Thu, 2 May 2024 12:47:12 +0100 Subject: [PATCH 2/9] [Build] Updated following initial comments from Eric. Change the `--linux[-static]-arch` option to `--linux[-static]-archs`, on the basis that it supports multiple values. Other tidying. rdar://123503470 --- CMakeLists.txt | 2 -- stdlib/cmake/modules/AddSwiftStdlib.cmake | 32 +++++++++++-------- utils/build-script | 10 +++--- utils/build-script-impl | 8 ++--- .../build_swift/driver_arguments.py | 4 +-- utils/build_swift/tests/expected_options.py | 4 +-- .../build_script_invocation.py | 8 ++--- 7 files changed, 34 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index df92f43770d5e..cc2272b43d831 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1107,7 +1107,6 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") set(SWIFT_HOST_VARIANT "linux" CACHE STRING "Deployment OS for Swift host tools (the compiler) [linux].") - # Should we build the standard library for the host? is_sdk_requested(LINUX swift_build_linux) if(swift_build_linux) if("${SWIFT_SDK_LINUX_ARCHITECTURES}" STREQUAL "") @@ -1117,7 +1116,6 @@ if("${SWIFT_HOST_VARIANT_SDK}" STREQUAL "LINUX") configure_sdk_unix("Linux" "${SWIFT_SDK_LINUX_ARCHITECTURES}") endif() - # Should we build it for LINUX_STATIC? is_sdk_requested(LINUX_STATIC swift_build_linux_static) if(swift_build_linux_static) if("${SWIFT_MUSL_PATH}" STREQUAL "") diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index 19a737724665b..90c2a8c5dc83e 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -2089,10 +2089,10 @@ function(add_swift_target_library name) elseif(sdk STREQUAL "LINUX" OR sdk STREQUAL "ANDROID") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX}) - elseif(${sdk} STREQUAL "LINUX_STATIC") + elseif(sdk STREQUAL "LINUX_STATIC") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_LINUX_STATIC}) - elseif(${sdk} STREQUAL "CYGWIN") + elseif(sdk STREQUAL "CYGWIN") list(APPEND swiftlib_module_depends_flattened ${SWIFTLIB_SWIFT_MODULE_DEPENDS_CYGWIN}) elseif(sdk STREQUAL "HAIKU") @@ -2349,19 +2349,19 @@ function(add_swift_target_library name) # If the SDK is static only, always build static instead of dynamic if(SWIFT_SDK_${sdk}_STATIC_ONLY AND SWIFTLIB_SHARED) - set(shared) - set(static STATIC) + set(shared_keyword) + set(static_keyword STATIC) else() - set(shared ${SWIFTLIB_SHARED_keyword}) - set(static ${SWIFTLIB_STATIC_keyword}) + set(shared_keyword ${SWIFTLIB_SHARED_keyword}) + set(static_keyword ${SWIFTLIB_STATIC_keyword}) endif() # Add this library variant. add_swift_target_library_single( ${variant_name} ${name} - ${shared} - ${static} + ${shared_keyword} + ${static_keyword} ${SWIFTLIB_NO_LINK_NAME_keyword} ${SWIFTLIB_OBJECT_LIBRARY_keyword} ${SWIFTLIB_INSTALL_WITH_SHARED_keyword} @@ -2420,19 +2420,23 @@ function(add_swift_target_library name) if(NOT SWIFTLIB_OBJECT_LIBRARY) # Add dependencies on the (not-yet-created) custom lipo target. - foreach(dep ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") + foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) + if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND + NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND + NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") add_dependencies(${VARIANT_NAME} - "${dep}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}") + "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}") endif() endforeach() if (SWIFTLIB_IS_STDLIB AND SWIFTLIB_STATIC) # Add dependencies on the (not-yet-created) custom lipo target. - foreach(dep ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${dep}" MATCHES "^(icucore|dispatch|BlocksRuntime)($|-.*)$") + foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) + if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND + NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND + NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") add_dependencies("${VARIANT_NAME}-static" - "${dep}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") + "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") endif() endforeach() endif() diff --git a/utils/build-script b/utils/build-script index bfe6c4c97257e..4d39db779e210 100755 --- a/utils/build-script +++ b/utils/build-script @@ -278,12 +278,13 @@ def default_stdlib_deployment_targets(args): if host_target is None: return None + targets = [host_target] + # OS X build machines configure all Darwin platforms by default. # Put iOS native targets last so that we test them last # (it takes a long time). if host_target == StdlibDeploymentTarget.OSX.x86_64 or \ host_target == StdlibDeploymentTarget.OSX.arm64: - targets = [host_target] if args.build_ios and args.build_ios_simulator: targets.extend(StdlibDeploymentTarget.iOSSimulator.targets) if args.build_ios and args.build_ios_device: @@ -305,14 +306,11 @@ def default_stdlib_deployment_targets(args): targets.extend(StdlibDeploymentTarget.XROS.targets) return targets elif host_target.platform.name == 'linux': - targets = [host_target] if args.build_linux_static: targets.append(getattr(StdlibDeploymentTarget.Musl, host_target.arch)) - return targets - else: - # All other machines only configure their host stdlib by default. - return [host_target] + + return targets def apply_default_arguments(toolchain, args): diff --git a/utils/build-script-impl b/utils/build-script-impl index f2fb5715aaa6d..3f340a67ef15b 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -95,8 +95,8 @@ KNOWN_SETTINGS=( ## Linux Options musl-path "/usr/local/musl" "The path to the Musl headers and libraries" - linux-arch "" "The Linux target architecture(s)" - linux-static-arch "" "The fully static Linux target architecture(s)" + linux-archs "" "The Linux target architecture(s)" + linux-static-archs "" "The fully static Linux target architecture(s)" ## Darwin Options darwin-crash-reporter-client "" "whether to enable CrashReporter integration, default is 1 on Darwin platforms, 0 otherwise" @@ -1729,7 +1729,7 @@ for host in "${ALL_HOSTS[@]}"; do if [[ ! "${SKIP_BUILD_LINUX}" ]]; then cmake_options=( "${cmake_options[@]}" - -DSWIFT_SDK_LINUX_ARCHITECTURES:STRING="${LINUX_ARCH}" + -DSWIFT_SDK_LINUX_ARCHITECTURES:STRING="${LINUX_ARCHS}" ) fi @@ -1737,7 +1737,7 @@ for host in "${ALL_HOSTS[@]}"; do cmake_options=( "${cmake_options[@]}" -DSWIFT_MUSL_PATH:STRING="${MUSL_PATH}" - -DSWIFT_SDK_LINUX_STATIC_ARCHITECTURES:STRING="${LINUX_STATIC_ARCH}" + -DSWIFT_SDK_LINUX_STATIC_ARCHITECTURES:STRING="${LINUX_STATIC_ARCHS}" ) fi diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index df4e85747a869..36411705919bd 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -1394,7 +1394,7 @@ def create_argument_parser(): # ------------------------------------------------------------------------- in_group('Build settings for Linux') - option('--linux-arch', store, + option('--linux-archs', store, type=argparse.ShellSplitType(), default=None, help='Semicolon-separated list of architectures to use when ' @@ -1407,7 +1407,7 @@ def create_argument_parser(): default='/usr/local/musl', help='The path to the Musl headers and libraries.') - option('--linux-static-arch', store, + option('--linux-static-archs', store, type=argparse.ShellSplitType(), default=None, help='Semicolon-separated list of architectures to use when ' diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 4200c2b2b8651..4537614063125 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -212,7 +212,7 @@ 'libdispatch_build_variant': 'Debug', 'libicu_build_variant': 'Debug', 'libxml2_build_variant': 'Debug', - 'linux_arch': None, + 'linux_archs': None, 'lit_jobs': multiprocessing.cpu_count(), 'zlib_build_variant': 'Debug', 'curl_build_variant': 'Debug', @@ -238,7 +238,7 @@ 'maccatalyst': False, 'maccatalyst_ios_tests': False, 'musl_path': '/usr/local/musl', - 'linux_static_arch': None, + 'linux_static_archs': None, 'native_clang_tools_path': None, 'native_llvm_tools_path': None, 'native_swift_tools_path': None, diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 6310f8ec96b70..91ffd6f325bff 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -523,13 +523,13 @@ def convert_to_impl_arguments(self): impl_args += [ "--musl-path=%s" % (args.musl_path, ) ] - if args.linux_static_arch: + if args.linux_static_archs: impl_args += [ - "--linux-static-arch=%s" % ';'.join(args.linux_static_arch) + "--linux-static-archs=%s" % ';'.join(args.linux_static_archs) ] - if args.linux_arch: + if args.linux_archs: impl_args += [ - "--linux-arch=%s" % ';'.join(args.linux_arch) + "--linux-archs=%s" % ';'.join(args.linux_archs) ] # Compute the set of host-specific variables, which we pass through to From 29e38af1fe71cc22aa4e60394a13b338a088d9c4 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Thu, 2 May 2024 12:57:27 +0100 Subject: [PATCH 3/9] [Build] Fix a conflict resolution mistake. Apparently I accidentally removed a `]` while resolving a conflict in `build_script_invocation.py.` rdar://123503470 --- .../swift_build_support/build_script_invocation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index 91ffd6f325bff..d6d5d73056fe3 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -518,6 +518,7 @@ def convert_to_impl_arguments(self): impl_args += [ "--extra-dsymutil-args=%s" % ' '.join( shlex.quote(opt) for opt in args.extra_dsymutil_args) + ] if args.musl_path: impl_args += [ From 68598514c2f9fdfc627dd52cf440feeb2198e391 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Thu, 2 May 2024 14:46:11 +0100 Subject: [PATCH 4/9] [Build] Fix Python linting issues. The Python linter was complaining about import orders. rdar://123503470 --- .../swift_build_support/swift_build_support/products/llvm.py | 1 - utils/swift_build_support/swift_build_support/targets.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/products/llvm.py b/utils/swift_build_support/swift_build_support/products/llvm.py index 1aab6a5dd5cb8..06a01067984f2 100644 --- a/utils/swift_build_support/swift_build_support/products/llvm.py +++ b/utils/swift_build_support/swift_build_support/products/llvm.py @@ -11,7 +11,6 @@ # ---------------------------------------------------------------------------- import os -import errno import shutil from platform import system diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index 4e233b0d467a9..5510d2bc5fd2f 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -8,11 +8,9 @@ # See https://swift.org/LICENSE.txt for license information # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors -import atexit -import contextlib import os -import platform import importlib.resources +import platform from . import cmake from . import shell From bac73d012afb52d643827129244781d107ccdc04 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Thu, 2 May 2024 14:46:43 +0100 Subject: [PATCH 5/9] [Build] Add options to the Python build script tests. Apparently I'd failed to add a couple of lines here. rdar://123503470 --- utils/build_swift/tests/expected_options.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index 4537614063125..a1d1a8714d4d6 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -780,6 +780,9 @@ class BuildScriptImplOption(_BaseOption): StrOption('--swift-darwin-supported-archs'), SetTrueOption('--swift-freestanding-is-darwin'), + StrOption('--linux-archs'), + StrOption('--linux-static-archs'), + PathOption('--android-deploy-device-path'), PathOption('--android-ndk'), PathOption('--build-subdir'), @@ -804,6 +807,7 @@ class BuildScriptImplOption(_BaseOption): PathOption('--cmake-cxx-launcher'), PathOption('--swift-profile-instr-use'), PathOption('--swift-runtime-fixed-backtracer-path'), + PathOption('--musl-path'), IntOption('--benchmark-num-o-iterations'), IntOption('--benchmark-num-onone-iterations'), From 3627507eb1877f2aa5ba1ffdb3fced0ab4aab186 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Fri, 3 May 2024 09:12:49 +0100 Subject: [PATCH 6/9] [Build] Further static Linux SDK support changes. Fix some indentation issues. Change `build-script-impl` to make `build-linux-static` a positive argument. Fix documentation for `--linux-archs` and `--linux-static-archs` (the options are comma separated for `build-script`, but semicolon separated for `build-script-impl`). Set the default for `linux-static-archs` to `x86_64, aarch64` so that we install the expected content in the toolchain. Add missing default for `test_linux_static`. Make sure to pass down `--skip-build-linux` and `--build-linux-static`. Factor out config file generation and call it from the install step in `llvm.py` as well as from the build step. rdar://123503470 --- stdlib/cmake/modules/AddSwiftStdlib.cmake | 6 +- utils/build-script-impl | 4 +- .../build_swift/driver_arguments.py | 6 +- utils/build_swift/tests/expected_options.py | 3 +- .../build_script_invocation.py | 5 ++ .../swift_build_support/products/llvm.py | 61 +++++++++++-------- 6 files changed, 52 insertions(+), 33 deletions(-) diff --git a/stdlib/cmake/modules/AddSwiftStdlib.cmake b/stdlib/cmake/modules/AddSwiftStdlib.cmake index 90c2a8c5dc83e..c3ad95c05c6a0 100644 --- a/stdlib/cmake/modules/AddSwiftStdlib.cmake +++ b/stdlib/cmake/modules/AddSwiftStdlib.cmake @@ -2432,9 +2432,9 @@ function(add_swift_target_library name) if (SWIFTLIB_IS_STDLIB AND SWIFTLIB_STATIC) # Add dependencies on the (not-yet-created) custom lipo target. foreach(DEP ${SWIFTLIB_LINK_LIBRARIES}) - if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND - NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND - NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") + if (NOT "${DEP}" MATCHES "^icucore($|-.*)$" AND + NOT "${DEP}" MATCHES "^dispatch($|-.*)$" AND + NOT "${DEP}" MATCHES "^BlocksRuntime($|-.*)$") add_dependencies("${VARIANT_NAME}-static" "${DEP}-${SWIFT_SDK_${sdk}_LIB_SUBDIR}-static") endif() diff --git a/utils/build-script-impl b/utils/build-script-impl index 3f340a67ef15b..bc1c15a4728f2 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -97,6 +97,7 @@ KNOWN_SETTINGS=( musl-path "/usr/local/musl" "The path to the Musl headers and libraries" linux-archs "" "The Linux target architecture(s)" linux-static-archs "" "The fully static Linux target architecture(s)" + build-linux-static "" "set to build Swift stdlibs for statically linked Linux" ## Darwin Options darwin-crash-reporter-client "" "whether to enable CrashReporter integration, default is 1 on Darwin platforms, 0 otherwise" @@ -136,7 +137,6 @@ KNOWN_SETTINGS=( skip-build "" "set to configure as usual while skipping the build step" skip-build-android "" "set to skip building Swift stdlibs for Android" skip-build-linux "" "set to skip building Swift stdlibs for Linux" - skip-build-linux-static "" "set to skip building Swift stdlibs for statically linked Linux" skip-build-wasm "" "set to skip building Swift stdlibs for WebAssembly" skip-build-benchmarks "" "set to skip building Swift Benchmark Suite" skip-build-clang-tools-extra "" "set to skip building clang-tools-extra as part of llvm" @@ -1733,7 +1733,7 @@ for host in "${ALL_HOSTS[@]}"; do ) fi - if [[ ! "${SKIP_BUILD_LINUX_STATIC}" ]]; then + if [[ "${BUILD_LINUX_STATIC}" ]]; then cmake_options=( "${cmake_options[@]}" -DSWIFT_MUSL_PATH:STRING="${MUSL_PATH}" diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index 36411705919bd..f2f285a85d51b 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -1397,7 +1397,7 @@ def create_argument_parser(): option('--linux-archs', store, type=argparse.ShellSplitType(), default=None, - help='Semicolon-separated list of architectures to use when ' + help='Comma-separated list of architectures to use when ' 'building for Linux.') # ------------------------------------------------------------------------- @@ -1409,8 +1409,8 @@ def create_argument_parser(): option('--linux-static-archs', store, type=argparse.ShellSplitType(), - default=None, - help='Semicolon-separated list of architectures to use when ' + default=['x86_64', 'aarch64'], + help='Comma-separated list of architectures to use when ' 'building for fully static Linux.') # ------------------------------------------------------------------------- diff --git a/utils/build_swift/tests/expected_options.py b/utils/build_swift/tests/expected_options.py index a1d1a8714d4d6..8e360ca20601b 100644 --- a/utils/build_swift/tests/expected_options.py +++ b/utils/build_swift/tests/expected_options.py @@ -238,7 +238,7 @@ 'maccatalyst': False, 'maccatalyst_ios_tests': False, 'musl_path': '/usr/local/musl', - 'linux_static_archs': None, + 'linux_static_archs': ['x86_64', 'aarch64'], 'native_clang_tools_path': None, 'native_llvm_tools_path': None, 'native_swift_tools_path': None, @@ -285,6 +285,7 @@ 'test_ios_host': False, 'test_ios_simulator': False, 'test_linux': False, + 'test_linux_static': False, 'test_optimize_for_size': None, 'test_optimize_none_with_implicit_dynamic': None, 'test_optimized': None, diff --git a/utils/swift_build_support/swift_build_support/build_script_invocation.py b/utils/swift_build_support/swift_build_support/build_script_invocation.py index d6d5d73056fe3..c8111bc6ad1ca 100644 --- a/utils/swift_build_support/swift_build_support/build_script_invocation.py +++ b/utils/swift_build_support/swift_build_support/build_script_invocation.py @@ -533,6 +533,11 @@ def convert_to_impl_arguments(self): "--linux-archs=%s" % ';'.join(args.linux_archs) ] + if not args.build_linux: + impl_args += ["--skip-build-linux"] + if args.build_linux_static: + impl_args += ["--build-linux-static"] + # Compute the set of host-specific variables, which we pass through to # the build script via environment variables. host_specific_variables = self.compute_host_specific_variables() diff --git a/utils/swift_build_support/swift_build_support/products/llvm.py b/utils/swift_build_support/swift_build_support/products/llvm.py index 06a01067984f2..96ff2e7fb5212 100644 --- a/utils/swift_build_support/swift_build_support/products/llvm.py +++ b/utils/swift_build_support/swift_build_support/products/llvm.py @@ -199,6 +199,35 @@ def copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain( elif self.args.verbose_build: print('no file exists at {}'.format(host_lib_path)) + def install_static_linux_config(self, arch, bin_dir): + """Install the .cfg files to set the relevant Clang options for the + fully static Linux SDK's -swift-linux-musl triple. + + Doing it this way means it's easier to modify the defaults without + having to change the compiler driver.""" + + try: + os.makedirs(bin_dir) + except FileExistsError: + pass + + musl_cfg = os.path.join(bin_dir, f'{arch}-swift-linux-musl-clang.cfg') + with open(musl_cfg, "wt") as f: + f.write(f""" +-target {arch}-swift-linux-musl +-rtlib=compiler-rt +-stdlib=libc++ +-fuse-ld=lld +-unwindlib=libunwind +-lc++abi +-static + """) + for name in (f'{arch}-swift-linux-musl-clang++.cfg', ): + try: + os.symlink(musl_cfg, os.path.join(bin_dir, name)) + except FileExistsError: + pass + def should_build(self, host_target): """should_build() -> Bool @@ -354,28 +383,7 @@ def build(self, host_target): # Install config files for linux-static bin_dir = os.path.join(host_build_dir, 'bin') - try: - os.makedirs(bin_dir) - except FileExistsError: - pass - - musl_cfg = os.path.join(bin_dir, f'{arch}-swift-linux-musl-clang.cfg') - with open(musl_cfg, "wt") as f: - f.write(f""" --target {arch}-swift-linux-musl --rtlib=compiler-rt --stdlib=libc++ --fuse-ld=lld --unwindlib=libunwind --lc++abi --funwind-tables --fasynchronous-unwind-tables - """) - for name in (f'{arch}-swift-linux-musl-clang++.cfg', ): - try: - os.symlink(musl_cfg, os.path.join(host_build_dir, 'bin', name)) - except FileExistsError: - pass + self.install_static_linux_config(arch, bin_dir) if self.is_cross_compile_target(host_target): build_root = os.path.dirname(self.build_dir) @@ -511,8 +519,13 @@ def install(self, host_target): self.install_with_cmake(install_targets, host_install_destdir) + clang_dest_dir = '{}{}'.format(host_install_destdir, + self.args.install_prefix) + + bin_dir = os.path.join(clang_dest_dir, 'bin') + for arch in self.args.linux_static_archs: + self.install_static_linux_config(arch, bin_dir) + if self.args.llvm_install_components and system() == 'Darwin': - clang_dest_dir = '{}{}'.format(host_install_destdir, - self.args.install_prefix) self.copy_embedded_compiler_rt_builtins_from_darwin_host_toolchain( clang_dest_dir) From 62c80a1073852e37096b2771c16c7dccfa33d8ae Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Fri, 3 May 2024 10:46:22 +0100 Subject: [PATCH 7/9] [Build] A couple more Python fixes. Remove unused import of `importlib.resources`. Set `test_linux_static` to `False` if tests are disabled. rdar://123503470 --- utils/build_swift/build_swift/driver_arguments.py | 1 + utils/swift_build_support/swift_build_support/targets.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build_swift/build_swift/driver_arguments.py b/utils/build_swift/build_swift/driver_arguments.py index f2f285a85d51b..446fe33009965 100644 --- a/utils/build_swift/build_swift/driver_arguments.py +++ b/utils/build_swift/build_swift/driver_arguments.py @@ -205,6 +205,7 @@ def _apply_default_arguments(args): # If none of tests specified skip swift stdlib test on all platforms if not args.test and not args.validation_test and not args.long_test: args.test_linux = False + args.test_linux_static = False args.test_freebsd = False args.test_cygwin = False args.test_osx = False diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index 5510d2bc5fd2f..9932b854cb692 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -9,7 +9,6 @@ # See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors import os -import importlib.resources import platform from . import cmake From 2297c3f5b6f6351b764bb2555473fa35f79bed0a Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Sun, 5 May 2024 20:50:41 +0100 Subject: [PATCH 8/9] [Build][Test] Fix TypeRoundTrip/round-trip.swift. `TypeRoundTrip/round-trip.swift` tries to link with `libswiftRemoteInspection`, but that library isn't in the default link path, so we need to add a `-L` option to the command line in the test. Also, we should check that `SWIFT_BUILD_REMOTE_MIRROR` is enabled and disable the test if not, because otherwise `libswiftRemoteInspection` won't have been built. rdar://123503470 --- test/CMakeLists.txt | 9 +++++++++ test/TypeRoundTrip/round-trip.swift | 3 ++- test/lit.cfg | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cd8fa93451cd5..3732b2c2f7c7c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -47,6 +47,10 @@ function(get_test_dependencies SDK result_var_name) list(APPEND deps sdk-overlay) endif() + if(SWIFT_BUILD_REMOTE_MIRROR) + list(APPEND deps swiftRemoteInspection) + endif() + set(deps_binaries) if (SWIFT_INCLUDE_TOOLS) @@ -215,6 +219,7 @@ normalize_boolean_spelling(SWIFT_STDLIB_ENABLE_OBJC_INTEROP) normalize_boolean_spelling(SWIFT_ENABLE_BACKTRACING) normalize_boolean_spelling(SWIFT_BUILD_SWIFT_SYNTAX) normalize_boolean_spelling(SWIFT_ENABLE_SYNCHRONIZATION) +normalize_boolean_spelling(SWIFT_BUILD_REMOTE_MIRROR) is_build_type_optimized("${SWIFT_STDLIB_BUILD_TYPE}" SWIFT_OPTIMIZED) # Get 'SWIFT_HOST_SDKROOT' for lit.site.cfg.in @@ -432,6 +437,10 @@ foreach(SDK ${SWIFT_SDKS}) list(APPEND LIT_ARGS "--param" "synchronization") endif() + if(SWIFT_BUILD_REMOTE_MIRROR) + list(APPEND LIT_ARGS "--param" "remote_mirror") + endif() + list(APPEND LIT_ARGS "--param" "threading=${SWIFT_SDK_${SDK}_THREADING_PACKAGE}") # Enable on-crash backtracing if supported diff --git a/test/TypeRoundTrip/round-trip.swift b/test/TypeRoundTrip/round-trip.swift index 1cc6e568edd5b..bbea5c9881db0 100644 --- a/test/TypeRoundTrip/round-trip.swift +++ b/test/TypeRoundTrip/round-trip.swift @@ -6,12 +6,13 @@ // RUN: echo "func runAllTests() throws {" >> %t/all-tests.swift // RUN: for module in %S/Inputs/testcases/*.swift; do modname=$(basename $module .swift); %target-build-swift -g -static -emit-module-path %t/$modname.swiftmodule -emit-module -emit-library -module-name $modname -o %t/%target-static-library-name($modname) -I%t -L%t $module -lRoundTrip; echo " print(\"--- $modname\")" >> %t/all-tests.swift; echo " $modname.test()" >> %t/all-tests.swift; echo " print(\"\")" >> %t/all-tests.swift; echo "-l$modname" >> %t/link.txt; done // RUN: echo "}" >> %t/all-tests.swift -// RUN: %target-build-swift -g -I%t -o %t/round-trip %s %t/all-tests.swift -L%t %target-cxx-lib $(cat %t/link.txt) -lm -lRoundTrip -lswiftRemoteInspection +// RUN: %target-build-swift -g -I%t -o %t/round-trip %s %t/all-tests.swift -L%t %target-cxx-lib $(cat %t/link.txt) -lm -lRoundTrip -L%swift-lib-dir/swift/%target-sdk-name/%target-arch -lswiftRemoteInspection // RUN: %target-codesign %t/round-trip // RUN: %target-run %t/round-trip | %FileCheck %s // REQUIRES: executable_test // REQUIRES: shell +// REQUIRES: remote_mirror // UNSUPPORTED: use_os_stdlib // UNSUPPORTED: back_deployment_runtime diff --git a/test/lit.cfg b/test/lit.cfg index c372541a50477..16ecf96b0d110 100644 --- a/test/lit.cfg +++ b/test/lit.cfg @@ -537,6 +537,10 @@ synchronization = lit_config.params.get('synchronization', None) if synchronization is not None: config.available_features.add('synchronization') +remote_mirror = lit_config.params.get('remote_mirror', None) +if remote_mirror is not None: + config.available_features.add('remote_mirror') + test_options = os.environ.get('SWIFT_TEST_OPTIONS') if test_options: config.swift_test_options += ' ' From 7431b96798a42fb6e520955e01b96b14028468a7 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Mon, 6 May 2024 10:54:36 +0100 Subject: [PATCH 9/9] [Build] Only add -lfts for linux-static-*. Looks like I'd unconditionally added -lfts to the Foundation build flags. rdar://123503470 --- utils/build-script-impl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/utils/build-script-impl b/utils/build-script-impl index bc1c15a4728f2..d52207f7c0039 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -1446,6 +1446,16 @@ function swift_c_flags() { esac } +function maybe_lfts() { + local host=$1 + + case $host in + linux-static-*) + echo -n "-lfts" + ;; + esac +} + function common_swift_flags() { if [ "${module_cache}" == "" ] ; then echo "error: a module cache path has not been set" @@ -2524,7 +2534,7 @@ for host in "${ALL_HOSTS[@]}"; do -DENABLE_TESTING:BOOL=NO -DBUILD_SHARED_LIBS=$([[ ${product} == foundation_static ]] && echo "NO" || echo "YES") - -DCMAKE_C_FLAGS="$(swift_c_flags ${host}) -lfts" + -DCMAKE_C_FLAGS="$(swift_c_flags ${host}) $(maybe_lfts ${host})" ) if [[ $(is_cross_tools_host ${host}) ]] ; then