From c03943ca23f5d046156ad02209d0733c83627047 Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 15:55:19 -0700 Subject: [PATCH 1/7] [build-script] Lift `--cross-compile-hosts` to build-script level. --- utils/build-script | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/utils/build-script b/utils/build-script index f6dc0b03c2ca0..3fc95c4349006 100755 --- a/utils/build-script +++ b/utils/build-script @@ -374,6 +374,10 @@ class BuildScriptInvocation(object): pipes.quote(arg) for arg in cmake.build_args()), ] + if args.cross_compile_hosts: + impl_args += [ + "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] + if toolchain.ninja: impl_args += ["--ninja-bin=%s" % toolchain.ninja] if args.distcc: @@ -815,6 +819,12 @@ details of the setups of other systems or automated environments.""") "target. The built LLVM and Clang will be used to compile Swift " "for the cross-compilation targets.", default=StdlibDeploymentTarget.host_target()) + targets_group.add_argument( + "--cross-compile-hosts", + help="A space separated list of targets to cross-compile host Swift " + "tools for. Can be used multiple times.", + action=arguments.action.concat, type=arguments.type.shell_split, + default=[]) targets_group.add_argument( "--stdlib-deployment-targets", help="list of targets to compile or cross-compile the Swift standard " From ca2d64a52cacab39605fabe389d1f9c4ee738a16 Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 16:22:24 -0700 Subject: [PATCH 2/7] [build-script] Add infrastructure for passing host-specific variables. - This isn't yet used, but we need an easy way for `build-script` to pass host-specific variables down to `build-script-impl`. - I don't think it is worth complicating the main argument parsing logic with a syntax for passing associative arrays on the command line, so this just passes them via environment variables. - Part of SR-237. --- utils/build-script | 37 +++++++++++++++++++++++++++++-------- utils/build-script-impl | 13 +++++++++++++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/utils/build-script b/utils/build-script index 3fc95c4349006..3eccd25971135 100755 --- a/utils/build-script +++ b/utils/build-script @@ -50,7 +50,7 @@ from swift_build_support.cmake import CMake # noqa (E402) import swift_build_support.workspace # noqa (E402) -def call_without_sleeping(command, dry_run=False): +def call_without_sleeping(command, env=None, dry_run=False): """ Execute a command during which system sleep is disabled. @@ -62,7 +62,7 @@ def call_without_sleeping(command, dry_run=False): # Don't mutate the caller's copy of the arguments. command = ["caffeinate"] + list(command) - shell.call(command, dry_run=dry_run, echo=False) + shell.call(command, env=env, dry_run=dry_run) class BuildScriptInvocation(object): @@ -322,10 +322,10 @@ class BuildScriptInvocation(object): self.toolchain.ninja = ninja_build.ninja_bin_path def convert_to_impl_arguments(self): - """convert_to_impl_arguments() -> args + """convert_to_impl_arguments() -> (env, args) - Convert the invocation to a list of arguments suitable for invoking - `build-script-impl`. + Convert the invocation to an environment and list of arguments suitable + for invoking `build-script-impl`. """ # Create local shadows, for convenience. @@ -540,7 +540,26 @@ class BuildScriptInvocation(object): if args.dry_run: impl_args += ["--dry-run"] - return impl_args + # Compute the set of host-specific variables, which we pass through to + # the build script via environment variables. + host_specific_variables = self.compute_host_specific_variables() + impl_env = {} + for (host_target, options) in host_specific_variables.items(): + for (name, value) in options.items(): + # We mangle into an environment variable we can easily evaluate + # from the `build-script-impl`. + impl_env["HOST_VARIABLE_{}__{}".format( + host_target.replace("-", "_"), name)] = value + + return (impl_env, impl_args) + + def compute_host_specific_variables(self): + """compute_host_specific_variables(args) -> dict + + Compute the host-specific options, organized as a dictionary keyed by + host of options. + """ + return {} # Main entry point for the preset mode. @@ -1551,10 +1570,12 @@ details of the setups of other systems or automated environments.""") invocation.build_ninja() # Convert to a build-script-impl invocation. - build_script_impl_args = invocation.convert_to_impl_arguments() + (build_script_impl_env,build_script_impl_args) = \ + invocation.convert_to_impl_arguments() # Execute the underlying build script implementation. - call_without_sleeping([build_script_impl] + build_script_impl_args) + call_without_sleeping([build_script_impl] + build_script_impl_args, + env=build_script_impl_env) if args.symbols_package: print('--- Creating symbols package ---') diff --git a/utils/build-script-impl b/utils/build-script-impl index eb1f31a47b565..985a63f0f23fc 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -1138,6 +1138,19 @@ if [[ ! "${SKIP_BUILD_SWIFTPM}" ]] ; then PRODUCTS=("${PRODUCTS[@]}" swiftpm) fi +# get_host_specific_variable(host, name) +# +# Get the value of a host-specific variable expected to have been passed by the +# `build-script`. +# +# This is a total hack, and is part of the SR-237 migration. +function get_host_specific_variable() { + local host="$1" + local name="$2" + local envvar_name="HOST_VARIABLE_${host//-/_}__${name}" + echo "${!envvar_name}" +} + function calculate_targets_for_host() { local host=$1 From bf75c8f50aafa0076339ecc2bfde668f1167f681 Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 17:18:31 -0700 Subject: [PATCH 3/7] [build-script] Create explicit Platform and Target types. - This is so that we can have a place to attach the additional metadata we need on platform or target specific behaviors. --- utils/SwiftBuildSupport.py | 16 +-- utils/build-script | 7 +- .../swift_build_support/targets.py | 117 +++++++++--------- 3 files changed, 72 insertions(+), 68 deletions(-) diff --git a/utils/SwiftBuildSupport.py b/utils/SwiftBuildSupport.py index 412f190d0196e..bf5a117108389 100644 --- a/utils/SwiftBuildSupport.py +++ b/utils/SwiftBuildSupport.py @@ -151,22 +151,22 @@ def get_preset_options(substitutions, preset_file_names, preset_name): from swift_build_support.targets import StdlibDeploymentTarget for sdk in sdks_to_configure: if sdk == "OSX": - tgts += StdlibDeploymentTarget.OSX.allArchs + tgts += StdlibDeploymentTarget.OSX.targets elif sdk == "IOS": - tgts += StdlibDeploymentTarget.iOS.allArchs + tgts += StdlibDeploymentTarget.iOS.targets elif sdk == "IOS_SIMULATOR": - tgts += StdlibDeploymentTarget.iOSSimulator.allArchs + tgts += StdlibDeploymentTarget.iOSSimulator.targets elif sdk == "TVOS": - tgts += StdlibDeploymentTarget.AppleTV.allArchs + tgts += StdlibDeploymentTarget.AppleTV.targets elif sdk == "TVOS_SIMULATOR": - tgts += StdlibDeploymentTarget.AppleTVSimulator.allArchs + tgts += StdlibDeploymentTarget.AppleTVSimulator.targets elif sdk == "WATCHOS": - tgts += StdlibDeploymentTarget.AppleWatch.allArchs + tgts += StdlibDeploymentTarget.AppleWatch.targets elif sdk == "WATCHOS_SIMULATOR": - tgts += StdlibDeploymentTarget.AppleWatchSimulator.allArchs + tgts += StdlibDeploymentTarget.AppleWatchSimulator.targets build_script_opts.append("--stdlib-deployment-targets=" + - " ".join(tgts)) + " ".join([tgt.name for tgt in tgts])) # Filter the swift-sdks parameter build_script_impl_opts = [opt for opt in build_script_impl_opts if not opt.startswith("--swift-sdks")] diff --git a/utils/build-script b/utils/build-script index 3eccd25971135..196f1d7fc73d7 100755 --- a/utils/build-script +++ b/utils/build-script @@ -837,7 +837,7 @@ details of the setups of other systems or automated environments.""") help="The host target. LLVM, Clang, and Swift will be built for this " "target. The built LLVM and Clang will be used to compile Swift " "for the cross-compilation targets.", - default=StdlibDeploymentTarget.host_target()) + default=StdlibDeploymentTarget.host_target().name) targets_group.add_argument( "--cross-compile-hosts", help="A space separated list of targets to cross-compile host Swift " @@ -849,7 +849,10 @@ details of the setups of other systems or automated environments.""") help="list of targets to compile or cross-compile the Swift standard " "library for. %(default)s by default.", nargs="*", - default=StdlibDeploymentTarget.default_stdlib_deployment_targets()) + default=[ + target.name + for target in + StdlibDeploymentTarget.default_stdlib_deployment_targets()]) projects_group = parser.add_argument_group( title="Options to select projects") diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index c2dd068f5530e..ca32c40d675a3 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -12,60 +12,61 @@ import platform +class Platform(object): + """ + Abstract representation of a platform Swift can run on. + """ + + def __init__(self, name, archs): + """ + Create a platform with the given name and list of architectures. + """ + self.name = name + self.targets = [Target(self, arch) for arch in archs] + + # Add a property for each arch. + for target in self.targets: + setattr(self, target.arch, target) + + +class Target(object): + """ + Abstract representation of a target Swift can run on. + """ + + def __init__(self, platform, arch): + self.platform = platform + self.arch = arch + + @property + def name(self): + return "{}-{}".format(self.platform.name, self.arch) + + class StdlibDeploymentTarget(object): + OSX = Platform("macosx", archs=["x86_64"]) + + iOS = Platform("iphoneos", archs=["armv7", "armv7s", "arm64"]) + iOSSimulator = Platform("iphonesimulator", archs=["i386", "x86_64"]) + + AppleTV = Platform("appletvos", archs=["arm64"]) + AppleTVSimulator = Platform("appletvsimulator", archs=["x86_64"]) + + AppleWatch = Platform("watchos", archs=["armv7k"]) + AppleWatchSimulator = Platform("watchsimulator", archs=["i386"]) + + Linux = Platform("linux", archs=[ + "x86_64", + "armv6", + "armv7", + "aarch64", + "ppc64", + "ppc64le", + "s390x"]) + + FreeBSD = Platform("freebsd", archs=["x86_64"]) - class OSX(object): - x86_64 = 'macosx-x86_64' - allArchs = [x86_64] - - class iOS(object): # noqa - armv7 = 'iphoneos-armv7' - armv7s = 'iphoneos-armv7s' - arm64 = 'iphoneos-arm64' - allArchs = [armv7, armv7s, arm64] - - class iOSSimulator(object): # noqa - i386 = 'iphonesimulator-i386' - x86_64 = 'iphonesimulator-x86_64' - allArchs = [i386, x86_64] - - class AppleTV(object): - arm64 = 'appletvos-arm64' - allArchs = [arm64] - - class AppleTVSimulator(object): - x86_64 = 'appletvsimulator-x86_64' - allArchs = [x86_64] - - class AppleWatch(object): - armv7k = 'watchos-armv7k' - allArchs = [armv7k] - - class AppleWatchSimulator(object): - i386 = 'watchsimulator-i386' - allArchs = [i386] - - class Linux(object): - x86_64 = 'linux-x86_64' - armv6 = 'linux-armv6' - armv7 = 'linux-armv7' - aarch64 = 'linux-aarch64' - ppc64 = 'linux-ppc64' - ppc64le = 'linux-ppc64le' - s390x = 'linux-s390x' - allArchs = [x86_64, armv6, armv7, aarch64, ppc64, ppc64le, s390x] - - class FreeBSD(object): - amd64 = 'freebsd-x86_64' - allArchs = [amd64] - - class Cygwin(object): - x86_64 = 'cygwin-x86_64' - allArchs = [x86_64] - - class Android(object): - armv7 = 'android-armv7' - allArchs = [armv7] + Cygwin = Platform("cygwin", archs=["x86_64"]) @staticmethod def host_target(): @@ -124,12 +125,12 @@ def default_stdlib_deployment_targets(): # (it takes a long time). if host_target == StdlibDeploymentTarget.OSX.x86_64: return [host_target] + \ - StdlibDeploymentTarget.iOSSimulator.allArchs + \ - StdlibDeploymentTarget.AppleTVSimulator.allArchs + \ - StdlibDeploymentTarget.AppleWatchSimulator.allArchs + \ - StdlibDeploymentTarget.iOS.allArchs + \ - StdlibDeploymentTarget.AppleTV.allArchs + \ - StdlibDeploymentTarget.AppleWatch.allArchs + StdlibDeploymentTarget.iOSSimulator.targets + \ + StdlibDeploymentTarget.AppleTVSimulator.targets + \ + StdlibDeploymentTarget.AppleWatchSimulator.targets + \ + StdlibDeploymentTarget.iOS.targets + \ + StdlibDeploymentTarget.AppleTV.targets + \ + StdlibDeploymentTarget.AppleWatch.targets else: # All other machines only configure their host stdlib by default. return [host_target] From 6df6ae707ac5f4ef210d9147f7f61ed3aad2793f Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 22:36:47 -0700 Subject: [PATCH 4/7] [build-script] Add a couple more options at the `build-script` layer. --- utils/build-script | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/utils/build-script b/utils/build-script index 196f1d7fc73d7..52a2784384825 100755 --- a/utils/build-script +++ b/utils/build-script @@ -374,6 +374,10 @@ class BuildScriptInvocation(object): pipes.quote(arg) for arg in cmake.build_args()), ] + if args.build_stdlib_deployment_targets: + impl_args += [ + "--build-stdlib-deployment-targets", " ".join( + args.build_stdlib_deployment_targets)] if args.cross_compile_hosts: impl_args += [ "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] @@ -419,6 +423,8 @@ class BuildScriptInvocation(object): impl_args += ["--skip-build-libdispatch"] if not args.build_swiftpm: impl_args += ["--skip-build-swiftpm"] + if args.build_swift_stdlib_unittest_extra: + impl_args += ["--build-swift-stdlib-unittest-extra"] if args.skip_build_linux: impl_args += ["--skip-build-linux"] @@ -853,6 +859,11 @@ details of the setups of other systems or automated environments.""") target.name for target in StdlibDeploymentTarget.default_stdlib_deployment_targets()]) + targets_group.add_argument( + "--build-stdlib-deployment-targets", + help="A space-separated list that filters which of the configured " + "targets to build the Swift standard library for, or 'all'.", + type=arguments.type.shell_split, default=["all"]) projects_group = parser.add_argument_group( title="Options to select projects") @@ -1152,6 +1163,10 @@ details of the setups of other systems or automated environments.""") help="Use the host compiler, not the self-built one to compile the " "Swift runtime", action="store_true") + parser.add_argument( + "--build-swift-stdlib-unittest-extra", + help="Build optional StdlibUnittest components", + action="store_true") run_build_group = parser.add_argument_group( title="Run build") From f407cdfb3039dffcbc3c0915e08d5c7c5e60a4db Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 22:44:07 -0700 Subject: [PATCH 5/7] [swift_build_support.targets] Add some additional metadata. - This adds several platform/target properties used to configure the default behavior for a build. - This also adds an API to find a target from a name. --- .../swift_build_support/targets.py | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index ca32c40d675a3..9e828023a121a 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -17,17 +17,47 @@ class Platform(object): Abstract representation of a platform Swift can run on. """ - def __init__(self, name, archs): + def __init__(self, name, archs, sdk_name=None): """ Create a platform with the given name and list of architectures. """ self.name = name self.targets = [Target(self, arch) for arch in archs] + # FIXME: Eliminate this argument; apparently the SDK names are + # internally a private implementation detail of the build script, so we + # should just make them the same as the platform name. + self.sdk_name = name.upper() if sdk_name is None else sdk_name # Add a property for each arch. for target in self.targets: setattr(self, target.arch, target) + @property + def is_darwin(self): + """Convenience function for checking if this is a Darwin platform.""" + return isinstance(self, DarwinPlatform) + + @property + def supports_benchmark(self): + # By default, we don't support benchmarks on most platforms. + return False + + +class DarwinPlatform(Platform): + def __init__(self, name, archs, sdk_name=None, is_simulator=False): + super(DarwinPlatform, self).__init__(name, archs, sdk_name) + self.is_simulator = is_simulator + + @property + def is_embedded(self): + """Check if this is a Darwin platform for embedded devices.""" + return self.name != "macosx" + + @property + def supports_benchmark(self): + # By default, on Darwin we support benchmarks. + return True + class Target(object): """ @@ -37,6 +67,8 @@ class Target(object): def __init__(self, platform, arch): self.platform = platform self.arch = arch + # Delegate to the platform, this is usually not arch specific. + self.supports_benchmark = self.platform.supports_benchmark @property def name(self): @@ -44,16 +76,29 @@ def name(self): class StdlibDeploymentTarget(object): - OSX = Platform("macosx", archs=["x86_64"]) - - iOS = Platform("iphoneos", archs=["armv7", "armv7s", "arm64"]) - iOSSimulator = Platform("iphonesimulator", archs=["i386", "x86_64"]) - - AppleTV = Platform("appletvos", archs=["arm64"]) - AppleTVSimulator = Platform("appletvsimulator", archs=["x86_64"]) - - AppleWatch = Platform("watchos", archs=["armv7k"]) - AppleWatchSimulator = Platform("watchsimulator", archs=["i386"]) + OSX = DarwinPlatform("macosx", archs=["x86_64"], + sdk_name="OSX") + + iOS = DarwinPlatform("iphoneos", archs=["armv7", "armv7s", "arm64"], + sdk_name="IOS") + iOSSimulator = DarwinPlatform("iphonesimulator", archs=["i386", "x86_64"], + sdk_name="IOS_SIMULATOR", + is_simulator=True) + + # Never build/test benchmarks on iOS armv7s. + iOS.armv7s.supports_benchmark = False + + AppleTV = DarwinPlatform("appletvos", archs=["arm64"], + sdk_name="TVOS") + AppleTVSimulator = DarwinPlatform("appletvsimulator", archs=["x86_64"], + sdk_name="TVOS_SIMULATOR", + is_simulator=True) + + AppleWatch = DarwinPlatform("watchos", archs=["armv7k"], + sdk_name="WATCHOS") + AppleWatchSimulator = DarwinPlatform("watchsimulator", archs=["i386"], + sdk_name="WATCHOS_SIMULATOR", + is_simulator=True) Linux = Platform("linux", archs=[ "x86_64", @@ -68,6 +113,24 @@ class StdlibDeploymentTarget(object): Cygwin = Platform("cygwin", archs=["x86_64"]) + Android = Platform("android", archs=["armv7"]) + + # The list of known platforms. + known_platforms = [ + OSX, + iOS, iOSSimulator, + AppleTV, AppleTVSimulator, + AppleWatch, AppleWatchSimulator, + Linux, + FreeBSD, + Cygwin, + Android] + + # Cache of targets by name. + _targets_by_name = dict((target.name, target) + for platform in known_platforms + for target in platform.targets) + @staticmethod def host_target(): """ @@ -135,6 +198,10 @@ def default_stdlib_deployment_targets(): # All other machines only configure their host stdlib by default. return [host_target] + @classmethod + def get_target_for_name(klass, name): + return klass._targets_by_name.get(name) + def install_prefix(): """ From bb9fc675c7411fe71c544359b57a5d115273ca57 Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Fri, 3 Jun 2016 22:47:08 -0700 Subject: [PATCH 6/7] [build-script] Switch over to computing targets in `build-script`. - Introduces a new `HostSpecificConfiguration` which holds the computed information for each host we need to use. - This relies on the functionality to pass host-specific variables to the script via environment variables. - The computation itself has been cleaned up a tiny bit (mostly via using the factored out platform/target properties), but is otherwise a straight port of the logic from `build-script-impl`. - This commit does not yet remove that code from `build-script-impl`; instead it validates that the two implementations are computing the same result. This is useful right now for testing the validity of the port. --- utils/build-script | 214 +++++++++++++++++++++++++++++++++++++++- utils/build-script-impl | 34 +++++++ 2 files changed, 244 insertions(+), 4 deletions(-) diff --git a/utils/build-script b/utils/build-script index 52a2784384825..80ca867197e18 100755 --- a/utils/build-script +++ b/utils/build-script @@ -65,6 +65,118 @@ def call_without_sleeping(command, env=None, dry_run=False): shell.call(command, env=env, dry_run=dry_run) +class HostSpecificConfiguration(object): + """Configuration information for an individual host.""" + + def __init__(self, host_target, invocation): + """Initialize for the given `host_target`.""" + + # Compute the set of deployment targets to configure/build. + args = invocation.args + if host_target == args.host_target: + # This host is the user's desired product, so honor the requested + # set of targets to configure/build. + stdlib_targets_to_configure = args.stdlib_deployment_targets + if "all" in args.build_stdlib_deployment_targets: + stdlib_targets_to_build = set(stdlib_targets_to_configure) + else: + stdlib_targets_to_build = set( + args.build_stdlib_deployment_targets).intersect( + set(args.stdlib_deployment_targets)) + else: + # Otherwise, this is a host we are building as part of + # cross-compiling, so we only need the target itself. + stdlib_targets_to_configure = [host_target] + stdlib_targets_to_build = set(stdlib_targets_to_configure) + + # Compute the lists of **CMake** targets for each use case (configure + # vs. build vs. run) and the SDKs to configure with. + self.sdks_to_configure = set() + self.swift_stdlib_build_targets = [] + self.swift_test_run_targets = [] + self.swift_benchmark_build_targets = [] + self.swift_benchmark_run_targets = [] + for deployment_target_name in stdlib_targets_to_configure: + # Get the target object. + deployment_target = StdlibDeploymentTarget.get_target_for_name( + deployment_target_name) + if deployment_target is None: + diagnostics.fatal("unknown target: %r" % ( + deployment_target_name,)) + + # Add the SDK to use. + deployment_platform = deployment_target.platform + self.sdks_to_configure.add(deployment_platform.sdk_name) + + # If we aren't actually building this target (only configuring + # it), do nothing else. + if deployment_target_name not in stdlib_targets_to_build: + continue + + # Compute which actions are desired. + build = deployment_platform not in \ + invocation.platforms_to_skip_build + test = deployment_platform not in \ + invocation.platforms_to_skip_test + test_host_only = None + build_benchmark = build and deployment_target.supports_benchmark + # FIXME: Note, `build-script-impl` computed a property here + # w.r.t. testing, but it was actually unused. + + # For platforms which normally require a connected device to + # test, the default behavior is to run tests that only require + # the host (i.e., they do not attempt to execute). + if deployment_platform.is_darwin and \ + deployment_platform.is_embedded and \ + not deployment_platform.is_simulator: + if deployment_platform not in \ + invocation.platforms_to_skip_test_host: + test_host_only = True + test = True + else: + test = False + + name = deployment_target.name + if build: + # Validation and long tests require building the full standard + # library, whereas the other targets can build a slightly + # smaller subset which is faster to build. + if args.build_swift_stdlib_unittest_extra or \ + args.validation_test or args.long_test: + self.swift_stdlib_build_targets.append( + "swift-stdlib-" + name) + else: + self.swift_stdlib_build_targets.append( + "swift-test-stdlib-" + name) + if build_benchmark: + self.swift_benchmark_build_targets.append( + "swift-benchmark-" + name) + # FIXME: This probably should respect `args.benchmark`, but + # a typo in build-script-impl meant we always would do this. + self.swift_benchmark_run_targets.append( + "check-swift-benchmark-" + name) + if test: + if test_host_only: + suffix = "-non-executable" + else: + suffix = "" + subset_suffix = "" + if args.validation_test and args.long_test: + subset_suffix = "-all" + elif args.validation_test: + subset_suffix = "-validation" + elif args.long_test: + subset_suffix = "-only_long" + else: + subset_suffix = "" + self.swift_test_run_targets.append("check-swift{}{}-{}".format( + subset_suffix, suffix, name)) + if args.test_optimized and not test_host_only: + self.swift_test_run_targets.append( + "check-swift{}-optimize-{}".format( + subset_suffix, name)) + + class BuildScriptInvocation(object): """Represent a single build script invocation.""" @@ -291,6 +403,77 @@ class BuildScriptInvocation(object): source_root=SWIFT_SOURCE_ROOT, build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir)) + # Compute derived information from the arguments. + # + # FIXME: We should move the platform-derived arguments to be entirely + # data driven, so that we can eliminate this code duplication and just + # iterate over all supported platforms. + + self.platforms_to_skip_build = set() + if args.skip_build_linux: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.Linux) + if args.skip_build_freebsd: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD) + if args.skip_build_cygwin: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.Cygwin) + if args.skip_build_osx: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.OSX) + if args.skip_build_ios_device: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.iOS) + if args.skip_build_ios_simulator: + self.platforms_to_skip_build.add( + StdlibDeploymentTarget.iOSSimulator) + if args.skip_build_tvos_device: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleTV) + if args.skip_build_tvos_simulator: + self.platforms_to_skip_build.add( + StdlibDeploymentTarget.AppleTVSimulator) + if args.skip_build_watchos_device: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleWatch) + if args.skip_build_watchos_simulator: + self.platforms_to_skip_build.add( + StdlibDeploymentTarget.AppleWatchSimulator) + if args.skip_build_android: + self.platforms_to_skip_build.add(StdlibDeploymentTarget.Android) + + self.platforms_to_skip_test = set() + if args.skip_test_linux: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.Linux) + if args.skip_test_freebsd: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD) + if args.skip_test_cygwin: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin) + if args.skip_test_osx: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.OSX) + if args.skip_test_ios_host: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS) + if args.skip_test_ios_simulator: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOSSimulator) + if args.skip_test_tvos_host: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV) + if args.skip_test_tvos_simulator: + self.platforms_to_skip_test.add( + StdlibDeploymentTarget.AppleTVSimulator) + if args.skip_test_watchos_host: + self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch) + if args.skip_test_watchos_simulator: + self.platforms_to_skip_test.add( + StdlibDeploymentTarget.AppleWatchSimulator) + # We never allow testing Android, currently. + # + # FIXME: Allow Android host tests to be enabled/disabled by the build + # script. + self.platforms_to_skip_test.add(StdlibDeploymentTarget.Android) + + self.platforms_to_skip_test_host = set() + if args.skip_test_ios_host: + self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS) + if args.skip_test_tvos_host: + self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.AppleTV) + if args.skip_test_watchos_host: + self.platforms_to_skip_test_host.add( + StdlibDeploymentTarget.AppleWatch) + def initialize_runtime_environment(self): """Change the program environment for building.""" @@ -381,7 +564,7 @@ class BuildScriptInvocation(object): if args.cross_compile_hosts: impl_args += [ "--cross-compile-hosts", " ".join(args.cross_compile_hosts)] - + if toolchain.ninja: impl_args += ["--ninja-bin=%s" % toolchain.ninja] if args.distcc: @@ -565,7 +748,29 @@ class BuildScriptInvocation(object): Compute the host-specific options, organized as a dictionary keyed by host of options. """ - return {} + + args = self.args + + options = {} + for host in [args.host_target] + args.cross_compile_hosts: + # Compute the host specific configuration. + config = HostSpecificConfiguration(host, self) + + # Convert into `build-script-impl` style variables. + options[host] = { + "SWIFT_SDKS": " ".join(sorted( + config.sdks_to_configure)), + "SWIFT_STDLIB_TARGETS": " ".join( + config.swift_stdlib_build_targets), + "SWIFT_BENCHMARK_TARGETS": " ".join( + config.swift_benchmark_build_targets), + "SWIFT_RUN_BENCHMARK_TARGETS": " ".join( + config.swift_benchmark_run_targets), + "SWIFT_TEST_TARGETS": " ".join( + config.swift_test_run_targets), + } + + return options # Main entry point for the preset mode. @@ -850,6 +1055,7 @@ details of the setups of other systems or automated environments.""") "tools for. Can be used multiple times.", action=arguments.action.concat, type=arguments.type.shell_split, default=[]) + stdlib_targets = StdlibDeploymentTarget.default_stdlib_deployment_targets() targets_group.add_argument( "--stdlib-deployment-targets", help="list of targets to compile or cross-compile the Swift standard " @@ -857,8 +1063,7 @@ details of the setups of other systems or automated environments.""") nargs="*", default=[ target.name - for target in - StdlibDeploymentTarget.default_stdlib_deployment_targets()]) + for target in stdlib_targets]) targets_group.add_argument( "--build-stdlib-deployment-targets", help="A space-separated list that filters which of the configured " @@ -1527,6 +1732,7 @@ details of the setups of other systems or automated environments.""") "--build-jobs", "--common-cmake-options", "--only-execute", + "--skip-test-optimized", action=arguments.action.unavailable) args = migration.parse_args(parser, sys.argv[1:]) diff --git a/utils/build-script-impl b/utils/build-script-impl index 985a63f0f23fc..2d6f1355b16ef 100755 --- a/utils/build-script-impl +++ b/utils/build-script-impl @@ -1306,6 +1306,40 @@ function calculate_targets_for_host() { # Filter duplicate SWIFT_SDKs # We will get them if building for multiple architecture variants SWIFT_SDKS=(`echo "${SWIFT_SDKS[@]}" | tr " " "\n" | sort -u | tr "\n" " "`) + + # Get the values passed by `build-script`. + LEGACY_SWIFT_STDLIB_TARGETS=(${SWIFT_STDLIB_TARGETS[@]}) + LEGACY_SWIFT_SDKS=(${SWIFT_SDKS[@]}) + LEGACY_SWIFT_BENCHMARK_TARGETS=(${SWIFT_BENCHMARK_TARGETS[@]}) + LEGACY_SWIFT_RUN_BENCHMARK_TARGETS=(${SWIFT_RUN_BENCHMARK_TARGETS[@]}) + LEGACY_SWIFT_TEST_TARGETS=(${SWIFT_TEST_TARGETS[@]}) + SWIFT_STDLIB_TARGETS=($(get_host_specific_variable ${host} SWIFT_STDLIB_TARGETS)) + SWIFT_SDKS=($(get_host_specific_variable ${host} SWIFT_SDKS)) + SWIFT_BENCHMARK_TARGETS=($(get_host_specific_variable ${host} SWIFT_BENCHMARK_TARGETS)) + SWIFT_RUN_BENCHMARK_TARGETS=($(get_host_specific_variable ${host} SWIFT_RUN_BENCHMARK_TARGETS)) + SWIFT_TEST_TARGETS=($(get_host_specific_variable ${host} SWIFT_TEST_TARGETS)) + + # Validate the parameters match. + if [[ "${SWIFT_STDLIB_TARGETS[*]}" != "${LEGACY_SWIFT_STDLIB_TARGETS[*]}" ]]; then + printf "error: invalid build-script refactor for 'SWIFT_STDLIB_TARGETS': '%s' vs '%s'\n" "${SWIFT_STDLIB_TARGETS[*]}" "${LEGACY_SWIFT_STDLIB_TARGETS[*]}" + exit 1 + fi + if [[ "${SWIFT_SDKS[*]}" != "${LEGACY_SWIFT_SDKS[*]}" ]]; then + printf "error: invalid build-script for 'SWIFT_SDKS' refactor: '%s' vs '%s'\n" "${SWIFT_SDKS[*]}" "${LEGACY_SWIFT_SDKS[*]}" + exit 1 + fi + if [[ "${SWIFT_BENCHMARK_TARGETS[*]}" != "${LEGACY_SWIFT_BENCHMARK_TARGETS[*]}" ]]; then + printf "error: invalid build-script refactor for 'SWIFT_BENCHMARK_TARGETS': '%s' vs '%s'\n" "${SWIFT_BENCHMARK_TARGETS[*]}" "${LEGACY_SWIFT_BENCHMARK_TARGETS[*]}" + exit 1 + fi + if [[ "${SWIFT_RUN_BENCHMARK_TARGETS[*]}" != "${LEGACY_SWIFT_RUN_BENCHMARK_TARGETS[*]}" ]]; then + printf "error: invalid build-script refactor for 'SWIFT_RUN_BENCHMARK_TARGETS': '%s' vs '%s'\n" "${SWIFT_RUN_BENCHMARK_TARGETS[*]}" "${LEGACY_SWIFT_RUN_BENCHMARK_TARGETS[*]}" + exit 1 + fi + if [[ "${SWIFT_TEST_TARGETS[*]}" != "${LEGACY_SWIFT_TEST_TARGETS[*]}" ]]; then + printf "error: invalid build-script refactor for 'SWIFT_TEST_TARGETS': '%s' vs '%s'\n" "${SWIFT_TEST_TARGETS[*]}" "${LEGACY_SWIFT_TEST_TARGETS[*]}" + exit 1 + fi } From d51507879aad4784246b887e16fd7b72bcdc4ae9 Mon Sep 17 00:00:00 2001 From: Daniel Dunbar Date: Wed, 8 Jun 2016 14:39:04 -0700 Subject: [PATCH 7/7] [build-script] Several PEP8 fixes. --- utils/build-script | 22 ++++++++++--------- .../swift_build_support/targets.py | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/utils/build-script b/utils/build-script index 80ca867197e18..abbe2344b3436 100755 --- a/utils/build-script +++ b/utils/build-script @@ -114,10 +114,10 @@ class HostSpecificConfiguration(object): continue # Compute which actions are desired. - build = deployment_platform not in \ - invocation.platforms_to_skip_build - test = deployment_platform not in \ - invocation.platforms_to_skip_test + build = ( + deployment_platform not in invocation.platforms_to_skip_build) + test = ( + deployment_platform not in invocation.platforms_to_skip_test) test_host_only = None build_benchmark = build and deployment_target.supports_benchmark # FIXME: Note, `build-script-impl` computed a property here @@ -448,7 +448,8 @@ class BuildScriptInvocation(object): if args.skip_test_ios_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS) if args.skip_test_ios_simulator: - self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOSSimulator) + self.platforms_to_skip_test.add( + StdlibDeploymentTarget.iOSSimulator) if args.skip_test_tvos_host: self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV) if args.skip_test_tvos_simulator: @@ -469,7 +470,8 @@ class BuildScriptInvocation(object): if args.skip_test_ios_host: self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS) if args.skip_test_tvos_host: - self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.AppleTV) + self.platforms_to_skip_test_host.add( + StdlibDeploymentTarget.AppleTV) if args.skip_test_watchos_host: self.platforms_to_skip_test_host.add( StdlibDeploymentTarget.AppleWatch) @@ -752,12 +754,12 @@ class BuildScriptInvocation(object): args = self.args options = {} - for host in [args.host_target] + args.cross_compile_hosts: + for host_target in [args.host_target] + args.cross_compile_hosts: # Compute the host specific configuration. - config = HostSpecificConfiguration(host, self) + config = HostSpecificConfiguration(host_target, self) # Convert into `build-script-impl` style variables. - options[host] = { + options[host_target] = { "SWIFT_SDKS": " ".join(sorted( config.sdks_to_configure)), "SWIFT_STDLIB_TARGETS": " ".join( @@ -1794,7 +1796,7 @@ details of the setups of other systems or automated environments.""") invocation.build_ninja() # Convert to a build-script-impl invocation. - (build_script_impl_env,build_script_impl_args) = \ + (build_script_impl_env, build_script_impl_args) = \ invocation.convert_to_impl_arguments() # Execute the underlying build script implementation. diff --git a/utils/swift_build_support/swift_build_support/targets.py b/utils/swift_build_support/swift_build_support/targets.py index 9e828023a121a..f66a60015db55 100644 --- a/utils/swift_build_support/swift_build_support/targets.py +++ b/utils/swift_build_support/swift_build_support/targets.py @@ -47,7 +47,7 @@ class DarwinPlatform(Platform): def __init__(self, name, archs, sdk_name=None, is_simulator=False): super(DarwinPlatform, self).__init__(name, archs, sdk_name) self.is_simulator = is_simulator - + @property def is_embedded(self): """Check if this is a Darwin platform for embedded devices."""