Merge pull request #2880 from ddunbar/build-script-compute-targets

[build-script] Compute targets in `build-script`

Author: ddunbar

utils/SwiftBuildSupport.py

@@ -151,22 +151,22 @@ def get_preset_options(substitutions, preset_file_names, preset_name):
             from swift_build_support.targets import StdlibDeploymentTarget
             for sdk in sdks_to_configure:
                 if sdk == "OSX":
-                    tgts += StdlibDeploymentTarget.OSX.allArchs
+                    tgts += StdlibDeploymentTarget.OSX.targets
                 elif sdk == "IOS":
-                    tgts += StdlibDeploymentTarget.iOS.allArchs
+                    tgts += StdlibDeploymentTarget.iOS.targets
                 elif sdk == "IOS_SIMULATOR":
-                    tgts += StdlibDeploymentTarget.iOSSimulator.allArchs
+                    tgts += StdlibDeploymentTarget.iOSSimulator.targets
                 elif sdk == "TVOS":
-                    tgts += StdlibDeploymentTarget.AppleTV.allArchs
+                    tgts += StdlibDeploymentTarget.AppleTV.targets
                 elif sdk == "TVOS_SIMULATOR":
-                    tgts += StdlibDeploymentTarget.AppleTVSimulator.allArchs
+                    tgts += StdlibDeploymentTarget.AppleTVSimulator.targets
                 elif sdk == "WATCHOS":
-                    tgts += StdlibDeploymentTarget.AppleWatch.allArchs
+                    tgts += StdlibDeploymentTarget.AppleWatch.targets
                 elif sdk == "WATCHOS_SIMULATOR":
-                    tgts += StdlibDeploymentTarget.AppleWatchSimulator.allArchs
+                    tgts += StdlibDeploymentTarget.AppleWatchSimulator.targets
 
             build_script_opts.append("--stdlib-deployment-targets=" +
-                                     " ".join(tgts))
+                                     " ".join([tgt.name for tgt in tgts]))
     # Filter the swift-sdks parameter
     build_script_impl_opts = [opt for opt in build_script_impl_opts
                               if not opt.startswith("--swift-sdks")]

utils/build-script

@@ -50,7 +50,7 @@ from swift_build_support.cmake import CMake  # noqa (E402)
 import swift_build_support.workspace    # noqa (E402)
 
 
-def call_without_sleeping(command, dry_run=False):
+def call_without_sleeping(command, env=None, dry_run=False):
     """
     Execute a command during which system sleep is disabled.
 
@@ -62,7 +62,119 @@ def call_without_sleeping(command, dry_run=False):
         # Don't mutate the caller's copy of the arguments.
         command = ["caffeinate"] + list(command)
 
-    shell.call(command, dry_run=dry_run, echo=False)
+    shell.call(command, env=env, dry_run=dry_run)
+
+
+class HostSpecificConfiguration(object):
+    """Configuration information for an individual host."""
+
+    def __init__(self, host_target, invocation):
+        """Initialize for the given `host_target`."""
+
+        # Compute the set of deployment targets to configure/build.
+        args = invocation.args
+        if host_target == args.host_target:
+            # This host is the user's desired product, so honor the requested
+            # set of targets to configure/build.
+            stdlib_targets_to_configure = args.stdlib_deployment_targets
+            if "all" in args.build_stdlib_deployment_targets:
+                stdlib_targets_to_build = set(stdlib_targets_to_configure)
+            else:
+                stdlib_targets_to_build = set(
+                    args.build_stdlib_deployment_targets).intersect(
+                        set(args.stdlib_deployment_targets))
+        else:
+            # Otherwise, this is a host we are building as part of
+            # cross-compiling, so we only need the target itself.
+            stdlib_targets_to_configure = [host_target]
+            stdlib_targets_to_build = set(stdlib_targets_to_configure)
+
+        # Compute the lists of **CMake** targets for each use case (configure
+        # vs. build vs. run) and the SDKs to configure with.
+        self.sdks_to_configure = set()
+        self.swift_stdlib_build_targets = []
+        self.swift_test_run_targets = []
+        self.swift_benchmark_build_targets = []
+        self.swift_benchmark_run_targets = []
+        for deployment_target_name in stdlib_targets_to_configure:
+            # Get the target object.
+            deployment_target = StdlibDeploymentTarget.get_target_for_name(
+                deployment_target_name)
+            if deployment_target is None:
+                diagnostics.fatal("unknown target: %r" % (
+                    deployment_target_name,))
+
+            # Add the SDK to use.
+            deployment_platform = deployment_target.platform
+            self.sdks_to_configure.add(deployment_platform.sdk_name)
+
+            # If we aren't actually building this target (only configuring
+            # it), do nothing else.
+            if deployment_target_name not in stdlib_targets_to_build:
+                continue
+
+            # Compute which actions are desired.
+            build = (
+                deployment_platform not in invocation.platforms_to_skip_build)
+            test = (
+                deployment_platform not in invocation.platforms_to_skip_test)
+            test_host_only = None
+            build_benchmark = build and deployment_target.supports_benchmark
+            # FIXME: Note, `build-script-impl` computed a property here
+            # w.r.t. testing, but it was actually unused.
+
+            # For platforms which normally require a connected device to
+            # test, the default behavior is to run tests that only require
+            # the host (i.e., they do not attempt to execute).
+            if deployment_platform.is_darwin and \
+               deployment_platform.is_embedded and \
+               not deployment_platform.is_simulator:
+                if deployment_platform not in \
+                       invocation.platforms_to_skip_test_host:
+                    test_host_only = True
+                    test = True
+                else:
+                    test = False
+
+            name = deployment_target.name
+            if build:
+                # Validation and long tests require building the full standard
+                # library, whereas the other targets can build a slightly
+                # smaller subset which is faster to build.
+                if args.build_swift_stdlib_unittest_extra or \
+                       args.validation_test or args.long_test:
+                    self.swift_stdlib_build_targets.append(
+                        "swift-stdlib-" + name)
+                else:
+                    self.swift_stdlib_build_targets.append(
+                        "swift-test-stdlib-" + name)
+            if build_benchmark:
+                self.swift_benchmark_build_targets.append(
+                    "swift-benchmark-" + name)
+                # FIXME: This probably should respect `args.benchmark`, but
+                # a typo in build-script-impl meant we always would do this.
+                self.swift_benchmark_run_targets.append(
+                    "check-swift-benchmark-" + name)
+            if test:
+                if test_host_only:
+                    suffix = "-non-executable"
+                else:
+                    suffix = ""
+                subset_suffix = ""
+                if args.validation_test and args.long_test:
+                    subset_suffix = "-all"
+                elif args.validation_test:
+                    subset_suffix = "-validation"
+                elif args.long_test:
+                    subset_suffix = "-only_long"
+                else:
+                    subset_suffix = ""
+                self.swift_test_run_targets.append("check-swift{}{}-{}".format(
+                    subset_suffix, suffix, name))
+                if args.test_optimized and not test_host_only:
+                    self.swift_test_run_targets.append(
+                        "check-swift{}-optimize-{}".format(
+                            subset_suffix, name))
 
 
 class BuildScriptInvocation(object):
@@ -291,6 +403,79 @@ class BuildScriptInvocation(object):
             source_root=SWIFT_SOURCE_ROOT,
             build_root=os.path.join(SWIFT_BUILD_ROOT, args.build_subdir))
 
+        # Compute derived information from the arguments.
+        #
+        # FIXME: We should move the platform-derived arguments to be entirely
+        # data driven, so that we can eliminate this code duplication and just
+        # iterate over all supported platforms.
+
+        self.platforms_to_skip_build = set()
+        if args.skip_build_linux:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.Linux)
+        if args.skip_build_freebsd:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.FreeBSD)
+        if args.skip_build_cygwin:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.Cygwin)
+        if args.skip_build_osx:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.OSX)
+        if args.skip_build_ios_device:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.iOS)
+        if args.skip_build_ios_simulator:
+            self.platforms_to_skip_build.add(
+                StdlibDeploymentTarget.iOSSimulator)
+        if args.skip_build_tvos_device:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleTV)
+        if args.skip_build_tvos_simulator:
+            self.platforms_to_skip_build.add(
+                StdlibDeploymentTarget.AppleTVSimulator)
+        if args.skip_build_watchos_device:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.AppleWatch)
+        if args.skip_build_watchos_simulator:
+            self.platforms_to_skip_build.add(
+                StdlibDeploymentTarget.AppleWatchSimulator)
+        if args.skip_build_android:
+            self.platforms_to_skip_build.add(StdlibDeploymentTarget.Android)
+
+        self.platforms_to_skip_test = set()
+        if args.skip_test_linux:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.Linux)
+        if args.skip_test_freebsd:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.FreeBSD)
+        if args.skip_test_cygwin:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.Cygwin)
+        if args.skip_test_osx:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.OSX)
+        if args.skip_test_ios_host:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.iOS)
+        if args.skip_test_ios_simulator:
+            self.platforms_to_skip_test.add(
+                StdlibDeploymentTarget.iOSSimulator)
+        if args.skip_test_tvos_host:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleTV)
+        if args.skip_test_tvos_simulator:
+            self.platforms_to_skip_test.add(
+                StdlibDeploymentTarget.AppleTVSimulator)
+        if args.skip_test_watchos_host:
+            self.platforms_to_skip_test.add(StdlibDeploymentTarget.AppleWatch)
+        if args.skip_test_watchos_simulator:
+            self.platforms_to_skip_test.add(
+                StdlibDeploymentTarget.AppleWatchSimulator)
+        # We never allow testing Android, currently.
+        #
+        # FIXME: Allow Android host tests to be enabled/disabled by the build
+        # script.
+        self.platforms_to_skip_test.add(StdlibDeploymentTarget.Android)
+
+        self.platforms_to_skip_test_host = set()
+        if args.skip_test_ios_host:
+            self.platforms_to_skip_test_host.add(StdlibDeploymentTarget.iOS)
+        if args.skip_test_tvos_host:
+            self.platforms_to_skip_test_host.add(
+                StdlibDeploymentTarget.AppleTV)
+        if args.skip_test_watchos_host:
+            self.platforms_to_skip_test_host.add(
+                StdlibDeploymentTarget.AppleWatch)
+
     def initialize_runtime_environment(self):
         """Change the program environment for building."""
 
@@ -322,10 +507,10 @@ class BuildScriptInvocation(object):
         self.toolchain.ninja = ninja_build.ninja_bin_path
 
     def convert_to_impl_arguments(self):
-        """convert_to_impl_arguments() -> args
+        """convert_to_impl_arguments() -> (env, args)
 
-        Convert the invocation to a list of arguments suitable for invoking
-        `build-script-impl`.
+        Convert the invocation to an environment and list of arguments suitable
+        for invoking `build-script-impl`.
         """
 
         # Create local shadows, for convenience.
@@ -374,6 +559,14 @@ class BuildScriptInvocation(object):
                 pipes.quote(arg) for arg in cmake.build_args()),
         ]
 
+        if args.build_stdlib_deployment_targets:
+            impl_args += [
+                "--build-stdlib-deployment-targets", " ".join(
+                    args.build_stdlib_deployment_targets)]
+        if args.cross_compile_hosts:
+            impl_args += [
+                "--cross-compile-hosts", " ".join(args.cross_compile_hosts)]
+
         if toolchain.ninja:
             impl_args += ["--ninja-bin=%s" % toolchain.ninja]
         if args.distcc:
@@ -415,6 +608,8 @@ class BuildScriptInvocation(object):
             impl_args += ["--skip-build-libdispatch"]
         if not args.build_swiftpm:
             impl_args += ["--skip-build-swiftpm"]
+        if args.build_swift_stdlib_unittest_extra:
+            impl_args += ["--build-swift-stdlib-unittest-extra"]
 
         if args.skip_build_linux:
             impl_args += ["--skip-build-linux"]
@@ -536,7 +731,48 @@ class BuildScriptInvocation(object):
         if args.dry_run:
             impl_args += ["--dry-run"]
 
-        return impl_args
+        # Compute the set of host-specific variables, which we pass through to
+        # the build script via environment variables.
+        host_specific_variables = self.compute_host_specific_variables()
+        impl_env = {}
+        for (host_target, options) in host_specific_variables.items():
+            for (name, value) in options.items():
+                # We mangle into an environment variable we can easily evaluate
+                # from the `build-script-impl`.
+                impl_env["HOST_VARIABLE_{}__{}".format(
+                    host_target.replace("-", "_"), name)] = value
+
+        return (impl_env, impl_args)
+
+    def compute_host_specific_variables(self):
+        """compute_host_specific_variables(args) -> dict
+
+        Compute the host-specific options, organized as a dictionary keyed by
+        host of options.
+        """
+
+        args = self.args
+
+        options = {}
+        for host_target in [args.host_target] + args.cross_compile_hosts:
+            # Compute the host specific configuration.
+            config = HostSpecificConfiguration(host_target, self)
+
+            # Convert into `build-script-impl` style variables.
+            options[host_target] = {
+                "SWIFT_SDKS": " ".join(sorted(
+                    config.sdks_to_configure)),
+                "SWIFT_STDLIB_TARGETS": " ".join(
+                    config.swift_stdlib_build_targets),
+                "SWIFT_BENCHMARK_TARGETS": " ".join(
+                    config.swift_benchmark_build_targets),
+                "SWIFT_RUN_BENCHMARK_TARGETS": " ".join(
+                    config.swift_benchmark_run_targets),
+                "SWIFT_TEST_TARGETS": " ".join(
+                    config.swift_test_run_targets),
+            }
+
+        return options
 
 
 # Main entry point for the preset mode.
@@ -814,13 +1050,27 @@ details of the setups of other systems or automated environments.""")
         help="The host target. LLVM, Clang, and Swift will be built for this "
              "target. The built LLVM and Clang will be used to compile Swift "
              "for the cross-compilation targets.",
-        default=StdlibDeploymentTarget.host_target())
+        default=StdlibDeploymentTarget.host_target().name)
+    targets_group.add_argument(
+        "--cross-compile-hosts",
+        help="A space separated list of targets to cross-compile host Swift "
+             "tools for. Can be used multiple times.",
+        action=arguments.action.concat, type=arguments.type.shell_split,
+        default=[])
+    stdlib_targets = StdlibDeploymentTarget.default_stdlib_deployment_targets()
     targets_group.add_argument(
         "--stdlib-deployment-targets",
         help="list of targets to compile or cross-compile the Swift standard "
              "library for. %(default)s by default.",
         nargs="*",
-        default=StdlibDeploymentTarget.default_stdlib_deployment_targets())
+        default=[
+            target.name
+            for target in stdlib_targets])
+    targets_group.add_argument(
+        "--build-stdlib-deployment-targets",
+        help="A space-separated list that filters which of the configured "
+             "targets to build the Swift standard library for, or 'all'.",
+        type=arguments.type.shell_split, default=["all"])
 
     projects_group = parser.add_argument_group(
         title="Options to select projects")
@@ -1120,6 +1370,10 @@ details of the setups of other systems or automated environments.""")
         help="Use the host compiler, not the self-built one to compile the "
              "Swift runtime",
         action="store_true")
+    parser.add_argument(
+        "--build-swift-stdlib-unittest-extra",
+        help="Build optional StdlibUnittest components",
+        action="store_true")
 
     run_build_group = parser.add_argument_group(
         title="Run build")
@@ -1480,6 +1734,7 @@ details of the setups of other systems or automated environments.""")
         "--build-jobs",
         "--common-cmake-options",
         "--only-execute",
+        "--skip-test-optimized",
         action=arguments.action.unavailable)
 
     args = migration.parse_args(parser, sys.argv[1:])
@@ -1541,10 +1796,12 @@ details of the setups of other systems or automated environments.""")
         invocation.build_ninja()
 
     # Convert to a build-script-impl invocation.
-    build_script_impl_args = invocation.convert_to_impl_arguments()
+    (build_script_impl_env, build_script_impl_args) = \
+        invocation.convert_to_impl_arguments()
 
     # Execute the underlying build script implementation.
-    call_without_sleeping([build_script_impl] + build_script_impl_args)
+    call_without_sleeping([build_script_impl] + build_script_impl_args,
+                          env=build_script_impl_env)
 
     if args.symbols_package:
         print('--- Creating symbols package ---')