Allow depending on local dependencies for CI

README.md

@@ -31,20 +31,23 @@ For local development you'll first need to download and install a recent [swift.
 
 To generate an Xcode project that's set up correctly, run `build-script-helper.py`, passing the path to the downloaded toolchain via the `--toolchain` option, the tool's package name in the `--package-dir` option, and the `generate-xcodeproj` action:
 ```
-$ ./build-script-helper.py --package-dir SourceKitStressTester --toolchain $TOOLCHAIN_DIR generate-xcodeproj
+$ ./build-script-helper.py --package-dir SourceKitStressTester --toolchain $TOOLCHAIN_DIR generate-xcodeproj --no-local-deps
 ```
+If you have the [SwiftSyntax](https://github.com/apple/swift-syntax) and [SwiftPM](https://github.com/apple/swift-package-manager) repositories already checked out next to the stress tester's repository, you can omit the `--no-local-deps` option to use the existing checkouts instead of fetching the dependencies using SwiftPM.
+
 This will generate `SourceKitStressTester/SourceKitStressTester.xcodeproj`. Open it and select the toolchain you installed from the Xcode > Toolchains menu, before building the `SourceKitStressTester-Package` scheme.
 
 #### Via command line
 
-To build, run `build-script-helper.py`, passing the path to the downloaded toolchain via the `--toolchain` option and the tool's package name in the `--package-dir` option:
+To build, run `build-script-helper.py`, passing the path to the downloaded toolchain via the `--toolchain` option and the tool's package name in the `--package-dir` option.
 ```
 $ ./build-script-helper.py --package-dir SourceKitStressTester --toolchain $TOOLCHAIN_DIR
 ```
+If you have the [SwiftSyntax](https://github.com/apple/swift-syntax) and [SwiftPM](https://github.com/apple/swift-package-manager) repositories already checked out next to the stress tester's repository, you can omit the `--no-local-deps` option to use the existing checkouts instead of fetching the dependencies using SwiftPM.
 
 To run the tests, repeat the above command, but additionally pass the `test` action:
 ```
-$ ./Utilities/build-script-helper.py --package-dir SourceKitStressTester --toolchain $TOOLCHAIN_DIR
+$ ./Utilities/build-script-helper.py test --package-dir SourceKitStressTester --toolchain $TOOLCHAIN_DIR
 ```
 
 ## Running

SourceKitStressTester/Package.swift

@@ -1,6 +1,11 @@
-// swift-tools-version:4.2
+// swift-tools-version:5.1
 
 import PackageDescription
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
 
 let package = Package(
   name: "SourceKitStressTester",
@@ -9,34 +14,57 @@ let package = Package(
     .executable(name: "sk-swiftc-wrapper", targets: ["sk-swiftc-wrapper"]),
   ],
   dependencies: [
-    .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
-    // FIXME: We should depend on master once master contains all the degybed files
-    .package(url: "https://github.com/apple/swift-syntax.git", .branch("master")),
-
+    // See dependencies added below.
   ],
   targets: [
     .target(
       name: "Common",
-      dependencies: ["TSCUtility"]),
+      dependencies: ["TSCUtility"]
+    ),
     .target(
       name: "StressTester",
-      dependencies: ["Common", "TSCUtility", "SwiftSyntax"]),
+      dependencies: ["Common", "TSCUtility", "SwiftSyntax"]
+    ),
     .target(
       name: "SwiftCWrapper",
-      dependencies: ["Common", "TSCUtility"]),
+      dependencies: ["Common", "TSCUtility"]
+    ),
 
     .target(
       name: "sk-stress-test",
-      dependencies: ["StressTester"]),
+      dependencies: ["StressTester"]
+    ),
     .target(
       name: "sk-swiftc-wrapper",
-      dependencies: ["SwiftCWrapper"]),
+      dependencies: ["SwiftCWrapper"]
+    ),
 
     .testTarget(
-        name: "StressTesterToolTests",
-        dependencies: ["StressTester"]),
+      name: "StressTesterToolTests",
+      dependencies: ["StressTester"],
+      // SwiftPM does not get the rpath for XCTests in multiroot packages right (rdar://56793593)
+      // Add the correct rpath here
+      linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@loader_path/../../../"])]
+    ),
     .testTarget(
       name: "SwiftCWrapperToolTests",
-      dependencies: ["SwiftCWrapper"])
+      dependencies: ["SwiftCWrapper"],
+      // SwiftPM does not get the rpath for XCTests in multiroot packages right (rdar://56793593)
+      // Add the correct rpath here
+      linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@loader_path/../../../"])]
+    )
   ]
 )
+
+if getenv("SWIFTCI_USE_LOCAL_DEPS") == nil {
+  // Building standalone.
+  package.dependencies += [
+    .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
+    .package(url: "https://github.com/apple/swift-syntax.git", .branch("master")),
+  ]
+} else {
+  package.dependencies += [
+    .package(path: "../../swiftpm"),
+    .package(path: "../../swift-syntax"),
+  ]
+}

SwiftEvolve/Package.swift

@@ -1,30 +1,52 @@
-// swift-tools-version:4.2
-// The swift-tools-version declares the minimum version of Swift required to build this package.
+// swift-tools-version:5.1
 
 import PackageDescription
 
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
+
 let package = Package(
     name: "SwiftEvolve",
     products: [
         .executable(name: "swift-evolve", targets: ["swift-evolve"]),
         .library(name: "SwiftEvolve", targets: ["SwiftEvolve"])
     ],
     dependencies: [
-        .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
-        // FIXME: We should depend on master once master contains all the degybed files
-        .package(url: "https://github.com/apple/swift-syntax.git", .branch("master")),
+        // See dependencies added below.
     ],
     targets: [
         // Targets are the basic building blocks of a package. A target can define a module or a test suite.
         // Targets can depend on other targets in this package, and on products in packages which this package depends on.
         .target(
             name: "swift-evolve",
-            dependencies: ["SwiftEvolve"]),
+            dependencies: ["SwiftEvolve"]
+        ),
         .target(
             name: "SwiftEvolve",
-            dependencies: ["TSCUtility", "SwiftSyntax"]),
+            dependencies: ["TSCUtility", "SwiftSyntax"]
+        ),
         .testTarget(
-          name: "SwiftEvolveTests",
-          dependencies: ["SwiftEvolve"])
+            name: "SwiftEvolveTests",
+            dependencies: ["SwiftEvolve"],
+            // SwiftPM does not get the rpath for XCTests in multiroot packages right (rdar://56793593)
+            // Add the correct rpath here
+            linkerSettings: [.unsafeFlags(["-Xlinker", "-rpath", "-Xlinker", "@loader_path/../../../"])]
+        )
     ]
 )
+
+if getenv("SWIFTCI_USE_LOCAL_DEPS") == nil {
+    // Building standalone.
+    package.dependencies += [
+        .package(url: "https://github.com/apple/swift-package-manager.git", .branch("master")),
+        .package(url: "https://github.com/apple/swift-syntax.git", .branch("master")),
+    ]
+} else {
+    package.dependencies += [
+        .package(path: "../../swiftpm"),
+        .package(path: "../../swift-syntax"),
+    ]
+}

build-script-helper.py

@@ -40,6 +40,7 @@ def parse_args(args):
   parser.add_argument('--build-dir', default='.build')
   parser.add_argument('--toolchain', required=True, help='the toolchain to use when building this package')
   parser.add_argument('--update', action='store_true', help='update all SwiftPM dependencies')
+  parser.add_argument('--no-local-deps', action='store_true', help='use normal remote dependencies when building')
   parser.add_argument('build_actions', help="Extra actions to perform. Can be any number of the following", choices=['all', 'build', 'test', 'install', 'generate-xcodeproj'], nargs="*", default=['build'])
 
   parsed = parser.parse_args(args)
@@ -64,12 +65,18 @@ def run(args):
   sourcekit_searchpath=args.sourcekitd_dir
   package_name = os.path.basename(args.package_dir)
 
+  env = dict(os.environ)
+  # Use local dependencies (i.e. checked out next sourcekit-lsp).
+  if not args.no_local_deps:
+    env['SWIFTCI_USE_LOCAL_DEPS'] = "1"
+
   if args.update:
     print("** Updating dependencies of %s **" % package_name)
     try:
       update_swiftpm_dependencies(package_dir=args.package_dir,
         swift_exec=args.swift_exec,
         build_dir=args.build_dir,
+        env=env,
         verbose=args.verbose)
     except subprocess.CalledProcessError as e:
       printerr('FAIL: Updating dependencies of %s failed' % package_name)
@@ -86,6 +93,7 @@ def run(args):
         sourcekit_searchpath=sourcekit_searchpath,
         build_dir=args.build_dir,
         config=args.config,
+        env=env,
         verbose=args.verbose)
     except subprocess.CalledProcessError as e:
       printerr('FAIL: Building %s failed' % package_name)
@@ -100,6 +108,7 @@ def run(args):
       generate_xcodeproj(args.package_dir,
         swift_exec=args.swift_exec,
         sourcekit_searchpath=sourcekit_searchpath,
+        env=env,
         verbose=args.verbose)
     except subprocess.CalledProcessError as e:
       printerr('FAIL: Generating the Xcode project failed')
@@ -119,6 +128,7 @@ def run(args):
         sourcekit_searchpath=sourcekit_searchpath,
         build_dir=args.build_dir,
         config=test_config,
+        env=env,
         verbose=args.verbose)
     except subprocess.CalledProcessError as e:
       printerr('FAIL: Testing %s failed' % package_name)
@@ -158,18 +168,22 @@ def should_run_action(action_name, selected_actions):
     return False
 
 
-def update_swiftpm_dependencies(package_dir, swift_exec, build_dir, verbose):
+def update_swiftpm_dependencies(package_dir, swift_exec, build_dir, env, verbose):
   args = [swift_exec, 'package', '--package-path', package_dir, '--build-path', build_dir, 'update']
-  check_call(args, verbose=verbose)
+  check_call(args, env=env, verbose=verbose)
 
 
-def invoke_swift(package_dir, action, swift_exec, sourcekit_searchpath, build_dir, config, verbose):
+def invoke_swift(package_dir, action, swift_exec, sourcekit_searchpath, build_dir, config, env, verbose):
   swiftc_args = ['-Fsystem', sourcekit_searchpath]
   linker_args = ['-rpath', sourcekit_searchpath]
 
-
   args = [swift_exec, action, '--package-path', package_dir, '-c', config, '--build-path', build_dir] + interleave('-Xswiftc', swiftc_args) + interleave('-Xlinker', linker_args)
-  check_call(args, verbose=verbose)
+
+  # Tell SwiftSyntax that we are building in a CI environment so that it does
+  # not need to rebuilt if it has already been built before.
+  env['SWIFT_BUILD_SCRIPT_ENVIRONMENT'] = '1'
+
+  check_call(args, env=env, verbose=verbose)
 
 
 def install_package(package_dir, install_dir, sourcekit_searchpath, build_dir, rpaths_to_delete, verbose):
@@ -203,7 +217,7 @@ def install(src, dest, rpaths_to_delete, rpaths_to_add, verbose):
   for rpath in rpaths_to_add:
     add_rpath(dest, rpath, verbose=verbose)
 
-def generate_xcodeproj(package_dir, swift_exec, sourcekit_searchpath, verbose):
+def generate_xcodeproj(package_dir, swift_exec, sourcekit_searchpath, env, verbose):
   package_name = os.path.basename(package_dir)
   config_path = os.path.join(package_dir, 'Config.xcconfig')
   with open(config_path, 'w') as config_file:
@@ -215,7 +229,7 @@ def generate_xcodeproj(package_dir, swift_exec, sourcekit_searchpath, verbose):
   xcodeproj_path = os.path.join(package_dir, '%s.xcodeproj' % package_name)
 
   args = [swift_exec, 'package', '--package-path', package_dir, 'generate-xcodeproj', '--xcconfig-overrides', config_path, '--output', xcodeproj_path]
-  check_call(args, verbose=verbose)
+  check_call(args, env=env, verbose=verbose)
 
 def add_rpath(binary, rpath, verbose):
   cmd = ['install_name_tool', '-add_rpath', rpath, binary]