From 5f48133a42f4ecd62138fb20be5ffe5797efc1a6 Mon Sep 17 00:00:00 2001 From: Boris Buegling Date: Tue, 22 Aug 2023 16:35:46 -0700 Subject: [PATCH] Implement `swift-get-version` in SwiftPM The changes in #6585 meant that we are no longer using LLBuild's built-in tracking of Swift compiler versions. We are lacking the infrastructure to really use that for the same purpose since we are now running the Swift compiler as a shell-tool, so this is adding a poor man's version of that. We have a task that writes the output of `swift -version` for a particular Swift compiler path to the build directory and all Swift tasks that are using that compiler depend on that file as an input. This should give us the desired behavior of the tasks re-running if the Swift version changes. rdar://114047018 --- Package.swift | 8 +++ ...dOperationBuildSystemDelegateHandler.swift | 4 +- Sources/Build/LLBuildManifestBuilder.swift | 24 +++++++++ Sources/LLBuildManifest/BuildManifest.swift | 34 ++++++++++++- Sources/LLBuildManifest/ManifestWriter.swift | 4 ++ Sources/LLBuildManifest/Tools.swift | 9 +++- Sources/SPMTestSupport/SwiftPMProduct.swift | 8 ++- Sources/dummy-swiftc/main.swift | 27 ++++++++++ Tests/BuildTests/BuildPlanTests.swift | 12 +++-- Tests/CommandsTests/BuildToolTests.swift | 49 +++++++++++++++++++ 10 files changed, 170 insertions(+), 9 deletions(-) create mode 100644 Sources/dummy-swiftc/main.swift diff --git a/Package.swift b/Package.swift index 36e264623c1..c669f9aaa2b 100644 --- a/Package.swift +++ b/Package.swift @@ -666,6 +666,13 @@ if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] == ] ), + .executableTarget( + name: "dummy-swiftc", + dependencies: [ + "Basics", + ] + ), + .testTarget( name: "CommandsTests", dependencies: [ @@ -681,6 +688,7 @@ if ProcessInfo.processInfo.environment["SWIFTCI_DISABLE_SDK_DEPENDENT_TESTS"] == "SourceControl", "SPMTestSupport", "Workspace", + "dummy-swiftc", ] ), ]) diff --git a/Sources/Build/BuildOperationBuildSystemDelegateHandler.swift b/Sources/Build/BuildOperationBuildSystemDelegateHandler.swift index 386fb27194f..d178589abdb 100644 --- a/Sources/Build/BuildOperationBuildSystemDelegateHandler.swift +++ b/Sources/Build/BuildOperationBuildSystemDelegateHandler.swift @@ -473,9 +473,11 @@ public final class BuildExecutionContext { final class WriteAuxiliaryFileCommand: CustomLLBuildCommand { override func getSignature(_ command: SPMLLBuild.Command) -> [UInt8] { guard let buildDescription = self.context.buildDescription else { + self.context.observabilityScope.emit(error: "unknown build description") return [] } - guard let tool = buildDescription.copyCommands[command.name] else { + guard let tool = buildDescription.writeCommands[command.name] else { + self.context.observabilityScope.emit(error: "command \(command.name) not registered") return [] } diff --git a/Sources/Build/LLBuildManifestBuilder.swift b/Sources/Build/LLBuildManifestBuilder.swift index b3f3d487922..db8c0a90a4c 100644 --- a/Sources/Build/LLBuildManifestBuilder.swift +++ b/Sources/Build/LLBuildManifestBuilder.swift @@ -53,6 +53,9 @@ public class LLBuildManifestBuilder { var buildParameters: BuildParameters { self.plan.buildParameters } var buildEnvironment: BuildEnvironment { self.buildParameters.buildEnvironment } + /// Mapping from Swift compiler path to Swift get version files. + var swiftGetVersionFiles = [AbsolutePath: AbsolutePath]() + /// Create a new builder with a build plan. public init( _ plan: BuildPlan, @@ -71,6 +74,8 @@ public class LLBuildManifestBuilder { /// Generate manifest at the given path. @discardableResult public func generateManifest(at path: AbsolutePath) throws -> BuildManifest { + self.swiftGetVersionFiles.removeAll() + self.manifest.createTarget(TargetKind.main.targetName) self.manifest.createTarget(TargetKind.test.targetName) self.manifest.defaultTarget = TargetKind.main.targetName @@ -608,6 +613,9 @@ extension LLBuildManifestBuilder { ) throws -> [Node] { var inputs = target.sources.map(Node.file) + let swiftVersionFilePath = addSwiftGetVersionCommand(buildParameters: target.buildParameters) + inputs.append(.file(swiftVersionFilePath)) + // Add resources node as the input to the target. This isn't great because we // don't need to block building of a module until its resources are assembled but // we don't currently have a good way to express that resources should be built @@ -733,6 +741,22 @@ extension LLBuildManifestBuilder { arguments: moduleWrapArgs ) } + + private func addSwiftGetVersionCommand(buildParameters: BuildParameters) -> AbsolutePath { + let swiftCompilerPath = buildParameters.toolchain.swiftCompilerPath + + // If we are already tracking this compiler, we can re-use the existing command by just returning the tracking file. + if let swiftVersionFilePath = swiftGetVersionFiles[swiftCompilerPath] { + return swiftVersionFilePath + } + + // Otherwise, come up with a path for the new file and generate a command to populate it. + let swiftCompilerPathHash = String(swiftCompilerPath.pathString.hash, radix: 16, uppercase: true) + let swiftVersionFilePath = buildParameters.buildPath.appending(component: "swift-version-\(swiftCompilerPathHash).txt") + self.manifest.addSwiftGetVersionCommand(swiftCompilerPath: swiftCompilerPath, swiftVersionFilePath: swiftVersionFilePath) + swiftGetVersionFiles[swiftCompilerPath] = swiftVersionFilePath + return swiftVersionFilePath + } } extension SwiftDriver.Job { diff --git a/Sources/LLBuildManifest/BuildManifest.swift b/Sources/LLBuildManifest/BuildManifest.swift index a75923f9ae4..59475b4fdff 100644 --- a/Sources/LLBuildManifest/BuildManifest.swift +++ b/Sources/LLBuildManifest/BuildManifest.swift @@ -12,6 +12,8 @@ import Basics +import class TSCBasic.Process + public protocol AuxiliaryFileType { static var name: String { get } @@ -19,7 +21,7 @@ public protocol AuxiliaryFileType { } public enum WriteAuxiliary { - public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self] + public static let fileTypes: [AuxiliaryFileType.Type] = [LinkFileList.self, SourcesFileList.self, SwiftGetVersion.self] public struct LinkFileList: AuxiliaryFileType { public static let name = "link-file-list" @@ -76,6 +78,26 @@ public enum WriteAuxiliary { return contents } } + + public struct SwiftGetVersion: AuxiliaryFileType { + public static let name = "swift-get-version" + + public static func computeInputs(swiftCompilerPath: AbsolutePath) -> [Node] { + return [.virtual(Self.name), .file(swiftCompilerPath)] + } + + public static func getFileContents(inputs: [Node]) throws -> String { + guard let swiftCompilerPathString = inputs.first(where: { $0.kind == .file })?.name else { + throw Error.unknownSwiftCompilerPath + } + let swiftCompilerPath = try AbsolutePath(validating: swiftCompilerPathString) + return try TSCBasic.Process.checkNonZeroExit(args: swiftCompilerPath.pathString, "-version") + } + + private enum Error: Swift.Error { + case unknownSwiftCompilerPath + } + } } public struct BuildManifest { @@ -173,6 +195,16 @@ public struct BuildManifest { commands[name] = Command(name: name, tool: tool) } + public mutating func addSwiftGetVersionCommand( + swiftCompilerPath: AbsolutePath, + swiftVersionFilePath: AbsolutePath + ) { + let inputs = WriteAuxiliary.SwiftGetVersion.computeInputs(swiftCompilerPath: swiftCompilerPath) + let tool = WriteAuxiliaryFile(inputs: inputs, outputFilePath: swiftVersionFilePath, alwaysOutOfDate: true) + let name = swiftVersionFilePath.pathString + commands[name] = Command(name: name, tool: tool) + } + public mutating func addPkgStructureCmd( name: String, inputs: [Node], diff --git a/Sources/LLBuildManifest/ManifestWriter.swift b/Sources/LLBuildManifest/ManifestWriter.swift index ab824d59395..b7dda66be89 100644 --- a/Sources/LLBuildManifest/ManifestWriter.swift +++ b/Sources/LLBuildManifest/ManifestWriter.swift @@ -72,6 +72,10 @@ public struct ManifestWriter { manifestToolWriter["inputs"] = tool.inputs manifestToolWriter["outputs"] = tool.outputs + if tool.alwaysOutOfDate { + manifestToolWriter["always-out-of-date"] = "true" + } + tool.write(to: manifestToolWriter) stream.send("\n") diff --git a/Sources/LLBuildManifest/Tools.swift b/Sources/LLBuildManifest/Tools.swift index 98702f48924..7b641d539d9 100644 --- a/Sources/LLBuildManifest/Tools.swift +++ b/Sources/LLBuildManifest/Tools.swift @@ -17,6 +17,9 @@ public protocol ToolProtocol: Codable { /// The name of the tool. static var name: String { get } + /// Whether or not the tool should run on every build instead of using dependency tracking. + var alwaysOutOfDate: Bool { get } + /// The list of inputs to declare. var inputs: [Node] { get } @@ -28,6 +31,8 @@ public protocol ToolProtocol: Codable { } extension ToolProtocol { + public var alwaysOutOfDate: Bool { return false } + public func write(to stream: ManifestToolStream) {} } @@ -155,10 +160,12 @@ public struct WriteAuxiliaryFile: ToolProtocol { public let inputs: [Node] private let outputFilePath: AbsolutePath + public let alwaysOutOfDate: Bool - public init(inputs: [Node], outputFilePath: AbsolutePath) { + public init(inputs: [Node], outputFilePath: AbsolutePath, alwaysOutOfDate: Bool = false) { self.inputs = inputs self.outputFilePath = outputFilePath + self.alwaysOutOfDate = alwaysOutOfDate } public var outputs: [Node] { diff --git a/Sources/SPMTestSupport/SwiftPMProduct.swift b/Sources/SPMTestSupport/SwiftPMProduct.swift index 9cc4e25292b..a3bb5b845b0 100644 --- a/Sources/SPMTestSupport/SwiftPMProduct.swift +++ b/Sources/SPMTestSupport/SwiftPMProduct.swift @@ -46,14 +46,18 @@ extension SwiftPM { /// Path to currently built binary. public var path: AbsolutePath { + return Self.testBinaryPath(for: self.executableName) + } + + public static func testBinaryPath(for executableName: RelativePath) -> AbsolutePath { #if canImport(Darwin) for bundle in Bundle.allBundles where bundle.bundlePath.hasSuffix(".xctest") { - return try! AbsolutePath(AbsolutePath(validating: bundle.bundlePath).parentDirectory, self.executableName) + return try! AbsolutePath(AbsolutePath(validating: bundle.bundlePath).parentDirectory, executableName) } fatalError() #else return try! AbsolutePath(validating: CommandLine.arguments.first!, relativeTo: localFileSystem.currentWorkingDirectory!) - .parentDirectory.appending(self.executableName) + .parentDirectory.appending(executableName) #endif } } diff --git a/Sources/dummy-swiftc/main.swift b/Sources/dummy-swiftc/main.swift new file mode 100644 index 00000000000..38d92f430e2 --- /dev/null +++ b/Sources/dummy-swiftc/main.swift @@ -0,0 +1,27 @@ +// This program can be used as `swiftc` in order to influence `-version` output + +import Foundation + +import class TSCBasic.Process + +let info = ProcessInfo.processInfo +let env = info.environment + +if info.arguments.last == "-version" { + if let customSwiftVersion = env["CUSTOM_SWIFT_VERSION"] { + print(customSwiftVersion) + } else { + print("999.0") + } +} else { + let swiftPath: String + if let swiftOriginalPath = env["SWIFT_ORIGINAL_PATH"] { + swiftPath = swiftOriginalPath + } else { + swiftPath = "/usr/bin/swiftc" + } + + let result = try Process.popen(arguments: [swiftPath] + info.arguments.dropFirst()) + print(try result.utf8Output()) + print(try result.utf8stderrOutput()) +} diff --git a/Tests/BuildTests/BuildPlanTests.swift b/Tests/BuildTests/BuildPlanTests.swift index a0815490830..7e3c256a4e9 100644 --- a/Tests/BuildTests/BuildPlanTests.swift +++ b/Tests/BuildTests/BuildPlanTests.swift @@ -867,8 +867,9 @@ final class BuildPlanTests: XCTestCase { let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope) try llbuild.generateManifest(at: yaml) let contents: String = try fs.readFileContents(yaml) + let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value) XCTAssertMatch(contents, .contains(""" - inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"] + inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "PkgLib.swiftmodule").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"] """)) } @@ -896,8 +897,9 @@ final class BuildPlanTests: XCTestCase { try llbuild.generateManifest(at: yaml) let contents: String = try fs.readFileContents(yaml) let buildPath = plan.buildParameters.dataPath.appending(components: "debug") + let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value) XCTAssertMatch(contents, .contains(""" - inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"] + inputs: ["\(Pkg.appending(components: "Sources", "exe", "main.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "exe.build", "sources").escapedPathString())"] """)) } } @@ -3894,6 +3896,7 @@ final class BuildPlanTests: XCTestCase { let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope) try llbuild.generateManifest(at: yaml) let contents: String = try fs.readFileContents(yaml) + let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value) #if os(Windows) let suffix = ".exe" @@ -3901,7 +3904,7 @@ final class BuildPlanTests: XCTestCase { let suffix = "" #endif XCTAssertMatch(contents, .contains(""" - inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"] + inputs: ["\(PkgA.appending(components: "Sources", "swiftlib", "lib.swift").escapedPathString())","\(swiftGetVersionFilePath.escapedPathString())","\(buildPath.appending(components: "exe\(suffix)").escapedPathString())","\(buildPath.appending(components: "swiftlib.build", "sources").escapedPathString())"] outputs: ["\(buildPath.appending(components: "swiftlib.build", "lib.swift.o").escapedPathString())","\(buildPath.escapedPathString()) """)) } @@ -4804,10 +4807,11 @@ final class BuildPlanTests: XCTestCase { let yaml = buildPath.appending("release.yaml") let llbuild = LLBuildManifestBuilder(plan, fileSystem: fs, observabilityScope: observability.topScope) try llbuild.generateManifest(at: yaml) + let swiftGetVersionFilePath = try XCTUnwrap(llbuild.swiftGetVersionFiles.first?.value) let yamlContents: String = try fs.readFileContents(yaml) let inputs: SerializedJSON = """ - inputs: ["\(AbsolutePath("/Pkg/Snippets/ASnippet.swift"))","\(AbsolutePath("/Pkg/.build/debug/Lib.swiftmodule"))" + inputs: ["\(AbsolutePath("/Pkg/Snippets/ASnippet.swift"))","\(swiftGetVersionFilePath.escapedPathString())","\(AbsolutePath("/Pkg/.build/debug/Lib.swiftmodule"))" """ XCTAssertMatch(yamlContents, .contains(inputs.underlying)) } diff --git a/Tests/CommandsTests/BuildToolTests.swift b/Tests/CommandsTests/BuildToolTests.swift index 59f2a56d64d..a1ffb6ce17d 100644 --- a/Tests/CommandsTests/BuildToolTests.swift +++ b/Tests/CommandsTests/BuildToolTests.swift @@ -400,4 +400,53 @@ final class BuildToolTests: CommandsTestCase { } } } + + func testSwiftGetVersion() throws { + try fixture(name: "Miscellaneous/Simple") { fixturePath in + func findSwiftGetVersionFile() throws -> AbsolutePath { + let buildArenaPath = fixturePath.appending(components: ".build", "debug") + let files = try localFileSystem.getDirectoryContents(buildArenaPath) + let filename = try XCTUnwrap(files.first { $0.hasPrefix("swift-version") }) + return buildArenaPath.appending(component: filename) + } + + let dummySwiftcPath = SwiftPM.testBinaryPath(for: "dummy-swiftc") + let swiftCompilerPath = try UserToolchain.default.swiftCompilerPath + + var environment = [ + "SWIFT_EXEC": dummySwiftcPath.pathString, + // Environment variables used by `dummy-swiftc.sh` + "SWIFT_ORIGINAL_PATH": swiftCompilerPath.pathString, + "CUSTOM_SWIFT_VERSION": "1.0", + ] + + // Build with a swiftc that returns version 1.0, we expect a successful build which compiles our one source file. + do { + let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath) + XCTAssertTrue(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task missing from build result: \(result.stdout)") + XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)") + let swiftGetVersionFilePath = try findSwiftGetVersionFile() + XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "1.0") + } + + // Build again with that same version, we do not expect any compilation tasks. + do { + let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath) + XCTAssertFalse(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task present in build result: \(result.stdout)") + XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)") + let swiftGetVersionFilePath = try findSwiftGetVersionFile() + XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "1.0") + } + + // Build again with a swiftc that returns version 2.0, we expect compilation happening once more. + do { + environment["CUSTOM_SWIFT_VERSION"] = "2.0" + let result = try execute(["--verbose"], environment: environment, packagePath: fixturePath) + XCTAssertTrue(result.stdout.contains("\(dummySwiftcPath.pathString) -module-name"), "compilation task missing from build result: \(result.stdout)") + XCTAssertTrue(result.stdout.contains("Build complete!"), "unexpected build result: \(result.stdout)") + let swiftGetVersionFilePath = try findSwiftGetVersionFile() + XCTAssertEqual(try String(contentsOfFile: swiftGetVersionFilePath.pathString).spm_chomp(), "2.0") + } + } + } }