From afb66114e2061c547e5ad0fe023f863ce680e95f Mon Sep 17 00:00:00 2001 From: Artem Chikin Date: Mon, 26 Feb 2024 17:44:02 -0800 Subject: [PATCH] [Dependency Scanning] Add and use API for per-query diagnostic gathering Adopts new API that produces per-scan diagnostics for each query of a dependency graph or an import set --- Sources/CSwiftScan/include/swiftscan_header.h | 4 + .../InterModuleDependencyOracle.swift | 25 +++- .../ModuleDependencyScanning.swift | 37 ++++-- Sources/SwiftDriver/SwiftScan/SwiftScan.swift | 56 ++++++-- .../SwiftDriverTests/CachingBuildTests.swift | 8 +- .../ExplicitModuleBuildTests.swift | 125 ++++++++++++++++-- 6 files changed, 212 insertions(+), 43 deletions(-) diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h index 9408f3fd5..d17b40b4c 100644 --- a/Sources/CSwiftScan/include/swiftscan_header.h +++ b/Sources/CSwiftScan/include/swiftscan_header.h @@ -97,6 +97,8 @@ typedef struct { (*swiftscan_dependency_graph_get_main_module_name)(swiftscan_dependency_graph_t); swiftscan_dependency_set_t * (*swiftscan_dependency_graph_get_dependencies)(swiftscan_dependency_graph_t); + swiftscan_diagnostic_set_t * + (*swiftscan_dependency_graph_get_diagnostics)(swiftscan_dependency_graph_t); //=== Dependency Module Info Functions ------------------------------------===// swiftscan_string_ref_t @@ -199,6 +201,8 @@ typedef struct { //=== Prescan Result Functions --------------------------------------------===// swiftscan_string_set_t * (*swiftscan_import_set_get_imports)(swiftscan_import_set_t); + swiftscan_diagnostic_set_t * + (*swiftscan_import_set_get_diagnostics)(swiftscan_import_set_t); //=== Scanner Invocation Functions ----------------------------------------===// swiftscan_scan_invocation_t diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift index e70de949b..27590f9ea 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift @@ -46,34 +46,40 @@ public class InterModuleDependencyOracle { @_spi(Testing) public func getDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]? = nil, - commandLine: [String]) + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { precondition(hasScannerInstance) return try swiftScanLibInstance!.scanDependencies(workingDirectory: workingDirectory, moduleAliases: moduleAliases, - invocationCommand: commandLine) + invocationCommand: commandLine, + diagnostics: &diagnostics) } @_spi(Testing) public func getBatchDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]? = nil, commandLine: [String], - batchInfos: [BatchScanModuleInfo]) + batchInfos: [BatchScanModuleInfo], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { precondition(hasScannerInstance) return try swiftScanLibInstance!.batchScanDependencies(workingDirectory: workingDirectory, moduleAliases: moduleAliases, invocationCommand: commandLine, - batchInfos: batchInfos) + batchInfos: batchInfos, + diagnostics: &diagnostics) } @_spi(Testing) public func getImports(workingDirectory: AbsolutePath, moduleAliases: [String: String]? = nil, - commandLine: [String]) + commandLine: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { precondition(hasScannerInstance) return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory, moduleAliases: moduleAliases, - invocationCommand: commandLine) + invocationCommand: commandLine, + diagnostics: &diagnostics) } /// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library @@ -147,6 +153,13 @@ public class InterModuleDependencyOracle { return swiftScan.supportsBridgingHeaderPCHCommand } + @_spi(Testing) public func supportsPerScanDiagnostics() throws -> Bool { + guard let swiftScan = swiftScanLibInstance else { + fatalError("Attempting to query supported scanner API with no scanner instance.") + } + return swiftScan.canQueryPerScanDiagnostics + } + @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? { guard let swiftScan = swiftScanLibInstance else { fatalError("Attempting to reset scanner cache with no scanner instance.") diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift index 74be48b69..0e0c77560 100644 --- a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift +++ b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift @@ -201,6 +201,7 @@ public extension Driver { let isSwiftScanLibAvailable = !(try initSwiftScanLib()) if isSwiftScanLibAvailable { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! var command = try Self.itemizedJobCommand(of: preScanJob, useResponseFiles: .disabled, @@ -209,12 +210,13 @@ public extension Driver { do { imports = try interModuleDependencyOracle.getImports(workingDirectory: cwd, moduleAliases: moduleOutputInfo.aliases, - commandLine: command) + commandLine: command, + diagnostics: &scanDiagnostics) } catch let DependencyScanningError.dependencyScanFailed(reason) { - try emitScannerDiagnostics() + try emitGlobalScannerDiagnostics() throw DependencyScanningError.dependencyScanFailed(reason) } - try emitScannerDiagnostics() + try emitGlobalScannerDiagnostics() } else { // Fallback to legacy invocation of the dependency scanner with // `swift-frontend -scan-dependencies -import-prescan` @@ -227,10 +229,8 @@ public extension Driver { return imports } - mutating internal func emitScannerDiagnostics() throws { - let possibleDiags = try interModuleDependencyOracle.getScannerDiagnostics() - if let diags = possibleDiags { - for diagnostic in diags { + internal func emitScannerDiagnostics(_ diagnostics: [ScannerDiagnosticPayload]) throws { + for diagnostic in diagnostics { switch diagnostic.severity { case .error: diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message)) @@ -244,6 +244,16 @@ public extension Driver { diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message)) } } + } + + mutating internal func emitGlobalScannerDiagnostics() throws { + // We only emit global scanner-collected diagnostics as a legacy flow + // when the scanner does not support per-scan diagnostic output + guard try !interModuleDependencyOracle.supportsPerScanDiagnostics() else { + return + } + if let diags = try interModuleDependencyOracle.getScannerDiagnostics() { + try emitScannerDiagnostics(diags) } } @@ -261,6 +271,7 @@ public extension Driver { let isSwiftScanLibAvailable = !(try initSwiftScanLib()) if isSwiftScanLibAvailable { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! var command = try Self.itemizedJobCommand(of: scannerJob, useResponseFiles: .disabled, @@ -269,12 +280,14 @@ public extension Driver { do { dependencyGraph = try interModuleDependencyOracle.getDependencies(workingDirectory: cwd, moduleAliases: moduleOutputInfo.aliases, - commandLine: command) + commandLine: command, + diagnostics: &scanDiagnostics) + try emitScannerDiagnostics(scanDiagnostics) } catch let DependencyScanningError.dependencyScanFailed(reason) { - try emitScannerDiagnostics() + try emitGlobalScannerDiagnostics() throw DependencyScanningError.dependencyScanFailed(reason) } - try emitScannerDiagnostics() + try emitGlobalScannerDiagnostics() } else { // Fallback to legacy invocation of the dependency scanner with // `swift-frontend -scan-dependencies` @@ -295,6 +308,7 @@ public extension Driver { let isSwiftScanLibAvailable = !(try initSwiftScanLib()) if isSwiftScanLibAvailable { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory! var command = try Self.itemizedJobCommand(of: batchScanningJob, useResponseFiles: .disabled, @@ -304,7 +318,8 @@ public extension Driver { try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd, moduleAliases: moduleOutputInfo.aliases, commandLine: command, - batchInfos: moduleInfos) + batchInfos: moduleInfos, + diagnostics: &scanDiagnostics) } else { // Fallback to legacy invocation of the dependency scanner with // `swift-frontend -scan-dependencies` diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift index 6d7b7f471..aeed424e3 100644 --- a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift +++ b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift @@ -126,7 +126,8 @@ internal extension swiftscan_diagnostic_severity_t { func preScanImports(workingDirectory: AbsolutePath, moduleAliases: [String: String]?, - invocationCommand: [String]) throws -> InterModuleDependencyImports { + invocationCommand: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports { // Create and configure the scanner invocation let invocation = api.swiftscan_scan_invocation_create() defer { api.swiftscan_scan_invocation_dispose(invocation) } @@ -144,6 +145,13 @@ internal extension swiftscan_diagnostic_severity_t { guard let importSetRef = importSetRefOrNull else { throw DependencyScanningError.dependencyScanFailed("Unable to produce import set") } + if canQueryPerScanDiagnostics { + let diagnosticsSetRefOrNull = api.swiftscan_import_set_get_diagnostics(importSetRef) + guard let diagnosticsSetRef = diagnosticsSetRefOrNull else { + throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics") + } + diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef) + } let importSet = try constructImportSet(from: importSetRef, with: moduleAliases) // Free the memory allocated for the in-memory representation of the import set @@ -154,7 +162,8 @@ internal extension swiftscan_diagnostic_severity_t { func scanDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]?, - invocationCommand: [String]) throws -> InterModuleDependencyGraph { + invocationCommand: [String], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph { // Create and configure the scanner invocation let invocation = api.swiftscan_scan_invocation_create() defer { api.swiftscan_scan_invocation_dispose(invocation) } @@ -172,6 +181,13 @@ internal extension swiftscan_diagnostic_severity_t { guard let graphRef = graphRefOrNull else { throw DependencyScanningError.dependencyScanFailed("Unable to produce dependency graph") } + if canQueryPerScanDiagnostics { + let diagnosticsSetRefOrNull = api.swiftscan_dependency_graph_get_diagnostics(graphRef) + guard let diagnosticsSetRef = diagnosticsSetRefOrNull else { + throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics") + } + diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef) + } let dependencyGraph = try constructGraph(from: graphRef, moduleAliases: moduleAliases) // Free the memory allocated for the in-memory representation of the dependency @@ -184,7 +200,8 @@ internal extension swiftscan_diagnostic_severity_t { func batchScanDependencies(workingDirectory: AbsolutePath, moduleAliases: [String: String]?, invocationCommand: [String], - batchInfos: [BatchScanModuleInfo]) + batchInfos: [BatchScanModuleInfo], + diagnostics: inout [ScannerDiagnosticPayload]) throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] { // Create and configure the scanner invocation let invocationRef = api.swiftscan_scan_invocation_create() @@ -315,6 +332,12 @@ internal extension swiftscan_diagnostic_severity_t { return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil } + + @_spi(Testing) public var canQueryPerScanDiagnostics : Bool { + return api.swiftscan_dependency_graph_get_diagnostics != nil && + api.swiftscan_import_set_get_diagnostics != nil + } + func serializeScannerCache(to path: AbsolutePath) { api.swiftscan_scanner_cache_serialize(scanner, path.description.cString(using: String.Encoding.utf8)) @@ -329,18 +352,10 @@ internal extension swiftscan_diagnostic_severity_t { api.swiftscan_scanner_cache_reset(scanner) } - @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { + internal func mapToDriverDiagnosticPayload(_ diagnosticSetRef: UnsafeMutablePointer) throws -> [ScannerDiagnosticPayload] { var result: [ScannerDiagnosticPayload] = [] - let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner) - guard let diagnosticSetRef = diagnosticSetRefOrNull else { - // Seems heavy-handed to fail here - // throw DependencyScanningError.dependencyScanFailed - return [] - } - defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) } let diagnosticRefArray = Array(UnsafeBufferPointer(start: diagnosticSetRef.pointee.diagnostics, count: Int(diagnosticSetRef.pointee.count))) - for diagnosticRefOrNull in diagnosticRefArray { guard let diagnosticRef = diagnosticRefOrNull else { throw DependencyScanningError.dependencyScanFailed("Unable to produce scanner diagnostics") @@ -352,6 +367,17 @@ internal extension swiftscan_diagnostic_severity_t { return result } + @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] { + let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner) + guard let diagnosticSetRef = diagnosticSetRefOrNull else { + // Seems heavy-handed to fail here + // throw DependencyScanningError.dependencyScanFailed + return [] + } + defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) } + return try mapToDriverDiagnosticPayload(diagnosticSetRef) + } + @_spi(Testing) public func resetScannerDiagnostics() throws { api.swiftscan_scanner_diagnostics_reset(scanner) } @@ -567,6 +593,12 @@ private extension swiftscan_functions_t { self.swiftscan_swift_binary_detail_get_header_dependencies = try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies") + // Per-scan-query diagnostic output + self.swiftscan_dependency_graph_get_diagnostics = + try loadOptional("swiftscan_dependency_graph_get_diagnostics") + self.swiftscan_import_set_get_diagnostics = + try loadOptional("swiftscan_import_set_get_diagnostics") + // MARK: Required Methods func loadRequired(_ symbol: String) throws -> T { guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else { diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift index 8ed718e95..27b38d046 100644 --- a/Tests/SwiftDriverTests/CachingBuildTests.swift +++ b/Tests/SwiftDriverTests/CachingBuildTests.swift @@ -666,9 +666,11 @@ final class CachingBuildTests: XCTestCase { // though the module-name above should be sufficient. "-I/tmp/foo/bar/\(index)"] do { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let dependencyGraph = try dependencyOracle.getDependencies(workingDirectory: path, - commandLine: iterationCommand) + commandLine: iterationCommand, + diagnostics: &scanDiagnostics) // The _Concurrency and _StringProcessing modules are automatically // imported in newer versions of the Swift compiler. If they happened to @@ -712,8 +714,10 @@ final class CachingBuildTests: XCTestCase { "-I/tmp/bad", "-cas-path", casPath2.nativePathString(escaped: true), ] + var scanDiagnostics: [ScannerDiagnosticPayload] = [] XCTAssertThrowsError(try dependencyOracle.getDependencies(workingDirectory: path, - commandLine: command)) { + commandLine: command, + diagnostics: &scanDiagnostics)) { XCTAssertTrue($0 is DependencyScanningError) } let diags = try XCTUnwrap(dependencyOracle.getScannerDiagnostics()) diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift index ba0271a38..73e0719d3 100644 --- a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift +++ b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift @@ -969,11 +969,12 @@ final class ExplicitModuleBuildTests: XCTestCase { "-module-alias", "Car=Bar", main.nativePathString(escaped: true)] + sdkArgumentsForTesting - + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let deps = try dependencyOracle.getImports(workingDirectory: path, moduleAliases: ["Car": "Bar"], - commandLine: scannerCommand) + commandLine: scannerCommand, + diagnostics: &scanDiagnostics) XCTAssertTrue(deps.imports.contains("Bar")) XCTAssertFalse(deps.imports.contains("Car")) @@ -1148,9 +1149,11 @@ final class ExplicitModuleBuildTests: XCTestCase { if scannerCommand.first == "-frontend" { scannerCommand.removeFirst() } + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let dependencyGraph = try dependencyOracle.getDependencies(workingDirectory: path, - commandLine: scannerCommand) + commandLine: scannerCommand, + diagnostics: &scanDiagnostics) let fooDependencyInfo = try XCTUnwrap(dependencyGraph.modules[.swiftPrebuiltExternal("Foo")]) guard case .swiftPrebuiltExternal(let fooDetails) = fooDependencyInfo.details else { @@ -1252,10 +1255,11 @@ final class ExplicitModuleBuildTests: XCTestCase { "-I", stdLibPath.nativePathString(escaped: true), "-I", shimsPath.nativePathString(escaped: true), main.nativePathString(escaped: true)] + sdkArgumentsForTesting - + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let imports = try dependencyOracle.getImports(workingDirectory: path, - commandLine: scannerCommand) + commandLine: scannerCommand, + diagnostics: &scanDiagnostics) let expectedImports = ["C", "E", "G", "Swift", "SwiftOnoneSupport"] // Dependnig on how recent the platform we are running on, the _Concurrency module may or may not be present. let expectedImports2 = ["C", "E", "G", "Swift", "SwiftOnoneSupport", "_Concurrency"] @@ -1362,10 +1366,19 @@ final class ExplicitModuleBuildTests: XCTestCase { if scannerCommand.first == "-frontend" { scannerCommand.removeFirst() } + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let _ = try dependencyOracle.getDependencies(workingDirectory: path, - commandLine: scannerCommand) - let potentialDiags = try dependencyOracle.getScannerDiagnostics() + commandLine: scannerCommand, + diagnostics: &scanDiagnostics) + let potentialDiags: [ScannerDiagnosticPayload]? + if try dependencyOracle.supportsPerScanDiagnostics() { + potentialDiags = scanDiagnostics + print("Using Per-Scan diagnostics") + } else { + potentialDiags = try dependencyOracle.getScannerDiagnostics() + print("Using Scanner-Global diagnostics") + } XCTAssertEqual(potentialDiags?.count, 5) let diags = try XCTUnwrap(potentialDiags) let error = diags[0] @@ -1463,9 +1476,11 @@ final class ExplicitModuleBuildTests: XCTestCase { // though the module-name above should be sufficient. "-I/tmp/foo/bar/\(index)"] do { + var scanDiagnostics: [ScannerDiagnosticPayload] = [] let dependencyGraph = try dependencyOracle.getDependencies(workingDirectory: path, - commandLine: iterationCommand) + commandLine: iterationCommand, + diagnostics: &scanDiagnostics) // The _Concurrency and _StringProcessing modules are automatically // imported in newer versions of the Swift compiler. If they happened to @@ -1502,6 +1517,87 @@ final class ExplicitModuleBuildTests: XCTestCase { } } + func testParallelDependencyScanningDiagnostics() throws { + let (stdlibPath, shimsPath, toolchain, _) = try getDriverArtifactsForScanning() + // The dependency oracle wraps an instance of libSwiftScan and ensures thread safety across + // queries. + let dependencyOracle = InterModuleDependencyOracle() + let scanLibPath = try XCTUnwrap(toolchain.lookupSwiftScanLib()) + try dependencyOracle.verifyOrCreateScannerInstance(fileSystem: localFileSystem, + swiftScanLibPath: scanLibPath) + if !(try dependencyOracle.supportsPerScanDiagnostics()) { + throw XCTSkip("Scanner does not support per-scan diagnostics") + } + // Create a simple test case. + try withTemporaryDirectory { path in + let cHeadersPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "CHeaders") + let swiftModuleInterfacesPath: AbsolutePath = + try testInputsPath.appending(component: "ExplicitModuleBuilds") + .appending(component: "Swift") + let sdkArgumentsForTesting = (try? Driver.sdkArgumentsForTesting()) ?? [] + let baseDriverArgs = ["swiftc", + "-I", cHeadersPath.nativePathString(escaped: true), + "-I", swiftModuleInterfacesPath.nativePathString(escaped: true), + "-I", stdlibPath.nativePathString(escaped: true), + "-I", shimsPath.nativePathString(escaped: true), + "/tmp/Foo.o", + "-explicit-module-build", + "-working-directory", path.nativePathString(escaped: true), + "-disable-clang-target"] + sdkArgumentsForTesting + let resolver = try ArgsResolver(fileSystem: localFileSystem) + let numFiles = 10 + var files: [AbsolutePath] = [] + var drivers: [Driver] = [] + var scannerCommands: [[String]] = [] + for fileIndex in 0..