[Dependency Scanning] Add and use API for per-query diagnostic gathering

artemcm · artemcm · commit a5af8f8e6e56 · 2024-02-27T12:09:15.000-08:00
Adopts new API that produces per-scan diagnostics for each query of a dependency graph or an import set
diff --git a/Sources/CSwiftScan/include/swiftscan_header.h b/Sources/CSwiftScan/include/swiftscan_header.h
@@ -97,6 +97,8 @@ typedef struct {
   (*swiftscan_dependency_graph_get_main_module_name)(swiftscan_dependency_graph_t);
   swiftscan_dependency_set_t *
   (*swiftscan_dependency_graph_get_dependencies)(swiftscan_dependency_graph_t);
+  swiftscan_diagnostic_set_t *
+  (*swiftscan_dependency_graph_get_diagnostics)(swiftscan_dependency_graph_t);
 
   //=== Dependency Module Info Functions ------------------------------------===//
   swiftscan_string_ref_t
@@ -199,6 +201,8 @@ typedef struct {
   //=== Prescan Result Functions --------------------------------------------===//
   swiftscan_string_set_t *
   (*swiftscan_import_set_get_imports)(swiftscan_import_set_t);
+  swiftscan_diagnostic_set_t *
+  (*swiftscan_import_set_get_diagnostics)(swiftscan_import_set_t);
 
   //=== Scanner Invocation Functions ----------------------------------------===//
   swiftscan_scan_invocation_t
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift
@@ -46,34 +46,40 @@ public class InterModuleDependencyOracle {
 
   @_spi(Testing) public func getDependencies(workingDirectory: AbsolutePath,
                                              moduleAliases: [String: String]? = nil,
-                                             commandLine: [String])
+                                             commandLine: [String],
+                                             diagnostics: inout [ScannerDiagnosticPayload])
   throws -> InterModuleDependencyGraph {
     precondition(hasScannerInstance)
     return try swiftScanLibInstance!.scanDependencies(workingDirectory: workingDirectory,
                                                       moduleAliases: moduleAliases,
-                                                      invocationCommand: commandLine)
+                                                      invocationCommand: commandLine,
+                                                      diagnostics: &diagnostics)
   }
 
   @_spi(Testing) public func getBatchDependencies(workingDirectory: AbsolutePath,
                                                   moduleAliases: [String: String]? = nil,
                                                   commandLine: [String],
-                                                  batchInfos: [BatchScanModuleInfo])
+                                                  batchInfos: [BatchScanModuleInfo],
+                                                  diagnostics: inout [ScannerDiagnosticPayload])
   throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
     precondition(hasScannerInstance)
     return try swiftScanLibInstance!.batchScanDependencies(workingDirectory: workingDirectory,
                                                            moduleAliases: moduleAliases,
                                                            invocationCommand: commandLine,
-                                                           batchInfos: batchInfos)
+                                                           batchInfos: batchInfos,
+                                                           diagnostics: &diagnostics)
   }
 
   @_spi(Testing) public func getImports(workingDirectory: AbsolutePath,
                                         moduleAliases: [String: String]? = nil,
-                                             commandLine: [String])
+                                        commandLine: [String],
+                                        diagnostics: inout [ScannerDiagnosticPayload])
   throws -> InterModuleDependencyImports {
     precondition(hasScannerInstance)
     return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory,
                                                     moduleAliases: moduleAliases,
-                                                    invocationCommand: commandLine)
+                                                    invocationCommand: commandLine,
+                                                    diagnostics: &diagnostics)
   }
 
   /// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library
@@ -147,6 +153,13 @@ public class InterModuleDependencyOracle {
     return swiftScan.supportsBridgingHeaderPCHCommand
   }
 
+  @_spi(Testing) public func supportsPerScanDiagnostics() throws -> Bool {
+    guard let swiftScan = swiftScanLibInstance else {
+      fatalError("Attempting to query supported scanner API with no scanner instance.")
+    }
+    return swiftScan.canQueryPerScanDiagnostics
+  }
+
   @_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
     guard let swiftScan = swiftScanLibInstance else {
       fatalError("Attempting to reset scanner cache with no scanner instance.")
diff --git a/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift b/Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift
@@ -201,6 +201,7 @@ public extension Driver {
 
     let isSwiftScanLibAvailable = !(try initSwiftScanLib())
     if isSwiftScanLibAvailable {
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
       let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
       var command = try Self.itemizedJobCommand(of: preScanJob,
                                                 useResponseFiles: .disabled,
@@ -209,12 +210,13 @@ public extension Driver {
       do {
         imports = try interModuleDependencyOracle.getImports(workingDirectory: cwd,
                                                              moduleAliases: moduleOutputInfo.aliases,
-                                                             commandLine: command)
+                                                             commandLine: command,
+                                                             diagnostics: &scanDiagnostics)
       } catch let DependencyScanningError.dependencyScanFailed(reason) {
-        try emitScannerDiagnostics()
+        try emitGlobalScannerDiagnostics()
         throw DependencyScanningError.dependencyScanFailed(reason)
       }
-      try emitScannerDiagnostics()
+      try emitGlobalScannerDiagnostics()
     } else {
       // Fallback to legacy invocation of the dependency scanner with
       // `swift-frontend -scan-dependencies -import-prescan`
@@ -227,10 +229,8 @@ public extension Driver {
     return imports
   }
 
-  mutating internal func emitScannerDiagnostics() throws {
-    let possibleDiags = try interModuleDependencyOracle.getScannerDiagnostics()
-    if let diags = possibleDiags {
-      for diagnostic in diags {
+  internal func emitScannerDiagnostics(_ diagnostics: [ScannerDiagnosticPayload]) throws {
+      for diagnostic in diagnostics {
         switch diagnostic.severity {
         case .error:
           diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
@@ -244,6 +244,16 @@ public extension Driver {
           diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
         }
       }
+  }
+
+  mutating internal func emitGlobalScannerDiagnostics() throws {
+    // We only emit global scanner-collected diagnostics as a legacy flow
+    // when the scanner does not support per-scan diagnostic output
+    guard try !interModuleDependencyOracle.supportsPerScanDiagnostics() else {
+      return
+    }
+    if let diags = try interModuleDependencyOracle.getScannerDiagnostics() {
+      try emitScannerDiagnostics(diags)
     }
   }
 
@@ -261,6 +271,7 @@ public extension Driver {
 
     let isSwiftScanLibAvailable = !(try initSwiftScanLib())
     if isSwiftScanLibAvailable {
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
       let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
       var command = try Self.itemizedJobCommand(of: scannerJob,
                                                 useResponseFiles: .disabled,
@@ -269,12 +280,14 @@ public extension Driver {
       do {
         dependencyGraph = try interModuleDependencyOracle.getDependencies(workingDirectory: cwd,
                                                                           moduleAliases: moduleOutputInfo.aliases,
-                                                                          commandLine: command)
+                                                                          commandLine: command,
+                                                                          diagnostics: &scanDiagnostics)
+        try emitScannerDiagnostics(scanDiagnostics)
       } catch let DependencyScanningError.dependencyScanFailed(reason) {
-        try emitScannerDiagnostics()
+        try emitGlobalScannerDiagnostics()
         throw DependencyScanningError.dependencyScanFailed(reason)
       }
-      try emitScannerDiagnostics()
+      try emitGlobalScannerDiagnostics()
     } else {
       // Fallback to legacy invocation of the dependency scanner with
       // `swift-frontend -scan-dependencies`
@@ -295,6 +308,7 @@ public extension Driver {
 
     let isSwiftScanLibAvailable = !(try initSwiftScanLib())
     if isSwiftScanLibAvailable {
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
       let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
       var command = try Self.itemizedJobCommand(of: batchScanningJob,
                                                 useResponseFiles: .disabled,
@@ -304,7 +318,8 @@ public extension Driver {
         try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd,
                                                              moduleAliases: moduleOutputInfo.aliases,
                                                              commandLine: command,
-                                                             batchInfos: moduleInfos)
+                                                             batchInfos: moduleInfos,
+                                                             diagnostics: &scanDiagnostics)
     } else {
       // Fallback to legacy invocation of the dependency scanner with
       // `swift-frontend -scan-dependencies`
diff --git a/Sources/SwiftDriver/SwiftScan/SwiftScan.swift b/Sources/SwiftDriver/SwiftScan/SwiftScan.swift
@@ -126,7 +126,8 @@ internal extension swiftscan_diagnostic_severity_t {
 
   func preScanImports(workingDirectory: AbsolutePath,
                       moduleAliases: [String: String]?,
-                      invocationCommand: [String]) throws -> InterModuleDependencyImports {
+                      invocationCommand: [String],
+                      diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports {
     // Create and configure the scanner invocation
     let invocation = api.swiftscan_scan_invocation_create()
     defer { api.swiftscan_scan_invocation_dispose(invocation) }
@@ -144,6 +145,13 @@ internal extension swiftscan_diagnostic_severity_t {
     guard let importSetRef = importSetRefOrNull else {
       throw DependencyScanningError.dependencyScanFailed("Unable to produce import set")
     }
+    if canQueryPerScanDiagnostics {
+      let diagnosticsSetRefOrNull = api.swiftscan_import_set_get_diagnostics(importSetRef)
+      guard let diagnosticsSetRef = diagnosticsSetRefOrNull else {
+        throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics")
+      }
+      diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef)
+    }
 
     let importSet = try constructImportSet(from: importSetRef, with: moduleAliases)
     // Free the memory allocated for the in-memory representation of the import set
@@ -154,7 +162,8 @@ internal extension swiftscan_diagnostic_severity_t {
 
   func scanDependencies(workingDirectory: AbsolutePath,
                         moduleAliases: [String: String]?,
-                        invocationCommand: [String]) throws -> InterModuleDependencyGraph {
+                        invocationCommand: [String],
+                        diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph {
     // Create and configure the scanner invocation
     let invocation = api.swiftscan_scan_invocation_create()
     defer { api.swiftscan_scan_invocation_dispose(invocation) }
@@ -172,6 +181,13 @@ internal extension swiftscan_diagnostic_severity_t {
     guard let graphRef = graphRefOrNull else {
       throw DependencyScanningError.dependencyScanFailed("Unable to produce dependency graph")
     }
+    if canQueryPerScanDiagnostics {
+      let diagnosticsSetRefOrNull = api.swiftscan_dependency_graph_get_diagnostics(graphRef)
+      guard let diagnosticsSetRef = diagnosticsSetRefOrNull else {
+        throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics")
+      }
+      diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef)
+    }
 
     let dependencyGraph = try constructGraph(from: graphRef, moduleAliases: moduleAliases)
     // Free the memory allocated for the in-memory representation of the dependency
@@ -184,7 +200,8 @@ internal extension swiftscan_diagnostic_severity_t {
   func batchScanDependencies(workingDirectory: AbsolutePath,
                              moduleAliases: [String: String]?,
                              invocationCommand: [String],
-                             batchInfos: [BatchScanModuleInfo])
+                             batchInfos: [BatchScanModuleInfo],
+                             diagnostics: inout [ScannerDiagnosticPayload])
   throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
     // Create and configure the scanner invocation
     let invocationRef = api.swiftscan_scan_invocation_create()
@@ -315,6 +332,12 @@ internal extension swiftscan_diagnostic_severity_t {
     return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil
   }
 
+
+  @_spi(Testing) public var canQueryPerScanDiagnostics : Bool {
+    return api.swiftscan_dependency_graph_get_diagnostics != nil &&
+           api.swiftscan_import_set_get_diagnostics != nil
+  }
+
   func serializeScannerCache(to path: AbsolutePath) {
     api.swiftscan_scanner_cache_serialize(scanner,
                                           path.description.cString(using: String.Encoding.utf8))
@@ -329,18 +352,10 @@ internal extension swiftscan_diagnostic_severity_t {
     api.swiftscan_scanner_cache_reset(scanner)
   }
 
-  @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
+  internal func mapToDriverDiagnosticPayload(_ diagnosticSetRef: UnsafeMutablePointer<swiftscan_diagnostic_set_t>) throws -> [ScannerDiagnosticPayload] {
     var result: [ScannerDiagnosticPayload] = []
-    let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner)
-    guard let diagnosticSetRef = diagnosticSetRefOrNull else {
-      // Seems heavy-handed to fail here
-      // throw DependencyScanningError.dependencyScanFailed
-      return []
-    }
-    defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) }
     let diagnosticRefArray = Array(UnsafeBufferPointer(start: diagnosticSetRef.pointee.diagnostics,
                                                        count: Int(diagnosticSetRef.pointee.count)))
-
     for diagnosticRefOrNull in diagnosticRefArray {
       guard let diagnosticRef = diagnosticRefOrNull else {
         throw DependencyScanningError.dependencyScanFailed("Unable to produce scanner diagnostics")
@@ -352,6 +367,17 @@ internal extension swiftscan_diagnostic_severity_t {
     return result
   }
 
+  @_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
+    let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner)
+    guard let diagnosticSetRef = diagnosticSetRefOrNull else {
+      // Seems heavy-handed to fail here
+      // throw DependencyScanningError.dependencyScanFailed
+      return []
+    }
+    defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) }
+    return try mapToDriverDiagnosticPayload(diagnosticSetRef)
+  }
+
   @_spi(Testing) public func resetScannerDiagnostics() throws {
     api.swiftscan_scanner_diagnostics_reset(scanner)
   }
@@ -567,6 +593,12 @@ private extension swiftscan_functions_t {
     self.swiftscan_swift_binary_detail_get_header_dependencies =
       try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies")
 
+    // Per-scan-query diagnostic output
+    self.swiftscan_dependency_graph_get_diagnostics =
+      try loadOptional("swiftscan_dependency_graph_get_diagnostics")
+    self.swiftscan_import_set_get_diagnostics =
+      try loadOptional("swiftscan_import_set_get_diagnostics")
+
     // MARK: Required Methods
     func loadRequired<T>(_ symbol: String) throws -> T {
       guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else {
diff --git a/Sources/SwiftDriver/Toolchains/Toolchain.swift b/Sources/SwiftDriver/Toolchains/Toolchain.swift
@@ -258,6 +258,7 @@ extension Toolchain {
     return try getToolPath(.swiftCompiler).parentDirectory // bin
                                           .appending(component: "_InternalSwiftScan.dll")
 #else
+    return try AbsolutePath(validating: "/Volumes/Data/GHWorkspace/build/Ninja-RelWithDebInfoAssert/swift-macosx-arm64/lib/lib_InternalSwiftScan.dylib")
     let libraryName = sharedLibraryName("lib_InternalSwiftScan")
     if let overrideString = env["SWIFT_DRIVER_SWIFTSCAN_LIB"],
        let path = try? AbsolutePath(validating: overrideString) {
diff --git a/Tests/SwiftDriverTests/CachingBuildTests.swift b/Tests/SwiftDriverTests/CachingBuildTests.swift
@@ -666,9 +666,11 @@ final class CachingBuildTests: XCTestCase {
                                                  // though the module-name above should be sufficient.
                                                  "-I/tmp/foo/bar/\(index)"]
         do {
+          var scanDiagnostics: [ScannerDiagnosticPayload] = []
           let dependencyGraph =
             try dependencyOracle.getDependencies(workingDirectory: path,
-                                                 commandLine: iterationCommand)
+                                                 commandLine: iterationCommand,
+                                                 diagnostics: &scanDiagnostics)
 
           // The _Concurrency and _StringProcessing modules are automatically
           // imported in newer versions of the Swift compiler. If they happened to
@@ -712,8 +714,10 @@ final class CachingBuildTests: XCTestCase {
                                       "-I/tmp/bad",
                                       "-cas-path", casPath2.nativePathString(escaped: true),
                                       ]
+      var scanDiagnostics: [ScannerDiagnosticPayload] = []
       XCTAssertThrowsError(try dependencyOracle.getDependencies(workingDirectory: path,
-                                                                commandLine: command)) {
+                                                                commandLine: command,
+                                                                diagnostics: &scanDiagnostics)) {
         XCTAssertTrue($0 is DependencyScanningError)
       }
       let diags = try XCTUnwrap(dependencyOracle.getScannerDiagnostics())
diff --git a/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift b/Tests/SwiftDriverTests/ExplicitModuleBuildTests.swift
