Skip to content

[Dependency Scanning] Add and use API for per-query diagnostic gathering #1552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 28, 2024
Merged

[Dependency Scanning] Add and use API for per-query diagnostic gathering #1552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions Sources/CSwiftScan/include/swiftscan_header.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ typedef struct {
(*swiftscan_dependency_graph_get_main_module_name)(swiftscan_dependency_graph_t);
swiftscan_dependency_set_t *
(*swiftscan_dependency_graph_get_dependencies)(swiftscan_dependency_graph_t);
swiftscan_diagnostic_set_t *
(*swiftscan_dependency_graph_get_diagnostics)(swiftscan_dependency_graph_t);

//=== Dependency Module Info Functions ------------------------------------===//
swiftscan_string_ref_t
Expand Down Expand Up @@ -199,6 +201,8 @@ typedef struct {
//=== Prescan Result Functions --------------------------------------------===//
swiftscan_string_set_t *
(*swiftscan_import_set_get_imports)(swiftscan_import_set_t);
swiftscan_diagnostic_set_t *
(*swiftscan_import_set_get_diagnostics)(swiftscan_import_set_t);

//=== Scanner Invocation Functions ----------------------------------------===//
swiftscan_scan_invocation_t
Expand Down
25 changes: 19 additions & 6 deletions ...wiftDriver/ExplicitModuleBuilds/InterModuleDependencies/InterModuleDependencyOracle.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,34 +46,40 @@ public class InterModuleDependencyOracle {

@_spi(Testing) public func getDependencies(workingDirectory: AbsolutePath,
moduleAliases: [String: String]? = nil,
commandLine: [String])
commandLine: [String],
diagnostics: inout [ScannerDiagnosticPayload])
throws -> InterModuleDependencyGraph {
precondition(hasScannerInstance)
return try swiftScanLibInstance!.scanDependencies(workingDirectory: workingDirectory,
moduleAliases: moduleAliases,
invocationCommand: commandLine)
invocationCommand: commandLine,
diagnostics: &diagnostics)
}

@_spi(Testing) public func getBatchDependencies(workingDirectory: AbsolutePath,
moduleAliases: [String: String]? = nil,
commandLine: [String],
batchInfos: [BatchScanModuleInfo])
batchInfos: [BatchScanModuleInfo],
diagnostics: inout [ScannerDiagnosticPayload])
throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
precondition(hasScannerInstance)
return try swiftScanLibInstance!.batchScanDependencies(workingDirectory: workingDirectory,
moduleAliases: moduleAliases,
invocationCommand: commandLine,
batchInfos: batchInfos)
batchInfos: batchInfos,
diagnostics: &diagnostics)
}

@_spi(Testing) public func getImports(workingDirectory: AbsolutePath,
moduleAliases: [String: String]? = nil,
commandLine: [String])
commandLine: [String],
diagnostics: inout [ScannerDiagnosticPayload])
throws -> InterModuleDependencyImports {
precondition(hasScannerInstance)
return try swiftScanLibInstance!.preScanImports(workingDirectory: workingDirectory,
moduleAliases: moduleAliases,
invocationCommand: commandLine)
invocationCommand: commandLine,
diagnostics: &diagnostics)
}

/// Given a specified toolchain path, locate and instantiate an instance of the SwiftScan library
Expand Down Expand Up @@ -147,6 +153,13 @@ public class InterModuleDependencyOracle {
return swiftScan.supportsBridgingHeaderPCHCommand
}

@_spi(Testing) public func supportsPerScanDiagnostics() throws -> Bool {
guard let swiftScan = swiftScanLibInstance else {
fatalError("Attempting to query supported scanner API with no scanner instance.")
}
return swiftScan.canQueryPerScanDiagnostics
}

@_spi(Testing) public func getScannerDiagnostics() throws -> [ScannerDiagnosticPayload]? {
guard let swiftScan = swiftScanLibInstance else {
fatalError("Attempting to reset scanner cache with no scanner instance.")
Expand Down
37 changes: 26 additions & 11 deletions Sources/SwiftDriver/ExplicitModuleBuilds/ModuleDependencyScanning.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,7 @@ public extension Driver {

let isSwiftScanLibAvailable = !(try initSwiftScanLib())
if isSwiftScanLibAvailable {
var scanDiagnostics: [ScannerDiagnosticPayload] = []
let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
var command = try Self.itemizedJobCommand(of: preScanJob,
useResponseFiles: .disabled,
Expand All @@ -209,12 +210,13 @@ public extension Driver {
do {
imports = try interModuleDependencyOracle.getImports(workingDirectory: cwd,
moduleAliases: moduleOutputInfo.aliases,
commandLine: command)
commandLine: command,
diagnostics: &scanDiagnostics)
} catch let DependencyScanningError.dependencyScanFailed(reason) {
try emitScannerDiagnostics()
try emitGlobalScannerDiagnostics()
throw DependencyScanningError.dependencyScanFailed(reason)
}
try emitScannerDiagnostics()
try emitGlobalScannerDiagnostics()
} else {
// Fallback to legacy invocation of the dependency scanner with
// `swift-frontend -scan-dependencies -import-prescan`
Expand All @@ -227,10 +229,8 @@ public extension Driver {
return imports
}

mutating internal func emitScannerDiagnostics() throws {
let possibleDiags = try interModuleDependencyOracle.getScannerDiagnostics()
if let diags = possibleDiags {
for diagnostic in diags {
internal func emitScannerDiagnostics(_ diagnostics: [ScannerDiagnosticPayload]) throws {
for diagnostic in diagnostics {
switch diagnostic.severity {
case .error:
diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
Expand All @@ -244,6 +244,16 @@ public extension Driver {
diagnosticEngine.emit(.scanner_diagnostic_error(diagnostic.message))
}
}
}

mutating internal func emitGlobalScannerDiagnostics() throws {
// We only emit global scanner-collected diagnostics as a legacy flow
// when the scanner does not support per-scan diagnostic output
guard try !interModuleDependencyOracle.supportsPerScanDiagnostics() else {
return
}
if let diags = try interModuleDependencyOracle.getScannerDiagnostics() {
try emitScannerDiagnostics(diags)
}
}

Expand All @@ -261,6 +271,7 @@ public extension Driver {

let isSwiftScanLibAvailable = !(try initSwiftScanLib())
if isSwiftScanLibAvailable {
var scanDiagnostics: [ScannerDiagnosticPayload] = []
let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
var command = try Self.itemizedJobCommand(of: scannerJob,
useResponseFiles: .disabled,
Expand All @@ -269,12 +280,14 @@ public extension Driver {
do {
dependencyGraph = try interModuleDependencyOracle.getDependencies(workingDirectory: cwd,
moduleAliases: moduleOutputInfo.aliases,
commandLine: command)
commandLine: command,
diagnostics: &scanDiagnostics)
try emitScannerDiagnostics(scanDiagnostics)
} catch let DependencyScanningError.dependencyScanFailed(reason) {
try emitScannerDiagnostics()
try emitGlobalScannerDiagnostics()
throw DependencyScanningError.dependencyScanFailed(reason)
}
try emitScannerDiagnostics()
try emitGlobalScannerDiagnostics()
} else {
// Fallback to legacy invocation of the dependency scanner with
// `swift-frontend -scan-dependencies`
Expand All @@ -295,6 +308,7 @@ public extension Driver {

let isSwiftScanLibAvailable = !(try initSwiftScanLib())
if isSwiftScanLibAvailable {
var scanDiagnostics: [ScannerDiagnosticPayload] = []
let cwd = workingDirectory ?? fileSystem.currentWorkingDirectory!
var command = try Self.itemizedJobCommand(of: batchScanningJob,
useResponseFiles: .disabled,
Expand All @@ -304,7 +318,8 @@ public extension Driver {
try interModuleDependencyOracle.getBatchDependencies(workingDirectory: cwd,
moduleAliases: moduleOutputInfo.aliases,
commandLine: command,
batchInfos: moduleInfos)
batchInfos: moduleInfos,
diagnostics: &scanDiagnostics)
} else {
// Fallback to legacy invocation of the dependency scanner with
// `swift-frontend -scan-dependencies`
Expand Down
56 changes: 44 additions & 12 deletions Sources/SwiftDriver/SwiftScan/SwiftScan.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,8 @@ internal extension swiftscan_diagnostic_severity_t {

func preScanImports(workingDirectory: AbsolutePath,
moduleAliases: [String: String]?,
invocationCommand: [String]) throws -> InterModuleDependencyImports {
invocationCommand: [String],
diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyImports {
// Create and configure the scanner invocation
let invocation = api.swiftscan_scan_invocation_create()
defer { api.swiftscan_scan_invocation_dispose(invocation) }
Expand All @@ -144,6 +145,13 @@ internal extension swiftscan_diagnostic_severity_t {
guard let importSetRef = importSetRefOrNull else {
throw DependencyScanningError.dependencyScanFailed("Unable to produce import set")
}
if canQueryPerScanDiagnostics {
let diagnosticsSetRefOrNull = api.swiftscan_import_set_get_diagnostics(importSetRef)
guard let diagnosticsSetRef = diagnosticsSetRefOrNull else {
throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics")
}
diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef)
}

let importSet = try constructImportSet(from: importSetRef, with: moduleAliases)
// Free the memory allocated for the in-memory representation of the import set
Expand All @@ -154,7 +162,8 @@ internal extension swiftscan_diagnostic_severity_t {

func scanDependencies(workingDirectory: AbsolutePath,
moduleAliases: [String: String]?,
invocationCommand: [String]) throws -> InterModuleDependencyGraph {
invocationCommand: [String],
diagnostics: inout [ScannerDiagnosticPayload]) throws -> InterModuleDependencyGraph {
// Create and configure the scanner invocation
let invocation = api.swiftscan_scan_invocation_create()
defer { api.swiftscan_scan_invocation_dispose(invocation) }
Expand All @@ -172,6 +181,13 @@ internal extension swiftscan_diagnostic_severity_t {
guard let graphRef = graphRefOrNull else {
throw DependencyScanningError.dependencyScanFailed("Unable to produce dependency graph")
}
if canQueryPerScanDiagnostics {
let diagnosticsSetRefOrNull = api.swiftscan_dependency_graph_get_diagnostics(graphRef)
guard let diagnosticsSetRef = diagnosticsSetRefOrNull else {
throw DependencyScanningError.dependencyScanFailed("Unable to query dependency diagnostics")
}
diagnostics = try mapToDriverDiagnosticPayload(diagnosticsSetRef)
}

let dependencyGraph = try constructGraph(from: graphRef, moduleAliases: moduleAliases)
// Free the memory allocated for the in-memory representation of the dependency
Expand All @@ -184,7 +200,8 @@ internal extension swiftscan_diagnostic_severity_t {
func batchScanDependencies(workingDirectory: AbsolutePath,
moduleAliases: [String: String]?,
invocationCommand: [String],
batchInfos: [BatchScanModuleInfo])
batchInfos: [BatchScanModuleInfo],
diagnostics: inout [ScannerDiagnosticPayload])
throws -> [ModuleDependencyId: [InterModuleDependencyGraph]] {
// Create and configure the scanner invocation
let invocationRef = api.swiftscan_scan_invocation_create()
Expand Down Expand Up @@ -315,6 +332,12 @@ internal extension swiftscan_diagnostic_severity_t {
return api.swiftscan_swift_textual_detail_get_bridging_pch_command_line != nil
}


@_spi(Testing) public var canQueryPerScanDiagnostics : Bool {
return api.swiftscan_dependency_graph_get_diagnostics != nil &&
api.swiftscan_import_set_get_diagnostics != nil
}

func serializeScannerCache(to path: AbsolutePath) {
api.swiftscan_scanner_cache_serialize(scanner,
path.description.cString(using: String.Encoding.utf8))
Expand All @@ -329,18 +352,10 @@ internal extension swiftscan_diagnostic_severity_t {
api.swiftscan_scanner_cache_reset(scanner)
}

@_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
internal func mapToDriverDiagnosticPayload(_ diagnosticSetRef: UnsafeMutablePointer<swiftscan_diagnostic_set_t>) throws -> [ScannerDiagnosticPayload] {
var result: [ScannerDiagnosticPayload] = []
let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner)
guard let diagnosticSetRef = diagnosticSetRefOrNull else {
// Seems heavy-handed to fail here
// throw DependencyScanningError.dependencyScanFailed
return []
}
defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) }
let diagnosticRefArray = Array(UnsafeBufferPointer(start: diagnosticSetRef.pointee.diagnostics,
count: Int(diagnosticSetRef.pointee.count)))

for diagnosticRefOrNull in diagnosticRefArray {
guard let diagnosticRef = diagnosticRefOrNull else {
throw DependencyScanningError.dependencyScanFailed("Unable to produce scanner diagnostics")
Expand All @@ -352,6 +367,17 @@ internal extension swiftscan_diagnostic_severity_t {
return result
}

@_spi(Testing) public func queryScannerDiagnostics() throws -> [ScannerDiagnosticPayload] {
let diagnosticSetRefOrNull = api.swiftscan_scanner_diagnostics_query(scanner)
guard let diagnosticSetRef = diagnosticSetRefOrNull else {
// Seems heavy-handed to fail here
// throw DependencyScanningError.dependencyScanFailed
return []
}
defer { api.swiftscan_diagnostics_set_dispose(diagnosticSetRef) }
return try mapToDriverDiagnosticPayload(diagnosticSetRef)
}

@_spi(Testing) public func resetScannerDiagnostics() throws {
api.swiftscan_scanner_diagnostics_reset(scanner)
}
Expand Down Expand Up @@ -567,6 +593,12 @@ private extension swiftscan_functions_t {
self.swiftscan_swift_binary_detail_get_header_dependencies =
try loadOptional("swiftscan_swift_binary_detail_get_header_dependencies")

// Per-scan-query diagnostic output
self.swiftscan_dependency_graph_get_diagnostics =
try loadOptional("swiftscan_dependency_graph_get_diagnostics")
self.swiftscan_import_set_get_diagnostics =
try loadOptional("swiftscan_import_set_get_diagnostics")

// MARK: Required Methods
func loadRequired<T>(_ symbol: String) throws -> T {
guard let sym: T = Loader.lookup(symbol: symbol, in: swiftscan) else {
Expand Down
8 changes: 6 additions & 2 deletions Tests/SwiftDriverTests/CachingBuildTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -666,9 +666,11 @@ final class CachingBuildTests: XCTestCase {
// though the module-name above should be sufficient.
"-I/tmp/foo/bar/\(index)"]
do {
var scanDiagnostics: [ScannerDiagnosticPayload] = []
let dependencyGraph =
try dependencyOracle.getDependencies(workingDirectory: path,
commandLine: iterationCommand)
commandLine: iterationCommand,
diagnostics: &scanDiagnostics)

// The _Concurrency and _StringProcessing modules are automatically
// imported in newer versions of the Swift compiler. If they happened to
Expand Down Expand Up @@ -712,8 +714,10 @@ final class CachingBuildTests: XCTestCase {
"-I/tmp/bad",
"-cas-path", casPath2.nativePathString(escaped: true),
]
var scanDiagnostics: [ScannerDiagnosticPayload] = []
XCTAssertThrowsError(try dependencyOracle.getDependencies(workingDirectory: path,
commandLine: command)) {
commandLine: command,
diagnostics: &scanDiagnostics)) {
XCTAssertTrue($0 is DependencyScanningError)
}
let diags = try XCTUnwrap(dependencyOracle.getScannerDiagnostics())
Expand Down
Loading