From 0dd0a7a94caba2987f424f2cd4824c2697e49698 Mon Sep 17 00:00:00 2001 From: Bassam Khouri Date: Thu, 30 Oct 2025 11:02:31 -0400 Subject: [PATCH] Add tags to swift-testing tests withKnownIssues Augment Swift Testing test that has "withKnownIssues" with tags. Also, augment tests to run against the SwiftBuild and Native build system, in addition to the debug and release build configuraitons. Relates to: #9283 --- .../SwiftTesting+Tags.swift | 29 +- .../SwiftTesting+TraitConditional.swift | 4 +- .../FileSystem/InMemoryFilesSystemTests.swift | 6 + Tests/BasicsTests/FileSystem/PathTests.swift | 6 + Tests/BasicsTests/FileSystem/VFSTests.swift | 208 +++--- .../BuildTests/BuildSystemDelegateTests.swift | 7 + Tests/BuildTests/PluginsBuildPlanTests.swift | 2 + .../FunctionalTests/CFamilyTargetTests.swift | 64 +- .../DependencyResolutionTests.swift | 4 +- .../ModuleAliasingFixtureTests.swift | 3 +- Tests/FunctionalTests/PluginTests.swift | 633 +++++++++--------- .../FunctionalTests/StaticBinaryLibrary.swift | 20 +- .../FunctionalTests/TestDiscoveryTests.swift | 214 ++++-- Tests/FunctionalTests/ToolsVersionTests.swift | 9 +- Tests/FunctionalTests/TraitTests.swift | 4 +- .../VersionSpecificTests.swift | 3 +- Tests/IntegrationTests/BasicTests.swift | 3 +- Tests/IntegrationTests/SwiftPMTests.swift | 9 + 18 files changed, 733 insertions(+), 495 deletions(-) diff --git a/Sources/_InternalTestSupport/SwiftTesting+Tags.swift b/Sources/_InternalTestSupport/SwiftTesting+Tags.swift index ccaf4b765a0..9873fdf111d 100644 --- a/Sources/_InternalTestSupport/SwiftTesting+Tags.swift +++ b/Sources/_InternalTestSupport/SwiftTesting+Tags.swift @@ -13,6 +13,7 @@ import Testing extension Tag { public enum TestSize {} public enum Feature {} + public enum Platform {} @Tag public static var UserWorkflow: Tag } @@ -22,10 +23,14 @@ extension Tag.TestSize { @Tag public static var large: Tag } +extension Tag.Platform { + @Tag public static var FileSystem: Tag +} + extension Tag.Feature { public enum Command {} public enum CommandLineArguments {} - public enum EnvironmentVariables {} + public enum EnvironmentVariables {} public enum PackageType {} public enum ProductType {} public enum TargetType {} @@ -33,14 +38,20 @@ extension Tag.Feature { @Tag public static var BuildCache: Tag @Tag public static var CodeCoverage: Tag + @Tag public static var CTargets: Tag + @Tag public static var DependencyResolution: Tag + @Tag public static var ModuleAliasing: Tag @Tag public static var Mirror: Tag @Tag public static var NetRc: Tag + @Tag public static var Plugin: Tag @Tag public static var Resource: Tag + @Tag public static var SourceGeneration: Tag @Tag public static var SpecialCharacters: Tag @Tag public static var Snippets: Tag + @Tag public static var TestDiscovery: Tag @Tag public static var Traits: Tag @Tag public static var TargetSettings: Tag - + @Tag public static var Version: Tag } extension Tag.Feature.Command { @@ -75,9 +86,10 @@ extension Tag.Feature.CommandLineArguments { @Tag public static var Verbose: Tag @Tag public static var VeryVerbose: Tag @Tag public static var Xlinker: Tag + @Tag public static var XbuildToolsSwiftc: Tag @Tag public static var Xcc: Tag @Tag public static var Xcxx: Tag - @Tag public static var SWIFT_ORIGINAL_PATH: Tag + @Tag public static var Xld: Tag @Tag public static var Xswiftc: Tag @Tag public static var TestParallel: Tag @Tag public static var TestNoParallel: Tag @@ -93,7 +105,8 @@ extension Tag.Feature.CommandLineArguments { } extension Tag.Feature.CommandLineArguments.Experimental { - @Tag public static var BuildDylibsAsFrameworks: Tag + @Tag public static var BuildDylibsAsFrameworks: Tag + @Tag public static var PruneUnusedDependencies: Tag } extension Tag.Feature.EnvironmentVariables { @Tag public static var CUSTOM_SWIFT_VERSION: Tag @@ -143,9 +156,16 @@ extension Tag.Feature.Command.PackageRegistry { } extension Tag.Feature.TargetType { + public enum BinaryTarget {} @Tag public static var Executable: Tag @Tag public static var Library: Tag @Tag public static var Macro: Tag + @Tag public static var Test: Tag +} + +extension Tag.Feature.TargetType.BinaryTarget { + @Tag public static var ArtifactBundle: Tag + @Tag public static var XCFramework: Tag } extension Tag.Feature.ProductType { @@ -157,6 +177,7 @@ extension Tag.Feature.ProductType { } extension Tag.Feature.PackageType { @Tag public static var Library: Tag + @Tag public static var Empty: Tag @Tag public static var Executable: Tag @Tag public static var Tool: Tag @Tag public static var Plugin: Tag diff --git a/Sources/_InternalTestSupport/SwiftTesting+TraitConditional.swift b/Sources/_InternalTestSupport/SwiftTesting+TraitConditional.swift index a46dfafe162..60b550e838f 100644 --- a/Sources/_InternalTestSupport/SwiftTesting+TraitConditional.swift +++ b/Sources/_InternalTestSupport/SwiftTesting+TraitConditional.swift @@ -18,8 +18,8 @@ import TSCclibc // for SPM_posix_spawn_file_actions_addchdir_np_supported extension Trait where Self == Testing.ConditionTrait { /// Skip test if the host operating system does not match the running OS. - public static func requireHostOS(_ os: OperatingSystem, when condition: Bool = true) -> Self { - enabled("This test requires a \(os) host OS.") { + public static func requireHostOS(_ os: OperatingSystem, _ comment: Comment? = nil, when condition: Bool = true) -> Self { + enabled(comment ?? "This test requires a \(os) host OS.") { ProcessInfo.hostOperatingSystem == os && condition } } diff --git a/Tests/BasicsTests/FileSystem/InMemoryFilesSystemTests.swift b/Tests/BasicsTests/FileSystem/InMemoryFilesSystemTests.swift index b2f50811f42..be08df26ab1 100644 --- a/Tests/BasicsTests/FileSystem/InMemoryFilesSystemTests.swift +++ b/Tests/BasicsTests/FileSystem/InMemoryFilesSystemTests.swift @@ -21,6 +21,12 @@ let isLinux = true let isLinux = false #endif +@Suite( + .tags( + .TestSize.small, + .Platform.FileSystem, + ), +) struct InMemoryFileSystemTests { @Test( arguments: [ diff --git a/Tests/BasicsTests/FileSystem/PathTests.swift b/Tests/BasicsTests/FileSystem/PathTests.swift index 58fe9cadd98..a4c15dac7de 100644 --- a/Tests/BasicsTests/FileSystem/PathTests.swift +++ b/Tests/BasicsTests/FileSystem/PathTests.swift @@ -19,6 +19,12 @@ private var windows: Bool { false } #endif +@Suite( + .tags( + .TestSize.small, + .Platform.FileSystem, + ), +) struct PathTests { struct AbsolutePathTests { private func pathStringIsSetCorrectlyTestImplementation(path: String, expected: String, label: String) { diff --git a/Tests/BasicsTests/FileSystem/VFSTests.swift b/Tests/BasicsTests/FileSystem/VFSTests.swift index 0f8b5efac2b..e2511af7637 100644 --- a/Tests/BasicsTests/FileSystem/VFSTests.swift +++ b/Tests/BasicsTests/FileSystem/VFSTests.swift @@ -37,110 +37,114 @@ func testWithTemporaryDirectory( struct VFSTests { - @Test + @Test( + .issue("https://github.com/swiftlang/swift-package-manager/issues/9310", relationship: .defect), + .skipHostOS(.windows, "Test currently fails with: you don't have permissions."), + .tags( + .TestSize.medium, + .Platform.FileSystem, + .UserWorkflow, + ) + ) func localBasics() throws { - try withKnownIssue("Permission issues on Windows") { - // tiny PE binary from: https://archive.is/w01DO - let contents: [UInt8] = [ - 0x4d, 0x5a, 0x00, 0x00, 0x50, 0x45, 0x00, 0x00, 0x4c, 0x01, 0x01, 0x00, - 0x6a, 0x2a, 0x58, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x03, 0x01, 0x0b, 0x01, 0x08, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, - 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x68, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x02 - ] - - let fs = localFileSystem - try withTemporaryFile { [contents] vfsPath in - try withTemporaryDirectory(removeTreeOnDeinit: true) { [contents] tempDirPath in - let file = tempDirPath.appending("best") - try fs.writeFileContents(file, string: "best") - - let sym = tempDirPath.appending("hello") - try fs.createSymbolicLink(sym, pointingAt: file, relative: false) - - let executable = tempDirPath.appending("exec-foo") - try fs.writeFileContents(executable, bytes: ByteString(contents)) - #if !os(Windows) - try fs.chmod(.executable, path: executable, options: []) - #endif - - let executableSym = tempDirPath.appending("exec-sym") - try fs.createSymbolicLink(executableSym, pointingAt: executable, relative: false) - - try fs.createDirectory(tempDirPath.appending("dir")) - try fs.writeFileContents(tempDirPath.appending(components: ["dir", "file"]), bytes: []) - - try VirtualFileSystem.serializeDirectoryTree(tempDirPath, into: AbsolutePath(vfsPath.path), fs: fs, includeContents: [executable]) - } - - let vfs = try VirtualFileSystem(path: vfsPath.path, fs: fs) - - // exists() - #expect(vfs.exists(AbsolutePath("/"))) - #expect(!vfs.exists(AbsolutePath("/does-not-exist"))) - - // isFile() - let filePath = AbsolutePath("/best") - #expect(vfs.exists(filePath)) - #expect(vfs.isFile(filePath)) - #expect(try vfs.getFileInfo(filePath).fileType == .typeRegular) - #expect(!vfs.isDirectory(filePath)) - #expect(!vfs.isFile(AbsolutePath("/does-not-exist"))) - #expect(!vfs.isSymlink(AbsolutePath("/does-not-exist"))) - #expect(throws: (any Error).self) { - try vfs.getFileInfo(AbsolutePath("/does-not-exist")) - } - - // isSymlink() - let symPath = AbsolutePath("/hello") - #expect(vfs.isSymlink(symPath)) - #expect(vfs.isFile(symPath)) - #expect(try vfs.getFileInfo(symPath).fileType == .typeSymbolicLink) - #expect(!vfs.isDirectory(symPath)) - - // isExecutableFile - let executablePath = AbsolutePath("/exec-foo") - let executableSymPath = AbsolutePath("/exec-sym") - #expect(vfs.isExecutableFile(executablePath)) - #expect(vfs.isExecutableFile(executableSymPath)) - #expect(vfs.isSymlink(executableSymPath)) - #expect(!vfs.isExecutableFile(symPath)) - #expect(!vfs.isExecutableFile(filePath)) - #expect(!vfs.isExecutableFile(AbsolutePath("/does-not-exist"))) - #expect(!vfs.isExecutableFile(AbsolutePath("/"))) - - // readFileContents - let execFileContents = try vfs.readFileContents(executablePath) - #expect(execFileContents == ByteString(contents)) - - // isDirectory() - #expect(vfs.isDirectory(AbsolutePath("/"))) - #expect(!vfs.isDirectory(AbsolutePath("/does-not-exist"))) - - // getDirectoryContents() - let dirContents = try vfs.getDirectoryContents(AbsolutePath("/")) - #expect(dirContents.sorted() == ["best", "dir", "exec-foo", "exec-sym", "hello"]) - #expect {try vfs.getDirectoryContents(AbsolutePath("/does-not-exist"))} throws: { error in - (error.localizedDescription == "no such file or directory: \(AbsolutePath("/does-not-exist"))") - } - - let thisDirectoryContents = try vfs.getDirectoryContents(AbsolutePath("/")) - #expect(!thisDirectoryContents.contains(where: { $0 == "." })) - #expect(!thisDirectoryContents.contains(where: { $0 == ".." })) - #expect(thisDirectoryContents.sorted() == ["best", "dir", "exec-foo", "exec-sym", "hello"]) - - let contents = try vfs.getDirectoryContents(AbsolutePath("/dir")) - #expect(contents == ["file"]) - - let fileContents = try vfs.readFileContents(AbsolutePath("/dir/file")) - #expect(fileContents == "") + // tiny PE binary from: https://archive.is/w01DO + let contents: [UInt8] = [ + 0x4d, 0x5a, 0x00, 0x00, 0x50, 0x45, 0x00, 0x00, 0x4c, 0x01, 0x01, 0x00, + 0x6a, 0x2a, 0x58, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x03, 0x01, 0x0b, 0x01, 0x08, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, + 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x68, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x02 + ] + + let fs = localFileSystem + try withTemporaryFile { [contents] vfsPath in + try withTemporaryDirectory(removeTreeOnDeinit: true) { [contents] tempDirPath in + let file = tempDirPath.appending("best") + try fs.writeFileContents(file, string: "best") + + let sym = tempDirPath.appending("hello") + try fs.createSymbolicLink(sym, pointingAt: file, relative: false) + + let executable = tempDirPath.appending("exec-foo") + try fs.writeFileContents(executable, bytes: ByteString(contents)) +#if !os(Windows) + try fs.chmod(.executable, path: executable, options: []) +#endif + + let executableSym = tempDirPath.appending("exec-sym") + try fs.createSymbolicLink(executableSym, pointingAt: executable, relative: false) + + try fs.createDirectory(tempDirPath.appending("dir")) + try fs.writeFileContents(tempDirPath.appending(components: ["dir", "file"]), bytes: []) + + try VirtualFileSystem.serializeDirectoryTree(tempDirPath, into: AbsolutePath(vfsPath.path), fs: fs, includeContents: [executable]) } - } when: { - ProcessInfo.hostOperatingSystem == .windows + + let vfs = try VirtualFileSystem(path: vfsPath.path, fs: fs) + + // exists() + #expect(vfs.exists(AbsolutePath("/"))) + #expect(!vfs.exists(AbsolutePath("/does-not-exist"))) + + // isFile() + let filePath = AbsolutePath("/best") + #expect(vfs.exists(filePath)) + #expect(vfs.isFile(filePath)) + #expect(try vfs.getFileInfo(filePath).fileType == .typeRegular) + #expect(!vfs.isDirectory(filePath)) + #expect(!vfs.isFile(AbsolutePath("/does-not-exist"))) + #expect(!vfs.isSymlink(AbsolutePath("/does-not-exist"))) + #expect(throws: (any Error).self) { + try vfs.getFileInfo(AbsolutePath("/does-not-exist")) + } + + // isSymlink() + let symPath = AbsolutePath("/hello") + #expect(vfs.isSymlink(symPath)) + #expect(vfs.isFile(symPath)) + #expect(try vfs.getFileInfo(symPath).fileType == .typeSymbolicLink) + #expect(!vfs.isDirectory(symPath)) + + // isExecutableFile + let executablePath = AbsolutePath("/exec-foo") + let executableSymPath = AbsolutePath("/exec-sym") + #expect(vfs.isExecutableFile(executablePath)) + #expect(vfs.isExecutableFile(executableSymPath)) + #expect(vfs.isSymlink(executableSymPath)) + #expect(!vfs.isExecutableFile(symPath)) + #expect(!vfs.isExecutableFile(filePath)) + #expect(!vfs.isExecutableFile(AbsolutePath("/does-not-exist"))) + #expect(!vfs.isExecutableFile(AbsolutePath("/"))) + + // readFileContents + let execFileContents = try vfs.readFileContents(executablePath) + #expect(execFileContents == ByteString(contents)) + + // isDirectory() + #expect(vfs.isDirectory(AbsolutePath("/"))) + #expect(!vfs.isDirectory(AbsolutePath("/does-not-exist"))) + + // getDirectoryContents() + let dirContents = try vfs.getDirectoryContents(AbsolutePath("/")) + #expect(dirContents.sorted() == ["best", "dir", "exec-foo", "exec-sym", "hello"]) + #expect {try vfs.getDirectoryContents(AbsolutePath("/does-not-exist"))} throws: { error in + (error.localizedDescription == "no such file or directory: \(AbsolutePath("/does-not-exist"))") + } + + let thisDirectoryContents = try vfs.getDirectoryContents(AbsolutePath("/")) + #expect(!thisDirectoryContents.contains(where: { $0 == "." })) + #expect(!thisDirectoryContents.contains(where: { $0 == ".." })) + #expect(thisDirectoryContents.sorted() == ["best", "dir", "exec-foo", "exec-sym", "hello"]) + + let contents = try vfs.getDirectoryContents(AbsolutePath("/dir")) + #expect(contents == ["file"]) + + let fileContents = try vfs.readFileContents(AbsolutePath("/dir/file")) + #expect(fileContents == "") } } } diff --git a/Tests/BuildTests/BuildSystemDelegateTests.swift b/Tests/BuildTests/BuildSystemDelegateTests.swift index a975f2db9b0..2f46834375f 100644 --- a/Tests/BuildTests/BuildSystemDelegateTests.swift +++ b/Tests/BuildTests/BuildSystemDelegateTests.swift @@ -63,6 +63,13 @@ struct BuildSystemDelegateTests { @Test( .issue("https://github.com/swiftlang/swift-package-manager/issues/8540", relationship: .defect), // Package fails to build when the test is being executed" .requiresSDKDependentTestsSupport, + .tags( + .Feature.Command.Build, + .Feature.TargetType.Executable, + .Feature.TargetType.Library, + .Feature.CommandLineArguments.BuildSystem, + .Feature.CommandLineArguments.Configuration, + ), arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), ) func filterNonFatalCodesignMessages( diff --git a/Tests/BuildTests/PluginsBuildPlanTests.swift b/Tests/BuildTests/PluginsBuildPlanTests.swift index 60e04740d1e..51bce014719 100644 --- a/Tests/BuildTests/PluginsBuildPlanTests.swift +++ b/Tests/BuildTests/PluginsBuildPlanTests.swift @@ -27,6 +27,8 @@ struct PluginsBuildPlanTests { @Test( .tags( .Feature.Command.Build, + .Feature.Plugin, + .Feature.SourceGeneration, ), .IssueWindowsPathTestsFailures, // Fails to build the project to due to incorrect Path handling arguments: BuildConfiguration.allCases, diff --git a/Tests/FunctionalTests/CFamilyTargetTests.swift b/Tests/FunctionalTests/CFamilyTargetTests.swift index ef26cd8aaf4..2156a85714a 100644 --- a/Tests/FunctionalTests/CFamilyTargetTests.swift +++ b/Tests/FunctionalTests/CFamilyTargetTests.swift @@ -43,6 +43,7 @@ fileprivate func expectDirectoryContainsFile( .serializedIfOnWindows, .tags( .TestSize.large, + .Feature.CTargets, ), ) struct CFamilyTargetTestCase { @@ -50,8 +51,10 @@ struct CFamilyTargetTestCase { .issue("https://github.com/swiftlang/swift-build/issues/333", relationship: .defect), .tags( .Feature.Command.Build, + .Feature.SpecialCharacters, ), - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func cLibraryWithSpaces( data: BuildData, @@ -75,14 +78,15 @@ struct CFamilyTargetTestCase { } @Test( - .tags( - .Feature.Command.Build, - ), .IssueWindowsLongPath, .IssueWindowsPathLastComponent, .IssueWindowsRelativePathAssert, .IssueWindowsCannotSaveAttachment, - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + .tags( + .Feature.Command.Build, + ), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func cUsingCAndSwiftDep( data: BuildData, @@ -110,14 +114,15 @@ struct CFamilyTargetTestCase { } @Test( - .tags( - .Feature.Command.Build, - ), .IssueWindowsLongPath, .IssueWindowsPathLastComponent, .IssueWindowsRelativePathAssert, .IssueWindowsCannotSaveAttachment, - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + .tags( + .Feature.Command.Build, + ), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func moduleMapGenerationCases( data: BuildData, @@ -146,7 +151,8 @@ struct CFamilyTargetTestCase { .tags( .Feature.Command.Build, ), - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func noIncludeDirCheck( data: BuildData, @@ -167,14 +173,16 @@ struct CFamilyTargetTestCase { } @Test( - .tags( - .Feature.Command.Build, - ), .IssueWindowsLongPath, .IssueWindowsPathLastComponent, .IssueWindowsRelativePathAssert, .IssueWindowsCannotSaveAttachment, - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.Xld, + ), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func canForwardExtraFlagsToClang( data: BuildData, @@ -198,17 +206,16 @@ struct CFamilyTargetTestCase { } @Test( + .requireHostOS(.macOS), .tags( .Feature.Command.Build, .Feature.Command.Test, ), - .requireHostOS(.macOS), - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), - + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func objectiveCPackageWithTestTarget( data: BuildData, - ) async throws { try await fixture(name: "CFamilyTargets/ObjCmacOSPackage") { fixturePath in // Build the package. @@ -217,10 +224,14 @@ struct CFamilyTargetTestCase { configuration: data.config, buildSystem: data.buildSystem, ) - if data.buildSystem == .native { + switch data.buildSystem { + case .native: let binPath = try fixturePath.appending(components: data.buildSystem.binPath(for: data.config)) expectDirectoryContainsFile(dir: binPath, filename: "HelloWorldExample.m.o") expectDirectoryContainsFile(dir: binPath, filename: "HelloWorldExample.m.o") + case .swiftbuild, .xcode: + // there aren't any specific expectations to look for + break } // Run swift-test on package. try await executeSwiftTest( @@ -233,14 +244,15 @@ struct CFamilyTargetTestCase { } @Test( - .tags( - .Feature.Command.Build, - ), .IssueWindowsLongPath, .IssueWindowsPathLastComponent, .IssueWindowsRelativePathAssert, .IssueWindowsCannotSaveAttachment, - arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), + .tags( + .Feature.Command.Build, + ), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) func canBuildRelativeHeaderSearchPaths( data: BuildData, @@ -253,9 +265,13 @@ struct CFamilyTargetTestCase { configuration: data.config, buildSystem: data.buildSystem, ) - if data.buildSystem == .native { + switch data.buildSystem { + case .native: let binPath = try fixturePath.appending(components: data.buildSystem.binPath(for: data.config)) expectDirectoryContainsFile(dir: binPath, filename: "HeaderInclude.swiftmodule") + case .swiftbuild, .xcode: + // there aren't any specific expectations to look for + break } } } when: { diff --git a/Tests/FunctionalTests/DependencyResolutionTests.swift b/Tests/FunctionalTests/DependencyResolutionTests.swift index 810d8957965..887a3adbcb6 100644 --- a/Tests/FunctionalTests/DependencyResolutionTests.swift +++ b/Tests/FunctionalTests/DependencyResolutionTests.swift @@ -25,7 +25,8 @@ import enum TSCUtility.Git @Suite( .serializedIfOnWindows, .tags( - Tag.TestSize.large, + .TestSize.large, + .Feature.DependencyResolution, ), ) struct DependencyResolutionTests { @@ -163,6 +164,7 @@ struct DependencyResolutionTests { @Test( .IssueWindowsLongPath, .IssueLdFailsUnexpectedly, + .issue("rdar://162339964", relationship: .defect), .tags( Tag.Feature.Command.Build, ), diff --git a/Tests/FunctionalTests/ModuleAliasingFixtureTests.swift b/Tests/FunctionalTests/ModuleAliasingFixtureTests.swift index db0b9d251e3..e2af71c7758 100644 --- a/Tests/FunctionalTests/ModuleAliasingFixtureTests.swift +++ b/Tests/FunctionalTests/ModuleAliasingFixtureTests.swift @@ -22,7 +22,8 @@ import Testing @Suite( .serializedIfOnWindows, .tags( - Tag.TestSize.large, + .TestSize.large, + .Feature.ModuleAliasing, ), ) struct ModuleAliasingFixtureTests { diff --git a/Tests/FunctionalTests/PluginTests.swift b/Tests/FunctionalTests/PluginTests.swift index b4eafeea5db..2040e57a6f4 100644 --- a/Tests/FunctionalTests/PluginTests.swift +++ b/Tests/FunctionalTests/PluginTests.swift @@ -26,76 +26,75 @@ import Foundation @Suite( .serializedIfOnWindows, .tags( - .Feature.Command.Package.Plugin, .TestSize.large, + .Feature.Command.Package.Plugin, + .Feature.Plugin, ) ) -final class PluginTests { +struct PluginTests { @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8791"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.Product, + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testUseOfBuildToolPluginTargetByExecutableInSamePackage() async throws { + func testUseOfBuildToolPluginTargetByExecutableInSamePackage( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("MySourceGenPlugin"), configuration: .debug, extraArgs: ["--product", "MyLocalTool"], - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows } - - try await withKnownIssue { - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("MySourceGenPlugin"), - configuration: .debug, - extraArgs: ["--product", "MyLocalTool", "--build-system", "swiftbuild"], - buildSystem: .native, - ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - } when: { ProcessInfo.hostOperatingSystem == .windows } } @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8786"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Test, + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testUseOfBuildToolPluginTargetNoPreBuildCommands() async throws { + func testUseOfBuildToolPluginTargetNoPreBuildCommands( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (_, stderr) = try await executeSwiftTest( fixturePath.appending("MySourceGenPluginNoPreBuildCommands"), - buildSystem: .native, + buildSystem: buildSystem, ) #expect(stderr.contains("file(s) which are unhandled; explicitly declare them as resources or exclude from the target"), "expected warning not emitted") } } when: { - ProcessInfo.hostOperatingSystem == .windows && CiEnvironment.runningInSelfHostedPipeline - } - - // Try again with the Swift Build build system - await withKnownIssue { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (_, stderr) = try await executeSwiftTest( - fixturePath.appending("MySourceGenPluginNoPreBuildCommands"), - extraArgs: ["--build-system", "swiftbuild"], - buildSystem: .native, - ) - #expect(stderr.contains("file(s) which are unhandled; explicitly declare them as resources or exclude from the target"), "expected warning not emitted") - } + (ProcessInfo.hostOperatingSystem == .windows && CiEnvironment.runningInSelfHostedPipeline && buildSystem == .native) + || (buildSystem == .swiftbuild) } } @@ -103,35 +102,35 @@ final class PluginTests { .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.BuildSystem, + .Feature.CommandLineArguments.Product, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testUseOfBuildToolPluginProductByExecutableAcrossPackages() async throws { + func testUseOfBuildToolPluginProductByExecutableAcrossPackages( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("MySourceGenClient"), configuration: .debug, extraArgs: ["--product", "MyTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MyTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyTool' complete!"), "stdout:\n\(stdout)") - } - } when: { - ProcessInfo.hostOperatingSystem == .windows - } - - try await withKnownIssue { - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("MySourceGenClient"), - configuration: .debug, - extraArgs: ["--build-system", "swiftbuild", "--product", "MyTool"], - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MyTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows @@ -142,35 +141,35 @@ final class PluginTests { .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.BuildSystem, + .Feature.CommandLineArguments.Product, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testUseOfPrebuildPluginTargetByExecutableAcrossPackages() async throws { + func testUseOfPrebuildPluginTargetByExecutableAcrossPackages( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("MySourceGenPlugin"), configuration: .debug, extraArgs: ["--product", "MyOtherLocalTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Compiling MyOtherLocalTool bar.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling MyOtherLocalTool baz.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MyOtherLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyOtherLocalTool' complete!"), "stdout:\n\(stdout)") - } - } when: { - ProcessInfo.hostOperatingSystem == .windows - } - - try await withKnownIssue { - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("MySourceGenPlugin"), - configuration: .debug, - extraArgs: ["--build-system", "swiftbuild", "--product", "MyOtherLocalTool"], - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling MyOtherLocalTool bar.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling MyOtherLocalTool baz.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MyOtherLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyOtherLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows @@ -181,30 +180,30 @@ final class PluginTests { .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + arguments: SupportedBuildSystemOnAllPlatforms ) - func testUseOfPluginWithInternalExecutable() async throws { + func testUseOfPluginWithInternalExecutable( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("ClientOfPluginWithInternalExecutable"), - buildSystem: .native, - ) - #expect(stdout.contains("Compiling PluginExecutable main.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking PluginExecutable"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling RootTarget foo.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking RootTarget"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("ClientOfPluginWithInternalExecutable"), - extraArgs: ["--build-system", "swiftbuild"], - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling PluginExecutable main.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking PluginExecutable"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling RootTarget foo.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking RootTarget"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows @@ -213,13 +212,20 @@ final class PluginTests { @Test( .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testInternalExecutableAvailableOnlyToPlugin() async throws { + func testInternalExecutableAvailableOnlyToPlugin( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let error = try await #require(throws: SwiftPMError.self, "Illegally used internal executable") { try await executeSwiftBuild( fixturePath.appending("InvalidUseOfInternalPluginExecutable"), - buildSystem: .native, + buildSystem: buildSystem, ) } @@ -229,56 +235,41 @@ final class PluginTests { } #expect( - stderr.contains("product 'PluginExecutable' required by package 'invaliduseofinternalpluginexecutable' target 'RootTarget' not found in package 'PluginWithInternalExecutable'."), "stderr:\n\(stderr)" + stderr.contains("product 'PluginExecutable' required by package 'invaliduseofinternalpluginexecutable' target 'RootTarget' not found in package 'PluginWithInternalExecutable'."), "stderr:\n\(stderr)" ) } - - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let error = try await #require(throws: SwiftPMError.self, "Illegally used internal executable") { - try await executeSwiftBuild( - fixturePath.appending("InvalidUseOfInternalPluginExecutable"), - buildSystem: .native, - ) - } - - guard case SwiftPMError.executionFailure(_, _, _) = error else { - Issue.record("Unexpected error type: \(error.interpolationDescription)") - return - } - } } - + @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testLocalBuildToolPluginUsingRemoteExecutable() async throws { - try await withKnownIssue { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("LibraryWithLocalBuildToolPluginUsingRemoteTool"), - buildSystem: .native, - ) - #expect(stdout.contains("Compiling MySourceGenBuildTool main.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating generated.swift from generated.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling MyLibrary generated.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - } when: { - ProcessInfo.hostOperatingSystem == .windows - } - + func testLocalBuildToolPluginUsingRemoteExecutable( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { - // Try again with the Swift Build build system try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("LibraryWithLocalBuildToolPluginUsingRemoteTool"), - extraArgs: ["--build-system", "swiftbuild"], - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling MySourceGenBuildTool main.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating generated.swift from generated.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling MyLibrary generated.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows @@ -290,69 +281,69 @@ final class PluginTests { .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .bug("https://github.com/swiftlang/swift-package-manager/issues/8791"), .requiresSwiftConcurrencySupport, + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testBuildToolPluginDependencies() async throws { + func testBuildToolPluginDependencies( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("MyBuildToolPluginDependencies"), - buildSystem: .native, + buildSystem: buildSystem, ) - #expect(stdout.contains("Compiling MySourceGenBuildTool main.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling MyLocalTool foo.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling MySourceGenBuildTool main.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling MyLocalTool foo.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows } - - try await withKnownIssue { - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("MyBuildToolPluginDependencies"), - extraArgs: ["--build-system", "swiftbuild"], - buildSystem: .native, - ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - } when: { ProcessInfo.hostOperatingSystem == .windows } } @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testContrivedTestCases() async throws { - try await withKnownIssue { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("ContrivedTestPlugin"), - configuration: .debug, - extraArgs: ["--product", "MyLocalTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") - } - } when: { - ProcessInfo.hostOperatingSystem == .windows - } - + func testContrivedTestCases( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in + let buildSpecificArgs: [String] = switch buildSystem { + case .native, .xcode: + [] + case .swiftbuild: + ["--disable-sandbox"] + } let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("ContrivedTestPlugin"), configuration: .debug, - extraArgs: ["--build-system", "swiftbuild", "--product", "MyLocalTool", "--disable-sandbox"], - buildSystem: .native, + extraArgs: ["--product", "MyLocalTool"] + buildSpecificArgs, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Generating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } when: { ProcessInfo.hostOperatingSystem == .windows @@ -361,29 +352,28 @@ final class PluginTests { @Test( .requiresSwiftConcurrencySupport, - .enabled(if: ProcessInfo.hostOperatingSystem == .macOS, "Test is only supported on macOS") + .requireHostOS(.macOS, "Test is only supported on macOS"), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testPluginScriptSandbox() async throws { + func testPluginScriptSandbox( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("SandboxTesterPlugin"), configuration: .debug, extraArgs: ["--product", "MyLocalTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") - } - - // Try again with Swift Build build system - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("SandboxTesterPlugin"), - configuration: .debug, - extraArgs: ["--build-system", "swiftbuild", "--product", "MyLocalTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + buildSystem: buildSystem, + ) + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } @@ -402,12 +392,13 @@ final class PluginTests { extraArgs: ["--product", "MyLocalTool"], buildSystem: buildSystem, ) - if buildSystem == .native { + switch buildSystem { + case .native: #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n(stdout)") - } else if buildSystem == .swiftbuild { + case .swiftbuild: #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } else { + case .xcode: Issue.record("Test has no expectation for \(buildSystem)") } } when: { ProcessInfo.hostOperatingSystem == .windows } @@ -416,18 +407,28 @@ final class PluginTests { @Test( .requiresSwiftConcurrencySupport, - .enabled(if: ProcessInfo.hostOperatingSystem == .macOS, "Test is only supported on macOS") + .requireHostOS(.macOS, "Test is only supported on macOS"), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testUseOfBinaryToolVendedAsProduct() async throws { + func testUseOfBinaryToolVendedAsProduct( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("BinaryToolProductPlugin"), configuration: .debug, extraArgs: ["--product", "MyLocalTool"], - buildSystem: .native, - ) - #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + buildSystem: buildSystem, + ) + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expectations must be defined.") + } } } @@ -712,7 +713,7 @@ final class PluginTests { customManifestLoader: ManifestLoader(toolchain: UserToolchain.default), delegate: MockWorkspaceDelegate() ) - + // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) let rootManifests = try await workspace.loadRootManifests( @@ -730,11 +731,11 @@ final class PluginTests { #expect(packageGraph.packages.count == 2, "\(packageGraph.packages)") #expect(packageGraph.rootPackages.count == 1, "\(packageGraph.rootPackages)") let package = try #require(packageGraph.rootPackages.first) - + // Find the regular target in our test package. let libraryTarget = try #require(package.modules.map(\.underlying).first{ $0.name == "MyLibrary" } as? SwiftModule) #expect(libraryTarget.type == .library) - + // Set up a delegate to handle callbacks from the command plugin. let delegateQueue = DispatchQueue(label: "plugin-invocation") class PluginDelegate: PluginInvocationDelegate { @@ -744,13 +745,13 @@ final class PluginTests { init(delegateQueue: DispatchQueue) { self.delegateQueue = delegateQueue } - + func pluginCompilationStarted(commandLine: [String], environment: [String: String]) { } - + func pluginCompilationEnded(result: PluginCompilationResult) { } - + func pluginCompilationWasSkipped(cachedResult: PluginCompilationResult) { } @@ -763,7 +764,7 @@ final class PluginTests { Basics.Diagnostic(severity: .info, message: String($0), metadata: .none) }) } - + func pluginEmittedDiagnostic(_ diagnostic: Basics.Diagnostic) { // Add the diagnostic as-is. dispatchPrecondition(condition: .onQueue(delegateQueue)) @@ -842,7 +843,7 @@ final class PluginTests { catch { Issue.record("error \(String(describing: error))", sourceLocation: sourceLocation) } - + // Check that we didn't end up with any completely empty diagnostics. #expect(observability.diagnostics.first{ $0.message.isEmpty } == nil) @@ -873,7 +874,7 @@ final class PluginTests { @Test( .IssueWindowsRelativePathAssert, .requiresSwiftConcurrencySupport, - arguments: [BuildSystemProvider.Kind.native, .swiftbuild] + arguments: SupportedBuildSystemOnAllPlatforms, ) func testLocalAndRemoteToolDependencies(buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue (isIntermittent: true) { @@ -884,12 +885,16 @@ final class PluginTests { extraArgs: ["plugin", "my-plugin"], buildSystem: buildSystem, ) - if buildSystem == .native { + switch buildSystem { + case .native: // Native build system is more explicit about what it's doing in stderr #expect(stderr.contains("Linking RemoteTool"), "stdout:\n\(stderr)\n\(stdout)") #expect(stderr.contains("Linking LocalTool"), "stdout:\n\(stderr)\n\(stdout)") #expect(stderr.contains("Linking ImpliedLocalTool"), "stdout:\n\(stderr)\n\(stdout)") #expect(stderr.contains("Build of product 'ImpliedLocalTool' complete!"), "stdout:\n\(stderr)\n\(stdout)") + case .swiftbuild, .xcode: + // There are nothing specific to expect + break } #expect(stdout.contains("A message from the remote tool."), "stdout:\n\(stderr)\n\(stdout)") #expect(stdout.contains("A message from the local tool."), "stdout:\n\(stderr)\n\(stdout)") @@ -1012,7 +1017,7 @@ final class PluginTests { customManifestLoader: ManifestLoader(toolchain: UserToolchain.default), delegate: MockWorkspaceDelegate() ) - + // Load the root manifest. let rootInput = PackageGraphRootInput(packages: [packageDir], dependencies: []) let rootManifests = try await workspace.loadRootManifests( @@ -1030,7 +1035,7 @@ final class PluginTests { #expect(packageGraph.packages.count == 1, "\(packageGraph.packages)") #expect(packageGraph.rootPackages.count == 1, "\(packageGraph.rootPackages)") let package: ResolvedPackage = try #require(packageGraph.rootPackages.first) - + // Find the regular target in our test package. let libraryTarget = try #require( package.modules @@ -1038,7 +1043,7 @@ final class PluginTests { .first{ $0.name == "MyLibrary" } as? SwiftModule ) #expect(libraryTarget.type == .library) - + // Set up a delegate to handle callbacks from the command plugin. In particular we want to know the process identifier. let delegateQueue = DispatchQueue(label: "plugin-invocation") class PluginDelegate: PluginInvocationDelegate { @@ -1049,16 +1054,16 @@ final class PluginTests { init(delegateQueue: DispatchQueue) { self.delegateQueue = delegateQueue } - + func pluginCompilationStarted(commandLine: [String], environment: [String: String]) { } - + func pluginCompilationEnded(result: PluginCompilationResult) { } - + func pluginCompilationWasSkipped(cachedResult: PluginCompilationResult) { } - + func pluginEmittedOutput(_ data: Data) { // Add each line of emitted output as a `.info` diagnostic. dispatchPrecondition(condition: .onQueue(delegateQueue)) @@ -1066,7 +1071,7 @@ final class PluginTests { diagnostics.append(contentsOf: textlines.map{ Basics.Diagnostic(severity: .info, message: String($0), metadata: .none) }) - + // If we don't already have the process identifier, we try to find it. if parsedProcessIdentifier == .none { func parseProcessIdentifier(_ string: String) -> Int? { @@ -1078,7 +1083,7 @@ final class PluginTests { parsedProcessIdentifier = textlines.compactMap{ parseProcessIdentifier(String($0)) }.first } } - + func pluginEmittedDiagnostic(_ diagnostic: Basics.Diagnostic) { // Add the diagnostic as-is. dispatchPrecondition(condition: .onQueue(delegateQueue)) @@ -1473,32 +1478,33 @@ final class PluginTests { } } } - + @Test( .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.BuildTests, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testIncorrectDependencies() async throws { - try await fixture(name: "Miscellaneous/Plugins") { path in - let (stdout, stderr) = try await executeSwiftBuild( - path.appending("IncorrectDependencies"), - extraArgs: ["--build-tests"], - buildSystem: .native, - ) - #expect(stdout.contains("Build complete!"), "output:\n\(stderr)\n\(stdout)") - } - + func testIncorrectDependencies( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue (isIntermittent: true) { - // Try again with the Swift Build build system try await fixture(name: "Miscellaneous/Plugins") { path in let (stdout, stderr) = try await executeSwiftBuild( path.appending("IncorrectDependencies"), extraArgs: ["--build-tests"], - buildSystem: .swiftbuild, + buildSystem: buildSystem, ) + #expect(stdout.contains("Build complete!"), "output:\n\(stderr)\n\(stdout)") } - } when: { ProcessInfo.hostOperatingSystem == .windows || (ProcessInfo.hostOperatingSystem == .linux && CiEnvironment.runningInSmokeTestPipeline) } + } when: { + (ProcessInfo.hostOperatingSystem == .windows && buildSystem == .swiftbuild) + || (ProcessInfo.hostOperatingSystem == .linux && buildSystem == .swiftbuild && CiEnvironment.runningInSmokeTestPipeline) + } } @Test( @@ -1523,7 +1529,7 @@ final class PluginTests { } @Test( - .enabled(if: ProcessInfo.hostOperatingSystem == .macOS, "sandboxing tests are only supported on macOS"), + .requireHostOS(.macOS, "sandboxing tests are only supported on macOS"), .requiresSwiftConcurrencySupport, arguments: SupportedBuildSystemOnAllPlatforms, ) @@ -1563,7 +1569,7 @@ final class PluginTests { } @Test( - .enabled(if: ProcessInfo.hostOperatingSystem == .macOS, "sandboxing tests are only supported on macOS"), + .requireHostOS(.macOS, "sandboxing tests are only supported on macOS"), arguments: SupportedBuildSystemOnAllPlatforms, ) func testBuildToolPluginSwiftFileExecutable( @@ -1576,10 +1582,13 @@ final class PluginTests { extraArgs: [ "--verbose"], buildSystem: buildSystem, ) - if buildSystem == .native { + switch buildSystem { + case .native: #expect(stdout.contains("Hello, Build Tool Plugin!"), "stdout:\n\(stdout)") - } else { + case .swiftbuild: #expect(stderr.contains("Hello, Build Tool Plugin!"), "stderr:\n\(stderr)") + case .xcode: + Issue.record("Test expected have not been considered") } } } @@ -1587,29 +1596,33 @@ final class PluginTests { @Test( .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testTransitivePluginOnlyDependency() async throws { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("TransitivePluginOnlyDependency"), - buildSystem: .native, - ) - #expect(stdout.contains("Compiling plugin MyPlugin"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling Library Library.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - + func testTransitivePluginOnlyDependency( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { - // Try again with Swift Build build system try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("TransitivePluginOnlyDependency"), - buildSystem: .swiftbuild, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling plugin MyPlugin"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling Library Library.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expected have not been considered") + } } } when: { - ProcessInfo.hostOperatingSystem == .windows + ProcessInfo.hostOperatingSystem == .windows && buildSystem == .swiftbuild } } @@ -1637,96 +1650,106 @@ final class PluginTests { @Test( .bug("https://github.com/swiftlang/swift-package-manager/issues/8774"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testPluginCanBeReferencedByProductName() async throws { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending("PluginCanBeReferencedByProductName"), - buildSystem: .native, - ) - #expect(stdout.contains("Compiling plugin MyPlugin"), "stdout:\n\(stdout)") - #expect(stdout.contains("Compiling PluginCanBeReferencedByProductName gen.swift"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - + func testPluginCanBeReferencedByProductName( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { - // Try again with the Swift Build build system try await fixture(name: "Miscellaneous/Plugins") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath.appending("PluginCanBeReferencedByProductName"), - buildSystem: .swiftbuild, + buildSystem: buildSystem, ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + switch buildSystem { + case .native: + #expect(stdout.contains("Compiling plugin MyPlugin"), "stdout:\n\(stdout)") + #expect(stdout.contains("Compiling PluginCanBeReferencedByProductName gen.swift"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") + case .xcode: + Issue.record("Test expected have not been considered") + } } - } when: { ProcessInfo.hostOperatingSystem == .windows } + + } when: { + ProcessInfo.hostOperatingSystem == .windows && buildSystem == .swiftbuild + } } @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8791"), .requiresSwiftConcurrencySupport, + .tags( + .Feature.Command.Build, + .Feature.CommandLineArguments.BuildSystem, + .Feature.CommandLineArguments.Product, + .Feature.CommandLineArguments.XbuildToolsSwiftc, + ), + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testPluginCanBeAffectedByXBuildToolsParameters() async throws { - try await withKnownIssue { - try await fixture(name: "Miscellaneous/Plugins") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath.appending(component: "MySourceGenPlugin"), - configuration: .debug, - extraArgs: ["--product", "MyLocalTool", "-Xbuild-tools-swiftc", "-DUSE_CREATE"], - buildSystem: .native, - ) - #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Creating foo.swift from foo.dat"), "stdout:\n\(stdout)") - #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") - #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") - } - } when: { ProcessInfo.hostOperatingSystem == .windows } - + func testPluginCanBeAffectedByXBuildToolsParameters( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins") { fixturePath in + let buildArgs: [String] = switch buildSystem { + case .native, .xcode: [] + case .swiftbuild: ["-v"] + } let (stdout, stderr) = try await executeSwiftBuild( fixturePath.appending(component: "MySourceGenPlugin"), configuration: .debug, - extraArgs: ["-v", "--product", "MyLocalTool", "-Xbuild-tools-swiftc", "-DUSE_CREATE",], - buildSystem: .swiftbuild, + extraArgs: ["--product", "MyLocalTool", "-Xbuild-tools-swiftc", "-DUSE_CREATE"] + buildArgs, + buildSystem: buildSystem, ) - #expect(stdout.contains("MySourceGenBuildTool-product"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") - #expect(stderr.contains("Creating foo.swift from foo.dat"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") + + switch buildSystem { + case .native: + #expect(stdout.contains("Linking MySourceGenBuildTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Creating foo.swift from foo.dat"), "stdout:\n\(stdout)") + #expect(stdout.contains("Linking MyLocalTool"), "stdout:\n\(stdout)") + #expect(stdout.contains("Build of product 'MyLocalTool' complete!"), "stdout:\n\(stdout)") + case .swiftbuild: + #expect(stdout.contains("MySourceGenBuildTool-product"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") + #expect(stderr.contains("Creating foo.swift from foo.dat"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") + #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)\nstderr:\n\(stderr)") + case .xcode: + Issue.record("Test expected have not been considered") + + } } - } when: { ProcessInfo.hostOperatingSystem == .windows } + } when: { + ProcessInfo.hostOperatingSystem == .windows + } } @Test( .IssueWindowsRelativePathAssert, .bug("https://github.com/swiftlang/swift-package-manager/issues/8791"), .requiresSwiftConcurrencySupport, + arguments: SupportedBuildSystemOnAllPlatforms, ) - func testURLBasedPluginAPI() async throws { + func testURLBasedPluginAPI( + buildSystem: BuildSystemProvider.Kind, + ) async throws { try await withKnownIssue { try await fixture(name: "Miscellaneous/Plugins/MySourceGenPluginUsingURLBasedAPI") { fixturePath in let (stdout, _) = try await executeSwiftBuild( fixturePath, configuration: .debug, - buildSystem: .native, + buildSystem: buildSystem, ) #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") } } when: { ProcessInfo.hostOperatingSystem == .windows } - - try await withKnownIssue { - // Try again with the Swift Build build system - try await fixture(name: "Miscellaneous/Plugins/MySourceGenPluginUsingURLBasedAPI") { fixturePath in - let (stdout, _) = try await executeSwiftBuild( - fixturePath, - configuration: .debug, - buildSystem: .swiftbuild, - ) - #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") - } - } when: { ProcessInfo.hostOperatingSystem == .windows } } @Test( @@ -1745,7 +1768,7 @@ final class PluginTests { ) #expect(stdout.contains("Build complete!"), "stdout:\n\(stdout)") } - } when: { + } when: { buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows } } diff --git a/Tests/FunctionalTests/StaticBinaryLibrary.swift b/Tests/FunctionalTests/StaticBinaryLibrary.swift index a3298ddf992..6006618f365 100644 --- a/Tests/FunctionalTests/StaticBinaryLibrary.swift +++ b/Tests/FunctionalTests/StaticBinaryLibrary.swift @@ -18,17 +18,30 @@ import _InternalTestSupport struct StaticBinaryLibraryTests { @Test( - .bug("https://github.com/swiftlang/swift-package-manager/issues/8657") + .issue("https://github.com/swiftlang/swift-package-manager/issues/8657", relationship: .defect), + .issue("https://github.com/swiftlang/swift-package-manager/issues/9311", relationship: .defect), + .tags( + .TestSize.large, + .Feature.Command.Run, + .Feature.CommandLineArguments.Experimental.PruneUnusedDependencies, + .Feature.TargetType.Library, + .Feature.TargetType.BinaryTarget.ArtifactBundle, + ), + buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.tags, + arguments: buildDataUsingBuildSystemAvailableOnAllPlatformsWithTags.buildData, ) - func staticLibrary() async throws { + func staticLibrary( + buildData: BuildData, + ) async throws { try await withKnownIssue { try await fixture(name: "BinaryLibraries") { fixturePath in let (stdout, _) = try await executeSwiftRun( fixturePath.appending("Static").appending("Package1"), "Example", + configuration: buildData.config, extraArgs: ["--experimental-prune-unused-dependencies"], - buildSystem: .native, + buildSystem: buildData.buildSystem, ) #expect(stdout == """ 42 @@ -38,6 +51,7 @@ struct StaticBinaryLibraryTests { } } when: { ProcessInfo.hostOperatingSystem == .windows + || buildData.buildSystem == .swiftbuild } } } diff --git a/Tests/FunctionalTests/TestDiscoveryTests.swift b/Tests/FunctionalTests/TestDiscoveryTests.swift index 2b77ffb1311..8a7064d1508 100644 --- a/Tests/FunctionalTests/TestDiscoveryTests.swift +++ b/Tests/FunctionalTests/TestDiscoveryTests.swift @@ -20,11 +20,21 @@ import class Foundation.ProcessInfo @Suite( .serializedIfOnWindows, + .tags( + .Feature.TestDiscovery, + ), ) struct TestDiscoveryTests { - static var buildSystems: [BuildSystemProvider.Kind] = [BuildSystemProvider.Kind.native, .swiftbuild] - @Test(arguments: buildSystems) + @Test( + .tags( + .Feature.Command.Build, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func build(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Simple") { fixturePath in @@ -37,7 +47,17 @@ struct TestDiscoveryTests { } } - @Test(arguments: buildSystems) + @Test( + .IssueWindowsLongPath, + .tags( + .Feature.Command.Test, + .Feature.CommandLineArguments.VeryVerbose, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func discovery(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Simple") { fixturePath in @@ -52,18 +72,43 @@ struct TestDiscoveryTests { } } - @Test(.bug("https://github.com/swiftlang/swift-build/issues/13"), arguments: [BuildSystemProvider.Kind.native]) + @Test( + .tags( + .Feature.Command.Test, + .Feature.SpecialCharacters, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + .IssueWindowsCannotSaveAttachment, + .issue("https://github.com/swiftlang/swift-build/issues/333", relationship: .defect), + .bug("https://github.com/swiftlang/swift-build/issues/13"), + // arguments: [BuildSystemProvider.Kind.native], + arguments: SupportedBuildSystemOnAllPlatforms, + ) func nonStandardName(_ buildSystem: BuildSystemProvider.Kind) async throws { - try await fixture(name: "Miscellaneous/TestDiscovery/hello world") { fixturePath in - let (stdout, stderr) = try await executeSwiftTest(fixturePath, buildSystem: buildSystem) - // in "swift test" build output goes to stderr - #expect(stderr.contains("Build complete!")) - // in "swift test" test output goes to stdout - #expect(stdout.contains("Executed 1 test")) + try await withKnownIssue(isIntermittent: true) { + try await fixture(name: "Miscellaneous/TestDiscovery/hello world") { fixturePath in + let (stdout, stderr) = try await executeSwiftTest(fixturePath, buildSystem: buildSystem) + // in "swift test" build output goes to stderr + #expect(stderr.contains("Build complete!")) + // in "swift test" test output goes to stdout + #expect(stdout.contains("Executed 1 test")) + } + } when: { + buildSystem == .swiftbuild && [.windows, .macOS].contains(ProcessInfo.hostOperatingSystem) } } - @Test(arguments: buildSystems) + @Test( + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func asyncMethods(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Async") { fixturePath in @@ -79,38 +124,81 @@ struct TestDiscoveryTests { } // FIXME: eliminate extraneous warnings with --build-system swiftbuild - @Test(.bug("https://github.com/swiftlang/swift-build/issues/573"), .skipHostOS(.macOS), arguments: [BuildSystemProvider.Kind.native]) + @Test( + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + .IssueWindowsCannotSaveAttachment, + .bug("https://github.com/swiftlang/swift-build/issues/573"), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func discovery_whenNoTests(_ buildSystem: BuildSystemProvider.Kind) async throws { - try await fixture(name: "Miscellaneous/TestDiscovery/NoTests") { fixturePath in - let (stdout, stderr) = try await executeSwiftTest(fixturePath, buildSystem: buildSystem) - // in "swift test" build output goes to stderr - #expect(stderr.contains("Build complete!")) - // we are expecting that no warning is produced - #expect(!stderr.contains("warning:")) - // in "swift test" test output goes to stdout - #expect(stdout.contains("Executed 0 tests")) - } + try await withKnownIssue(isIntermittent: true) { + try await fixture(name: "Miscellaneous/TestDiscovery/NoTests") { fixturePath in + let (stdout, stderr) = try await executeSwiftTest(fixturePath, buildSystem: buildSystem) + // in "swift test" build output goes to stderr + #expect(stderr.contains("Build complete!")) + // we are expecting that no warning is produced + withKnownIssue(isIntermittent: true) { + #expect(!stderr.contains("warning:")) + } when: { + buildSystem == .swiftbuild + } + // in "swift test" test output goes to stdout + #expect(stdout.contains("Executed 0 tests")) + } + } when: { + buildSystem == .swiftbuild && ProcessInfo.hostOperatingSystem == .windows + } } // FIXME: --build-system swiftbuild should support hand-authored entry points. - @Test(.bug("https://github.com/swiftlang/swift-build/issues/572"), .skipHostOS(.macOS), arguments: [BuildSystemProvider.Kind.native]) - func entryPointOverride(_ buildSystem: BuildSystemProvider.Kind) async throws { - for name in SwiftModule.testEntryPointNames { + @Test( + .bug("https://github.com/swiftlang/swift-build/issues/572"), + .IssueWindowsCannotSaveAttachment, + .skipHostOS(.macOS), + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, SwiftModule.testEntryPointNames + ) + func entryPointOverride( + _ buildSystem: BuildSystemProvider.Kind, + testEntrypointName: String + ) async throws { + try await withKnownIssue(isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Simple") { fixturePath in let random = UUID().uuidString - let manifestPath = fixturePath.appending(components: "Tests", name) + let manifestPath = fixturePath.appending(components: "Tests", testEntrypointName) try localFileSystem.writeFileContents(manifestPath, string: "print(\"\(random)\")") let (stdout, stderr) = try await executeSwiftTest(fixturePath, buildSystem: buildSystem) // in "swift test" build output goes to stderr #expect(stderr.contains("Build complete!")) // in "swift test" test output goes to stdout #expect(!stdout.contains("Executed 1 test")) - #expect(stdout.contains(random)) + #expect(stdout.contains(random), "stderr: \(stderr)") } + } when: { + [.windows, .linux].contains(ProcessInfo.hostOperatingSystem) && buildSystem == .swiftbuild } } - @Test(.skipHostOS(.macOS), arguments: buildSystems) + @Test( + .skipHostOS(.macOS), + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func entryPointOverrideIgnored(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Simple") { fixturePath in @@ -127,7 +215,15 @@ struct TestDiscoveryTests { } } - @Test(.skipHostOS(.macOS), arguments: buildSystems) + @Test( + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func testExtensions(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Extensions") { fixturePath in @@ -135,13 +231,19 @@ struct TestDiscoveryTests { // in "swift test" build output goes to stderr #expect(stderr.contains("Build complete!")) // in "swift test" test output goes to stdout - #expect(stdout.contains("SimpleTests1.testExample1")) - #expect(stdout.contains("SimpleTests1.testExample1_a")) - #expect(stdout.contains("SimpleTests2.testExample2")) - #expect(stdout.contains("SimpleTests2.testExample2_a")) - #expect(stdout.contains("SimpleTests4.testExample")) - #expect(stdout.contains("SimpleTests4.testExample1")) - #expect(stdout.contains("SimpleTests4.testExample2")) + let delimiter: String + #if os(macOS) + delimiter = " " + #else + delimiter = "." + #endif + #expect(stdout.contains("SimpleTests1\(delimiter)testExample1")) + #expect(stdout.contains("SimpleTests1\(delimiter)testExample1_a")) + #expect(stdout.contains("SimpleTests2\(delimiter)testExample2")) + #expect(stdout.contains("SimpleTests2\(delimiter)testExample2_a")) + #expect(stdout.contains("SimpleTests4\(delimiter)testExample")) + #expect(stdout.contains("SimpleTests4\(delimiter)testExample1")) + #expect(stdout.contains("SimpleTests4\(delimiter)testExample2")) #expect(stdout.contains("Executed 7 tests")) } } when: { @@ -149,7 +251,15 @@ struct TestDiscoveryTests { } } - @Test(.skipHostOS(.macOS), arguments: buildSystems) + @Test( + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func deprecatedTests(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Deprecation") { fixturePath in @@ -163,7 +273,15 @@ struct TestDiscoveryTests { } } - @Test(.skipHostOS(.macOS), arguments: buildSystems) + @Test( + .tags( + .Feature.Command.Test, + ), + .tags( + .Feature.CommandLineArguments.BuildSystem, + ), + arguments: SupportedBuildSystemOnAllPlatforms, + ) func testSubclassedTestClassTests(_ buildSystem: BuildSystemProvider.Kind) async throws { try await withKnownIssue("Windows builds encounter long path handling issues", isIntermittent: true) { try await fixture(name: "Miscellaneous/TestDiscovery/Subclass") { fixturePath in @@ -171,21 +289,27 @@ struct TestDiscoveryTests { // in "swift test" build output goes to stderr #expect(stderr.contains("Build complete!")) // in "swift test" test output goes to stdout - #expect(stdout.contains("Tests3.test11")) + let delimiter: String + #if os(macOS) + delimiter = " " + #else + delimiter = "." + #endif + #expect(stdout.contains("Tests3\(delimiter)test11")) #expect(stdout.contains("->Module1::Tests1::test11")) - #expect(stdout.contains("Tests3.test12")) + #expect(stdout.contains("Tests3\(delimiter)test12")) #expect(stdout.contains("->Module1::Tests1::test12")) - #expect(stdout.contains("Tests3.test13")) + #expect(stdout.contains("Tests3\(delimiter)test13")) #expect(stdout.contains("->Module1::Tests1::test13")) - #expect(stdout.contains("Tests3.test21")) + #expect(stdout.contains("Tests3\(delimiter)test21")) #expect(stdout.contains("->Module1::Tests2::test21")) - #expect(stdout.contains("Tests3.test22")) + #expect(stdout.contains("Tests3\(delimiter)test22")) #expect(stdout.contains("->Module1::Tests2::test22")) - #expect(stdout.contains("Tests3.test31")) + #expect(stdout.contains("Tests3\(delimiter)test31")) #expect(stdout.contains("->Module1::Tests3::test31")) - #expect(stdout.contains("Tests3.test32")) + #expect(stdout.contains("Tests3\(delimiter)test32")) #expect(stdout.contains("->Module1::Tests3::test32")) - #expect(stdout.contains("Tests3.test33")) + #expect(stdout.contains("Tests3\(delimiter)test33")) #expect(stdout.contains("->Module1::Tests3::test33")) #expect(stdout.contains("->Module2::Tests1::test11")) diff --git a/Tests/FunctionalTests/ToolsVersionTests.swift b/Tests/FunctionalTests/ToolsVersionTests.swift index 7adb72851b5..c6da1339922 100644 --- a/Tests/FunctionalTests/ToolsVersionTests.swift +++ b/Tests/FunctionalTests/ToolsVersionTests.swift @@ -31,14 +31,15 @@ struct ToolsVersionTests { .TestSize.large, .Feature.Command.Build, .Feature.Command.Package.ToolsVersion, + .Feature.ProductType.Library, ), - // arguments: [BuildSystemProvider.Kind.swiftbuild], [BuildConfiguration.release], - arguments: SupportedBuildSystemOnAllPlatforms, BuildConfiguration.allCases, + arguments: getBuildData(for: SupportedBuildSystemOnAllPlatforms), ) func toolsVersion( - buildSystem: BuildSystemProvider.Kind, - configuration: BuildConfiguration, + buildData: BuildData, ) async throws { + let buildSystem = buildData.buildSystem + let configuration = buildData.config try await withKnownIssue("https://github.com/swiftlang/swift-build/issues/609", isIntermittent: true) { try await testWithTemporaryDirectory{ path in let fs = localFileSystem diff --git a/Tests/FunctionalTests/TraitTests.swift b/Tests/FunctionalTests/TraitTests.swift index 7138b7522a8..b5db4e797f0 100644 --- a/Tests/FunctionalTests/TraitTests.swift +++ b/Tests/FunctionalTests/TraitTests.swift @@ -22,8 +22,8 @@ import _InternalTestSupport @Suite( .serializedIfOnWindows, .tags( - Tag.TestSize.large, - Tag.Feature.Traits, + .TestSize.large, + .Feature.Traits, ), ) struct TraitTests { diff --git a/Tests/FunctionalTests/VersionSpecificTests.swift b/Tests/FunctionalTests/VersionSpecificTests.swift index 341909a5b70..e60443390b3 100644 --- a/Tests/FunctionalTests/VersionSpecificTests.swift +++ b/Tests/FunctionalTests/VersionSpecificTests.swift @@ -31,6 +31,7 @@ struct VersionSpecificTests { .tags( .Feature.Command.Build, .Feature.Command.Package.Reset, + .Feature.Version, ), arguments: SupportedBuildSystemOnAllPlatforms, BuildConfiguration.allCases, ) @@ -38,7 +39,7 @@ struct VersionSpecificTests { buildSystem: BuildSystemProvider.Kind, configuration: BuildConfiguration, ) async throws { - try await withKnownIssue(isIntermittent: true) { + try await withKnownIssue(isIntermittent: true) { // Test passed on Windows at-desk try await testWithTemporaryDirectory{ path in let fs = localFileSystem diff --git a/Tests/IntegrationTests/BasicTests.swift b/Tests/IntegrationTests/BasicTests.swift index 1f44d060556..a7ed24883f7 100644 --- a/Tests/IntegrationTests/BasicTests.swift +++ b/Tests/IntegrationTests/BasicTests.swift @@ -20,7 +20,7 @@ import Basics private struct BasicTests { @Test( - .bug("https://github.com/swiftlang/swift-package-manager/issues/8409"), + .issue("https://github.com/swiftlang/swift-package-manager/issues/8409", relationship: .defect), .requireUnrestrictedNetworkAccess("Test requires access to https://github.com"), .tags( Tag.UserWorkflow, @@ -175,6 +175,7 @@ private struct BasicTests { Tag.Feature.Command.Package.Init, Tag.Feature.Command.Test, Tag.Feature.PackageType.Executable, + .Feature.CommandLineArguments.VeryVerbose, ), ) func testSwiftPackageInitExecTests() async throws { diff --git a/Tests/IntegrationTests/SwiftPMTests.swift b/Tests/IntegrationTests/SwiftPMTests.swift index b3b7cad57ee..b8cf10ecc85 100644 --- a/Tests/IntegrationTests/SwiftPMTests.swift +++ b/Tests/IntegrationTests/SwiftPMTests.swift @@ -242,6 +242,15 @@ private struct SwiftPMTests { @Test( .requireSwift6_2, + .tags( + .UserWorkflow, + .Feature.CodeCoverage, + .Feature.Command.Package.Init, + .Feature.Command.Package.AddTarget, + .Feature.Command.Test, + .Feature.PackageType.Empty, + .Feature.TargetType.Test, + ), arguments: SupportedBuildSystemOnAllPlatforms ) func testCodeCoverageMergedAcrossSubprocesses(