diff --git a/README.md b/README.md index 6aee3d9c6..a33da010d 100644 --- a/README.md +++ b/README.md @@ -24,13 +24,15 @@ The rest of this document will focus on how this version of XCTest differs from ## Working on XCTest +### On Linux + XCTest can be built as part of the overall Swift package. When following [the instructions for building Swift](http://www.github.com/apple/swift), pass the `--xctest` option to the build script: ```sh swift/utils/build-script --xctest ``` -If you want to build just XCTest, use the `build_script.py` script at the root of the project. The `master` version of XCTest must be built with the `master` version of Swift. +If you want to build just XCTest, use the `build_script.py` script at the root of the project. The `master` version of XCTest must be built with the `master` version of Swift. XCTest has a dependency upon Foundation, so you must have built the `master` version of that as well. If your install of Swift is located at `/swift` and you wish to install XCTest into that same location, here is a sample invocation of the build script: @@ -38,6 +40,7 @@ If your install of Swift is located at `/swift` and you wish to install XCTest i ./build_script.py \ --swiftc="/swift/usr/bin/swiftc" \ --build-dir="/tmp/XCTest_build" \ + --foundation-build-dir "/swift//usr/lib/swift/linux" \ --library-install-path="/swift/usr/lib/swift/linux" \ --module-install-path="/swift/usr/lib/swift/linux/x86_64" ``` @@ -45,8 +48,26 @@ If your install of Swift is located at `/swift` and you wish to install XCTest i To run the tests on Linux, use the `--test` option: ```sh -./build_script.py --swiftc="/swift/usr/bin/swiftc" --test +./build_script.py \ + --swiftc="/swift/usr/bin/swiftc" \ + --foundation-build-dir "/swift/usr/lib/swift/linux" \ + --test +``` + +You may add tests for XCTest by including them in the `Tests/Functional/` directory. For an example, see `Tests/Functional/SingleFailingTestCase`. + +### On OS X + +You may build XCTest via the "SwiftXCTest" scheme in `XCTest.xcworkspace`. The workspace assumes that Foundation and XCTest are checked out from GitHub in sibling directories. For example: + ``` +% cd Development +% ls +swift-corelibs-foundation swift-corelibs-xctest +% +``` + +Unlike on Linux, you do not need to build Foundation prior to building XCTest. The "SwiftXCTest" Xcode scheme takes care of that for you. To run the tests on OS X, build and run the `SwiftXCTestFunctionalTests` target in the Xcode workspace. You may also run them via the command line: @@ -54,7 +75,7 @@ To run the tests on OS X, build and run the `SwiftXCTestFunctionalTests` target xcodebuild -workspace XCTest.xcworkspace -scheme SwiftXCTestFunctionalTests ``` -You may add tests for XCTest by including them in the `Tests/Functional/` directory. For an example, see `Tests/Functional/SingleFailingTestCase`. +When adding tests to the `Tests/Functional` directory, make sure they can be opened in the `XCTest.xcworkspace` by adding references to them, but do not add them to any of the targets. ### Additional Considerations for Swift on Linux diff --git a/Sources/XCTest/XCTestCase.swift b/Sources/XCTest/XCTestCase.swift index 41b2ff0c2..e15957df8 100644 --- a/Sources/XCTest/XCTestCase.swift +++ b/Sources/XCTest/XCTestCase.swift @@ -11,6 +11,12 @@ // Base class for test cases // +#if os(Linux) || os(FreeBSD) + import Foundation +#else + import SwiftFoundation +#endif + /// This is a compound type used by `XCTMain` to represent tests to run. It combines an /// `XCTestCase` subclass type with the list of test methods to invoke on the test case. /// This type is intended to be produced by the `testCase` helper function. @@ -48,6 +54,12 @@ private func test(testFunc: T -> () throws -> Void) -> XCTestCase } } +// FIXME: Expectations should be stored in an instance variable defined on +// XCTestCase, but when so defined Linux tests fail with "hidden symbol +// isn't defined". Use a global for the time being, as this seems to +// appease the Linux compiler. +private var XCTAllExpectations = [XCTestExpectation]() + extension XCTestCase { public var continueAfterFailure: Bool { @@ -92,6 +104,8 @@ extension XCTestCase { } testCase.tearDown() + testCase.failIfExpectationsNotWaitedFor(XCTAllExpectations) + XCTAllExpectations = [] totalDuration += duration @@ -122,4 +136,161 @@ extension XCTestCase { XCTPrint("Executed \(tests.count) test\(testCountSuffix), with \(totalFailures) failure\(failureSuffix) (\(unexpectedFailures) unexpected) in \(printableStringForTimeInterval(totalDuration)) (\(printableStringForTimeInterval(overallDuration))) seconds") } + + /// It is an API violation to create expectations but not wait for them to + /// be completed. Notify the user of a mistake via a test failure. + private func failIfExpectationsNotWaitedFor(expectations: [XCTestExpectation]) { + if expectations.count > 0 { + let failure = XCTFailure( + message: "Failed due to unwaited expectations.", + failureDescription: "", + expected: false, + file: expectations.last!.file, + line: expectations.last!.line) + if let failureHandler = XCTFailureHandler { + failureHandler(failure) + } + } + } + + /// Creates and returns an expectation associated with the test case. + /// + /// - Parameter description: This string will be displayed in the test log + /// to help diagnose failures. + /// - Parameter file: The file name to use in the error message if + /// this expectation is not waited for. Default is the file + /// containing the call to this method. It is rare to provide this + /// parameter when calling this method. + /// - Parameter line: The line number to use in the error message if the + /// this expectation is not waited for. Default is the line + /// number of the call to this method in the calling file. It is rare to + /// provide this parameter when calling this method. + /// + /// - Note: Whereas Objective-C XCTest determines the file and line + /// number of expectations that are created by using symbolication, this + /// implementation opts to take `file` and `line` as parameters instead. + /// As a result, the interface to these methods are not exactly identical + /// between these environments. To ensure compatibility of tests between + /// swift-corelibs-xctest and Apple XCTest, it is not recommended to pass + /// explicit values for `file` and `line`. + public func expectationWithDescription(description: String, file: StaticString = #file, line: UInt = #line) -> XCTestExpectation { + let expectation = XCTestExpectation( + description: description, + file: file, + line: line, + testCase: self) + XCTAllExpectations.append(expectation) + return expectation + } + + /// Creates a point of synchronization in the flow of a test. Only one + /// "wait" can be active at any given time, but multiple discrete sequences + /// of { expectations -> wait } can be chained together. + /// + /// - Parameter timeout: The amount of time within which all expectation + /// must be fulfilled. + /// - Parameter file: The file name to use in the error message if + /// expectations are not met before the given timeout. Default is the file + /// containing the call to this method. It is rare to provide this + /// parameter when calling this method. + /// - Parameter line: The line number to use in the error message if the + /// expectations are not met before the given timeout. Default is the line + /// number of the call to this method in the calling file. It is rare to + /// provide this parameter when calling this method. + /// - Parameter handler: If provided, the handler will be invoked both on + /// timeout or fulfillment of all expectations. Timeout is always treated + /// as a test failure. + /// + /// - Note: Whereas Objective-C XCTest determines the file and line + /// number of the "wait" call using symbolication, this implementation + /// opts to take `file` and `line` as parameters instead. As a result, + /// the interface to these methods are not exactly identical between + /// these environments. To ensure compatibility of tests between + /// swift-corelibs-xctest and Apple XCTest, it is not recommended to pass + /// explicit values for `file` and `line`. + public func waitForExpectationsWithTimeout(timeout: NSTimeInterval, file: StaticString = #file, line: UInt = #line, handler: XCWaitCompletionHandler?) { + // Mirror Objective-C XCTest behavior; display an unexpected test + // failure when users wait without having first set expectations. + // FIXME: Objective-C XCTest raises an exception for most "API + // violation" failures, including this one. Normally this causes + // the test to stop cold. swift-corelibs-xctest does not stop, + // and executes the rest of the test. This discrepancy should be + // fixed. + if XCTAllExpectations.count == 0 { + let failure = XCTFailure( + message: "call made to wait without any expectations having been set.", + failureDescription: "API violation", + expected: false, + file: file, + line: line) + if let failureHandler = XCTFailureHandler { + failureHandler(failure) + } + return + } + + // Objective-C XCTest outputs the descriptions of every unfulfilled + // expectation. We gather them into this array, which is also used + // to determine failure--a non-empty array meets expectations weren't + // met. + var unfulfilledDescriptions = [String]() + + // We continue checking whether expectations have been fulfilled until + // the specified timeout has been reached. + // FIXME: Instead of polling the expectations to check whether they've + // been fulfilled, it would be more efficient to use a runloop + // source that can be signaled to wake up when an expectation is + // fulfilled. + let runLoop = NSRunLoop.currentRunLoop() + let timeoutDate = NSDate(timeIntervalSinceNow: timeout) + repeat { + unfulfilledDescriptions = [] + for expectation in XCTAllExpectations { + if !expectation.fulfilled { + unfulfilledDescriptions.append(expectation.description) + } + } + + // If we've met all expectations, then break out of the specified + // timeout loop early. + if unfulfilledDescriptions.count == 0 { + break + } + + // Otherwise, wait another fraction of a second. + runLoop.runUntilDate(NSDate(timeIntervalSinceNow: 0.01)) + } while NSDate().compare(timeoutDate) == NSComparisonResult.OrderedAscending + + if unfulfilledDescriptions.count > 0 { + // Not all expectations were fulfilled. Append a failure + // to the array of expectation-based failures. + let descriptions = unfulfilledDescriptions.joinWithSeparator(", ") + let failure = XCTFailure( + message: "Exceeded timeout of \(timeout) seconds, with unfulfilled expectations: \(descriptions)", + failureDescription: "Asynchronous wait failed", + expected: true, + file: file, + line: line) + if let failureHandler = XCTFailureHandler { + failureHandler(failure) + } + } + + // We've recorded all the failures; clear the expectations that + // were set for this test case. + XCTAllExpectations = [] + + // The handler is invoked regardless of whether the test passed. + if let completionHandler = handler { + var error: NSError? = nil + if unfulfilledDescriptions.count > 0 { + // If the test failed, send an error object. + error = NSError( + domain: "org.swift.XCTestErrorDomain", + code: 0, + userInfo: [:]) + } + completionHandler(error) + } + } } diff --git a/Sources/XCTest/XCTestExpectation.swift b/Sources/XCTest/XCTestExpectation.swift new file mode 100644 index 000000000..dc2051241 --- /dev/null +++ b/Sources/XCTest/XCTestExpectation.swift @@ -0,0 +1,71 @@ +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +// +// XCTestExpectation.swift +// Expectations represent specific conditions in asynchronous testing. +// + +/// Expectations represent specific conditions in asynchronous testing. +public class XCTestExpectation { + internal let description: String + internal let file: StaticString + internal let line: UInt + + internal var fulfilled = false + internal weak var testCase: XCTestCase? + + internal init(description: String, file: StaticString, line: UInt, testCase: XCTestCase) { + self.description = description + self.file = file + self.line = line + self.testCase = testCase + } + + /// Marks an expectation as having been met. It's an error to call this + /// method on an expectation that has already been fulfilled, or when the + /// test case that vended the expectation has already completed. + /// + /// - Parameter file: The file name to use in the error message if + /// expectations are not met before the given timeout. Default is the file + /// containing the call to this method. It is rare to provide this + /// parameter when calling this method. + /// - Parameter line: The line number to use in the error message if the + /// expectations are not met before the given timeout. Default is the line + /// number of the call to this method in the calling file. It is rare to + /// provide this parameter when calling this method. + /// + /// - Note: Whereas Objective-C XCTest determines the file and line + /// number the expectation was fulfilled using symbolication, this + /// implementation opts to take `file` and `line` as parameters instead. + /// As a result, the interface to these methods are not exactly identical + /// between these environments. To ensure compatibility of tests between + /// swift-corelibs-xctest and Apple XCTest, it is not recommended to pass + /// explicit values for `file` and `line`. + public func fulfill(file: StaticString = #file, line: UInt = #line) { + // FIXME: Objective-C XCTest emits failures when expectations are + // fulfilled after the test cases that generated those + // expectations have completed. Similarly, this should cause an + // error as well. + if fulfilled { + // Mirror Objective-C XCTest behavior: treat multiple calls to + // fulfill() as an unexpected failure. + let failure = XCTFailure( + message: "multiple calls made to XCTestExpectation.fulfill() for \(description).", + failureDescription: "API violation", + expected: false, + file: file, + line: line) + if let failureHandler = XCTFailureHandler { + failureHandler(failure) + } + } else { + fulfilled = true + } + } +} diff --git a/Sources/XCTest/XCWaitCompletionHandler.swift b/Sources/XCTest/XCWaitCompletionHandler.swift new file mode 100644 index 000000000..e65ec23c7 --- /dev/null +++ b/Sources/XCTest/XCWaitCompletionHandler.swift @@ -0,0 +1,27 @@ +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2015 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See http://swift.org/LICENSE.txt for license information +// See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +// +// XCWaitCompletionHandler.swift +// A closure invoked by XCTestCase when a wait for expectations to be +// fulfilled times out. +// + +#if os(Linux) || os(FreeBSD) + import Foundation +#else + import SwiftFoundation +#endif + +/// A block to be invoked when a call to wait times out or has had all +/// associated expectations fulfilled. +/// +/// - Parameter error: If the wait timed out or a failure was raised while +/// waiting, the error's code will specify the type of failure. Otherwise +/// error will be nil. +public typealias XCWaitCompletionHandler = (NSError?) -> () diff --git a/Tests/Functional/Asynchronous/Expectations/main.swift b/Tests/Functional/Asynchronous/Expectations/main.swift new file mode 100644 index 000000000..0f6249675 --- /dev/null +++ b/Tests/Functional/Asynchronous/Expectations/main.swift @@ -0,0 +1,94 @@ +// RUN: %{swiftc} %s -o %{built_tests_dir}/Asynchronous +// RUN: %{built_tests_dir}/Asynchronous > %t || true +// RUN: %{xctest_checker} %t %s + +#if os(Linux) || os(FreeBSD) + import XCTest + import Foundation +#else + import SwiftXCTest + import SwiftFoundation +#endif + +class ExpectationsTestCase: XCTestCase { +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnUnfulfilledExpectation_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Expectations/main.swift:19: error: ExpectationsTestCase.test_waitingForAnUnfulfilledExpectation_fails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: foo +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnUnfulfilledExpectation_fails' failed \(\d+\.\d+ seconds\). + func test_waitingForAnUnfulfilledExpectation_fails() { + expectationWithDescription("foo") + waitForExpectationsWithTimeout(0.2, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Expectations/main.swift:28: error: ExpectationsTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: bar, baz +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails' failed \(\d+\.\d+ seconds\). + func test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails() { + expectationWithDescription("bar") + expectationWithDescription("baz") + waitForExpectationsWithTimeout(0.2, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnImmediatelyFulfilledExpectation_passes' started. +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnImmediatelyFulfilledExpectation_passes' passed \(\d+\.\d+ seconds\). + func test_waitingForAnImmediatelyFulfilledExpectation_passes() { + let expectation = expectationWithDescription("flim") + expectation.fulfill() + waitForExpectationsWithTimeout(0.2, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnEventuallyFulfilledExpectation_passes' started. +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnEventuallyFulfilledExpectation_passes' passed \(\d+\.\d+ seconds\). + func test_waitingForAnEventuallyFulfilledExpectation_passes() { + let expectation = expectationWithDescription("flam") + let timer = NSTimer.scheduledTimer(0.1, repeats: false) { _ in + expectation.fulfill() + } + NSRunLoop.currentRunLoop().addTimer(timer, forMode: NSDefaultRunLoopMode) + waitForExpectationsWithTimeout(1.0, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Expectations/main.swift:59: error: ExpectationsTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails : Asynchronous wait failed - Exceeded timeout of 0.1 seconds, with unfulfilled expectations: hog +// CHECK: Test Case 'ExpectationsTestCase.test_waitingForAnExpectationFulfilledAfterTheTimeout_fails' failed \(\d+\.\d+ seconds\). + func test_waitingForAnExpectationFulfilledAfterTheTimeout_fails() { + let expectation = expectationWithDescription("hog") + let timer = NSTimer.scheduledTimer(1.0, repeats: false) { _ in + expectation.fulfill() + } + NSRunLoop.currentRunLoop().addTimer(timer, forMode: NSDefaultRunLoopMode) + waitForExpectationsWithTimeout(0.1, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_whenTimeoutIsImmediate_andAllExpectationsAreFulfilled_passes' started. +// CHECK: Test Case 'ExpectationsTestCase.test_whenTimeoutIsImmediate_andAllExpectationsAreFulfilled_passes' passed \(\d+\.\d+ seconds\). + func test_whenTimeoutIsImmediate_andAllExpectationsAreFulfilled_passes() { + let expectation = expectationWithDescription("smog") + expectation.fulfill() + waitForExpectationsWithTimeout(0.0, handler: nil) + } + +// CHECK: Test Case 'ExpectationsTestCase.test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Expectations/main.swift:75: error: ExpectationsTestCase.test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails : Asynchronous wait failed - Exceeded timeout of -1.0 seconds, with unfulfilled expectations: dog +// CHECK: Test Case 'ExpectationsTestCase.test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails' failed \(\d+\.\d+ seconds\). + func test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails() { + expectationWithDescription("dog") + waitForExpectationsWithTimeout(-1.0, handler: nil) + } + + static var allTests: [(String, ExpectationsTestCase -> () throws -> Void)] { + return [ + ("test_waitingForAnUnfulfilledExpectation_fails", test_waitingForAnUnfulfilledExpectation_fails), + ("test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails", test_waitingForUnfulfilledExpectations_outputsAllExpectations_andFails), + ("test_waitingForAnImmediatelyFulfilledExpectation_passes", test_waitingForAnImmediatelyFulfilledExpectation_passes), + ("test_waitingForAnEventuallyFulfilledExpectation_passes", test_waitingForAnEventuallyFulfilledExpectation_passes), + ("test_waitingForAnExpectationFulfilledAfterTheTimeout_fails", test_waitingForAnExpectationFulfilledAfterTheTimeout_fails), + ("test_whenTimeoutIsImmediate_andAllExpectationsAreFulfilled_passes", test_whenTimeoutIsImmediate_andAllExpectationsAreFulfilled_passes), + ("test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails", test_whenTimeoutIsImmediate_butNotAllExpectationsAreFulfilled_fails), + ] + } +} + +XCTMain([testCase(ExpectationsTestCase.allTests)]) + +// CHECK: Executed 7 tests, with 4 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: Total executed 7 tests, with 4 failures \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds diff --git a/Tests/Functional/Asynchronous/Handler/main.swift b/Tests/Functional/Asynchronous/Handler/main.swift new file mode 100644 index 000000000..5ff628fc7 --- /dev/null +++ b/Tests/Functional/Asynchronous/Handler/main.swift @@ -0,0 +1,55 @@ +// RUN: %{swiftc} %s -o %{built_tests_dir}/Handler +// RUN: %{built_tests_dir}/Handler > %t || true +// RUN: %{xctest_checker} %t %s + +#if os(Linux) || os(FreeBSD) + import XCTest + import Foundation +#else + import SwiftXCTest + import SwiftFoundation +#endif + +class HandlerTestCase: XCTestCase { +// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Handler/main.swift:21: error: HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails : Asynchronous wait failed - Exceeded timeout of 0.2 seconds, with unfulfilled expectations: fog +// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreNotFulfilled_handlerCalled_andFails' failed \(\d+\.\d+ seconds\). + func test_whenExpectationsAreNotFulfilled_handlerCalled_andFails() { + self.expectationWithDescription("fog") + + var handlerWasCalled = false + self.waitForExpectationsWithTimeout(0.2) { error in + XCTAssertNotNil(error, "Expectation handlers for unfulfilled expectations should not be nil.") + XCTAssertTrue(error!.domain.hasSuffix("XCTestErrorDomain"), "The last component of the error domain should match Objective-C XCTest.") + XCTAssertEqual(error!.code, 0, "The error code should match Objective-C XCTest.") + handlerWasCalled = true + } + XCTAssertTrue(handlerWasCalled) + } + +// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreFulfilled_handlerCalled_andPasses' started. +// CHECK: Test Case 'HandlerTestCase.test_whenExpectationsAreFulfilled_handlerCalled_andPasses' passed \(\d+\.\d+ seconds\). + func test_whenExpectationsAreFulfilled_handlerCalled_andPasses() { + let expectation = self.expectationWithDescription("bog") + expectation.fulfill() + + var handlerWasCalled = false + self.waitForExpectationsWithTimeout(0.2) { error in + XCTAssertNil(error, "Expectation handlers for fulfilled expectations should be nil.") + handlerWasCalled = true + } + XCTAssertTrue(handlerWasCalled) + } + + static var allTests: [(String, HandlerTestCase -> () throws -> Void)] { + return [ + ("test_whenExpectationsAreNotFulfilled_handlerCalled_andFails", test_whenExpectationsAreNotFulfilled_handlerCalled_andFails), + ("test_whenExpectationsAreFulfilled_handlerCalled_andPasses", test_whenExpectationsAreFulfilled_handlerCalled_andPasses), + ] + } +} + +XCTMain([testCase(HandlerTestCase.allTests)]) + +// CHECK: Executed 2 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: Total executed 2 tests, with 1 failure \(0 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds diff --git a/Tests/Functional/Asynchronous/Misuse/main.swift b/Tests/Functional/Asynchronous/Misuse/main.swift new file mode 100644 index 000000000..c0a6c8d14 --- /dev/null +++ b/Tests/Functional/Asynchronous/Misuse/main.swift @@ -0,0 +1,60 @@ +// RUN: %{swiftc} %s -o %{built_tests_dir}/Misuse +// RUN: %{built_tests_dir}/Misuse > %t || true +// RUN: %{xctest_checker} %t %s + +#if os(Linux) || os(FreeBSD) + import XCTest +#else + import SwiftXCTest +#endif + +class MisuseTestCase: XCTestCase { +// CHECK: Test Case 'MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Misuse/main.swift:17: unexpected error: MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails : - Failed due to unwaited expectations. +// CHECK: Test Case 'MisuseTestCase.test_whenExpectationsAreMade_butNotWaitedFor_fails' failed \(\d+\.\d+ seconds\). + func test_whenExpectationsAreMade_butNotWaitedFor_fails() { + self.expectationWithDescription("the first expectation") + self.expectationWithDescription("the second expectation (the file and line number for this one are included in the failure message") + } + +// CHECK: Test Case 'MisuseTestCase.test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Misuse/main.swift:24: unexpected error: MisuseTestCase.test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails : API violation - call made to wait without any expectations having been set. +// CHECK: Test Case 'MisuseTestCase.test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails' failed \(\d+\.\d+ seconds\). + func test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails() { + self.waitForExpectationsWithTimeout(0.1, handler: nil) + } + +// CHECK: Test Case 'MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails' started. +// CHECK: .*/Tests/Functional/Asynchronous/Misuse/main.swift:34: unexpected error: MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob. +// CHECK: .*/Tests/Functional/Asynchronous/Misuse/main.swift:44: unexpected error: MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails : API violation - multiple calls made to XCTestExpectation.fulfill\(\) for rob. +// CHECK: Test Case 'MisuseTestCase.test_whenExpectationIsFulfilledMultipleTimes_fails' failed \(\d+\.\d+ seconds\). + func test_whenExpectationIsFulfilledMultipleTimes_fails() { + let expectation = self.expectationWithDescription("rob") + expectation.fulfill() + expectation.fulfill() + // FIXME: The behavior here is subtly different from Objective-C XCTest. + // Objective-C XCTest would stop executing the test on the line + // above, and so would not report a failure for this line below. + // In total, it would highlight one line as a failure in this + // test. + // + // swift-corelibs-xctest continues to execute the test, and so + // highlights both the lines above and below as failures. + // This should be fixed such that the behavior is identical. + expectation.fulfill() + self.waitForExpectationsWithTimeout(0.1, handler: nil) + } + + static var allTests: [(String, MisuseTestCase -> () throws -> Void)] { + return [ + ("test_whenExpectationsAreMade_butNotWaitedFor_fails", test_whenExpectationsAreMade_butNotWaitedFor_fails), + ("test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails", test_whenNoExpectationsAreMade_butTheyAreWaitedFor_fails), + ("test_whenExpectationIsFulfilledMultipleTimes_fails", test_whenExpectationIsFulfilledMultipleTimes_fails), + ] + } +} + +XCTMain([testCase(MisuseTestCase.allTests)]) + +// CHECK: Executed 3 tests, with 4 failures \(4 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds +// CHECK: Total executed 3 tests, with 4 failures \(4 unexpected\) in \d+\.\d+ \(\d+\.\d+\) seconds diff --git a/Tests/Functional/lit.cfg b/Tests/Functional/lit.cfg index b0cb24c1d..364112262 100644 --- a/Tests/Functional/lit.cfg +++ b/Tests/Functional/lit.cfg @@ -22,10 +22,8 @@ def _getenv(name): value = os.getenv(name, None) if value is None: lit_config.fatal( - 'Environment variable ${} is not set.\n' - '$SWIFT_EXEC, $SDKROOT, $BUILT_PRODUCTS_DIR, $PLATFORM_NAME, and ' - '$MACOSX_DEPLOYMENT_TARGET must all be set for lit tests to ' - 'run.'.format(name)) + 'Environment variable ${} is required to run tests on this ' + 'platform, but it is not set.'.format(name)) return value built_products_dir = _getenv('BUILT_PRODUCTS_DIR') @@ -51,6 +49,22 @@ if platform.system() == 'Darwin': '-sdk', sdk_root, '-target', target, '-F', built_products_dir, + # FIXME: We must include the C header dependencies of any module we wish + # to use, due to a limitation in the Swift compiler. See SR-655 + # for details. Here, we include the headers from CoreFoundation. + '-I', os.path.join(built_products_dir, 'usr', 'local', 'include'), + ]) +else: + # On Linux, we need to jump through extra hoops to link + # swift-corelibs-foundation. + foundation_dir = _getenv('FOUNDATION_BUILT_PRODUCTS_DIR') + core_foundation_dir = _getenv('CORE_FOUNDATION_BUILT_PRODUCTS_DIR') + swift_exec.extend([ + '-Xlinker', '-rpath', + '-Xlinker', foundation_dir, + '-L', foundation_dir, + '-I', foundation_dir, + '-I', core_foundation_dir, ]) # Having prepared the swiftc command, we set the substitution. diff --git a/XCTest.xcodeproj/project.pbxproj b/XCTest.xcodeproj/project.pbxproj index c21fed6e9..39256cccc 100644 --- a/XCTest.xcodeproj/project.pbxproj +++ b/XCTest.xcodeproj/project.pbxproj @@ -11,6 +11,9 @@ C265F6701C3AEB6A00520CF9 /* XCTestCase.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F66A1C3AEB6A00520CF9 /* XCTestCase.swift */; }; C265F6721C3AEB6A00520CF9 /* XCTestMain.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F66C1C3AEB6A00520CF9 /* XCTestMain.swift */; }; C265F6731C3AEB6A00520CF9 /* XCTimeUtilities.swift in Sources */ = {isa = PBXBuildFile; fileRef = C265F66D1C3AEB6A00520CF9 /* XCTimeUtilities.swift */; }; + DA7805FA1C6704A2003C6636 /* SwiftFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = DA7805F91C6704A2003C6636 /* SwiftFoundation.framework */; }; + DACC94421C8B87B900EC85F5 /* XCWaitCompletionHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = DACC94411C8B87B900EC85F5 /* XCWaitCompletionHandler.swift */; }; + DADB979C1C51BDA2005E68B6 /* XCTestExpectation.swift in Sources */ = {isa = PBXBuildFile; fileRef = DADB979B1C51BDA2005E68B6 /* XCTestExpectation.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -33,6 +36,9 @@ C265F66A1C3AEB6A00520CF9 /* XCTestCase.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestCase.swift; sourceTree = ""; }; C265F66C1C3AEB6A00520CF9 /* XCTestMain.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestMain.swift; sourceTree = ""; }; C265F66D1C3AEB6A00520CF9 /* XCTimeUtilities.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTimeUtilities.swift; sourceTree = ""; }; + DA7805F91C6704A2003C6636 /* SwiftFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = SwiftFoundation.framework; path = "../swift-corelibs-foundation/build/Debug/SwiftFoundation.framework"; sourceTree = ""; }; + DACC94411C8B87B900EC85F5 /* XCWaitCompletionHandler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCWaitCompletionHandler.swift; sourceTree = ""; }; + DADB979B1C51BDA2005E68B6 /* XCTestExpectation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = XCTestExpectation.swift; sourceTree = ""; }; EA3E74BB1BF2B6D500635A73 /* build_script.py */ = {isa = PBXFileReference; lastKnownFileType = text.script.python; path = build_script.py; sourceTree = ""; }; /* End PBXFileReference section */ @@ -41,6 +47,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + DA7805FA1C6704A2003C6636 /* SwiftFoundation.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -53,6 +60,7 @@ C265F6661C3AEB6A00520CF9 /* Sources */, DA78F7E71C4039410082E15B /* Tests */, 5B5D86DC1BBC74AD00234F36 /* Products */, + DA7805FB1C6704CC003C6636 /* Linked Libraries */, EA3E74BC1BF2B6D700635A73 /* Linux Build */, B1384A401C1B3E6A00EDF031 /* Documentation */, ); @@ -92,11 +100,21 @@ C265F6691C3AEB6A00520CF9 /* XCTAssert.swift */, C265F66A1C3AEB6A00520CF9 /* XCTestCase.swift */, C265F66C1C3AEB6A00520CF9 /* XCTestMain.swift */, + DADB979B1C51BDA2005E68B6 /* XCTestExpectation.swift */, C265F66D1C3AEB6A00520CF9 /* XCTimeUtilities.swift */, + DACC94411C8B87B900EC85F5 /* XCWaitCompletionHandler.swift */, ); path = XCTest; sourceTree = ""; }; + DA7805FB1C6704CC003C6636 /* Linked Libraries */ = { + isa = PBXGroup; + children = ( + DA7805F91C6704A2003C6636 /* SwiftFoundation.framework */, + ); + name = "Linked Libraries"; + sourceTree = ""; + }; DA78F7E71C4039410082E15B /* Tests */ = { isa = PBXGroup; children = ( @@ -214,8 +232,10 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + DACC94421C8B87B900EC85F5 /* XCWaitCompletionHandler.swift in Sources */, C265F6731C3AEB6A00520CF9 /* XCTimeUtilities.swift in Sources */, C265F6701C3AEB6A00520CF9 /* XCTestCase.swift in Sources */, + DADB979C1C51BDA2005E68B6 /* XCTestExpectation.swift in Sources */, C265F66F1C3AEB6A00520CF9 /* XCTAssert.swift in Sources */, C265F6721C3AEB6A00520CF9 /* XCTestMain.swift in Sources */, ); @@ -331,9 +351,11 @@ FRAMEWORK_VERSION = A; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_NAME = SwiftXCTest; SKIP_INSTALL = YES; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + USER_HEADER_SEARCH_PATHS = $BUILT_PRODUCTS_DIR/usr/local/include/CoreFoundation; }; name = Debug; }; @@ -350,8 +372,10 @@ FRAMEWORK_VERSION = A; INSTALL_PATH = "$(LOCAL_LIBRARY_DIR)/Frameworks"; LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/../Frameworks @loader_path/Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 10.11; PRODUCT_NAME = SwiftXCTest; SKIP_INSTALL = YES; + USER_HEADER_SEARCH_PATHS = $BUILT_PRODUCTS_DIR/usr/local/include/CoreFoundation; }; name = Release; }; @@ -362,6 +386,7 @@ DEBUG_INFORMATION_FORMAT = dwarf; GCC_GENERATE_DEBUGGING_SYMBOLS = YES; GCC_OPTIMIZATION_LEVEL = 0; + MACOSX_DEPLOYMENT_TARGET = 10.11; OTHER_CFLAGS = ""; OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -373,6 +398,7 @@ isa = XCBuildConfiguration; buildSettings = { DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + MACOSX_DEPLOYMENT_TARGET = 10.11; OTHER_CFLAGS = ""; OTHER_LDFLAGS = ""; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/XCTest.xcworkspace/contents.xcworkspacedata b/XCTest.xcworkspace/contents.xcworkspacedata index 5e3543e23..fc9680164 100644 --- a/XCTest.xcworkspace/contents.xcworkspacedata +++ b/XCTest.xcworkspace/contents.xcworkspacedata @@ -4,4 +4,7 @@ + + diff --git a/build_script.py b/build_script.py index a15c0be93..ef10ea53a 100755 --- a/build_script.py +++ b/build_script.py @@ -34,6 +34,20 @@ def _mkdirp(path): run("mkdir -p {}".format(path)) +def _core_foundation_build_dir(foundation_build_dir): + """ + Given the path to a swift-corelibs-foundation built product directory, + return the path to CoreFoundation built products. + + When specifying a built Foundation dir such as + '/build/foundation-linux-x86_64/Foundation', CoreFoundation dependencies + are placed in 'usr/lib/swift'. Note that it's technically not necessary to + include this extra path when linking the installed Swift's + 'usr/lib/swift/linux/libFoundation.so'. + """ + return os.path.join(foundation_build_dir, 'usr', 'lib', 'swift') + + def _build(args): """ Build XCTest and place the built products in the given 'build_dir'. @@ -41,6 +55,10 @@ def _build(args): """ swiftc = os.path.abspath(args.swiftc) build_dir = os.path.abspath(args.build_dir) + foundation_build_dir = os.path.abspath(args.foundation_build_dir) + core_foundation_build_dir = _core_foundation_build_dir( + foundation_build_dir) + _mkdirp(build_dir) sourcePaths = glob.glob(os.path.join( @@ -53,14 +71,31 @@ def _build(args): # Not incremental.. # Build library - run("{0} -c {1} -emit-object {2} -module-name XCTest -parse-as-library -emit-module " - "-emit-module-path {3}/XCTest.swiftmodule -o {3}/XCTest.o -force-single-frontend-invocation " - "-module-link-name XCTest".format(swiftc, style_options, " ".join(sourcePaths), build_dir)) - run("{0} -emit-library {1}/XCTest.o -o {1}/libXCTest.so -lswiftGlibc -lswiftCore -lm".format(swiftc, build_dir)) + run("{swiftc} -c {style_options} -emit-object -emit-module " + "-module-name XCTest -module-link-name XCTest -parse-as-library " + "-emit-module-path {build_dir}/XCTest.swiftmodule " + "-force-single-frontend-invocation " + "-I {foundation_build_dir} -I {core_foundation_build_dir} " + "{source_paths} -o {build_dir}/XCTest.o".format( + swiftc=swiftc, + style_options=style_options, + build_dir=build_dir, + foundation_build_dir=foundation_build_dir, + core_foundation_build_dir=core_foundation_build_dir, + source_paths=" ".join(sourcePaths))) + run("{swiftc} -emit-library {build_dir}/XCTest.o " + "-L {foundation_build_dir} -lswiftGlibc -lswiftCore -lFoundation -lm " + "-o {build_dir}/libXCTest.so".format( + swiftc=swiftc, + build_dir=build_dir, + foundation_build_dir=foundation_build_dir)) if args.test: # Execute main() using the arguments necessary to run the tests. - main(args=["test", "--swiftc", swiftc, build_dir]) + main(args=["test", + "--swiftc", swiftc, + "--foundation-build-dir", foundation_build_dir, + build_dir]) # If --module-install-path and --library-install-path were specified, # we also install the built XCTest products. @@ -85,14 +120,22 @@ def _test(args): # FIXME: Allow these to be specified by the Swift build script. lit_flags = "-sv --no-progress-bar" tests_path = os.path.join(SOURCE_DIR, "Tests", "Functional") - run("SWIFT_EXEC={swiftc} " - "BUILT_PRODUCTS_DIR={build_dir} " - "{lit_path} {lit_flags} " - "{tests_path}".format(swiftc=args.swiftc, - build_dir=args.build_dir, - lit_path=lit_path, - lit_flags=lit_flags, - tests_path=tests_path)) + core_foundation_build_dir = _core_foundation_build_dir( + args.foundation_build_dir) + + run('SWIFT_EXEC={swiftc} ' + 'BUILT_PRODUCTS_DIR={built_products_dir} ' + 'FOUNDATION_BUILT_PRODUCTS_DIR={foundation_build_dir} ' + 'CORE_FOUNDATION_BUILT_PRODUCTS_DIR={core_foundation_build_dir} ' + '{lit_path} {lit_flags} ' + '{tests_path}'.format( + swiftc=args.swiftc, + built_products_dir=args.build_dir, + foundation_build_dir=args.foundation_build_dir, + core_foundation_build_dir=core_foundation_build_dir, + lit_path=lit_path, + lit_flags=lit_flags, + tests_path=tests_path)) def _install(args): @@ -164,7 +207,7 @@ def main(args=sys.argv[1:]): help="Path to swift-corelibs-foundation build products, which " "the built XCTest.so will be linked against.", metavar="PATH", - required=False) + required=True) build_parser.add_argument("--swift-build-dir", help="deprecated, do not use") build_parser.add_argument("--arch", help="deprecated, do not use") @@ -219,7 +262,7 @@ def main(args=sys.argv[1:]): help="Path to swift-corelibs-foundation build products, which the " "tests will be linked against.", metavar="PATH", - required=False) + required=True) install_parser = subparsers.add_parser( "install",