From f1bfb02af62d255ac0a65dce3b6b985a45651e9e Mon Sep 17 00:00:00 2001 From: Brian Gesiak Date: Fri, 8 Apr 2016 00:16:51 -0400 Subject: [PATCH] [xctest_checker] Inline expectations Enhance xctest_checker such that it can parse "CHECK" prefixes that are in the middle of a line (instead of just at the very beginning). In addition, use a common `XCTestCheckerError` to ensure all functional test suite failures are displayed inline in Xcode. --- .../SingleFailingTestCase/main.swift | 6 +- .../xctest_checker/tests/test_compare.py | 35 +++++++++-- .../xctest_checker/xctest_checker/compare.py | 61 ++++++++++++++----- .../xctest_checker/xctest_checker/error.py | 19 ++++++ 4 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 Tests/Functional/xctest_checker/xctest_checker/error.py diff --git a/Tests/Functional/SingleFailingTestCase/main.swift b/Tests/Functional/SingleFailingTestCase/main.swift index 5c560d999..315cc69d2 100644 --- a/Tests/Functional/SingleFailingTestCase/main.swift +++ b/Tests/Functional/SingleFailingTestCase/main.swift @@ -19,9 +19,9 @@ class SingleFailingTestCase: XCTestCase { ] } -// CHECK: Test Case 'SingleFailingTestCase.test_fails' started at \d+:\d+:\d+\.\d+ -// CHECK: .*/SingleFailingTestCase/main.swift:26: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed - -// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\). + // CHECK: Test Case 'SingleFailingTestCase.test_fails' started at \d+:\d+:\d+\.\d+ + // CHECK: .*/SingleFailingTestCase/main.swift:26: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed - + // CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\). func test_fails() { XCTAssert(false) } diff --git a/Tests/Functional/xctest_checker/tests/test_compare.py b/Tests/Functional/xctest_checker/tests/test_compare.py index e64836ebd..3d33c442f 100644 --- a/Tests/Functional/xctest_checker/tests/test_compare.py +++ b/Tests/Functional/xctest_checker/tests/test_compare.py @@ -12,6 +12,7 @@ import unittest from xctest_checker import compare +from xctest_checker.error import XCTestCheckerError def _tmpfile(content): @@ -26,30 +27,52 @@ class CompareTestCase(unittest.TestCase): def test_no_match_raises(self): actual = _tmpfile('foo\nbar\nbaz\n') expected = _tmpfile('c: foo\nc: baz\nc: bar\n') - with self.assertRaises(AssertionError): + with self.assertRaises(XCTestCheckerError): compare.compare(actual, expected, check_prefix='c: ') - def test_too_few_expected_raises(self): + def test_too_few_expected_raises_and_first_line_in_error(self): actual = _tmpfile('foo\nbar\nbaz\n') expected = _tmpfile('c: foo\nc: bar\n') - with self.assertRaises(AssertionError): + with self.assertRaises(XCTestCheckerError) as cm: compare.compare(actual, expected, check_prefix='c: ') - def test_too_many_expected_raises(self): + self.assertIn('{}:{}'.format(expected, 1), cm.exception.message) + + def test_too_many_expected_raises_and_excess_check_line_in_error(self): actual = _tmpfile('foo\nbar\n') expected = _tmpfile('c: foo\nc: bar\nc: baz\n') - with self.assertRaises(AssertionError): + with self.assertRaises(XCTestCheckerError) as cm: compare.compare(actual, expected, check_prefix='c: ') + self.assertIn('{}:{}'.format(expected, 3), cm.exception.message) + def test_match_does_not_raise(self): actual = _tmpfile('foo\nbar\nbaz\n') expected = _tmpfile('c: foo\nc: bar\nc: baz\n') compare.compare(actual, expected, check_prefix='c: ') + def test_match_with_inline_check_does_not_raise(self): + actual = _tmpfile('bling\nblong\n') + expected = _tmpfile('meep meep // c: bling\nmeep\n// c: blong\n') + compare.compare(actual, expected, check_prefix='// c: ') + + def test_check_prefix_twice_in_the_same_line_raises_with_line(self): + actual = _tmpfile('blorp\nbleep\n') + expected = _tmpfile('c: blorp\nc: bleep c: blammo\n') + with self.assertRaises(XCTestCheckerError) as cm: + compare.compare(actual, expected, check_prefix='c: ') + + self.assertIn('{}:{}'.format(expected, 2), cm.exception.message) + + def test_check_prefix_in_run_line_ignored(self): + actual = _tmpfile('flim\n') + expected = _tmpfile('// RUN: xctest_checker --prefix "c: "\nc: flim\n') + compare.compare(actual, expected, check_prefix='c: ') + def test_includes_file_name_and_line_of_expected_in_error(self): actual = _tmpfile('foo\nbar\nbaz\n') expected = _tmpfile('c: foo\nc: baz\nc: bar\n') - with self.assertRaises(AssertionError) as cm: + with self.assertRaises(XCTestCheckerError) as cm: compare.compare(actual, expected, check_prefix='c: ') self.assertIn("{}:{}:".format(expected, 2), cm.exception.message) diff --git a/Tests/Functional/xctest_checker/xctest_checker/compare.py b/Tests/Functional/xctest_checker/xctest_checker/compare.py index cc9e4f44f..a77da7b49 100644 --- a/Tests/Functional/xctest_checker/xctest_checker/compare.py +++ b/Tests/Functional/xctest_checker/xctest_checker/compare.py @@ -10,6 +10,8 @@ import re +from .error import XCTestCheckerError + def _actual_lines(path): """ @@ -26,13 +28,33 @@ def _expected_lines_and_line_numbers(path, check_prefix): that begins with the given prefix. """ with open(path) as f: - for line_number, line in enumerate(f): - if line.startswith(check_prefix): - yield line[len(check_prefix):].strip(), line_number+1 + for index, line in enumerate(f): + if 'RUN:' in line: + # Ignore lit directives, which may include a call to + # xctest_checker that specifies a check prefix. + continue + + # Note that line numbers are not zero-indexed; we must add one to + # the loop index. + line_number = index + 1 + + components = line.split(check_prefix) + if len(components) == 2: + yield components[1].strip(), line_number + elif len(components) > 2: + # Include a newline, then the file name and line number in the + # exception in order to have it appear as an inline failure in + # Xcode. + raise XCTestCheckerError( + path, line_number, + 'Usage violation: prefix "{}" appears twice in the same ' + 'line.'.format(check_prefix)) + def _add_whitespace_leniency(original_regex): return "^ *" + original_regex + " *$" + def compare(actual, expected, check_prefix): """ Compares each line in the two given files. @@ -40,23 +62,30 @@ def compare(actual, expected, check_prefix): file, raises an AssertionError. Also raises an AssertionError if the number of lines in the two files differ. """ - for actual_line, expected_line_and_line_number in map( + for actual_line, expected_line_and_number in map( None, _actual_lines(actual), _expected_lines_and_line_numbers(expected, check_prefix)): - if actual_line is None: - raise AssertionError('There were more lines expected to appear ' - 'than there were lines in the actual input.') - if expected_line_and_line_number is None: - raise AssertionError('There were more lines than expected to ' - 'appear.') + if expected_line_and_number is None: + raise XCTestCheckerError( + expected, 1, + 'The actual output contained more lines of text than the ' + 'expected output. First unexpected line: {}'.format( + repr(actual_line))) - (expected_line, expectation_source_line_number) = expected_line_and_line_number + (expected_line, expectation_line_number) = expected_line_and_number + + if actual_line is None: + raise XCTestCheckerError( + expected, expectation_line_number, + 'There were more lines expected to appear than there were ' + 'lines in the actual input. Unmet expectation: {}'.format( + repr(expected_line))) if not re.match(_add_whitespace_leniency(expected_line), actual_line): - raise AssertionError('Actual line did not match the expected ' - 'regular expression.\n' - '{}:{}: Actual: {}\n' - 'Expected: {}\n'.format( - expected, expectation_source_line_number, repr(actual_line), repr(expected_line))) + raise XCTestCheckerError( + expected, expectation_line_number, + 'Actual line did not match the expected regular expression.\n' + 'Actual: {}\nExpected: {}'.format( + repr(actual_line), repr(expected_line))) diff --git a/Tests/Functional/xctest_checker/xctest_checker/error.py b/Tests/Functional/xctest_checker/xctest_checker/error.py new file mode 100644 index 000000000..68c44145b --- /dev/null +++ b/Tests/Functional/xctest_checker/xctest_checker/error.py @@ -0,0 +1,19 @@ +# xctest_checker/error.py - Errors that display nicely in Xcode -*- python -*- +# +# This source file is part of the Swift.org open source project +# +# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors +# Licensed under Apache License v2.0 with Runtime Library Exception +# +# See http://swift.org/LICENSE.txt for license information +# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors + + +class XCTestCheckerError(Exception): + """ + An exception that indicates an xctest_checker-based functional test should + fail. Formats exception messages such that they render inline in Xcode. + """ + def __init__(self, path, line_number, message): + super(XCTestCheckerError, self).__init__( + '\n{}:{}: {}'.format(path, line_number, message))