[xctest_checker] Inline expectations

Tests/Functional/SingleFailingTestCase/main.swift

@@ -19,9 +19,9 @@ class SingleFailingTestCase: XCTestCase {
         ]
     }
 
-// CHECK: Test Case 'SingleFailingTestCase.test_fails' started at \d+:\d+:\d+\.\d+
-// CHECK: .*/SingleFailingTestCase/main.swift:26: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed -
-// CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\).
+    // CHECK: Test Case 'SingleFailingTestCase.test_fails' started at \d+:\d+:\d+\.\d+
+    // CHECK: .*/SingleFailingTestCase/main.swift:26: error: SingleFailingTestCase.test_fails : XCTAssertTrue failed -
+    // CHECK: Test Case 'SingleFailingTestCase.test_fails' failed \(\d+\.\d+ seconds\).
     func test_fails() {
         XCTAssert(false)
     }

Tests/Functional/xctest_checker/tests/test_compare.py

@@ -12,6 +12,7 @@
 import unittest
 
 from xctest_checker import compare
+from xctest_checker.error import XCTestCheckerError
 
 
 def _tmpfile(content):
@@ -26,30 +27,52 @@ class CompareTestCase(unittest.TestCase):
     def test_no_match_raises(self):
         actual = _tmpfile('foo\nbar\nbaz\n')
         expected = _tmpfile('c: foo\nc: baz\nc: bar\n')
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(XCTestCheckerError):
             compare.compare(actual, expected, check_prefix='c: ')
 
-    def test_too_few_expected_raises(self):
+    def test_too_few_expected_raises_and_first_line_in_error(self):
         actual = _tmpfile('foo\nbar\nbaz\n')
         expected = _tmpfile('c: foo\nc: bar\n')
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(XCTestCheckerError) as cm:
             compare.compare(actual, expected, check_prefix='c: ')
 
-    def test_too_many_expected_raises(self):
+        self.assertIn('{}:{}'.format(expected, 1), cm.exception.message)
+
+    def test_too_many_expected_raises_and_excess_check_line_in_error(self):
         actual = _tmpfile('foo\nbar\n')
         expected = _tmpfile('c: foo\nc: bar\nc: baz\n')
-        with self.assertRaises(AssertionError):
+        with self.assertRaises(XCTestCheckerError) as cm:
             compare.compare(actual, expected, check_prefix='c: ')
 
+        self.assertIn('{}:{}'.format(expected, 3), cm.exception.message)
+
     def test_match_does_not_raise(self):
         actual = _tmpfile('foo\nbar\nbaz\n')
         expected = _tmpfile('c: foo\nc: bar\nc: baz\n')
         compare.compare(actual, expected, check_prefix='c: ')
 
+    def test_match_with_inline_check_does_not_raise(self):
+        actual = _tmpfile('bling\nblong\n')
+        expected = _tmpfile('meep meep // c: bling\nmeep\n// c: blong\n')
+        compare.compare(actual, expected, check_prefix='// c: ')
+
+    def test_check_prefix_twice_in_the_same_line_raises_with_line(self):
+        actual = _tmpfile('blorp\nbleep\n')
+        expected = _tmpfile('c: blorp\nc: bleep c: blammo\n')
+        with self.assertRaises(XCTestCheckerError) as cm:
+            compare.compare(actual, expected, check_prefix='c: ')
+
+        self.assertIn('{}:{}'.format(expected, 2), cm.exception.message)
+
+    def test_check_prefix_in_run_line_ignored(self):
+        actual = _tmpfile('flim\n')
+        expected = _tmpfile('// RUN: xctest_checker --prefix "c: "\nc: flim\n')
+        compare.compare(actual, expected, check_prefix='c: ')
+
     def test_includes_file_name_and_line_of_expected_in_error(self):
         actual = _tmpfile('foo\nbar\nbaz\n')
         expected = _tmpfile('c: foo\nc: baz\nc: bar\n')
-        with self.assertRaises(AssertionError) as cm:
+        with self.assertRaises(XCTestCheckerError) as cm:
             compare.compare(actual, expected, check_prefix='c: ')
 
         self.assertIn("{}:{}:".format(expected, 2), cm.exception.message)

Tests/Functional/xctest_checker/xctest_checker/compare.py

@@ -10,6 +10,8 @@
 
 import re
 
+from .error import XCTestCheckerError
+
 
 def _actual_lines(path):
     """
@@ -26,37 +28,64 @@ def _expected_lines_and_line_numbers(path, check_prefix):
     that begins with the given prefix.
     """
     with open(path) as f:
-        for line_number, line in enumerate(f):
-            if line.startswith(check_prefix):
-                yield line[len(check_prefix):].strip(), line_number+1
+        for index, line in enumerate(f):
+            if 'RUN:' in line:
+                # Ignore lit directives, which may include a call to
+                # xctest_checker that specifies a check prefix.
+                continue
+
+            # Note that line numbers are not zero-indexed; we must add one to
+            # the loop index.
+            line_number = index + 1
+
+            components = line.split(check_prefix)
+            if len(components) == 2:
+                yield components[1].strip(), line_number
+            elif len(components) > 2:
+                # Include a newline, then the file name and line number in the
+                # exception in order to have it appear as an inline failure in
+                # Xcode.
+                raise XCTestCheckerError(
+                    path, line_number,
+                    'Usage violation: prefix "{}" appears twice in the same '
+                    'line.'.format(check_prefix))
+
 
 def _add_whitespace_leniency(original_regex):
     return "^ *" + original_regex + " *$"
 
+
 def compare(actual, expected, check_prefix):
     """
     Compares each line in the two given files.
     If any line in the 'actual' file doesn't match the regex in the 'expected'
     file, raises an AssertionError. Also raises an AssertionError if the number
     of lines in the two files differ.
     """
-    for actual_line, expected_line_and_line_number in map(
+    for actual_line, expected_line_and_number in map(
             None,
             _actual_lines(actual),
             _expected_lines_and_line_numbers(expected, check_prefix)):
 
-        if actual_line is None:
-            raise AssertionError('There were more lines expected to appear '
-                                 'than there were lines in the actual input.')
-        if expected_line_and_line_number is None:
-            raise AssertionError('There were more lines than expected to '
-                                 'appear.')
+        if expected_line_and_number is None:
+            raise XCTestCheckerError(
+                expected, 1,
+                'The actual output contained more lines of text than the '
+                'expected output. First unexpected line: {}'.format(
+                    repr(actual_line)))
 
-        (expected_line, expectation_source_line_number) = expected_line_and_line_number
+        (expected_line, expectation_line_number) = expected_line_and_number
+
+        if actual_line is None:
+            raise XCTestCheckerError(
+                expected, expectation_line_number,
+                'There were more lines expected to appear than there were '
+                'lines in the actual input. Unmet expectation: {}'.format(
+                    repr(expected_line)))
 
         if not re.match(_add_whitespace_leniency(expected_line), actual_line):
-            raise AssertionError('Actual line did not match the expected '
-                                 'regular expression.\n'
-                                 '{}:{}: Actual: {}\n'
-                                 'Expected: {}\n'.format(
-                                     expected, expectation_source_line_number, repr(actual_line), repr(expected_line)))
+            raise XCTestCheckerError(
+                expected, expectation_line_number,
+                'Actual line did not match the expected regular expression.\n'
+                'Actual: {}\nExpected: {}'.format(
+                    repr(actual_line), repr(expected_line)))

Tests/Functional/xctest_checker/xctest_checker/error.py

@@ -0,0 +1,19 @@
+# xctest_checker/error.py - Errors that display nicely in Xcode -*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2016 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See http://swift.org/LICENSE.txt for license information
+# See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+
+
+class XCTestCheckerError(Exception):
+    """
+    An exception that indicates an xctest_checker-based functional test should
+    fail. Formats exception messages such that they render inline in Xcode.
+    """
+    def __init__(self, path, line_number, message):
+        super(XCTestCheckerError, self).__init__(
+            '\n{}:{}: {}'.format(path, line_number, message))