feat(pytest): parse output sections

Allows ultest to determine the suitable stack trace to use such as
for hypothesis, the second trace should be used.

Author: rcarriga

rplugin/python3/ultest/handler/parsers/output/python/pytest.py

@@ -1,7 +1,18 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
 from .. import parsec as p
 from ..base import ParsedOutput, ParseResult
 from ..parsec import generate
-from ..util import join_chars, until_eol
+from ..util import eol, join_chars, until_eol
+
+
+@dataclass
+class PytestCodeTrace:
+    code: List[str]
+    file: str
+    line: int
+    message: Optional[List[str]] = None
 
 
 @generate
@@ -16,18 +27,19 @@ def pytest_output():
 def failed_test_section():
     namespaces, test_name = yield failed_test_section_title
     yield until_eol
-    yield failed_test_section_code
-    trace_origin = yield p.optional(until_eol >> failed_test_stacktrace)
-    error_message = yield failed_test_section_error_message
-    yield until_eol
-    error_file, error_line_no = yield failed_test_error_location
-    yield p.optional(failed_test_captured_stdout, [])
-    if trace_origin:
-        test_file = trace_origin[0]
-        test_line_no = trace_origin[1]
-    else:
-        test_file = error_file
-        test_line_no = error_line_no
+    traces: List[PytestCodeTrace]
+    traces = yield failed_test_code_sections
+    sections = yield failed_test_captured_output_sections
+    trace = traces[0]
+    # Hypothesis traces provide the test definition as the first layer of trace
+    if "Hypothesis" in sections and len(traces) > 1:
+        trace = traces[1]
+    test_file = trace.file
+    test_line_no = trace.line
+    error_message = None
+    for trace in traces:
+        if trace.message:
+            error_message = trace.message
     return ParseResult(
         name=test_name,
         namespaces=namespaces,
@@ -37,14 +49,6 @@ def failed_test_section():
     )
 
 
-@generate
-def failed_test_stacktrace():
-    file_name, line_no = yield failed_test_error_location
-    yield p.string("_ _") >> until_eol
-    yield p.many1(p.exclude(failed_test_code_line, failed_test_section_error_message))
-    return file_name, line_no
-
-
 @generate
 def failed_test_section_title():
     yield p.many1(p.string("_")) >> p.space()
@@ -60,22 +64,45 @@ def failed_test_section_title():
 
 
 @generate
-def failed_test_error_message_line():
-    yield p.string("E")
-    yield p.many(p.string(" "))
-    error_text = yield until_eol
-    return error_text
+def failed_test_captured_output_sections():
+    sections = yield p.many(failed_test_captured_output)
+    return dict(sections)
+
+
+failed_test_section_sep = p.many1(p.one_of("_ ")) >> eol
 
 
 @generate
-def failed_test_section_code():
-    code = yield p.many1(
+def failed_test_code_sections():
+
+    sections = yield p.sepBy(failed_test_code_section, failed_test_section_sep)
+    return sections
+
+
+@generate
+def failed_test_code_section():
+    code = yield p.many(
         p.exclude(
             failed_test_code_line,
-            failed_test_section_error_message ^ failed_test_stacktrace,
+            failed_test_section_error_message ^ failed_test_error_location,
+        )
+    )
+    message = yield p.optional(failed_test_section_error_message)
+    if code or message:
+        yield until_eol
+    trace = yield failed_test_error_location
+    yield p.many(
+        p.exclude(
+            until_eol,
+            failed_test_section_sep
+            ^ failed_test_captured_output_title
+            ^ failed_test_section_title
+            ^ pytest_summary_info_title,
         )
     )
-    return code
+    return PytestCodeTrace(
+        code=code, file=trace and trace[0], line=trace and trace[1], message=message
+    )
 
 
 @generate
@@ -90,19 +117,33 @@ def failed_test_section_error_message():
     return lines
 
 
+@generate
+def failed_test_error_message_line():
+    yield p.string("E")
+    yield p.many(p.string(" "))
+    error_text = yield until_eol
+    return error_text
+
+
 @generate
 def pytest_summary_info():
+    yield pytest_summary_info_title
+    summary = yield p.many(until_eol)
+    return summary
+
+
+@generate
+def pytest_summary_info_title():
     yield p.many1(p.string("="))
     yield p.string(" short test summary info ")
     yield until_eol
-    summary = yield p.many(until_eol)
-    return summary
 
 
 @generate
 def pytest_test_results_summary():
-    summary = yield p.many(p.exclude(until_eol, pytest_failed_tests_title))
-    yield pytest_failed_tests_title
+    failures_title = p.many1(p.string("=")) >> p.string(" FAILURES ") >> until_eol
+    summary = yield p.many(p.exclude(until_eol, failures_title))
+    yield failures_title
     return summary
 
 
@@ -117,18 +158,24 @@ def failed_test_error_location():
 
 
 @generate
-def failed_test_captured_stdout():
-    yield p.many1(p.string("-"))
-    yield p.string(" Captured stdout call ")
-    yield until_eol
+def failed_test_captured_output():
+    title = yield failed_test_captured_output_title.parsecmap(join_chars)
     stdout = yield p.many(
-        p.exclude(until_eol, failed_test_section_title ^ pytest_summary_info)
+        p.exclude(
+            until_eol,
+            failed_test_section_title
+            ^ pytest_summary_info_title
+            ^ failed_test_captured_output_title,
+        )
     )
-    return stdout
+    return title, stdout
 
 
 @generate
-def pytest_failed_tests_title():
-    yield p.many1(p.string("="))
-    yield p.string(" FAILURES ")
+def failed_test_captured_output_title():
+    yield p.many1(p.string("-"))
+    yield p.string(" ")
+    title = yield p.many1(p.exclude(p.any(), p.string(" ---")))
+    yield p.string(" ")
     yield until_eol
+    return title

tests/mocks/test_outputs/pytest_hypothesis

@@ -0,0 +1,46 @@
+================================================= test session starts ==================================================
+platform linux -- Python 3.8.6, pytest-6.2.5, py-1.10.0, pluggy-1.0.0
+rootdir: /home/ronan/Dev/repos/vim-ultest, configfile: pyproject.toml
+plugins: asyncio-0.16.0, hypothesis-6.23.3, cov-3.0.0
+collecting ... collected 4 items                                                                                                      
+
+tests/unit/models/test_tree.py F...                                                                              [100%]
+
+======================================================= FAILURES =======================================================
+__________________________________________ test_get_nearest_from_strict_match __________________________________________
+
+    @given(sorted_tests())
+>   def test_get_nearest_from_strict_match(tests: List[Union[Test, Namespace]]):
+
+tests/unit/models/test_tree.py:29: 
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
+
+tests = [Test(id='', name='', file='', line=2, col=0, running=0, namespaces=[], type='test'), Test(id='', name='', file='', li...namespaces=[], type='test'), Test(id='', name='', file='', line=12, col=0, running=0, namespaces=[], type='test'), ...]
+
+    @given(sorted_tests())
+    def test_get_nearest_from_strict_match(tests: List[Union[Test, Namespace]]):
+        test_i = int(random.random() * len(tests))
+        expected = tests[test_i]
+        tree = Tree[Position].from_list([File(file="", name="", id=""), *tests])
+        result = tree.sorted_search(expected.line, lambda test: test.line, strict=True)
+>       assert expected != result.data
+E       AssertionError: assert Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test') != Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test')
+E        +  where Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test') = Tree(data=Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test'), children=[]).data
+
+tests/unit/models/test_tree.py:34: AssertionError
+------------------------------------------------------ Hypothesis ------------------------------------------------------
+Falsifying example: test_get_nearest_from_strict_match(
+    tests=[Test(id='', name='', file='', line=2, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=4, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=6, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=8, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=10, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=12, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=14, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=16, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=20, col=0, running=0, namespaces=[], type='test')],
+)
+=============================================== short test summary info ================================================
+FAILED tests/unit/models/test_tree.py::test_get_nearest_from_strict_match - AssertionError: assert Test(id='', name='...
+============================================= 1 failed, 3 passed in 20.07s =============================================

tests/unit/handler/parsers/output/python/test_pytest.py

@@ -1,11 +1,9 @@
-from dataclasses import asdict
 from unittest import TestCase
 
 from rplugin.python3.ultest.handler.parsers.output import OutputParser
 from rplugin.python3.ultest.handler.parsers.output.python.pytest import (
     ParseResult,
     failed_test_section,
-    failed_test_section_code,
     failed_test_section_error_message,
     failed_test_section_title,
 )
@@ -86,6 +84,27 @@ def test_parse_file(self):
             ],
         )
 
+    def test_parse_hypothesis_file(self):
+        output = get_output("pytest_hypothesis")
+        parser = OutputParser([])
+        result = parser.parse_failed("python#pytest", output)
+        self.assertEqual(
+            result,
+            [
+                ParseResult(
+                    name="test_get_nearest_from_strict_match",
+                    namespaces=[],
+                    file="tests/unit/models/test_tree.py",
+                    message=[
+                        "AssertionError: assert Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test') != Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test')",
+                        "+  where Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test') = Tree(data=Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test'), children=[]).data",
+                    ],
+                    output=None,
+                    line=34,
+                )
+            ],
+        )
+
     def test_parse_failed_test_section_title(self):
         raw = "_____ MyClass.test_a ______"
         result = failed_test_section_title.parse(raw)
@@ -111,36 +130,6 @@ def test_parse_failed_test_section_error(self):
         ]
         self.assertEqual(expected, result)
 
-    def test_parse_failed_test_section_code(self):
-        self.maxDiff = None
-        raw = """self = <test_a.TestClass testMethod=test_b>
-
-    def test_b(self):
->       self.assertEqual({
-            "a": 1,
-         "b": 2,
-         "c": 3},
-         {"a": 1,
-         "b": 5,
-         "c": 3,
-         "d": 4})
-E This should not be parsed"""
-        result, _ = failed_test_section_code.parse_partial(raw)
-        expected = [
-            "self = <test_a.TestClass testMethod=test_b>",
-            "",
-            "    def test_b(self):",
-            ">       self.assertEqual({",
-            '            "a": 1,',
-            '         "b": 2,',
-            '         "c": 3},',
-            '         {"a": 1,',
-            '         "b": 5,',
-            '         "c": 3,',
-            '         "d": 4})',
-        ]
-        self.assertEqual(expected, result)
-
     def test_parse_failed_test_section(self):
         raw = """_____________________________________________________________________ MyClass.test_b _____________________________________________________________________
 
@@ -197,14 +186,61 @@ def a_function():
 """
         result = failed_test_section.parse(raw)
         self.assertEqual(
-            asdict(result),
-            asdict(
-                ParseResult(
-                    file="test_a.py",
-                    name="test_c",
-                    namespaces=["TestClass"],
-                    message=["Exception: OH NO"],
-                    line=39,
-                )
+            result,
+            ParseResult(
+                file="test_a.py",
+                name="test_c",
+                namespaces=["TestClass"],
+                message=["Exception: OH NO"],
+                line=39,
+            ),
+        )
+
+    def test_parse_failed_test_with_code_below_trace_location(self):
+        raw = """__________________________________________ test_get_nearest_from_strict_match __________________________________________
+
+    @given(sorted_tests())
+>   def test_get_nearest_from_strict_match(tests: List[Union[Test, Namespace]]):
+
+tests/unit/models/test_tree.py:30:
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+tests/unit/models/test_tree.py:35: in test_get_nearest_from_strict_match
+    logging.warn("AAAAAAH")
+_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
+
+msg = 'AAAAAAH', args = (), kwargs = {}
+
+    def warn(msg, *args, **kwargs):
+>       warnings.warn("The 'warn' function is deprecated, "
+            "use 'warning' instead", DeprecationWarning, 2)
+E       DeprecationWarning: The 'warn' function is deprecated, use 'warning' instead
+
+../../../.pyenv/versions/3.8.6/lib/python3.8/logging/__init__.py:2058: DeprecationWarning
+------------------------------------------------------ Hypothesis ------------------------------------------------------
+Falsifying example: test_get_nearest_from_strict_match(
+    tests=[Test(id='', name='', file='', line=2, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='0', file='', line=4, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=6, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=8, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=10, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=12, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=14, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=16, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=18, col=0, running=0, namespaces=[], type='test'),
+     Test(id='', name='', file='', line=514, col=0, running=0, namespaces=[], type='test')],
+)"""
+
+        result = failed_test_section.parse(raw)
+        self.assertEqual(
+            result,
+            ParseResult(
+                name="test_get_nearest_from_strict_match",
+                namespaces=[],
+                file="tests/unit/models/test_tree.py",
+                message=[
+                    "DeprecationWarning: The 'warn' function is deprecated, use 'warning' instead"
+                ],
+                output=None,
+                line=35,
             ),
         )

tests/unit/models/test_tree.py

@@ -1,3 +1,4 @@
+import logging
 import random
 from typing import List, Union