Merge pull request #3110 from nicoddemus/progress-teardown-3088

Fix progress report when tests fail during teardown

Author: RonnyPfannschmidt

_pytest/terminal.py

@@ -152,9 +152,18 @@ def __init__(self, config, file=None):
         self.reportchars = getreportopt(config)
         self.hasmarkup = self._tw.hasmarkup
         self.isatty = file.isatty()
-        self._progress_items_reported = 0
-        self._show_progress_info = (self.config.getoption('capture') != 'no' and
-                                    self.config.getini('console_output_style') == 'progress')
+        self._progress_nodeids_reported = set()
+        self._show_progress_info = self._determine_show_progress_info()
+
+    def _determine_show_progress_info(self):
+        """Return True if we should display progress information based on the current config"""
+        # do not show progress if we are not capturing output (#3038)
+        if self.config.getoption('capture') == 'no':
+            return False
+        # do not show progress if we are showing fixture setup/teardown
+        if self.config.getoption('setupshow'):
+            return False
+        return self.config.getini('console_output_style') == 'progress'
 
     def hasopt(self, char):
         char = {'xfailed': 'x', 'skipped': 's'}.get(char, char)
@@ -179,7 +188,6 @@ def write_ensure_prefix(self, prefix, extra="", **kwargs):
         if extra:
             self._tw.write(extra, **kwargs)
             self.currentfspath = -2
-            self._write_progress_information_filling_space()
 
     def ensure_newline(self):
         if self.currentfspath:
@@ -269,14 +277,13 @@ def pytest_runtest_logreport(self, report):
             # probably passed setup/teardown
             return
         running_xdist = hasattr(rep, 'node')
-        self._progress_items_reported += 1
         if self.verbosity <= 0:
             if not running_xdist and self.showfspath:
                 self.write_fspath_result(rep.nodeid, letter)
             else:
                 self._tw.write(letter)
-            self._write_progress_if_past_edge()
         else:
+            self._progress_nodeids_reported.add(rep.nodeid)
             if markup is None:
                 if rep.passed:
                     markup = {'green': True}
@@ -289,6 +296,8 @@ def pytest_runtest_logreport(self, report):
             line = self._locationline(rep.nodeid, *rep.location)
             if not running_xdist:
                 self.write_ensure_prefix(line, word, **markup)
+                if self._show_progress_info:
+                    self._write_progress_information_filling_space()
             else:
                 self.ensure_newline()
                 self._tw.write("[%s]" % rep.node.gateway.id)
@@ -300,31 +309,28 @@ def pytest_runtest_logreport(self, report):
                 self._tw.write(" " + line)
                 self.currentfspath = -2
 
-    def _write_progress_if_past_edge(self):
-        if not self._show_progress_info:
-            return
-        last_item = self._progress_items_reported == self._session.testscollected
-        if last_item:
-            self._write_progress_information_filling_space()
-            return
-
-        past_edge = self._tw.chars_on_current_line + self._PROGRESS_LENGTH + 1 >= self._screen_width
-        if past_edge:
-            msg = self._get_progress_information_message()
-            self._tw.write(msg + '\n', cyan=True)
+    def pytest_runtest_logfinish(self, nodeid):
+        if self.verbosity <= 0 and self._show_progress_info:
+            self._progress_nodeids_reported.add(nodeid)
+            last_item = len(self._progress_nodeids_reported) == self._session.testscollected
+            if last_item:
+                self._write_progress_information_filling_space()
+            else:
+                past_edge = self._tw.chars_on_current_line + self._PROGRESS_LENGTH + 1 >= self._screen_width
+                if past_edge:
+                    msg = self._get_progress_information_message()
+                    self._tw.write(msg + '\n', cyan=True)
 
     _PROGRESS_LENGTH = len(' [100%]')
 
     def _get_progress_information_message(self):
         collected = self._session.testscollected
         if collected:
-            progress = self._progress_items_reported * 100 // collected
+            progress = len(self._progress_nodeids_reported) * 100 // collected
             return ' [{:3d}%]'.format(progress)
         return ' [100%]'
 
     def _write_progress_information_filling_space(self):
-        if not self._show_progress_info:
-            return
         msg = self._get_progress_information_message()
         fill = ' ' * (self._tw.fullwidth - self._tw.chars_on_current_line - len(msg) - 1)
         self.write(fill + msg, cyan=True)

changelog/3088.bugfix

@@ -0,0 +1 @@
+Fix progress percentage reported when tests fail during teardown.

testing/test_terminal.py

@@ -969,7 +969,7 @@ def test_no_trailing_whitespace_after_inifile_word(testdir):
 class TestProgress:
 
     @pytest.fixture
-    def many_tests_file(self, testdir):
+    def many_tests_files(self, testdir):
         testdir.makepyfile(
             test_bar="""
                 import pytest
@@ -1006,30 +1006,30 @@ def pytest_collection_modifyitems(items, config):
             '=* 2 passed in *=',
         ])
 
-    def test_normal(self, many_tests_file, testdir):
+    def test_normal(self, many_tests_files, testdir):
         output = testdir.runpytest()
         output.stdout.re_match_lines([
             r'test_bar.py \.{10} \s+ \[ 50%\]',
             r'test_foo.py \.{5} \s+ \[ 75%\]',
             r'test_foobar.py \.{5} \s+ \[100%\]',
         ])
 
-    def test_verbose(self, many_tests_file, testdir):
+    def test_verbose(self, many_tests_files, testdir):
         output = testdir.runpytest('-v')
         output.stdout.re_match_lines([
             r'test_bar.py::test_bar\[0\] PASSED \s+ \[  5%\]',
             r'test_foo.py::test_foo\[4\] PASSED \s+ \[ 75%\]',
             r'test_foobar.py::test_foobar\[4\] PASSED \s+ \[100%\]',
         ])
 
-    def test_xdist_normal(self, many_tests_file, testdir):
+    def test_xdist_normal(self, many_tests_files, testdir):
         pytest.importorskip('xdist')
         output = testdir.runpytest('-n2')
         output.stdout.re_match_lines([
             r'\.{20} \s+ \[100%\]',
         ])
 
-    def test_xdist_verbose(self, many_tests_file, testdir):
+    def test_xdist_verbose(self, many_tests_files, testdir):
         pytest.importorskip('xdist')
         output = testdir.runpytest('-n2', '-v')
         output.stdout.re_match_lines_random([
@@ -1038,10 +1038,85 @@ def test_xdist_verbose(self, many_tests_file, testdir):
             r'\[gw\d\] \[\s*\d+%\] PASSED test_foobar.py::test_foobar\[1\]',
         ])
 
-    def test_capture_no(self, many_tests_file, testdir):
+    def test_capture_no(self, many_tests_files, testdir):
         output = testdir.runpytest('-s')
         output.stdout.re_match_lines([
             r'test_bar.py \.{10}',
             r'test_foo.py \.{5}',
             r'test_foobar.py \.{5}',
         ])
+
+
+class TestProgressWithTeardown:
+    """Ensure we show the correct percentages for tests that fail during teardown (#3088)"""
+
+    @pytest.fixture
+    def contest_with_teardown_fixture(self, testdir):
+        testdir.makeconftest('''
+            import pytest
+
+            @pytest.fixture
+            def fail_teardown():
+                yield
+                assert False
+        ''')
+
+    @pytest.fixture
+    def many_files(self, testdir, contest_with_teardown_fixture):
+        testdir.makepyfile(
+            test_bar='''
+                import pytest
+                @pytest.mark.parametrize('i', range(5))
+                def test_bar(fail_teardown, i):
+                    pass
+            ''',
+            test_foo='''
+                import pytest
+                @pytest.mark.parametrize('i', range(15))
+                def test_foo(fail_teardown, i):
+                    pass
+            ''',
+        )
+
+    def test_teardown_simple(self, testdir, contest_with_teardown_fixture):
+        testdir.makepyfile('''
+            def test_foo(fail_teardown):
+                pass
+        ''')
+        output = testdir.runpytest()
+        output.stdout.re_match_lines([
+            r'test_teardown_simple.py \.E\s+\[100%\]',
+        ])
+
+    def test_teardown_with_test_also_failing(self, testdir, contest_with_teardown_fixture):
+        testdir.makepyfile('''
+            def test_foo(fail_teardown):
+                assert False
+        ''')
+        output = testdir.runpytest()
+        output.stdout.re_match_lines([
+            r'test_teardown_with_test_also_failing.py FE\s+\[100%\]',
+        ])
+
+    def test_teardown_many(self, testdir, many_files):
+        output = testdir.runpytest()
+        output.stdout.re_match_lines([
+            r'test_bar.py (\.E){5}\s+\[ 25%\]',
+            r'test_foo.py (\.E){15}\s+\[100%\]',
+        ])
+
+    def test_teardown_many_verbose(self, testdir, many_files):
+        output = testdir.runpytest('-v')
+        output.stdout.re_match_lines([
+            r'test_bar.py::test_bar\[0\] PASSED\s+\[  5%\]',
+            r'test_bar.py::test_bar\[0\] ERROR\s+\[  5%\]',
+            r'test_bar.py::test_bar\[4\] PASSED\s+\[ 25%\]',
+            r'test_bar.py::test_bar\[4\] ERROR\s+\[ 25%\]',
+        ])
+
+    def test_xdist_normal(self, many_files, testdir):
+        pytest.importorskip('xdist')
+        output = testdir.runpytest('-n2')
+        output.stdout.re_match_lines([
+            r'[\.E]{40} \s+ \[100%\]',
+        ])