gh-40814: Rerun plural and singular/function on failure
    
This pull request:

* add new feature `--all-except` to `sage -t` (does what you expect)
* modify `ci-meson.yml` to workaround
#29528 , the cause of which is
yet unknown. (I also tried porting an old pull request that purportedly
fix the issue at #39628, but the
result is even worse.)

controlling this in bash seems easier than
#39539 , for now. I suspect testing
these files separately will make it stop failing however (doesn't really
matter, the bug remains).

#40729 (comment)
contains a traceback, but I think it isn't of too much help.

(Thought? Is `--all --exclude=a --exclude=b` better?)

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [ ] The title is concise and informative.
- [ ] The description explains in detail what this PR is about.
- [ ] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation and checked the documentation
preview.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - #12345: short description why this is a dependency -->
<!-- - #34567: ... -->
    
URL: #40814
Reported by: user202729
Reviewer(s): Tobias Diez

Author: Release Manager

.github/workflows/ci-meson.yml

@@ -21,6 +21,8 @@ jobs:
   test:
     name: Conda (${{ matrix.os }}, Python ${{ matrix.python }}, ${{ matrix.tests }}${{ matrix.editable && ', editable' || '' }})
     runs-on: ${{ matrix.os }}-latest
+    env:
+      SEPARATELY_TESTED_FLAKY_FILES: "src/sage/libs/singular/function.pyx src/sage/rings/polynomial/plural.pyx"
 
     strategy:
       fail-fast: false
@@ -168,9 +170,22 @@ jobs:
             pytest --doctest-ignore-import-errors --doctest -rfEs -s src || true
           else
             pytest -rfEs -s src
-            ./sage -t ${{ matrix.tests == 'all' && '--all' || '--new --long' }} -p4 --format github
+            ./sage -t ${{
+              matrix.tests == 'all' &&
+              '--all-except="$SEPARATELY_TESTED_FLAKY_FILES"' ||
+              '--new --long' }} -p4 --format github
           fi
 
+      - name: Test flaky files
+        # unknown issues with plural.pyx causes sporadic failure: https://github.com/sagemath/sage/issues/29528
+        # we rerun a few times, this step succeeds if any of the 5 runs succeed
+        if: runner.os != 'windows' && matrix.tests == 'all'
+        shell: bash -l {0}
+        run: |
+          for i in {1..5}; do
+            ./sage -t -p4 --format github $SEPARATELY_TESTED_FLAKY_FILES && break
+          done
+
       - name: Check that all modules can be imported
         shell: bash -l {0}
         run: |

src/sage/doctest/__main__.py

@@ -1,6 +1,7 @@
 import argparse
 import os
 import sys
+import shlex
 
 # Note: the DOT_SAGE and SAGE_STARTUP_FILE environment variables have already been set by sage-env
 DOT_SAGE = os.environ.get('DOT_SAGE', os.path.join(os.environ.get('HOME'),
@@ -53,6 +54,9 @@ def _make_parser():
     parser.add_argument("-T", "--timeout", type=int, default=-1, help="timeout (in seconds) for doctesting one file, 0 for no timeout")
     what = parser.add_mutually_exclusive_group()
     what.add_argument("-a", "--all", action="store_true", default=False, help="test all files in the Sage library")
+    what.add_argument("--all-except", type=shlex.split, default=None,
+                      help="test all files in the Sage library except the specified space-separated list "
+                      "(backslash or quote are needed to escape spaces or backslashes or quotes)")
     what.add_argument("--installed", action="store_true", default=False, help="test all installed modules of the Sage library")
     parser.add_argument("--logfile", type=argparse.FileType('a'), metavar="FILE", help="log all output to FILE")
 
@@ -163,7 +167,7 @@ def main():
     in_filenames = False
     afterlog = False
     for arg in sys.argv[1:]:
-        if arg in ('-n', '--new', '-a', '--all', '--installed'):
+        if arg in ('-n', '--new', '-a', '--all', '--installed') or arg.startswith("--all-except"):
             need_filenames = False
         elif need_filenames and not (afterlog or in_filenames) and os.path.exists(arg):
             in_filenames = True
@@ -174,8 +178,8 @@ def main():
 
     args = parser.parse_args(new_arguments)
 
-    if not args.filenames and not (args.all or args.new or args.installed):
-        print('either use --new, --all, --installed, or some filenames')
+    if not args.filenames and not (args.all or args.new or args.installed) and args.all_except is None:
+        print('either use --new, --all, --all-except=..., --installed, or some filenames')
         return 2
 
     # Limit the number of threads to 2 to save system resources.

src/sage/doctest/control.py

@@ -66,6 +66,11 @@ class DocTestDefaults(SageObject):
     """
     This class is used for doctesting the Sage doctest module.
 
+    The interface of this object should be compatible with the ``options`` input
+    to :class:`DocTestController`, that is, the same interface as the
+    argument object parsed by the :class:`argparse.ArgumentParser` in
+    :func:`sage.doctest.__main__._make_parser`.
+
     INPUT:
 
     - ``runtest_default`` -- boolean (default: ``False``); if ``True``,
@@ -118,6 +123,7 @@ def __init__(self, runtest_default=False, **kwds):
         self.timeout = -1
         self.die_timeout = -1
         self.all = False
+        self.all_except = None
         self.installed = False
         self.logfile = None
         self.long = False
@@ -410,7 +416,9 @@ def __init__(self, options, args):
         INPUT:
 
         - ``options`` -- either options generated from the command line by sage-runtests
-          or a DocTestDefaults object (possibly with some entries modified)
+          or a :class:`DocTestDefaults` object (possibly with some entries modified).
+          The attributes available in this object are defined by the :class:`argparse.ArgumentParser`
+          in :func:`sage.doctest.__main__._make_parser`.
         - ``args`` -- list of filenames to doctest
 
         EXAMPLES::
@@ -913,7 +921,7 @@ def all_doc_sources():
             all_installed_modules()
             all_installed_doc()
 
-        elif self.options.all or (self.options.new and not have_git):
+        elif self.options.all or self.options.all_except is not None or (self.options.new and not have_git):
             all_files()
             all_doc_sources()
 
@@ -1007,7 +1015,14 @@ def expand():
                                   if_installed=self.options.if_installed,
                                   log=self.log):  # log when directly specified filenames are skipped
                     yield path
-        self.sources = [FileDocTestSource(path, self.options) for path in expand()]
+        paths = list(expand())
+        if self.options.all_except is not None:
+            paths_to_remove = set(os.path.abspath(x) for x in self.options.all_except)
+            if not paths_to_remove.issubset(paths):
+                raise ValueError(f"--all-except includes {paths_to_remove - set(paths)}, "
+                                 f"which are not found in {paths}")
+            paths = [path for path in paths if path not in paths_to_remove]  # keep duplicates
+        self.sources = [FileDocTestSource(path, self.options) for path in paths]
 
     def filter_sources(self):
         """