Doctest hide option: Better detection of hidden packages

src/sage/doctest/control.py

@@ -416,7 +416,7 @@ def __init__(self, options, args):
             if options.gc:
                 options.timeout *= 2
         if options.nthreads == 0:
-            options.nthreads = int(os.getenv('SAGE_NUM_THREADS_PARALLEL',1))
+            options.nthreads = int(os.getenv('SAGE_NUM_THREADS_PARALLEL', 1))
         if options.failed and not (args or options.new):
             # If the user doesn't specify any files then we rerun all failed files.
             options.all = True
@@ -450,15 +450,11 @@ def __init__(self, options, args):
                 options.hide.discard('all')
                 from sage.features.all import all_features
                 feature_names = {f.name for f in all_features() if not f.is_standard()}
-                from sage.doctest.external import external_software
-                feature_names.difference_update(external_software)
                 options.hide = options.hide.union(feature_names)
             if 'optional' in options.hide:
                 options.hide.discard('optional')
                 from sage.features.all import all_features
                 feature_names = {f.name for f in all_features() if f.is_optional()}
-                from sage.doctest.external import external_software
-                feature_names.difference_update(external_software)
                 options.hide = options.hide.union(feature_names)
 
         options.disabled_optional = set()
@@ -1008,7 +1004,7 @@ def expand():
                 if os.path.isdir(path):
                     for root, dirs, files in os.walk(path):
                         for dir in list(dirs):
-                            if dir[0] == "." or skipdir(os.path.join(root,dir)):
+                            if dir[0] == "." or skipdir(os.path.join(root, dir)):
                                 dirs.remove(dir)
                         for file in files:
                             if not skipfile(os.path.join(root, file),
@@ -1345,9 +1341,9 @@ def run_val_gdb(self, testing=False):
                 flags = os.getenv("SAGE_MEMCHECK_FLAGS")
                 if flags is None:
                     flags = "--leak-resolution=high --leak-check=full --num-callers=25 "
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "pyalloc.supp"))
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "sage.supp"))
-                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE,"valgrind", "sage-additional.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "pyalloc.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "sage.supp"))
+                    flags += '''--suppressions="%s" ''' % (os.path.join(SAGE_EXTCODE, "valgrind", "sage-additional.supp"))
             elif opt.massif:
                 toolname = "massif"
                 flags = os.getenv("SAGE_MASSIF_FLAGS", "--depth=6 ")
@@ -1362,7 +1358,7 @@ def run_val_gdb(self, testing=False):
             if opt.omega:
                 toolname = "omega"
             if "%s" in flags:
-                flags %= toolname + ".%p" # replace %s with toolname
+                flags %= toolname + ".%p"  # replace %s with toolname
         cmd += flags + sage_cmd
 
         sys.stdout.flush()
@@ -1489,6 +1485,25 @@ def run(self):
                 cumulative wall time: ... seconds
             Features detected...
             0
+
+        Test *Features that have been hidden* message::
+
+            sage: DC.run()                              # optional - meataxe
+            Running doctests with ID ...
+            Using --optional=sage
+            Features to be detected: ...
+            Doctesting 1 file.
+            sage -t ....py
+                [4 tests, ... s]
+            ----------------------------------------------------------------------
+            All tests passed!
+            ----------------------------------------------------------------------
+            Total time for all tests: ... seconds
+                cpu time: ... seconds
+                cumulative wall time: ... seconds
+            Features detected...
+            Features that have been hidden: ...meataxe...
+            0
         """
         opt = self.options
         L = (opt.gdb, opt.lldb, opt.valgrind, opt.massif, opt.cachegrind, opt.omega)
@@ -1516,10 +1531,10 @@ def run(self):
                     pass
                 try:
                     ref = subprocess.check_output(["git",
-                                                      "--git-dir=" + SAGE_ROOT_GIT,
-                                                      "describe",
-                                                      "--always",
-                                                      "--dirty"])
+                                                   "--git-dir=" + SAGE_ROOT_GIT,
+                                                   "describe",
+                                                   "--always",
+                                                   "--dirty"])
                     ref = ref.decode('utf-8')
                     self.log("Git ref: " + ref, end="")
                 except subprocess.CalledProcessError:
@@ -1537,12 +1552,11 @@ def run(self):
                     pass
                 else:
                     f = available_software._features[i]
-                    if f.is_present():
-                        f.hide()
-                        self.options.hidden_features.add(f)
-                        for g in f.joined_features():
-                            if g.name in self.options.optional:
-                                self.options.optional.discard(g.name)
+                    f.hide()
+                    self.options.hidden_features.add(f)
+                    for g in f.joined_features():
+                        if g.name in self.options.optional:
+                            self.options.optional.discard(g.name)
 
             for o in self.options.disabled_optional:
                 try:
@@ -1553,8 +1567,6 @@ def run(self):
                     available_software._seen[i] = -1
 
             self.log("Features to be detected: " + ','.join(available_software.detectable()))
-            if self.options.hidden_features:
-                self.log("Hidden features: " + ','.join([f.name for f in self.options.hidden_features]))
             if self.options.probe:
                 self.log("Features to be probed: " + ('all' if self.options.probe is True
                                                       else ','.join(self.options.probe)))
@@ -1564,11 +1576,11 @@ def run(self):
             self.sort_sources()
             self.run_doctests()
 
-            for f in self.options.hidden_features:
-                f.unhide()
-
             self.log("Features detected for doctesting: "
                      + ','.join(available_software.seen()))
+            if self.options.hidden_features:
+                features_hidden = [f.name for f in self.options.hidden_features if f.unhide()]
+                self.log("Features that have been hidden: " + ','.join(features_hidden))
             self.cleanup()
             return self.reporter.error_status
 

src/sage/features/__init__.py

@@ -28,7 +28,7 @@
 Here we test whether the grape GAP package is available::
 
     sage: from sage.features.gap import GapPackage
-    sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_packages
+    sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_package_grape
     FeatureTestResult('gap_package_grape', True)
 
 Note that a :class:`FeatureTestResult` acts like a bool in most contexts::
@@ -158,6 +158,12 @@ def __init__(self, name, spkg=None, url=None, description=None, type='optional')
         self._hidden = False
         self._type = type
 
+        # For multiprocessing of doctests, the data self._num_hidings should be
+        # shared among subprocesses. Thus we use the Value class from the
+        # multiprocessing module (cf. self._seen of class AvailableSoftware)
+        from multiprocessing import Value
+        self._num_hidings = Value('i', 0)
+
         try:
             from sage.misc.package import spkg_type
         except ImportError:  # may have been surgically removed in a downstream distribution
@@ -182,7 +188,7 @@ def is_present(self):
         EXAMPLES::
 
             sage: from sage.features.gap import GapPackage
-            sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_packages
+            sage: GapPackage("grape", spkg="gap_packages").is_present()  # optional - gap_package_grape
             FeatureTestResult('gap_package_grape', True)
             sage: GapPackage("NOT_A_PACKAGE", spkg="gap_packages").is_present()
             FeatureTestResult('gap_package_NOT_A_PACKAGE', False)
@@ -205,15 +211,21 @@ def is_present(self):
             sage: TestFeature("other").is_present()
             FeatureTestResult('other', True)
         """
-        if self._hidden:
-            return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name))
         # We do not use @cached_method here because we wish to use
         # Feature early in the build system of sagelib.
         if self._cache_is_present is None:
             res = self._is_present()
             if not isinstance(res, FeatureTestResult):
                 res = FeatureTestResult(self, res)
             self._cache_is_present = res
+
+        if self._hidden:
+            if self._num_hidings.value > 0:
+                self._num_hidings.value += 1
+            elif self._cache_is_present:
+                self._num_hidings.value = 1
+            return FeatureTestResult(self, False, reason="Feature `{name}` is hidden.".format(name=self.name))
+
         return self._cache_is_present
 
     def _is_present(self):
@@ -381,40 +393,34 @@ def hide(self):
             Feature `benzene` is hidden.
             Use method `unhide` to make it available again.
 
-            sage: Benzene().unhide()
+            sage: Benzene().unhide()            # optional - benzene, needs sage.graphs
+            1
             sage: len(list(graphs.fusenes(2)))  # optional - benzene, needs sage.graphs
             1
         """
         self._hidden = True
 
     def unhide(self):
         r"""
-        Revert what :meth:`hide` does.
-
-        EXAMPLES:
+        Revert what :meth:`hide` did.
 
-        PolyCyclic is an optional GAP package. The following test
-        fails if it is hidden, regardless of whether it is installed
-        or not::
+        OUTPUT: The number of events a present feature has been hidden.
 
-            sage: from sage.features.gap import GapPackage
-            sage: Polycyclic = GapPackage("polycyclic", spkg="gap_packages")
-            sage: Polycyclic.hide()
-            sage: libgap(AbelianGroup(3, [0,3,4], names="abc"))                         # needs sage.libs.gap  # optional - gap_packages_polycyclic
-            Traceback (most recent call last):
-            ...
-            FeatureNotPresentError: gap_package_polycyclic is not available.
-            Feature `gap_package_polycyclic` is hidden.
-            Use method `unhide` to make it available again.
-
-        After unhiding the feature, the test should pass again if PolyCyclic
-        is installed and loaded::
+        EXAMPLES:
 
-            sage: Polycyclic.unhide()
-            sage: libgap(AbelianGroup(3, [0,3,4], names="abc"))                         # needs sage.libs.gap  # optional - gap_packages_polycyclic
-            Pcp-group with orders [ 0, 3, 4 ]
+            sage: from sage.features.sagemath import sage__plot
+            sage: sage__plot().hide()
+            sage: sage__plot().is_present()
+            FeatureTestResult('sage.plot', False)
+            sage: sage__plot().unhide()                                                 # needs sage.plot
+            1
+            sage: sage__plot().is_present()                                             # needs sage.plot
+            FeatureTestResult('sage.plot', True)
         """
+        num_hidings = self._num_hidings.value
+        self._num_hidings.value = 0
         self._hidden = False
+        return int(num_hidings)
 
 
 class FeatureNotPresentError(RuntimeError):
@@ -802,7 +808,7 @@ class StaticFile(FileFeature):
         To install no_such_file...you can try to run...sage -i some_spkg...
         Further installation instructions might be available at http://rand.om.
     """
-    def __init__(self, name, filename, *, search_path=None, **kwds):
+    def __init__(self, name, filename, *, search_path=None, type='optional', **kwds):
         r"""
         TESTS::
 
@@ -817,7 +823,7 @@ def __init__(self, name, filename, *, search_path=None, **kwds):
             '/bin/sh'
 
         """
-        Feature.__init__(self, name, **kwds)
+        Feature.__init__(self, name, type=type, **kwds)
         self.filename = filename
         if search_path is None:
             self.search_path = [SAGE_SHARE]

src/sage/features/databases.py

@@ -52,12 +52,12 @@ class DatabaseCremona(StaticFile):
     EXAMPLES::
 
         sage: from sage.features.databases import DatabaseCremona
-        sage: DatabaseCremona('cremona_mini').is_present()
+        sage: DatabaseCremona('cremona_mini', type='standard').is_present()
         FeatureTestResult('database_cremona_mini_ellcurve', True)
         sage: DatabaseCremona().is_present()                                    # optional - database_cremona_ellcurve
         FeatureTestResult('database_cremona_ellcurve', True)
     """
-    def __init__(self, name="cremona"):
+    def __init__(self, name="cremona", spkg="database_cremona_ellcurve", type='optional'):
         r"""
         TESTS::
 
@@ -290,7 +290,7 @@ def __init__(self, name='polytopes_db'):
 def all_features():
     return [PythonModule('conway_polynomials', spkg='conway_polynomials', type='standard'),
             DatabaseCremona(),
-            DatabaseCremona('cremona_mini'),
+            DatabaseCremona('cremona_mini', type='standard'),
             DatabaseEllcurves(),
             DatabaseGraphs(),
             DatabaseJones(),

src/sage/features/gap.py

@@ -53,7 +53,7 @@ def _is_present(self):
         EXAMPLES::
 
             sage: from sage.features.gap import GapPackage
-            sage: GapPackage("grape", spkg="gap_packages")._is_present()  # optional - gap_packages
+            sage: GapPackage("grape", spkg="gap_packages")._is_present()  # optional - gap_package_grape
             FeatureTestResult('gap_package_grape', True)
         """
         try:

src/sage/features/join_feature.py

@@ -152,7 +152,9 @@ def hide(self):
 
     def unhide(self):
         r"""
-        Revert what :meth:`hide` does.
+        Revert what :meth:`hide` did.
+
+        OUTPUT: The number of events a present feature has been hidden.
 
         EXAMPLES::
 
@@ -165,11 +167,14 @@ def unhide(self):
             FeatureTestResult('sage.groups.perm_gps.permgroup', False)
 
             sage: f.unhide()
+            4
             sage: f.is_present()    # optional sage.groups
             FeatureTestResult('sage.groups', True)
             sage: f._features[0].is_present() # optional sage.groups
             FeatureTestResult('sage.groups.perm_gps.permgroup', True)
         """
+        num_hidings = 0
         for f in self._features:
-            f.unhide()
-        super().unhide()
+            num_hidings += f.unhide()
+        num_hidings += super().unhide()
+        return num_hidings