gh-36393: `sage --package list`: Sort output, add switches `--{in,ex}clude-dependencies`
    
We enhance the command `sage --package list`:
- New switches `--{in,ex}clude-dependencies` walk the tree of
dependencies declared in `dependencies` files
- Output is always (uniquified and) sorted, eliminating the use of
`sort` in various scripts
- Handling of `--exclude` is moved to the end, which improves its
applicability.

The new parsing of the `dependencies*` files in `sage_bootstrap` is
generic preparation for handling more of the bootstrapping phase in
Python, as sketched in #33860 (part of #29146).

This PR is immediate preparation for #35593, where it is desired to
create "minimized" lists of system packages that do not include all
standard packages that are really only needed as dependencies of "top-
level" packages.

<!-- ^^^^^
Please provide a concise, informative and self-explanatory title.
Don't put issue numbers in there, do this in the PR body below.
For example, instead of "Fixes #1234" use "Introduce new method to
calculate 1+1"
-->
<!-- Describe your changes here in detail -->

<!-- Why is this change required? What problem does it solve? -->
<!-- If this PR resolves an open issue, please link to it here. For
example "Fixes #12345". -->
<!-- If your change requires a documentation PR, please link it
appropriately. -->

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->
<!-- If your change requires a documentation PR, please link it
appropriately -->
<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
<!-- Feel free to remove irrelevant items. -->

- [x] The title is concise, informative, and self-explanatory.
- [x] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on
- #12345: short description why this is a dependency
- #34567: ...
-->

<!-- If you're unsure about any of these, don't hesitate to ask. We're
here to help! -->
    
URL: #36393
Reported by: Matthias Köppe
Reviewer(s): Michael Orlitzky

Author: Release Manager

bootstrap

@@ -47,12 +47,12 @@ bootstrap () {
     done
     spkg_configures=""
     # initialize SAGE_ENABLE... options for standard packages
-    for pkgname in $(sage-package list :standard: | sort); do
+    for pkgname in $(sage-package list :standard:); do
         spkg_configures="$spkg_configures
 AS_VAR_SET_IF([SAGE_ENABLE_$pkgname], [], [AS_VAR_SET([SAGE_ENABLE_$pkgname], [yes])])"
     done
     # --enable-SPKG options
-    for pkgname in $(sage-package list :optional: :experimental: | sort); do
+    for pkgname in $(sage-package list :optional: :experimental:); do
                 # Issue #29629: Temporary solution for Sage 9.1: Do not provide
                 # --enable-SPKG options for installing pip packages
                 if [ ! -f build/pkgs/$pkgname/requirements.txt ]; then
@@ -71,7 +71,7 @@ SAGE_SPKG_ENABLE([$pkgname], [$pkgtype], [$(grep -v ^= build/pkgs/$pkgname/SPKG.
                     esac
                 fi
     done
-    for pkgname in $(sage-package list --has-file spkg-configure.m4 | sort); do
+    for pkgname in $(sage-package list --has-file spkg-configure.m4); do
         echo "m4_sinclude([build/pkgs/$pkgname/spkg-configure.m4])"
         config="SAGE_SPKG_CONFIGURE_$(echo ${pkgname} | tr '[a-z]' '[A-Z]')"
         if grep -q SAGE_PYTHON_PACKAGE_CHECK build/pkgs/$pkgname/spkg-configure.m4; then
@@ -86,7 +86,7 @@ $config"
 $spkg_configures
 $spkg_configures_python
 EOF
-    for pkgname in $(sage-package list | sort); do
+    for pkgname in $(sage-package list); do
         DIR=build/pkgs/$pkgname
         pkgtype="$(cat $DIR/type)"
         if test -f "$DIR/requirements.txt"; then

build/bin/sage-package

@@ -13,7 +13,7 @@
 #
 # * Print a list of all available packages
 #
-#   $ sage-package list | sort
+#   $ sage-package list
 #   4ti2
 #   arb
 #   autotools

build/sage_bootstrap/app.py

@@ -53,7 +53,7 @@ def list_cls(self, *package_classes, **filters):
         """
         Print a list of all available packages
 
-        $ sage --package list | sort
+        $ sage --package list
         4ti2
         arb
         autotools
@@ -63,7 +63,7 @@ def list_cls(self, *package_classes, **filters):
         $ sage -package list --has-file=spkg-configure.m4 :experimental:
         perl_term_readline_gnu
 
-        $ sage -package list --has-file=spkg-configure.m4 --has-file=distros/debian.txt | sort
+        $ sage -package list --has-file=spkg-configure.m4 --has-file=distros/debian.txt
         arb
         boost_cropped
         brial

build/sage_bootstrap/cmdline.py

@@ -64,18 +64,18 @@
 
 epilog_list = \
 """
-Print a list of packages known to Sage
+Print a list of packages known to Sage (sorted alphabetically)
 
 EXAMPLE:
 
-    $ sage --package list | sort
+    $ sage --package list
     4ti2
     arb
     autotools
     [...]
     zlib
 
-    $ sage --package list :standard: | sort
+    $ sage --package list :standard:
     arb
     backports_ssl_match_hostname
     [...]
@@ -214,10 +214,10 @@ def make_parser():
         help='Print a list of packages known to Sage')
     parser_list.add_argument(
         'package_class', metavar='[package_name|:package_type:]',
-        type=str, default=[':all:'], nargs='*',
+        type=str, default=[':all-or-nothing:'], nargs='*',
         help=('package name or designator for all packages of a given type '
               '(one of :all:, :standard:, :optional:, and :experimental:); '
-              'default: :all:'))
+              'default: :all: (or nothing when --include-dependencies or --exclude-dependencies is given'))
     parser_list.add_argument(
         '--has-file', action='append', default=[], metavar='FILENAME', dest='has_files',
         help=('only include packages that have this file in their metadata directory '
@@ -227,8 +227,15 @@ def make_parser():
         help=('only include packages that do not have this file in their metadata directory '
               '(examples: huge, patches, huge|has_nonfree_dependencies)'))
     parser_list.add_argument(
-        '--exclude', action='append', default=[], metavar='PACKAGE_NAME',
+        '--exclude', nargs='*', action='append', default=[], metavar='PACKAGE_NAME',
         help='exclude package from list')
+    parser_list.add_argument(
+        '--include-dependencies', action='store_true',
+        help='include (ordinary) dependencies of the packages recursively')
+    parser_list.add_argument(
+        '--exclude-dependencies', action='store_true',
+        help='exclude (ordinary) dependencies of the packages recursively')
+
     parser_name = subparsers.add_parser(
         'name', epilog=epilog_name,
         formatter_class=argparse.RawDescriptionHelpFormatter,
@@ -353,7 +360,16 @@ def run():
     if args.subcommand == 'config':
         app.config()
     elif args.subcommand == 'list':
-        app.list_cls(*args.package_class, has_files=args.has_files, no_files=args.no_files, exclude=args.exclude)
+        if args.package_class == [':all-or-nothing:']:
+            if args.include_dependencies or args.exclude_dependencies:
+                args.package_class = []
+            else:
+                args.package_class = [':all:']
+        app.list_cls(*args.package_class,
+                     has_files=args.has_files, no_files=args.no_files,
+                     exclude=args.exclude,
+                     include_dependencies=args.include_dependencies,
+                     exclude_dependencies=args.exclude_dependencies)
     elif args.subcommand == 'name':
         app.name(args.tarball_filename)
     elif args.subcommand == 'tarball':

build/sage_bootstrap/expand_class.py

@@ -21,17 +21,19 @@
 
 class PackageClass(object):
 
-    def __init__(self, *package_names_or_classes, **filters):
-        self.names = []
+    def __init__(self, *package_names_or_classes, exclude=(),
+                 include_dependencies=False, exclude_dependencies=False,
+                 **filters):
+        self.__names = set()
         filenames = filters.pop('has_files', [])
         no_filenames = filters.pop('no_files', [])
-        excluded = filters.pop('exclude', [])
+        excluded = []
+        for package_names in exclude:
+            excluded.extend(package_names)
         if filters:
             raise ValueError('filter not supported')
 
         def included_in_filter(pkg):
-            if pkg.name in excluded:
-                return False
             if not all(any(pkg.has_file(filename)
                            for filename in filename_disjunction.split('|'))
                        for filename_disjunction in filenames):
@@ -55,19 +57,58 @@ def included_in_filter(pkg):
                                      'which must be one of '
                                      ':all:, :standard:, :optional:, or :experimental:'
                                      'got {}'.format(package_name_or_class))
-                self.names.append(package_name_or_class)
+                self.__names.add(package_name_or_class)
+
+        def include_recursive_dependencies(names, package_name):
+            if package_name in names:
+                return
+            try:
+                pkg = Package(package_name)
+            except FileNotFoundError:
+                # Silently ignore unknown packages,
+                # substitutions such as $(BLAS) $(PYTHON),
+                # and optional dependencies of the form $(find-string ...).
+                return
+            names.add(package_name)
+            for dependency in pkg.dependencies:
+                include_recursive_dependencies(names, dependency)
+
+        if include_dependencies:
+            package_names = set()
+            for name in self.__names:
+                include_recursive_dependencies(package_names, name)
+            self.__names = package_names
+
+        def exclude_recursive_dependencies(names, package_name):
+            try:
+                pkg = Package(package_name)
+            except FileNotFoundError:
+                return
+            for dependency in pkg.dependencies:
+                names.discard(dependency)
+                exclude_recursive_dependencies(names, dependency)
+
+        if exclude_dependencies:
+            for name in list(self.__names):
+                exclude_recursive_dependencies(self.__names, name)
+
+        self.__names.difference_update(excluded)
+
+    @property
+    def names(self):
+        return sorted(self.__names)
 
     def _init_all(self, predicate):
-        self.names.extend(pkg.name for pkg in Package.all() if predicate(pkg))
+        self.__names.update(pkg.name for pkg in Package.all() if predicate(pkg))
 
     def _init_standard(self, predicate):
-        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'standard' and predicate(pkg))
+        self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'standard' and predicate(pkg))
 
     def _init_optional(self, predicate):
-        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'optional' and predicate(pkg))
+        self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'optional' and predicate(pkg))
 
     def _init_experimental(self, predicate):
-        self.names.extend(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg))
+        self.__names.update(pkg.name for pkg in Package.all() if pkg.type == 'experimental' and predicate(pkg))
 
     def apply(self, function, *args, **kwds):
         for package_name in self.names:

build/sage_bootstrap/package.py

@@ -47,6 +47,7 @@ def __init__(self, package_name):
         self._init_version()
         self._init_type()
         self._init_install_requires()
+        self._init_dependencies()
 
     def __repr__(self):
         return 'Package {0}'.format(self.name)
@@ -245,6 +246,28 @@ def distribution_name(self):
                 return part
         return None
 
+    @property
+    def dependencies(self):
+        """
+        Return a list of strings, the package names of the (ordinary) dependencies
+        """
+        # after a '|', we have order-only dependencies
+        return self.__dependencies.partition('|')[0].strip().split()
+
+    @property
+    def dependencies_order_only(self):
+        """
+        Return a list of strings, the package names of the order-only dependencies
+        """
+        return self.__dependencies.partition('|')[2].strip().split() + self.__dependencies_order_only.strip().split()
+
+    @property
+    def dependencies_check(self):
+        """
+        Return a list of strings, the package names of the check dependencies
+        """
+        return self.__dependencies_order_only.strip().split()
+
     def __eq__(self, other):
         return self.tarball == other.tarball
 
@@ -335,3 +358,20 @@ def _init_install_requires(self):
                 self.__install_requires = f.read().strip()
         except IOError:
             self.__install_requires = None
+
+    def _init_dependencies(self):
+        try:
+            with open(os.path.join(self.path, 'dependencies')) as f:
+                self.__dependencies = f.readline().strip()
+        except IOError:
+            self.__dependencies = ''
+        try:
+            with open(os.path.join(self.path, 'dependencies_check')) as f:
+                self.__dependencies_check = f.readline().strip()
+        except IOError:
+            self.__dependencies_check = ''
+        try:
+            with open(os.path.join(self.path, 'dependencies_order_only')) as f:
+                self.__dependencies_order_only = f.readline()
+        except IOError:
+            self.__dependencies_order_only = ''

src/doc/bootstrap

@@ -100,7 +100,7 @@ Sage depends.  It installs them automatically if it does not find
 equivalent system packages.
 
 EOF
-for PKG_BASE in $(sage-package list --has-file SPKG.rst :standard: | grep -v '^sagemath_' | sort); do
+for PKG_BASE in $(sage-package list --has-file SPKG.rst :standard: | grep -v '^sagemath_'); do
     echo "* :ref:\`spkg_$PKG_BASE\`"
 done >> "$OUTPUT_INDEX"
 cat >> "$OUTPUT_INDEX" <<EOF
@@ -112,7 +112,7 @@ For additional functionality, you can install some of the following
 optional packages.
 
 EOF
-for PKG_BASE in $(sage-package list --has-file SPKG.rst :optional: | grep -v '^sagemath_' | sort); do
+for PKG_BASE in $(sage-package list --has-file SPKG.rst :optional: | grep -v '^sagemath_'); do
     echo "* :ref:\`spkg_$PKG_BASE\`"
 done >> "$OUTPUT_INDEX"
 cat >> "$OUTPUT_INDEX" <<EOF
@@ -160,7 +160,7 @@ Distribution Packages of the Sage Library
 -----------------------------------------
 
 EOF
-for PKG_BASE in $(sage-package list --has-file SPKG.rst | grep '^sagemath_' | sort); do
+for PKG_BASE in $(sage-package list --has-file SPKG.rst | grep '^sagemath_'); do
     echo "* :ref:\`spkg_$PKG_BASE\`"
 done >> "$OUTPUT_INDEX"
 cat >> "$OUTPUT_INDEX" <<EOF
@@ -173,7 +173,7 @@ Some packages that provide additional functionality are marked as
 integration of these packages into the Sage distribution.
 
 EOF
-for PKG_BASE in $(sage-package list --has-file SPKG.rst :experimental: | grep -v '^sagemath_' | sort); do
+for PKG_BASE in $(sage-package list --has-file SPKG.rst :experimental: | grep -v '^sagemath_'); do
     echo "* :ref:\`spkg_$PKG_BASE\`"
 done >> "$OUTPUT_INDEX"
 
@@ -203,7 +203,7 @@ Packages are in alphabetical order.
    :maxdepth: 1
 
 EOF
-for PKG_BASE in $(sage-package list --has-file SPKG.rst | sort); do
+for PKG_BASE in $(sage-package list --has-file SPKG.rst); do
             PKG_SCRIPTS=build/pkgs/$PKG_BASE
             # Instead of just copying, we may want to call
             # a version of sage-spkg-info to format extra information.