gh-37762: `dist.yml`: Download optional/experimental tarballs for GitHub Release assets
    
<!-- ^ Please provide a concise and informative title. -->
<!-- ^ Don't put issue numbers in the title, do this in the PR
description below. -->
<!-- ^ For example, instead of "Fixes #12345" use "Introduce new method
to calculate 1 + 2". -->
<!-- v Describe your changes below in detail. -->
<!-- v Why is this change required? What problem does it solve? -->
<!-- v If this PR resolves an open issue, please link to it here. For
example, "Fixes #12345". -->

- Fixes #37752, at least for the non-"huge" tarballs (we skip the "huge"
tarballs here).
- More options for `sage -package download` (a subset of what `sage
-package list` supports)
- Also remove some zombie package files

Test run: https://github.com/mkoeppe/sage/actions/workflows/dist.yml

### 📝 Checklist

<!-- Put an `x` in all the boxes that apply. -->

- [x] The title is concise and informative.
- [ ] The description explains in detail what this PR is about.
- [x] I have linked a relevant issue or discussion.
- [ ] I have created tests covering the changes.
- [ ] I have updated the documentation accordingly.

### ⌛ Dependencies

<!-- List all open PRs that this PR logically depends on. For example,
-->
<!-- - #12345: short description why this is a dependency -->
<!-- - #34567: ... -->

- Depends on #37099 (merged to resolve merge conflict)
    
URL: #37762
Reported by: Matthias Köppe
Reviewer(s): Kwankyu Lee, Matthias Köppe

Author: Release Manager

.github/workflows/dist.yml

@@ -37,16 +37,37 @@ jobs:
           sudo DEBIAN_FRONTEND=noninteractive apt-get update
           sudo DEBIAN_FRONTEND=noninteractive apt-get install $(build/bin/sage-get-system-packages debian _bootstrap)
       - name: make dist (--disable-download-from-upstream-url)
+        id: make_dist
         run: |
           ./bootstrap -D && ./configure --disable-download-from-upstream-url && make dist
         env:
           MAKE: make -j8
+      - name: make download (--disable-download-from-upstream-url)
+        id: make_download
+        run: |
+          make -k download DOWNLOAD_PACKAGES=":all: --no-file huge"
+        env:
+          MAKE: make -j8
+      - name: Reconfigure with --enable-download-from-upstream-url
+        if: (success() || failure()) && (steps.make_dist.outcome != 'success' || steps.make_download.outcome != 'success')
+        run: |
+          ./configure
       - name: make dist (--enable-download-from-upstream-url)
-        if: failure()
+        if: (success() || failure()) && steps.make_dist.outcome != 'success'
         run: |
-          ./configure && make dist
+          make dist
         env:
           MAKE: make -j8
+      - name: make download (--enable-download-from-upstream-url)
+        if: (success() || failure()) && steps.make_download.outcome != 'success'
+        run: |
+          make -k download DOWNLOAD_PACKAGES=":all: --no-file huge --allow-upstream"
+        env:
+          MAKE: make -j8
+      - name: Remove what cannot be distributed
+        if: success() || failure()
+        run: |
+          rm -f upstream/*do-not-distribute*
       - uses: actions/upload-artifact@v4
         if: success() || failure()
         with:

Makefile

@@ -79,10 +79,11 @@ reconfigure:
 	fi
 
 # Preemptively download all source tarballs of normal packages.
+DOWNLOAD_PACKAGES=:all:
 download:
 	export SAGE_ROOT=$$(pwd) && \
 	export PATH=$$SAGE_ROOT/build/bin:$$PATH && \
-	sage-package download :all:
+	sage-package download $(DOWNLOAD_PACKAGES)
 
 dist: build/make/Makefile
 	./sage --sdist