.ci/merge-fixes.sh: Obtain patches via URL, make customizable by repository variable

.ci/merge-fixes.sh

@@ -1,36 +1,82 @@
 #!/bin/sh
-# Merge open PRs from sagemath/sage labeled "blocker".
-REPO="sagemath/sage"
-GH="gh -R $REPO"
-PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')"
-if [ -z "$PRs" ]; then
-    echo 'Nothing to do: Found no open PRs with "blocker" status.'
-else
-    echo "Found PRs: " $PRs
-    export GIT_AUTHOR_NAME="ci-sage workflow"
-    export GIT_AUTHOR_EMAIL="[email protected]"
-    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
-    export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
-    git tag -f test_base
-    git commit -q -m "Uncommitted changes" --no-allow-empty -a
-    for a in $PRs; do
-        git fetch --unshallow --all > /dev/null 2>&1 && echo "Unshallowed."
-        echo "::group::Merging PR https://github.com/$REPO/pull/$a"
-        git tag -f test_head
-        $GH pr checkout -b pr-$a $a
-        git checkout -q test_head
-        if git merge --no-edit --squash -q pr-$a; then
-            echo "::endgroup::"
-            if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then
-                echo "Merged #$a"
+# Apply open PRs labeled "blocker" from sagemath/sage as patches.
+# This script is invoked by various workflows in .github/workflows
+#
+# The repository variable SAGE_CI_FIXES_FROM_REPOS can be set
+# to customize this for CI runs in forks:
+#
+# - If set to a whitespace-separated list of repositories, use them instead of sagemath/sage.
+# - If set to "none", do not apply any PRs.
+#
+# https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-a-repository
+export GIT_AUTHOR_NAME="ci-sage workflow"
+export GIT_AUTHOR_EMAIL="[email protected]"
+export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
+export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"
+mkdir -p upstream
+for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do
+    case $REPO in
+        none)
+            echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES"
+            ;;
+        */*)
+            echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22"
+            GH="gh -R $REPO"
+            REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}"
+            PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)"
+            date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date  # Record the date, for future reference
+            if [ -z "$PRs" ]; then
+                echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO."
             else
-                echo "Empty, skipped"
+                echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)"
+                git tag -f test_base
+                git commit -q -m "Uncommitted changes" --no-allow-empty -a
+                for a in $PRs; do
+                    # We used to pull the branch and merge it (after unshallowing), but when run on PRs that are
+                    # based on older releases, it has the side effect of updating to this release,
+                    # which may be confusing.
+                    #
+                    # Here, we obtain the "git am"-able patch of the PR branch.
+                    # This also makes it unnecessary to unshallow the repository.
+                    #
+                    # Considered alternative: Use https://github.com/$REPO/pull/$a.diff,
+                    # which squashes everything into one diff without commit metadata.
+                    PULL_URL="https://github.com/$REPO/pull/$a"
+                    PULL_FILE="$REPO_FILE-$a"
+                    PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch
+                    date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date  # Record the date, for future reference
+                    LAST_SHA=$(sed -n -E '/^From [0-9a-f]{40}/s/^From ([0-9a-f]{40}).*/\1/p' $PULL_FILE.patch | tail -n 1)
+                    COMMITS_URL="https://github.com/$REPO/commits/$LAST_SHA"
+                    echo "::group::Applying PR $PULL_URL @ $COMMITS_URL as a patch"
+                    export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME applying $PULL_URL @ $COMMITS_URL"
+                    if git am --signoff --empty=keep < $PULL_FILE.patch; then
+                        echo "---- Applied patch --------------------------------------------------------------------------------"
+                        cat $PULL_FILE.patch
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                    elif git am --abort \
+                            && if git fetch --unshallow --all > /dev/null 2>&1; then echo "Unshallowed"; fi \
+                            && echo "Retrying with 3-way merge" \
+                            && git am --empty=keep --3way < $PULL_FILE.patch; then
+                        echo "---- Applied patch --------------------------------------------------------------------------------"
+                        cat $PULL_FILE.patch
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                    else
+                        echo "---- Failure applying patch -----------------------------------------------------------------------"
+                        git am --signoff --show-current-patch=diff
+                        echo "--------------------------------------------------------------------8<-----------------------------"
+                        echo "::endgroup::"
+                        echo "Failure applying $PULL_URL as a patch, resetting"
+                        git am --signoff --abort
+                    fi
+                done
             fi
-        else
-            echo "::endgroup::"
-            echo "Failure merging #$a, resetting"
-            git reset --hard
-        fi
-    done
-    git log test_base..HEAD
-fi
+            ;;
+        *)
+            echo "Repository variable SAGE_CI_FIXES_FROM_REPOSITORIES, if set, should be a whitespace-separated list of repositories or 'none'"
+            echo "Got: $SAGE_CI_FIXES_FROM_REPOSITORIES"
+            exit 1
+            ;;
+    esac
+done

.github/workflows/build.yml

@@ -38,6 +38,7 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
       - name: Store CI fixes in upstream artifact
         run: |
           mkdir -p upstream

.github/workflows/ci-conda.yml

@@ -38,6 +38,7 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
 
       - name: Create conda environment files
         run: ./bootstrap-conda

.github/workflows/doc-build-pdf.yml

@@ -32,6 +32,7 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
       - name: Store CI fixes in upstream artifact
         run: |
           mkdir -p upstream

.github/workflows/doc-build.yml

@@ -27,6 +27,7 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
       - name: Store CI fixes in upstream artifact
         run: |
           mkdir -p upstream

.github/workflows/docker.yml

@@ -218,6 +218,8 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
+
       - name: Show disk space
         run: |
           df -h

.github/workflows/lint.yml

@@ -26,6 +26,7 @@ jobs:
         .ci/merge-fixes.sh
       env:
         GH_TOKEN: ${{ github.token }}
+        SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
 
     - name: Set up Python
       uses: actions/setup-python@v4

.github/workflows/macos.yml

@@ -95,6 +95,7 @@ jobs:
           .ci/merge-fixes.sh
         env:
           GH_TOKEN: ${{ github.token }}
+          SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
 
       - uses: actions/download-artifact@v3
         with: