diff --git a/.ci/merge-fixes.sh b/.ci/merge-fixes.sh index 8ee138660ec..73b4c665caf 100755 --- a/.ci/merge-fixes.sh +++ b/.ci/merge-fixes.sh @@ -1,36 +1,82 @@ #!/bin/sh -# Merge open PRs from sagemath/sage labeled "blocker". -REPO="sagemath/sage" -GH="gh -R $REPO" -PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number')" -if [ -z "$PRs" ]; then - echo 'Nothing to do: Found no open PRs with "blocker" status.' -else - echo "Found PRs: " $PRs - export GIT_AUTHOR_NAME="ci-sage workflow" - export GIT_AUTHOR_EMAIL="ci-sage@example.com" - export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" - export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" - git tag -f test_base - git commit -q -m "Uncommitted changes" --no-allow-empty -a - for a in $PRs; do - git fetch --unshallow --all > /dev/null 2>&1 && echo "Unshallowed." - echo "::group::Merging PR https://github.com/$REPO/pull/$a" - git tag -f test_head - $GH pr checkout -b pr-$a $a - git checkout -q test_head - if git merge --no-edit --squash -q pr-$a; then - echo "::endgroup::" - if git commit -q -m "Merge https://github.com/$REPO/pull/$a" -a --no-allow-empty; then - echo "Merged #$a" +# Apply open PRs labeled "blocker" from sagemath/sage as patches. +# This script is invoked by various workflows in .github/workflows +# +# The repository variable SAGE_CI_FIXES_FROM_REPOS can be set +# to customize this for CI runs in forks: +# +# - If set to a whitespace-separated list of repositories, use them instead of sagemath/sage. +# - If set to "none", do not apply any PRs. +# +# https://docs.github.com/en/actions/learn-github-actions/variables#creating-configuration-variables-for-a-repository +export GIT_AUTHOR_NAME="ci-sage workflow" +export GIT_AUTHOR_EMAIL="ci-sage@example.com" +export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" +export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" +mkdir -p upstream +for REPO in ${SAGE_CI_FIXES_FROM_REPOSITORIES:-sagemath/sage}; do + case $REPO in + none) + echo "Nothing to do for 'none' in SAGE_CI_FIXES_FROM_REPOSITORIES" + ;; + */*) + echo "Getting open PRs with 'blocker' status from https://github.com/$REPO/pulls?q=is%3Aopen+label%3A%22p%3A+blocker+%2F+1%22" + GH="gh -R $REPO" + REPO_FILE="upstream/ci-fixes-${REPO%%/*}-${REPO##*/}" + PRs="$($GH pr list --label "p: blocker / 1" --json number --jq '.[].number' | tee $REPO_FILE)" + date -u +"%Y-%m-%dT%H:%M:%SZ" > $REPO_FILE.date # Record the date, for future reference + if [ -z "$PRs" ]; then + echo "Nothing to do: Found no open PRs with 'blocker' status in $REPO." else - echo "Empty, skipped" + echo "Found open PRs with 'blocker' status in $REPO: $(echo $PRs)" + git tag -f test_base + git commit -q -m "Uncommitted changes" --no-allow-empty -a + for a in $PRs; do + # We used to pull the branch and merge it (after unshallowing), but when run on PRs that are + # based on older releases, it has the side effect of updating to this release, + # which may be confusing. + # + # Here, we obtain the "git am"-able patch of the PR branch. + # This also makes it unnecessary to unshallow the repository. + # + # Considered alternative: Use https://github.com/$REPO/pull/$a.diff, + # which squashes everything into one diff without commit metadata. + PULL_URL="https://github.com/$REPO/pull/$a" + PULL_FILE="$REPO_FILE-$a" + PATH=build/bin:$PATH build/bin/sage-download-file --quiet "$PULL_URL.patch" $PULL_FILE.patch + date -u +"%Y-%m-%dT%H:%M:%SZ" > $PULL_FILE.date # Record the date, for future reference + LAST_SHA=$(sed -n -E '/^From [0-9a-f]{40}/s/^From ([0-9a-f]{40}).*/\1/p' $PULL_FILE.patch | tail -n 1) + COMMITS_URL="https://github.com/$REPO/commits/$LAST_SHA" + echo "::group::Applying PR $PULL_URL @ $COMMITS_URL as a patch" + export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME applying $PULL_URL @ $COMMITS_URL" + if git am --signoff --empty=keep < $PULL_FILE.patch; then + echo "---- Applied patch --------------------------------------------------------------------------------" + cat $PULL_FILE.patch + echo "--------------------------------------------------------------------8<-----------------------------" + echo "::endgroup::" + elif git am --abort \ + && if git fetch --unshallow --all > /dev/null 2>&1; then echo "Unshallowed"; fi \ + && echo "Retrying with 3-way merge" \ + && git am --empty=keep --3way < $PULL_FILE.patch; then + echo "---- Applied patch --------------------------------------------------------------------------------" + cat $PULL_FILE.patch + echo "--------------------------------------------------------------------8<-----------------------------" + echo "::endgroup::" + else + echo "---- Failure applying patch -----------------------------------------------------------------------" + git am --signoff --show-current-patch=diff + echo "--------------------------------------------------------------------8<-----------------------------" + echo "::endgroup::" + echo "Failure applying $PULL_URL as a patch, resetting" + git am --signoff --abort + fi + done fi - else - echo "::endgroup::" - echo "Failure merging #$a, resetting" - git reset --hard - fi - done - git log test_base..HEAD -fi + ;; + *) + echo "Repository variable SAGE_CI_FIXES_FROM_REPOSITORIES, if set, should be a whitespace-separated list of repositories or 'none'" + echo "Got: $SAGE_CI_FIXES_FROM_REPOSITORIES" + exit 1 + ;; + esac +done diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f466d3a285..644ad35f801 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -38,6 +38,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | mkdir -p upstream diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 2240c224a52..7e6282b92cc 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -38,6 +38,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Create conda environment files run: ./bootstrap-conda diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 1d14d0c29f7..97fc4a9767a 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -32,6 +32,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | mkdir -p upstream diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 3660ff3a947..1a645a7aa46 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -27,6 +27,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Store CI fixes in upstream artifact run: | mkdir -p upstream diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3f5bf2e592c..6a8275f0394 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -218,6 +218,8 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} + - name: Show disk space run: | df -h diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 41560674b49..69abb95deb4 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,6 +26,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - name: Set up Python uses: actions/setup-python@v4 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 91c38807e76..0399fd1a3bb 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -95,6 +95,7 @@ jobs: .ci/merge-fixes.sh env: GH_TOKEN: ${{ github.token }} + SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }} - uses: actions/download-artifact@v3 with: