diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index dc80c9bfe72..ea2cb1aa79d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -32,50 +32,82 @@ jobs: id: checkout uses: actions/checkout@v3 - - name: Prepare + - name: Update system packages id: prepare run: | - # Install test tools. - if apt-get update && apt-get install -y git python3-venv; then - # Debian-specific temporary code: - # Installation of python3-venv can be removed as soon as a - # base image with a release including #33822 is available - : - else - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git) - fi - # Reuse built SAGE_LOCAL contained in the Docker image - ./bootstrap - ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-editable --enable-download-from-upstream-url - + export PATH="build/bin:$PATH" + eval $(sage-print-system-package-command auto update) + eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git) + + - name: Add prebuilt tree as a worktree + id: worktree + run: | + set -ex + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" + git config --global --add safe.directory $(pwd) + # If actions/checkout downloaded our source tree using the GitHub REST API + # instead of with git (because do not have git installed in our image), + # we first make the source tree a repo. + if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi + # Tag this state of the source tree "new". This is what we want to build and test. + git tag -f new + # Our container image contains a source tree in /sage with a full build of Sage. + # But /sage is not a git repository. + # We make /sage a worktree whose index is at tag "new". + # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) + # Then we update worktree and index with "git reset --hard new". + # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) + # Finally we reset the index to "old". (This keeps all mtimes unchanged.) + # The changed files now show up as uncommitted changes. + git worktree add --detach worktree-image + rm -rf /sage/.git && mv worktree-image/.git /sage/ + rm -rf worktree-image && ln -s /sage worktree-image + if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi + (cd worktree-image && git add -A && git commit --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset old) + + - name: Incremental build, test changed files (sage -t --new) + id: incremental + run: | + # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. + # We run tests with "sage -t --new"; this only tests the uncommitted changes. + ./bootstrap && make build && ./sage -t --new -p2 + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + - name: Build and test modularized distributions - if: always() && steps.prepare.outcome == 'success' + if: always() && steps.worktree.outcome == 'success' run: make V=0 tox && make pypi-wheels + working-directory: ./worktree-image env: MAKE: make -j2 SAGE_NUM_THREADS: 2 - name: Set up node to install pyright - if: always() && steps.prepare.outcome == 'success' + if: always() && steps.worktree.outcome == 'success' uses: actions/setup-node@v3 with: node-version: '12' - name: Install pyright - if: always() && steps.prepare.outcome == 'success' + if: always() && steps.worktree.outcome == 'success' # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239 run: npm install -g pyright@1.1.232 - name: Static code check with pyright - if: always() && steps.prepare.outcome == 'success' - run: pyright - - - name: Build + if: always() && steps.worktree.outcome == 'success' + run: pyright + working-directory: ./worktree-image + + - name: Build (fallback to non-incremental) id: build - if: always() && steps.prepare.outcome == 'success' - run: make build + if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + run: | + set -ex + ./bootstrap && make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build + working-directory: ./worktree-image env: MAKE: make -j2 SAGE_NUM_THREADS: 2 @@ -85,27 +117,28 @@ jobs: run: | ../sage -python -m pip install coverage pytest-xdist ../sage -python -m coverage run -m pytest -c tox.ini --doctest-modules || true - working-directory: ./src + working-directory: ./worktree-image/src env: # Increase the length of the lines in the "short summary" COLUMNS: 120 - - name: Test - if: always() && steps.build.outcome == 'success' + - name: Test all files (sage -t --all --long) + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') run: | ../sage -python -m pip install coverage - ../sage -python -m coverage run ./bin/sage-runtests --all -p2 --random-seed=286735480429121101562228604801325644303 - working-directory: ./src + ../sage -python -m coverage run ./bin/sage-runtests --all --long -p2 --random-seed=286735480429121101562228604801325644303 + working-directory: ./worktree-image/src - name: Prepare coverage results - if: always() && steps.build.outcome == 'success' + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') run: | ./venv/bin/python3 -m coverage combine src/.coverage/ ./venv/bin/python3 -m coverage xml find . -name *coverage* - + working-directory: ./worktree-image + - name: Upload coverage to codecov - if: always() && steps.build.outcome == 'success' + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') uses: codecov/codecov-action@v3 with: - files: ./coverage.xml + files: ./worktree-image/coverage.xml diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 4be22210eee..01748c3c823 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -19,30 +19,94 @@ jobs: - name: Checkout uses: actions/checkout@v3 - - name: Prepare + - name: Update system packages run: | - apt-get update && apt-get install -y zip - # Reuse built SAGE_LOCAL contained in the Docker image - ./bootstrap - ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv --enable-download-from-upstream-url + apt-get update && apt-get install -y git zip - - name: Build - run: make doc-html + - name: Add prebuilt tree as a worktree + id: worktree + run: | + set -ex + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" + git config --global --add safe.directory $(pwd) + # If actions/checkout downloaded our source tree using the GitHub REST API + # instead of with git (because do not have git installed in our image), + # we first make the source tree a repo. + if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi + # Tag this state of the source tree "new". This is what we want to build and test. + git tag -f new + # Our container image contains a source tree in /sage with a full build of Sage. + # But /sage is not a git repository. + # We make /sage a worktree whose index is at tag "new". + # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) + # Then we update worktree and index with "git reset --hard new". + # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) + # Finally we reset the index to "old". (This keeps all mtimes unchanged.) + # The changed files now show up as uncommitted changes. + git worktree add --detach worktree-image + rm -rf /sage/.git && mv worktree-image/.git /sage/ + rm -rf worktree-image && ln -s /sage worktree-image + if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi + (cd worktree-image && git add -A && git commit --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset old) + # Keep track of changes to built HTML + (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage dev /'; git init && (echo ".buildinfo"; echo ".inv") > .gitignore; git add -A && git commit --quiet -m "old") + + - name: Incremental build + id: incremental + run: | + # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. + ./bootstrap && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + + - name: Build (fallback to non-incremental) + id: build + if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + run: | + set -ex + make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + + - name: Build docs + id: docbuild + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') + # Always non-incremental because of the concern that + # incremental docbuild may introduce broken links (inter-file references) though build succeeds + run: | + set -ex + make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make doc-html + working-directory: ./worktree-image env: MAKE: make -j2 SAGE_NUM_THREADS: 2 - name: Copy docs + id: copy + if: always() && steps.docbuild.outcome == 'success' run: | + set -ex + mkdir -p ./docs + # Create changelog + echo '## Preview of CHANGES.html' + (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage dev /'; git diff --name-only) | tee ./docs/CHANGES.txt + (cd /sage/local/share/doc/sage/html/en && git diff; rm -rf .git) > ./docs/html.diff + echo '## Preview of html.diff'; head -n 400 ./docs/html.diff + (echo '

HTML diff'; sed -E 's,(.*),

\1,' ./docs/CHANGES.txt) > ./docs/CHANGES.html # For some reason the deploy step below cannot find /sage/... # So copy everything from there to local folder # We also need to replace the symlinks because netlify is not following them - mkdir -p ./docs cp -r -L /sage/local/share/doc/sage/html/en/* ./docs # Zip everything for increased performance zip -r docs.zip docs - name: Upload docs + if: always() && steps.copy.outcome == 'success' uses: actions/upload-artifact@v3 with: name: docs diff --git a/.github/workflows/doc-publish.yml b/.github/workflows/doc-publish.yml index 538e77322fc..e86727e73ef 100644 --- a/.github/workflows/doc-publish.yml +++ b/.github/workflows/doc-publish.yml @@ -72,7 +72,7 @@ jobs: header: preview-comment recreate: true message: | - [Documentation preview for this PR](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}) (built with commit ${{ steps.source-run-info.outputs.sourceHeadSha }}) is ready! :tada: + [Documentation preview for this PR](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}) (built with commit ${{ steps.source-run-info.outputs.sourceHeadSha }}; [changes](${{ steps.deploy-netlify.outputs.NETLIFY_URL }}/CHANGES.html)) is ready! :tada: - name: Update deployment status PR check uses: myrotvorets/set-commit-status-action@v1.1.7 diff --git a/build/pkgs/ecl/dependencies b/build/pkgs/ecl/dependencies index 51a953403e9..cda6316bf5a 100644 --- a/build/pkgs/ecl/dependencies +++ b/build/pkgs/ecl/dependencies @@ -1,4 +1,4 @@ -$(MP_LIBRARY) readline gc libffi info +$(MP_LIBRARY) readline gc libffi ---------- All lines of this file are ignored except the first. diff --git a/build/pkgs/ecl/dependencies_order_only b/build/pkgs/ecl/dependencies_order_only new file mode 100644 index 00000000000..55021245387 --- /dev/null +++ b/build/pkgs/ecl/dependencies_order_only @@ -0,0 +1 @@ +info diff --git a/build/pkgs/maxima/dependencies b/build/pkgs/maxima/dependencies index 55c7e0d8d14..fffb89e2050 100644 --- a/build/pkgs/maxima/dependencies +++ b/build/pkgs/maxima/dependencies @@ -1,4 +1,4 @@ -ecl info +ecl ---------- All lines of this file are ignored except the first. diff --git a/build/pkgs/maxima/dependencies_order_only b/build/pkgs/maxima/dependencies_order_only new file mode 100644 index 00000000000..55021245387 --- /dev/null +++ b/build/pkgs/maxima/dependencies_order_only @@ -0,0 +1 @@ +info diff --git a/src/sage/doctest/control.py b/src/sage/doctest/control.py index 450d9b607da..abaa33f6239 100644 --- a/src/sage/doctest/control.py +++ b/src/sage/doctest/control.py @@ -791,7 +791,7 @@ def add_files(self): # SAGE_ROOT_GIT can be None on distributions which typically # only have the SAGE_LOCAL install tree but not SAGE_ROOT if SAGE_ROOT_GIT is not None: - have_git = os.path.isdir(SAGE_ROOT_GIT) + have_git = os.path.exists(SAGE_ROOT_GIT) else: have_git = False @@ -866,11 +866,12 @@ def all_doc_sources(): data = line.strip().split(' ') status, filename = data[0], data[-1] if (set(status).issubset("MARCU") - and filename.startswith("src/sage") - and (filename.endswith(".py") or - filename.endswith(".pyx") or - filename.endswith(".rst"))): - self.files.append(os.path.relpath(opj(SAGE_ROOT,filename))) + and filename.startswith("src/sage") + and (filename.endswith(".py") or + filename.endswith(".pyx") or + filename.endswith(".rst")) + and not skipfile(opj(SAGE_ROOT, filename), self.options.optional)): + self.files.append(os.path.relpath(opj(SAGE_ROOT, filename))) def expand_files_into_sources(self): r""" diff --git a/src/sage/doctest/test.py b/src/sage/doctest/test.py index a36b3caf64f..60075a53578 100644 --- a/src/sage/doctest/test.py +++ b/src/sage/doctest/test.py @@ -251,17 +251,24 @@ Even though the doctester master process has exited, the child process is still alive, but it should be killed automatically -after the `die_timeout` given above (10 seconds):: +after the ``die_timeout`` given above (10 seconds):: sage: pid = int(open(F).read()) # long time sage: time.sleep(2) # long time sage: os.kill(pid, signal.SIGQUIT) # long time; 2 seconds passed => still alive sage: time.sleep(8) # long time - sage: os.kill(pid, signal.SIGQUIT) # long time; 10 seconds passed => dead + sage: os.kill(pid, signal.SIGQUIT) # long time; 10 seconds passed => dead # random Traceback (most recent call last): ... ProcessLookupError: ... +If the child process is dead and removed, the last output should be as above. +However, the child process interrupted its parent process (see +``"interrupt_diehard.rst"``), and became an orphan process. Depending on the +system, an orphan process may eventually become a zombie process instead of +being removed, and then the last output would just be a blank. Hence the ``# +random`` tag. + Test a doctest failing with ``abort()``:: sage: subprocess.call(["sage", "-t", "--warn-long", "0", # long time