From 6127016ec90f4c2f66a99b6e6966cc05df9eeb6c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 4 Oct 2023 21:23:47 -0700 Subject: [PATCH 01/32] bootstrap-conda: Generate environment files with hard-coded Python minor versions (environment-3.9.yml etc.) This is convenient for conda-lock. Partial cherry-pick from "Use conda-lock for reproducible conda env" --- .devcontainer/onCreate-conda.sh | 2 +- .gitignore | 18 ++++ .gitpod.yml | 2 +- bootstrap-conda | 134 ++++++++++++------------ build/pkgs/matplotlib/distros/conda.txt | 2 +- 5 files changed, 89 insertions(+), 69 deletions(-) diff --git a/.devcontainer/onCreate-conda.sh b/.devcontainer/onCreate-conda.sh index eaa045cd27b..2cbb94e3492 100755 --- a/.devcontainer/onCreate-conda.sh +++ b/.devcontainer/onCreate-conda.sh @@ -4,7 +4,7 @@ set -e # Create conda environment ./bootstrap-conda conda install mamba -n base -c conda-forge -y -mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml +mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml conda init bash # Build sage diff --git a/.gitignore b/.gitignore index 2cec8a0cf62..43f58abcafe 100644 --- a/.gitignore +++ b/.gitignore @@ -27,11 +27,29 @@ # no longer generated, but may still be in user worktrees /src/lib/pkgconfig +# Environment files generated by bootstrap-conda. +# The files without Python version are no longer generated +# but may still be in users' directories. /environment.yml +/environment-3.9.yml +/environment-3.10.yml +/environment-3.11.yml /environment-optional.yml +/environment-optional-3.9.yml +/environment-optional-3.10.yml +/environment-optional-3.11.yml /src/environment.yml +/src/environment-3.9.yml +/src/environment-3.10.yml +/src/environment-3.11.yml /src/environment-dev.yml +/src/environment-dev-3.9.yml +/src/environment-dev-3.10.yml +/src/environment-dev-3.11.yml /src/environment-optional.yml +/src/environment-optional-3.9.yml +/src/environment-optional-3.10.yml +/src/environment-optional-3.11.yml /src/setup.cfg /src/requirements.txt diff --git a/.gitpod.yml b/.gitpod.yml index 52ac8d7184c..51c2687c5d3 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -8,7 +8,7 @@ tasks: # Create conda environment, then configure and build sage init: >- ./bootstrap-conda - && mamba env create --file src/environment-dev.yml --prefix venv + && mamba env create --file src/environment-dev-3.11.yml --prefix venv && conda config --append envs_dirs $(pwd) && conda activate $(pwd)/venv && ./bootstrap diff --git a/bootstrap-conda b/bootstrap-conda index ed4bb9e0d08..811dcbf43b0 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -51,72 +51,74 @@ for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sage fi done echo >&2 $0:$LINENO: generate conda environment files -( - echo "name: sage-build" - echo "channels:" - echo " - conda-forge" - echo " - nodefaults" - echo "dependencies:" - for pkg in $SYSTEM_PACKAGES; do - echo " - $pkg" - done - echo " # Packages needed for ./bootstrap" - for pkg in $BOOTSTRAP_PACKAGES; do - echo " - $pkg" - done -) > environment.yml +for python_version in 3.9 3.10 3.11; do + ( + echo "name: sage-build" + echo "channels:" + echo " - conda-forge" + echo " - nodefaults" + echo "dependencies:" + echo " - python=$python_version" + for pkg in $SYSTEM_PACKAGES; do + echo " - $pkg" + done + echo " # Packages needed for ./bootstrap" + for pkg in $BOOTSTRAP_PACKAGES; do + echo " - $pkg" + done + ) > environment-$python_version.yml + ( + sed 's/name: sage-build/name: sage/' environment-$python_version.yml + echo " # Additional packages providing all dependencies for the Sage library" + for pkg in $SAGELIB_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-$python_version.yml -( - sed 's/name: sage-build/name: sage/' environment.yml - echo " # Additional packages providing all dependencies for the Sage library" - for pkg in $SAGELIB_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment.yml + ( + cat environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > environment-optional-$python_version.yml -( - sed 's/name: sage/name: sage-dev/' src/environment.yml - echo " # Additional dev tools" - for pkg in $DEVELOP_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-dev.yml - -( - cat environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > environment-optional.yml - -( - cat src/environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-optional.yml -( - echo >&4 " - pip:" - echo >&5 " - pip:" - for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do - PKG_SCRIPTS=build/pkgs/$PKG_BASE - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt - if [ ! -f $SYSTEM_PACKAGES_FILE ]; then - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt - fi - PKG_TYPE=$(cat $PKG_SCRIPTS/type) - if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then - : # cannot install packages that depend on the Sage library - else - case "$PKG_BASE:$PKG_TYPE" in - $DEVELOP_SPKG_PATTERN:*) FD=4;; - *) FD=5;; - esac - ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do - [ -n "$line" ] && echo >&$FD " - $line" + ( + ( + sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + echo " # Additional dev tools" + for pkg in $DEVELOP_SYSTEM_PACKAGES; do + echo " - $pkg" done - fi - done -) 4>> src/environment-dev.yml 5>> src/environment-optional.yml + ) >&4 + ( + cat src/environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) >&5 + echo >&4 " - pip:" + echo >&5 " - pip:" + for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do + PKG_SCRIPTS=build/pkgs/$PKG_BASE + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt + if [ ! -f $SYSTEM_PACKAGES_FILE ]; then + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt + fi + PKG_TYPE=$(cat $PKG_SCRIPTS/type) + if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then + : # cannot install packages that depend on the Sage library + else + case "$PKG_BASE:$PKG_TYPE" in + $DEVELOP_SPKG_PATTERN:*) FD=4;; + *:standard) FD="4 5";; + *) FD=5;; + esac + ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do + [ -n "$line" ] && for fd in $FD; do echo >&$fd " - $line"; done + done + fi + done + ) 4>> src/environment-dev-$python_version.yml 5>> src/environment-optional-$python_version.yml +done diff --git a/build/pkgs/matplotlib/distros/conda.txt b/build/pkgs/matplotlib/distros/conda.txt index 8b3901c7c9a..9fdcdfeb47f 100644 --- a/build/pkgs/matplotlib/distros/conda.txt +++ b/build/pkgs/matplotlib/distros/conda.txt @@ -1,2 +1,2 @@ # Trac #33642: For unknown reasons, without the version constraint, conda installs only 3.3.2 -"matplotlib>=3.5.1" +matplotlib>=3.5.1 From 03a2acfeade61d5bc57c1bfb79be802b8b10b7f2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 8 Oct 2023 13:07:57 -0700 Subject: [PATCH 02/32] src/doc/en/installation/conda.rst: Update for versioned environment-... files --- src/doc/en/installation/conda.rst | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst index 6b4cb871dd7..14fe52ad2e2 100644 --- a/src/doc/en/installation/conda.rst +++ b/src/doc/en/installation/conda.rst @@ -75,13 +75,16 @@ from source as follows: - Create a new conda environment including all standard packages recognized by sage, and activate it:: - $ conda env create --file environment.yml --name sage-build + $ conda env create --file environment-3.11.yml --name sage-build $ conda activate sage-build - Alternatively, use ``environment-optional.yml`` in place of + Alternatively, use ``environment-optional-3.11.yml`` in place of ``environment.yml`` to create an environment with all standard and optional packages recognized by sage. + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. + - Then the SageMath distribution will be built using the compilers provided by Conda and using many packages installed by Conda:: @@ -123,16 +126,15 @@ Here we assume that you are using a git checkout. - Create and activate a new conda environment with the dependencies of Sage and a few additional developer tools:: - $ mamba env create --file src/environment-dev.yml --name sage-dev + $ mamba env create --file src/environment-dev-3.11.yml --name sage-dev $ conda activate sage-dev - Alternatively, you can use ``src/environment.yml`` or - ``src/environment-optional.yml``, which will only install standard + Alternatively, you can use ``src/environment-3.11.yml`` or + ``src/environment-optional-3.11.yml``, which will only install standard (and optional) packages without any additional developer tools. - By default, the most recent version of Python supported by Sage is - installed. You can use the additional option ``python=3.9`` in the above - ``env create`` command to select another Python version (here 3.9). + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. - Install the build prerequisites and the Sage library:: @@ -142,7 +144,7 @@ Here we assume that you are using a git checkout. - Verify that Sage has been installed:: $ sage -c 'print(version())' - SageMath version 9.6.beta5, Release Date: 2022-03-12 + SageMath version 10.2.beta4, Release Date: 2023-09-24 Note that ``make`` is not used at all. All dependencies (including all Python packages) are provided by conda. @@ -162,7 +164,7 @@ After editing any Cython files, rebuild the Sage library using:: In order to update the conda environment later, you can run:: - $ mamba env update --file src/environment-dev.yml --name sage-dev + $ mamba env update --file src/environment-dev-3.11.yml --name sage-dev To build the documentation, use:: From 5c0bac856fb04cf589209ca5059a02f121e4b3ff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 12:37:35 -0700 Subject: [PATCH 03/32] bootstrap, Makefile (bootstrap-clean): Update for versioned environment-... files --- Makefile | 6 +----- bootstrap | 8 ++++---- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 11a7b77665a..ae90db4e9e2 100644 --- a/Makefile +++ b/Makefile @@ -158,11 +158,7 @@ bootstrap-clean: rm -rf config/install-sh config/compile config/config.guess config/config.sub config/missing configure build/make/Makefile-auto.in rm -f src/doc/en/installation/*.txt rm -rf src/doc/en/reference/spkg/*.rst - rm -f environment.yml - rm -f src/environment.yml - rm -f src/environment-dev.yml - rm -f environment-optional.yml - rm -f src/environment-optional.yml + for a in environment environment-optional src/environment src/environment-dev src/environment-optional; do rm -f $$a.yml $$a-3.[89].yml $$a-3.1[0-9].yml; done rm -f src/Pipfile rm -f src/pyproject.toml rm -f src/requirements.txt diff --git a/bootstrap b/bootstrap index 54d0a156239..2ae99949025 100755 --- a/bootstrap +++ b/bootstrap @@ -226,10 +226,10 @@ save () { build/make/Makefile-auto.in \ src/doc/en/installation/*.txt \ src/doc/en/reference/spkg/*.rst \ - environment.yml \ - src/environment.yml \ - environment-optional.yml \ - src/environment-optional.yml \ + environment-3.[89].yml environment-3.1[0-9].yml \ + src/environment-3.[89].yml src/environment-3.1[0-9].yml \ + environment-optional-3.[89].yml environment-optional-3.1[0-9].yml \ + src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \ src/Pipfile \ src/pyproject.toml \ src/requirements.txt \ From c2be172347daec2c6c292e248eb598b406776577 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 12:43:59 -0700 Subject: [PATCH 04/32] tox.ini (local-conda-environment): Update for versioned environment-... files --- tox.ini | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index c084e4c821e..9e124a281a6 100644 --- a/tox.ini +++ b/tox.ini @@ -527,9 +527,14 @@ setenv = local-conda-miniconda: CONDA_INSTALLER_FILE=Miniconda3-latest-{env:CONDA_OS}-x86_64.sh local-conda: SETENV=. {env:CONDA_PREFIX}/bin/activate base local-conda-environment: CONDA_SAGE_ENVIRONMENT=sage-build - local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment.yml - local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional.yml + local-conda-environment: CONDA_SAGE_ENVIRONMENT_DIR= + local-conda-environment-src: CONDA_SAGE_ENVIRONMENT=sage local-conda-environment-src: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT=sage-dev + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_FILE=environment-dev-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml local-conda-environment: SETENV_CONFIGURE=( {env:CONDA_PREFIX}/bin/conda env create -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} || {env:CONDA_PREFIX}/bin/conda env update -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} ) && . {env:CONDA_PREFIX}/bin/activate {env:CONDA_SAGE_ENVIRONMENT} # # Configuration factors From b3ba591faf23946ff7bdd777a8c865bd474f039c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 13:17:26 -0700 Subject: [PATCH 05/32] .github/workflows/ci-conda.yml: Update for versioned environment*.yml --- .github/workflows/ci-conda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 75babf3ab8c..231f0572f75 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -59,7 +59,7 @@ jobs: with: path: ~/conda_pkgs_dir key: - ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }} + ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }} - name: Setup Conda uses: conda-incubator/setup-miniconda@v2 @@ -69,7 +69,7 @@ jobs: channels: conda-forge,defaults channel-priority: true activate-environment: sage-build - environment-file: src/${{ matrix.conda-env }}.yml + environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml - name: Print Conda environment shell: bash -l {0} From 018de2b76b3fefb561460e3feef3a69fa1c5e3be Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 3 Oct 2023 15:47:29 -0700 Subject: [PATCH 06/32] .github/workflows/ci-conda.yml: Fix environment name --- .github/workflows/ci-conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 231f0572f75..81ab9fe3a61 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -68,7 +68,7 @@ jobs: mamba-version: "*" channels: conda-forge,defaults channel-priority: true - activate-environment: sage-build + activate-environment: sage environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml - name: Print Conda environment From fc9bab5e35bf54388d9f5edd5e9e0d9668d4755c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 14:01:14 -0700 Subject: [PATCH 07/32] tox.ini (conda-environment-{src,dev}), .github/workflows/ci-conda.yml: Use configure --enable-system-site-packages --- .github/workflows/ci-conda.yml | 2 +- tox.ini | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 81ab9fe3a61..e520fa17104 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -83,7 +83,7 @@ jobs: run: | ./bootstrap echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json" - ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) + ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX --enable-system-site-packages $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) echo "::remove-matcher owner=configure-system-package-warning::" echo "::remove-matcher owner=configure-system-package-error::" diff --git a/tox.ini b/tox.ini index 9e124a281a6..0f4540605e2 100644 --- a/tox.ini +++ b/tox.ini @@ -191,6 +191,7 @@ setenv = sitepackages: ENABLE_SYSTEM_SITE_PACKAGES=yes sitepackages: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages conda-environment: SAGE_PACKAGE_LIST_ARGS=_prereq + conda-environment-{src,dev}: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages # Whether to add the system packages needed for bootstrapping EXTRA_SAGE_PACKAGES_0=_bootstrap nobootstrap: EXTRA_SAGE_PACKAGES_0= From b02ef8eb62678bde34550ab1eb9ed8458ed9b099 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 4 Oct 2023 21:24:22 -0700 Subject: [PATCH 08/32] bootstrap-conda: Refactor using sage-get-system-packages --- bootstrap-conda | 90 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 29 deletions(-) diff --git a/bootstrap-conda b/bootstrap-conda index 811dcbf43b0..90d27ab7be7 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -11,45 +11,75 @@ STRIP_COMMENTS="sed s/#.*//;" shopt -s extglob DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))" -BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt)) -SYSTEM_PACKAGES= -OPTIONAL_SYSTEM_PACKAGES= -SAGELIB_SYSTEM_PACKAGES= -SAGELIB_OPTIONAL_SYSTEM_PACKAGES= -DEVELOP_SYSTEM_PACKAGES= +BOOTSTRAP_PACKAGES=_bootstrap +PACKAGES= +OPTIONAL_PACKAGES= +SAGELIB_PACKAGES= +SAGELIB_OPTIONAL_PACKAGES= +DEVELOP_PACKAGES= + for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do PKG_SCRIPTS=build/pkgs/$PKG_BASE SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt PKG_TYPE=$(cat $PKG_SCRIPTS/type) PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)) - if [ -n "PKG_SYSTEM_PACKAGES" ]; then + if [ -n "$PKG_SYSTEM_PACKAGES" ]; then if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then + if grep -q SAGE_PYTHON_PACKAGE_CHECK $PKG_SCRIPTS/spkg-configure.m4; then + # Python package that would need --enable-system-site-packages to be used + # with the Sage distribution, but we do not recommend that for conda. + PKG_SAGELIB_ONLY=yes + else + PKG_SAGELIB_ONLY=no + fi + else + # No spkg-configure, so the Sage distribution is not able to make use of this package. + PKG_SAGELIB_ONLY=yes + fi + [ -n "$BOOTSTRAP_VERBOSE" ] && echo "$PKG_BASE:$PKG_TYPE:$PKG_SAGELIB_ONLY" + if [ $PKG_SAGELIB_ONLY = no ]; then case "$PKG_BASE:$PKG_TYPE" in *:standard) - SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac else case "$PKG_BASE:$PKG_TYPE" in *:standard) - SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac fi fi done +unset PKG_SYSTEM_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Collected:" && set | grep PACKAGES= + +# Translate to system packages +export ENABLE_SYSTEM_SITE_PACKAGES=yes # Disable filtering in sage-get-system-packages +SYSTEM_PACKAGES=$(sage-get-system-packages conda $PACKAGES) +BOOTSTRAP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $BOOTSTRAP_PACKAGES) +OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $OPTIONAL_PACKAGES) +SAGELIB_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_PACKAGES) +SAGELIB_OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_OPTIONAL_PACKAGES) +DEVELOP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $DEVELOP_PACKAGES) +unset ENABLE_SYSTEM_SITE_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES= + echo >&2 $0:$LINENO: generate conda environment files for python_version in 3.9 3.10 3.11; do ( @@ -63,7 +93,7 @@ for python_version in 3.9 3.10 3.11; do echo " - $pkg" done echo " # Packages needed for ./bootstrap" - for pkg in $BOOTSTRAP_PACKAGES; do + for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do echo " - $pkg" done ) > environment-$python_version.yml @@ -84,23 +114,25 @@ for python_version in 3.9 3.10 3.11; do ) > environment-optional-$python_version.yml ( - ( - sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml - echo " # Additional dev tools" - for pkg in $DEVELOP_SYSTEM_PACKAGES; do - echo " - $pkg" - done - ) >&4 - ( - cat src/environment-$python_version.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done - ) >&5 + sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + echo " # Additional dev tools" + for pkg in $DEVELOP_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-dev-$python_version.yml + + ( + cat src/environment-$python_version.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-optional-$python_version.yml + + ( echo >&4 " - pip:" echo >&5 " - pip:" - for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do + for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do PKG_SCRIPTS=build/pkgs/$PKG_BASE SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt if [ ! -f $SYSTEM_PACKAGES_FILE ]; then From 235ce91795439949e3b86f49f37bc0a72dba740b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Sun, 8 Oct 2023 23:23:18 -0700 Subject: [PATCH 09/32] bootstrap-conda: Only one 'python' line + speed up --- bootstrap-conda | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/bootstrap-conda b/bootstrap-conda index 90d27ab7be7..19e56e3c5fa 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -81,14 +81,13 @@ unset ENABLE_SYSTEM_SITE_PACKAGES [ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES= echo >&2 $0:$LINENO: generate conda environment files -for python_version in 3.9 3.10 3.11; do + ( echo "name: sage-build" echo "channels:" echo " - conda-forge" echo " - nodefaults" echo "dependencies:" - echo " - python=$python_version" for pkg in $SYSTEM_PACKAGES; do echo " - $pkg" done @@ -96,38 +95,38 @@ for python_version in 3.9 3.10 3.11; do for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > environment-$python_version.yml + ) > environment-template.yml ( - sed 's/name: sage-build/name: sage/' environment-$python_version.yml + sed 's/name: sage-build/name: sage/' environment-template.yml echo " # Additional packages providing all dependencies for the Sage library" for pkg in $SAGELIB_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-$python_version.yml + ) > src/environment-template.yml ( - cat environment-$python_version.yml + cat environment-template.yml echo " # optional packages" for pkg in $OPTIONAL_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > environment-optional-$python_version.yml + ) > environment-optional-template.yml ( - sed 's/name: sage/name: sage-dev/' src/environment-$python_version.yml + sed 's/name: sage/name: sage-dev/' src/environment-template.yml echo " # Additional dev tools" for pkg in $DEVELOP_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-dev-$python_version.yml + ) > src/environment-dev-template.yml ( - cat src/environment-$python_version.yml + cat src/environment-template.yml echo " # optional packages" for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do echo " - $pkg" done - ) > src/environment-optional-$python_version.yml + ) > src/environment-optional-template.yml ( echo >&4 " - pip:" @@ -152,5 +151,11 @@ for python_version in 3.9 3.10 3.11; do done fi done - ) 4>> src/environment-dev-$python_version.yml 5>> src/environment-optional-$python_version.yml + ) 4>> src/environment-dev-template.yml 5>> src/environment-optional-template.yml + +for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do + for python_version in 3.9 3.10 3.11; do + sed -E 's/^( *- *)python *$/\1python='$python_version'/' $f-template.yml > $f-$python_version.yml + done + rm -f $f-template.yml done From 3332a899abd86dbe03b9c7834eb898cb5d4f86ff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 10:39:48 -0700 Subject: [PATCH 10/32] build/sage_bootstrap/download/mirror_list.py: Remove sagepad.org --- build/sage_bootstrap/download/mirror_list.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 12868bf6084..3a4b9d22fc5 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -200,8 +200,6 @@ def __iter__(self): pass for mirror in self.mirrors: yield mirror - # If all else fails: Try the packages we host ourselves - yield 'http://sagepad.org/' @property def fastest(self): From dd939ff74e271282a217ce5bcbeb8122672bdc0d Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:25:44 -0700 Subject: [PATCH 11/32] sage_bootstrap.{tarball, download.mirror_list}: Lower layout of the mirror to mirror_list --- build/sage_bootstrap/download/mirror_list.py | 4 +++- build/sage_bootstrap/tarball.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 3a4b9d22fc5..62f622b1392 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -199,7 +199,9 @@ def __iter__(self): except KeyError: pass for mirror in self.mirrors: - yield mirror + if not mirror.endswith('/'): + mirror += '/' + yield mirror + '/'.join(['spkg', 'upstream', '${SPKG}']) @property def fastest(self): diff --git a/build/sage_bootstrap/tarball.py b/build/sage_bootstrap/tarball.py index 18e3da97af8..e868c5fa51c 100644 --- a/build/sage_bootstrap/tarball.py +++ b/build/sage_bootstrap/tarball.py @@ -159,7 +159,10 @@ def download(self, allow_upstream=False): successful_download = False log.info('Attempting to download package {0} from mirrors'.format(self.filename)) for mirror in MirrorList(): - url = mirror + '/'.join(['spkg', 'upstream', self.package.name, self.filename]) + url = mirror.replace('${SPKG}', self.package.name) + if not url.endswith('/'): + url += '/' + url += self.filename log.info(url) try: Download(url, destination).run() From 2dfe081632fd4ba254cc6d3e34647f198151ab18 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:28:18 -0700 Subject: [PATCH 12/32] sage_bootstrap.download.mirror_list: Delay downloading/reading/ranking to first use --- build/sage_bootstrap/download/mirror_list.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index 62f622b1392..f03d03aabb4 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -51,7 +51,12 @@ class MirrorList(object): def __init__(self): self.filename = MIRRORLIST_FILENAME - self.mirrors = None + self._mirrors = None + + @property + def mirrors(self): + if self._mirrors is not None: + return self._mirrors try: self.mirrorfile = open(self.filename, 'r+t') @@ -67,8 +72,10 @@ def __init__(self): # process while we waited for the lock? Check again. if self._must_refresh(): self._refresh() - if self.mirrors is None: - self.mirrors = self._load() + if self._mirrors is None: + self._mirrors = self._load() + + return self._mirrors def _load(self, mirror_list=None): """ @@ -147,7 +154,7 @@ def _rank_mirrors(self): log.info('Cannot time mirrors via proxy, using default order') else: timed_mirrors.sort() - self.mirrors = [m[1] for m in timed_mirrors] + self._mirrors = [m[1] for m in timed_mirrors] log.info('Fastest mirror: ' + self.fastest) def _age(self): @@ -176,12 +183,12 @@ def _refresh(self): """ log.info('Downloading the Sage mirror list') try: - with contextlib.closing(urllib.urlopen(self.URL)) as f: + with contextlib.closing(urllib.urlopen(self.url)) as f: mirror_list = f.read().decode("ascii") except IOError: log.critical('Downloading the mirror list failed, using cached version') else: - self.mirrors = self._load(mirror_list) + self._mirrors = self._load(mirror_list) self._rank_mirrors() self._save() From a37f4fdd8261e1871f446593d62cb51903a5dae8 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 12:32:03 -0700 Subject: [PATCH 13/32] sage_bootstrap.download.mirror_list: Use master list of download sources in SAGE_ROOT/.upstream.d --- .upstream.d/10-SAGE_SERVER | 2 + .../20-github.com-sagemath-sage-releases | 2 + .upstream.d/30-www.sagemath.org-mirror_list | 1 + build/sage_bootstrap/download/mirror_list.py | 56 +++++++++++++++++-- 4 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 .upstream.d/10-SAGE_SERVER create mode 100644 .upstream.d/20-github.com-sagemath-sage-releases create mode 100644 .upstream.d/30-www.sagemath.org-mirror_list diff --git a/.upstream.d/10-SAGE_SERVER b/.upstream.d/10-SAGE_SERVER new file mode 100644 index 00000000000..40ee9e19450 --- /dev/null +++ b/.upstream.d/10-SAGE_SERVER @@ -0,0 +1,2 @@ +# When SAGE_SERVER is set, it should be an https/https server in the format of Sage mirrors. +${SAGE_SERVER}/spkg/upstream/${SPKG}/ diff --git a/.upstream.d/20-github.com-sagemath-sage-releases b/.upstream.d/20-github.com-sagemath-sage-releases new file mode 100644 index 00000000000..a9986bf3cce --- /dev/null +++ b/.upstream.d/20-github.com-sagemath-sage-releases @@ -0,0 +1,2 @@ +# Upstream packages as uploaded as GitHub release assets. +https://github.com/sagemath/sage/releases/download/10.1/ diff --git a/.upstream.d/30-www.sagemath.org-mirror_list b/.upstream.d/30-www.sagemath.org-mirror_list new file mode 100644 index 00000000000..951c1df6cfa --- /dev/null +++ b/.upstream.d/30-www.sagemath.org-mirror_list @@ -0,0 +1 @@ +https://www.sagemath.org/mirror_list diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index f03d03aabb4..f18c7b3e4b2 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -19,7 +19,7 @@ log = logging.getLogger() from sage_bootstrap.compat import urllib, urlparse -from sage_bootstrap.env import SAGE_DISTFILES +from sage_bootstrap.env import SAGE_DISTFILES, SAGE_ROOT from fcntl import flock, LOCK_SH, LOCK_EX from errno import ENOLCK @@ -41,16 +41,60 @@ class MirrorListException(RuntimeError): pass -MIRRORLIST_FILENAME = os.path.join(SAGE_DISTFILES, 'mirror_list') +class MirrorList(object): + def __init__(self): + self.sources = [] + upstream_d = os.path.join(SAGE_ROOT, '.upstream.d') + for fname in sorted(os.listdir(upstream_d)): + if '~' in fname or '#' in fname: + # Ignore auto-save and backup files + continue + try: + with open(os.path.join(upstream_d, fname), 'r') as f: + for line in f: + line = line.replace('${SAGE_ROOT}', SAGE_ROOT) + line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES) + if '${SAGE_SERVER}' in line: + SAGE_SERVER = os.environ.get("SAGE_SERVER", "") + if not SAGE_SERVER: + continue + line = line.replace('${SAGE_SERVER}',) + line = line.strip() + if line.startswith('#'): + continue + if not line: + continue + if line.endswith('mirror_list'): + cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2]) + self.sources.append(MirrorList_from_url(line, cache_filename)) + else: + self.sources.append([line]) + except IOError: + # Silently ignore files that do not exist + pass -class MirrorList(object): + def __iter__(self): + """ + Iterate through the list of mirrors. + + This is the main entry point into the mirror list. Every + script should just use this function to try mirrors in order + of preference. This will not just yield the official mirrors, + but also urls for packages that are currently being tested. + """ + for source in self.sources: + for mirror in source: + yield mirror + + +class MirrorList_from_url(object): - URL = 'http://www.sagemath.org/mirror_list' MAXAGE = 24*60*60 # seconds - def __init__(self): - self.filename = MIRRORLIST_FILENAME + def __init__(self, url, filename): + self.url = url + self.filename = filename self._mirrors = None @property From 2323480b507a760d568727b11935949c52ed2d00 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 13:09:38 -0700 Subject: [PATCH 14/32] src/bin/sage-update-version: Update .upstream.d/20-github.com-sagemath-sage-releases --- .../20-github.com-sagemath-sage-releases | 1 + src/bin/sage-update-version | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/.upstream.d/20-github.com-sagemath-sage-releases b/.upstream.d/20-github.com-sagemath-sage-releases index a9986bf3cce..6bccdd7ff0a 100644 --- a/.upstream.d/20-github.com-sagemath-sage-releases +++ b/.upstream.d/20-github.com-sagemath-sage-releases @@ -1,2 +1,3 @@ # Upstream packages as uploaded as GitHub release assets. +# This file is automatically updated by the sage-update-version script. https://github.com/sagemath/sage/releases/download/10.1/ diff --git a/src/bin/sage-update-version b/src/bin/sage-update-version index 29d8c794375..5ace38182ba 100755 --- a/src/bin/sage-update-version +++ b/src/bin/sage-update-version @@ -87,6 +87,25 @@ EOF # Create a top-level VERSION.txt file, which some external utilities rely on echo "$SAGE_VERSION_BANNER" > "$SAGE_ROOT/VERSION.txt" +# Add version to the front of GitHub release assets URLs. +SAGE_MINOR_VERSION=${SAGE_VERSION//.alpha*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.beta*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.dev*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.post*/} +SAGE_MINOR_VERSION=${SAGE_MINOR_VERSION//.rc*/} +( echo "https://github.com/sagemath/sage/releases/download/$SAGE_MINOR_VERSION/" + sed '/^#/d' "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" +) | uniq | head -n 3 > "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" +( cat < "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" +rm -f "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" + +exit + # Regenerate auto-generated files tarball "$SAGE_ROOT/bootstrap" -s From 7e6526068c86068d840651c97593ddd445f38336 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 14:53:41 -0700 Subject: [PATCH 15/32] sage_bootstrap.download.mirror_list: Skip empty lines earlier --- build/sage_bootstrap/download/mirror_list.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build/sage_bootstrap/download/mirror_list.py b/build/sage_bootstrap/download/mirror_list.py index f18c7b3e4b2..a8baa66da2d 100644 --- a/build/sage_bootstrap/download/mirror_list.py +++ b/build/sage_bootstrap/download/mirror_list.py @@ -53,18 +53,18 @@ def __init__(self): try: with open(os.path.join(upstream_d, fname), 'r') as f: for line in f: + line = line.strip() + if line.startswith('#'): + continue + if not line: + continue line = line.replace('${SAGE_ROOT}', SAGE_ROOT) line = line.replace('${SAGE_DISTFILES}', SAGE_DISTFILES) if '${SAGE_SERVER}' in line: SAGE_SERVER = os.environ.get("SAGE_SERVER", "") if not SAGE_SERVER: continue - line = line.replace('${SAGE_SERVER}',) - line = line.strip() - if line.startswith('#'): - continue - if not line: - continue + line = line.replace('${SAGE_SERVER}', SAGE_SERVER) if line.endswith('mirror_list'): cache_filename = os.path.join(SAGE_DISTFILES, line.rpartition('/')[2]) self.sources.append(MirrorList_from_url(line, cache_filename)) From 371be50a9df9e2ed693a6c4fafaace67f9308e3b Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 14:55:32 -0700 Subject: [PATCH 16/32] src/bin/sage-update-version: Also commit --- src/bin/sage-update-version | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/bin/sage-update-version b/src/bin/sage-update-version index 5ace38182ba..3b7358d2929 100755 --- a/src/bin/sage-update-version +++ b/src/bin/sage-update-version @@ -104,8 +104,6 @@ EOF ) > "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" rm -f "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases.tmp" -exit - # Regenerate auto-generated files tarball "$SAGE_ROOT/bootstrap" -s @@ -125,6 +123,7 @@ git commit -m "Updated SageMath version to $SAGE_VERSION" -- \ "$SAGE_ROOT/build/pkgs/configure/package-version.txt" \ "$SAGE_ROOT/build/pkgs/*/install-requires.txt" \ "$SAGE_ROOT"/pkgs/*/VERSION.txt \ + "$SAGE_ROOT/.upstream.d/20-github.com-sagemath-sage-releases" \ || die "Error committing to the repository." git tag -a "$SAGE_VERSION" -m "$SAGE_VERSION_BANNER" \ From 423e48a3f54de03df73afe1d7fa33e8394af8d3a Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 15:28:13 -0700 Subject: [PATCH 17/32] build/bin/write-dockerfile.sh: ADD .upstream.d --- build/bin/write-dockerfile.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 530c2c6ec63..5e5eb522de5 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -223,6 +223,7 @@ $ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg. $ADD m4 ./m4 $ADD pkgs pkgs $ADD build ./build +$ADD .upstream.d ./.upstream.d ARG BOOTSTRAP=./bootstrap $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN From 73ab316abb4ffef3953b46bf0620d4a4cb2866e1 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 9 Oct 2023 19:47:55 -0700 Subject: [PATCH 18/32] bootstrap-conda (src/environment-dev*.yml): Suppress pip section, for now --- bootstrap-conda | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-conda b/bootstrap-conda index 19e56e3c5fa..faa29513db4 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -151,7 +151,7 @@ echo >&2 $0:$LINENO: generate conda environment files done fi done - ) 4>> src/environment-dev-template.yml 5>> src/environment-optional-template.yml + ) 4>> /dev/null 5>> src/environment-optional-template.yml for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do for python_version in 3.9 3.10 3.11; do From f4299939f1636afe30f5fca76547e4fde5942985 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 10:23:38 -0700 Subject: [PATCH 19/32] .ci/retrofit-worktree.sh: New, factored out from .github/workflows/build*.yml --- .ci/retrofit-worktree.sh | 39 +++++++++++++++++++++++++++++ .github/workflows/build.yml | 24 +----------------- .github/workflows/doc-build-pdf.yml | 25 +----------------- .github/workflows/doc-build.yml | 25 +----------------- 4 files changed, 42 insertions(+), 71 deletions(-) create mode 100755 .ci/retrofit-worktree.sh diff --git a/.ci/retrofit-worktree.sh b/.ci/retrofit-worktree.sh new file mode 100755 index 00000000000..f926e59059e --- /dev/null +++ b/.ci/retrofit-worktree.sh @@ -0,0 +1,39 @@ +#!/bin/sh +if [ $# != 2 ]; then + echo >&2 "usage: $0 WORKTREE_NAME WORKTREE_DIRECTORY" + echo >&2 "Ensures that the current working directory is a git repository," + echo >&2 "then makes WORKTREE_DIRECTORY a git worktree named WORKTREE_NAME." +fi +WORKTREE_NAME="$1" +WORKTREE_DIRECTORY="$2" + +export GIT_AUTHOR_NAME="ci-sage workflow" +export GIT_AUTHOR_EMAIL="ci-sage@example.com" +export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME" +export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL" + +set -ex + +# If actions/checkout downloaded our source tree using the GitHub REST API +# instead of with git (because do not have git installed in our image), +# we first make the source tree a repo. +if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi + +# Tag this state of the source tree "new". This is what we want to build and test. +git tag -f new + +# Our container image contains a source tree in $WORKTREE_DIRECTORY with a full build of Sage. +# But $WORKTREE_DIRECTORY is not a git repository. +# We make $WORKTREE_DIRECTORY a worktree whose index is at tag "new". +# We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) +# Then we update worktree and index with "git reset --hard new". +# (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) +# Finally we reset the index to "old". (This keeps all mtimes unchanged.) +# The changed files now show up as uncommitted changes. +# The final "git add -N" makes sure that files that were added in "new" do not show +# as untracked files, which would be removed by "git clean -fx". +git worktree add --detach $WORKTREE_NAME +rm -rf $WORKTREE_DIRECTORY/.git && mv $WORKTREE_NAME/.git $WORKTREE_DIRECTORY/ +rm -rf $WORKTREE_NAME && ln -s $WORKTREE_DIRECTORY $WORKTREE_NAME +if [ ! -f $WORKTREE_NAME/.gitignore ]; then cp .gitignore $WORKTREE_NAME/; fi +(cd $WORKTREE_NAME && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 062fa973ac3..89ed87ecf91 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,30 +69,8 @@ jobs: id: worktree run: | set -ex - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" git config --global --add safe.directory $(pwd) - # If actions/checkout downloaded our source tree using the GitHub REST API - # instead of with git (because do not have git installed in our image), - # we first make the source tree a repo. - if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi - # Tag this state of the source tree "new". This is what we want to build and test. - git tag -f new - # Our container image contains a source tree in /sage with a full build of Sage. - # But /sage is not a git repository. - # We make /sage a worktree whose index is at tag "new". - # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) - # Then we update worktree and index with "git reset --hard new". - # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) - # Finally we reset the index to "old". (This keeps all mtimes unchanged.) - # The changed files now show up as uncommitted changes. - # The final "git add -N" makes sure that files that were added in "new" do not show - # as untracked files, which would be removed by "git clean -fx". - git worktree add --detach worktree-image - rm -rf /sage/.git && mv worktree-image/.git /sage/ - rm -rf worktree-image && ln -s /sage worktree-image - if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi - (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status) + .ci/retrofit-worktree.sh worktree-image /sage - name: Download upstream artifact uses: actions/download-artifact@v3 diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 0413954210b..34c96e70a40 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -62,31 +62,8 @@ jobs: - name: Add prebuilt tree as a worktree id: worktree run: | - set -ex - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" git config --global --add safe.directory $(pwd) - # If actions/checkout downloaded our source tree using the GitHub REST API - # instead of with git (because do not have git installed in our image), - # we first make the source tree a repo. - if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi - # Tag this state of the source tree "new". This is what we want to build and test. - git tag -f new - # Our container image contains a source tree in /sage with a full build of Sage. - # But /sage is not a git repository. - # We make /sage a worktree whose index is at tag "new". - # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) - # Then we update worktree and index with "git reset --hard new". - # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) - # Finally we reset the index to "old". (This keeps all mtimes unchanged.) - # The changed files now show up as uncommitted changes. - # The final "git add -N" makes sure that files that were added in "new" do not show - # as untracked files, which would be removed by "git clean -fx". - git worktree add --detach worktree-image - rm -rf /sage/.git && mv worktree-image/.git /sage/ - rm -rf worktree-image && ln -s /sage worktree-image - if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi - (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status) + .ci/retrofit-worktree.sh worktree-image /sage # Keep track of changes to built HTML new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 355e07ab78e..31354c8e531 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -53,31 +53,8 @@ jobs: - name: Add prebuilt tree as a worktree id: worktree run: | - set -ex - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" git config --global --add safe.directory $(pwd) - # If actions/checkout downloaded our source tree using the GitHub REST API - # instead of with git (because do not have git installed in our image), - # we first make the source tree a repo. - if [ ! -d .git ]; then git init && git add -A && git commit --quiet -m "new"; fi - # Tag this state of the source tree "new". This is what we want to build and test. - git tag -f new - # Our container image contains a source tree in /sage with a full build of Sage. - # But /sage is not a git repository. - # We make /sage a worktree whose index is at tag "new". - # We then commit the current sources and set the tag "old". (This keeps all mtimes unchanged.) - # Then we update worktree and index with "git reset --hard new". - # (This keeps mtimes of unchanged files unchanged and mtimes of changed files newer than unchanged files.) - # Finally we reset the index to "old". (This keeps all mtimes unchanged.) - # The changed files now show up as uncommitted changes. - # The final "git add -N" makes sure that files that were added in "new" do not show - # as untracked files, which would be removed by "git clean -fx". - git worktree add --detach worktree-image - rm -rf /sage/.git && mv worktree-image/.git /sage/ - rm -rf worktree-image && ln -s /sage worktree-image - if [ ! -f worktree-image/.gitignore ]; then cp .gitignore worktree-image/; fi - (cd worktree-image && git add -A && git commit --quiet --allow-empty -m "old" -a && git tag -f old && git reset --hard new && git reset --quiet old && git add -N . && git status) + .ci/retrofit-worktree.sh worktree-image /sage # Keep track of changes to built HTML new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") From 47f6a987b914833c259c95992933e0e13903cae0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 12:38:50 -0700 Subject: [PATCH 20/32] src/tox.ini (pyright): Remove old version pin, do not rely on global node installation --- src/tox.ini | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/tox.ini b/src/tox.ini index 5021d0aa691..81ba69cc25d 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -78,12 +78,13 @@ commands = [testenv:pyright] description = run the static typing checker pyright -deps = pyright +deps = + pyright setenv = {[sagedirect]setenv} HOME={envdir} - # Fix version, see .github/workflows/build.yml - PYRIGHT_PYTHON_FORCE_VERSION=1.1.232 + PYRIGHT_PYTHON_FORCE_VERSION=latest + PYRIGHT_PYTHON_GLOBAL_NODE=false allowlist_externals = {[sagedirect]allowlist_externals} ## We run pyright from within the sage-env so that SAGE_LOCAL is available. ## pyright is already configured via SAGE_ROOT/pyrightconfig.json to use our venv. From 47e27832cdea3c282eae14171127d7ccd169d629 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 12:40:24 -0700 Subject: [PATCH 21/32] .github/workflows/build.yml: Simplify use of pyright by going through 'sage -tox' --- .github/workflows/build.yml | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 062fa973ac3..f8f852d6483 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -126,20 +126,9 @@ jobs: MAKE: make -j2 --output-sync=recurse SAGE_NUM_THREADS: 2 - - name: Set up node to install pyright - if: always() && steps.worktree.outcome == 'success' - uses: actions/setup-node@v3 - with: - node-version: '12' - - - name: Install pyright - if: always() && steps.worktree.outcome == 'success' - # Fix to v232 due to bug https://github.com/microsoft/pyright/issues/3239 - run: npm install -g pyright@1.1.232 - - name: Static code check with pyright if: always() && steps.worktree.outcome == 'success' - run: pyright + run: ./sage -tox -e pyright working-directory: ./worktree-image - name: Clean (fallback to non-incremental) From 94c8b8855aefaf5695fccda84bd47d57b8a4eacc Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 21:06:19 -0700 Subject: [PATCH 22/32] Fixup --- .github/workflows/doc-build-pdf.yml | 2 ++ .github/workflows/doc-build.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml index 34c96e70a40..50a836b3b10 100644 --- a/.github/workflows/doc-build-pdf.yml +++ b/.github/workflows/doc-build-pdf.yml @@ -63,6 +63,8 @@ jobs: id: worktree run: | git config --global --add safe.directory $(pwd) + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" .ci/retrofit-worktree.sh worktree-image /sage # Keep track of changes to built HTML new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml index 31354c8e531..f8b338f50d6 100644 --- a/.github/workflows/doc-build.yml +++ b/.github/workflows/doc-build.yml @@ -54,6 +54,8 @@ jobs: id: worktree run: | git config --global --add safe.directory $(pwd) + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" .ci/retrofit-worktree.sh worktree-image /sage # Keep track of changes to built HTML new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") From 8bf3bcbb2b3e50b95587af7673dc6e5e9f95c237 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 11 Oct 2023 09:33:04 -0700 Subject: [PATCH 23/32] src/tox.ini (pyright): Do not restrict to subdirectories by default --- src/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tox.ini b/src/tox.ini index 81ba69cc25d..f0e848be647 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -93,7 +93,7 @@ allowlist_externals = {[sagedirect]allowlist_externals} ## and may run out of memory. When no files/directories are given, just run it ## on the packages that already have typing annotations. commands = - {env:SAGE} -sh -c 'pyright {posargs:{toxinidir}/sage/combinat {toxinidir}/sage/manifolds}' + {env:SAGE} -sh -c 'pyright {posargs:}' [testenv:pycodestyle] description = From 81e1afa176c3906a26b8515632b0a0c937602fb2 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 11 Oct 2023 09:54:22 -0700 Subject: [PATCH 24/32] src/tox.ini (pyright): Pin to 1.1.331 (latest) --- src/tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tox.ini b/src/tox.ini index f0e848be647..3b935f9b9b7 100644 --- a/src/tox.ini +++ b/src/tox.ini @@ -83,7 +83,7 @@ deps = setenv = {[sagedirect]setenv} HOME={envdir} - PYRIGHT_PYTHON_FORCE_VERSION=latest + PYRIGHT_PYTHON_FORCE_VERSION=1.1.331 PYRIGHT_PYTHON_GLOBAL_NODE=false allowlist_externals = {[sagedirect]allowlist_externals} ## We run pyright from within the sage-env so that SAGE_LOCAL is available. From e3dce3f4841721316aaa475b4fba5e20fca319b0 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 12 Oct 2023 09:11:39 -0700 Subject: [PATCH 25/32] conda: Do not pin setuptools, instead add legacy editable to install instructions --- .github/workflows/ci-conda.yml | 2 +- build/pkgs/setuptools/distros/conda.txt | 5 +---- src/doc/en/installation/conda.rst | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 75babf3ab8c..2477d39794e 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -92,7 +92,7 @@ jobs: run: | # Use --no-deps and pip check below to verify that all necessary dependencies are installed via conda. pip install --no-build-isolation --no-deps -v -v -e ./pkgs/sage-conf ./pkgs/sage-setup - pip install --no-build-isolation --no-deps -v -v -e ./src + pip install --no-build-isolation --no-deps --config-settings editable_mode=compat -v -v -e ./src env: SAGE_NUM_THREADS: 2 diff --git a/build/pkgs/setuptools/distros/conda.txt b/build/pkgs/setuptools/distros/conda.txt index 2602d0f6344..49fe098d9e6 100644 --- a/build/pkgs/setuptools/distros/conda.txt +++ b/build/pkgs/setuptools/distros/conda.txt @@ -1,4 +1 @@ -# Set this bound until https://github.com/sagemath/sage/issues/34209 adds support for PEP660 editable builds -# By setting this version bound, we avoid having to include the following in our installation instructions. -# export SETUPTOOLS_ENABLE_FEATURES=legacy-editable -"setuptools<64" +setuptools diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst index 6b4cb871dd7..1b12e2d5057 100644 --- a/src/doc/en/installation/conda.rst +++ b/src/doc/en/installation/conda.rst @@ -158,7 +158,7 @@ suffices to restart Sage. After editing any Cython files, rebuild the Sage library using:: - $ pip install --no-build-isolation -v -v --editable src + $ pip install --no-build-isolation --config-settings editable_mode=compat -v -v --editable src In order to update the conda environment later, you can run:: From c60de9a4944212184d77034baf5a07f37727ee29 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 27 Sep 2023 11:44:56 -0700 Subject: [PATCH 26/32] build/pkgs/setuptools_wheel: Make distros a symlink too --- build/pkgs/setuptools_wheel/distros | 1 + build/pkgs/setuptools_wheel/distros/repology.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) create mode 120000 build/pkgs/setuptools_wheel/distros delete mode 100644 build/pkgs/setuptools_wheel/distros/repology.txt diff --git a/build/pkgs/setuptools_wheel/distros b/build/pkgs/setuptools_wheel/distros new file mode 120000 index 00000000000..b22be5c01a6 --- /dev/null +++ b/build/pkgs/setuptools_wheel/distros @@ -0,0 +1 @@ +../setuptools/distros \ No newline at end of file diff --git a/build/pkgs/setuptools_wheel/distros/repology.txt b/build/pkgs/setuptools_wheel/distros/repology.txt deleted file mode 100644 index 845a263c318..00000000000 --- a/build/pkgs/setuptools_wheel/distros/repology.txt +++ /dev/null @@ -1 +0,0 @@ -python:setuptools From 756e5c431386cba8a23d096330cac7d5bf036f2f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 11 Oct 2023 09:20:58 -0700 Subject: [PATCH 27/32] src/doc/en/installation/conda.rst: Restore 'bootstrap' in instructions, lost in #36367 --- src/doc/en/installation/conda.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst index 1b12e2d5057..cb2095bdd0b 100644 --- a/src/doc/en/installation/conda.rst +++ b/src/doc/en/installation/conda.rst @@ -134,8 +134,9 @@ Here we assume that you are using a git checkout. installed. You can use the additional option ``python=3.9`` in the above ``env create`` command to select another Python version (here 3.9). - - Install the build prerequisites and the Sage library:: + - Bootstrap the source tree and install the build prerequisites and the Sage library:: + $ ./bootstrap $ pip install --no-build-isolation -v -v --editable ./pkgs/sage-conf_conda ./pkgs/sage-setup $ pip install --no-build-isolation --config-settings editable_mode=compat -v -v --editable ./src From ddbbdc5fb4263681ac7026d6d0b4e5de0bd63c26 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Wed, 11 Oct 2023 14:30:32 -0700 Subject: [PATCH 28/32] pkgs/sage-conf_conda/setup.py: Always run configure --- pkgs/sage-conf_conda/setup.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pkgs/sage-conf_conda/setup.py b/pkgs/sage-conf_conda/setup.py index 9e8ac353f4f..bc46900e2d7 100644 --- a/pkgs/sage-conf_conda/setup.py +++ b/pkgs/sage-conf_conda/setup.py @@ -30,22 +30,22 @@ def run(self): if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')): print(f'Reusing configured SAGE_ROOT={SAGE_ROOT}') - else: - cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc" - cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"' - cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)' - print(f"Running {cmd}") - sys.stdout.flush() - if os.system(cmd) != 0: - if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')): - print("Warning: A configuration has been written, but the configure script has exited with an error. " - "Carefully check any messages above before continuing.") - else: - print(f"Error: The configure script has failed; this may be caused by missing build prerequisites.") - sys.stdout.flush() - PREREQ_SPKG = "_prereq bzip2 xz libffi" # includes python3 SPKG_DEPCHECK packages - os.system(f'cd {SAGE_ROOT} && export PACKAGES="$(build/bin/sage-get-system-packages conda {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command conda --verbose --sudo install $PACKAGES && echo ""') - raise SetupError("configure failed") + + cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc" + cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"' + cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)' + print(f"Running {cmd}") + sys.stdout.flush() + if os.system(cmd) != 0: + if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')): + print("Warning: A configuration has been written, but the configure script has exited with an error. " + "Carefully check any messages above before continuing.") + else: + print(f"Error: The configure script has failed; this may be caused by missing build prerequisites.") + sys.stdout.flush() + PREREQ_SPKG = "_prereq bzip2 xz libffi" # includes python3 SPKG_DEPCHECK packages + os.system(f'cd {SAGE_ROOT} && export PACKAGES="$(build/bin/sage-get-system-packages conda {PREREQ_SPKG})" && [ -n "$PACKAGES" ] && echo "You can install the required build prerequisites using the following shell command" && echo "" && build/bin/sage-print-system-package-command conda --verbose --sudo install $PACKAGES && echo ""') + raise SetupError("configure failed") # In this mode, we never run "make". From d0cae8008f7b56f8fc752a5e297c785f3150e0d3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Thu, 12 Oct 2023 09:10:12 -0700 Subject: [PATCH 29/32] pkgs/sage-conf_conda/setup.py: Remove 'reusing configured SAGE_ROOT' message --- pkgs/sage-conf_conda/setup.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkgs/sage-conf_conda/setup.py b/pkgs/sage-conf_conda/setup.py index bc46900e2d7..96f163dfe5a 100644 --- a/pkgs/sage-conf_conda/setup.py +++ b/pkgs/sage-conf_conda/setup.py @@ -28,9 +28,6 @@ def run(self): raise SetupError('No conda environment is active. ' 'See https://doc.sagemath.org/html/en/installation/conda.html on how to get started.') - if os.path.exists(os.path.join(SAGE_ROOT, 'config.status')): - print(f'Reusing configured SAGE_ROOT={SAGE_ROOT}') - cmd = f"cd {SAGE_ROOT} && ./configure --enable-build-as-root --with-system-python3=force --disable-notebook --disable-sagelib --disable-sage_conf --disable-doc" cmd += ' --with-python=$CONDA_PREFIX/bin/python --prefix="$CONDA_PREFIX"' cmd += ' $(for pkg in $(PATH="build/bin:$PATH" build/bin/sage-package list :standard: --exclude rpy2 --has-file spkg-configure.m4 --has-file distros/conda.txt); do echo --with-system-$pkg=force; done)' From 0bfb487e1a4dc78569bb523802ecc65606a23454 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 17:55:16 -0700 Subject: [PATCH 30/32] build/bin/write-dockerfile.sh, tox.ini (docker-incremental): Make incremental build more incremental via git, via git, .ci/retrofit-worktree.sh --- build/bin/write-dockerfile.sh | 49 +++++++++++++++++++++++++++-------- tox.ini | 6 +++-- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/build/bin/write-dockerfile.sh b/build/bin/write-dockerfile.sh index 5e5eb522de5..8e4ad8a1a54 100755 --- a/build/bin/write-dockerfile.sh +++ b/build/bin/write-dockerfile.sh @@ -213,17 +213,34 @@ cat <> .gitignore && \ + ./.ci/retrofit-worktree.sh worktree-image /sage); \ + else \ + for a in local logs; do \ + if [ -d /sage/\$a ]; then mv /sage/\$a /new/; fi; \ + done; \ + rm -rf /sage; \ + mv /new /sage; \ + fi; \ + else \ + mv /new /sage; \ + fi WORKDIR /sage -$ADD Makefile VERSION.txt COPYING.txt condarc.yml README.md bootstrap bootstrap-conda configure.ac sage .homebrew-build-env tox.ini Pipfile.m4 ./ -$ADD config/config.rpath config/config.rpath -$ADD src/doc/bootstrap src/doc/bootstrap -$ADD src/bin src/bin -$ADD src/Pipfile.m4 src/pyproject.toml.m4 src/requirements.txt.m4 src/setup.cfg.m4 src/VERSION.txt src/ -$ADD m4 ./m4 -$ADD pkgs pkgs -$ADD build ./build -$ADD .upstream.d ./.upstream.d + ARG BOOTSTRAP=./bootstrap $RUN sh -x -c "\${BOOTSTRAP}" $ENDRUN @@ -269,7 +286,17 @@ ENV MAKE="make -j\${NUMPROC}" ARG USE_MAKEFLAGS="-k V=0" ENV SAGE_CHECK=warn ENV SAGE_CHECK_PACKAGES="!cython,!r,!python3,!gap,!cysignals,!linbox,!git,!ppl,!cmake,!rpy2,!sage_sws2rst" -$ADD src src +$ADD .gitignore /new +$ADD src /new/src +RUN if command -v git; then \ + rm -f /sage/.git && \ + cd /new && \ + ./.ci/retrofit-worktree.sh worktree-pre /sage; \ + else \ + rm -rf /sage/src; \ + mv /new/src /sage/src; \ + fi + ARG TARGETS="build" $RUN make SAGE_SPKG="sage-spkg -y -o" \${USE_MAKEFLAGS} \${TARGETS} $ENDRUN diff --git a/tox.ini b/tox.ini index 0f4540605e2..eed3d1310a1 100644 --- a/tox.ini +++ b/tox.ini @@ -185,6 +185,7 @@ setenv = # What system packages should be installed. Default: All standard packages with spkg-configure. SAGE_PACKAGE_LIST_ARGS=--has-file=spkg-configure.m4 :standard: recommended: EXTRA_SAGE_PACKAGES_3=_recommended $(head -n 1 build/pkgs/_recommended/dependencies) + incremental: EXTRA_SAGE_PACKAGES_4=git develop: EXTRA_SAGE_PACKAGES_4=_develop $(head -n 1 build/pkgs/_develop/dependencies) minimal: SAGE_PACKAGE_LIST_ARGS=_prereq maximal: SAGE_PACKAGE_LIST_ARGS=:standard: :optional: @@ -489,8 +490,9 @@ setenv = # docker: FULL_BASE_IMAGE_AND_TAG={env:ARCH_IMAGE_PREFIX:}{env:BASE_IMAGE}{env:ARCH_IMAGE_SUFFIX:}:{env:ARCH_TAG_PREFIX:}{env:BASE_TAG}{env:ARCH_TAG_SUFFIX:} docker-incremental: FULL_BASE_IMAGE_AND_TAG={env:FROM_DOCKER_REPOSITORY:ghcr.io/sagemath/sage/}sage-$(echo {envname} | sed -E "s/(docker-|-incremental|-sitepackages)//g")-{env:FROM_DOCKER_TARGET:with-targets}:{env:FROM_DOCKER_TAG:dev} - docker-incremental: SKIP_SYSTEM_PKG_INSTALL=yes - docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no + # Can SKIP_SYSTEM_PKG_INSTALL if the base image already has git + docker-incremental-{develop,recommended,maximal}: SKIP_SYSTEM_PKG_INSTALL=yes + docker-incremental-sitepackages: SKIP_SYSTEM_PKG_INSTALL=no # docker-nobootstrap: BOOTSTRAP=./bootstrap -D ### From 8e25cbdbbac72902b987ad1c4f82bda784b0fe36 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 20:38:52 -0700 Subject: [PATCH 31/32] .github/workflows/build.yml: Merge doc-build.yml, doc-build-pdf.yml --- .github/workflows/build.yml | 185 ++++++++++++++++++++++++++++ .github/workflows/doc-build-pdf.yml | 130 ------------------- .github/workflows/doc-build.yml | 138 --------------------- 3 files changed, 185 insertions(+), 268 deletions(-) delete mode 100644 .github/workflows/doc-build-pdf.yml delete mode 100644 .github/workflows/doc-build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 87ca884a71b..e5a8c164ea8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -162,3 +162,188 @@ jobs: uses: codecov/codecov-action@v3 with: files: ./worktree-image/coverage.xml + + build-docs: + runs-on: ubuntu-latest + container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev + needs: [get_ci_fixes] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Update system packages + run: | + apt-get update && apt-get install -y git zip + + - name: Add prebuilt tree as a worktree + id: worktree + run: | + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" + git config --global --add safe.directory $(pwd) + .ci/retrofit-worktree.sh worktree-image /sage + # Keep track of changes to built HTML + new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") + + - name: Download upstream artifact + uses: actions/download-artifact@v3 + with: + path: upstream + name: upstream + + - name: Apply CI fixes from sagemath/sage + # After applying the fixes, make sure all changes are marked as uncommitted changes. + run: | + if [ -r upstream/ci_fixes.patch ]; then + (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch + fi + + - name: Incremental build + id: incremental + run: | + # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. + ./bootstrap && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + + - name: Build (fallback to non-incremental) + id: build + if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + run: | + set -ex + make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + + - name: Build docs + id: docbuild + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') + # Always non-incremental because of the concern that + # incremental docbuild may introduce broken links (inter-file references) though build succeeds + run: | + set -ex + export SAGE_USE_CDNS=yes + mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc + make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage + mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git + ./config.status && make doc-html + working-directory: ./worktree-image + env: + MAKE: make -j2 --output-sync=recurse + SAGE_NUM_THREADS: 2 + + - name: Copy docs + id: copy + if: always() && steps.docbuild.outcome == 'success' + run: | + set -ex + mkdir -p ./docs + # Create changelog + echo '## Preview of CHANGES.html' + (cd /sage/local/share/doc/sage/html/en && git diff --name-only) | tee ./docs/CHANGES.txt + (cd /sage/local/share/doc/sage/html/en && git diff; rm -rf .git) > ./docs/html.diff + echo '## Preview of html.diff'; head -n 400 ./docs/html.diff + (echo '

HTML diff'; sed -E 's,(.*),

\1,' ./docs/CHANGES.txt) > ./docs/CHANGES.html + # For some reason the deploy step below cannot find /sage/... + # So copy everything from there to local folder + # We also need to replace the symlinks because netlify is not following them + cp -r -L /sage/local/share/doc/sage/html/en/* ./docs + # Zip everything for increased performance + zip -r docs.zip docs + + - name: Upload docs + if: always() && steps.copy.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: docs + path: docs.zip + + build-docs-pdf: + runs-on: ubuntu-latest + container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} + needs: [get_ci_fixes] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Update system packages + run: | + export PATH="build/bin:$PATH" + eval $(sage-print-system-package-command auto update) + eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) + eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive) + + + - name: Add prebuilt tree as a worktree + id: worktree + run: | + git config --global user.email "ci-sage@example.com" + git config --global user.name "Build & Test workflow" + git config --global --add safe.directory $(pwd) + .ci/retrofit-worktree.sh worktree-image /sage + # Keep track of changes to built HTML + new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") + + - name: Download upstream artifact + uses: actions/download-artifact@v3 + with: + path: upstream + name: upstream + + - name: Apply CI fixes from sagemath/sage + # After applying the fixes, make sure all changes are marked as uncommitted changes. + run: | + if [ -r upstream/ci_fixes.patch ]; then + (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch + fi + + - name: Incremental build + id: incremental + run: | + # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. + ./bootstrap && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + + - name: Build (fallback to non-incremental) + id: build + if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' + run: | + set -ex + make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + + - name: Build docs (PDF) + id: docbuild + if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') + run: make build V=0 && make doc-pdf + working-directory: ./worktree-image + env: + MAKE: make -j2 + SAGE_NUM_THREADS: 2 + + - name: Copy docs + if: always() && steps.docbuild.outcome == 'success' + run: | + # For some reason the deploy step below cannot find /sage/... + # So copy everything from there to local folder + mkdir -p ./docs + cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs + # Zip everything for increased performance + zip -r docs-pdf.zip docs + + - name: Upload docs + if: always() && steps.copy.outcome == 'success' + uses: actions/upload-artifact@v3 + with: + name: docs-pdf + path: docs-pdf.zip diff --git a/.github/workflows/doc-build-pdf.yml b/.github/workflows/doc-build-pdf.yml deleted file mode 100644 index 50a836b3b10..00000000000 --- a/.github/workflows/doc-build-pdf.yml +++ /dev/null @@ -1,130 +0,0 @@ -name: Build documentation (PDF) - -on: - pull_request: - push: - workflow_dispatch: - # Allow to run manually - inputs: - platform: - description: 'Platform' - required: true - default: 'ubuntu-focal-standard' - docker_tag: - description: 'Docker tag' - required: true - default: 'dev' - -concurrency: - # Cancel previous runs of this workflow for the same branch - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - get_ci_fixes: - runs-on: ubuntu-latest - steps: - - name: Checkout - id: checkout - uses: actions/checkout@v4 - - name: Merge CI fixes from sagemath/sage - run: | - .ci/merge-fixes.sh - env: - GH_TOKEN: ${{ github.token }} - - name: Store CI fixes in upstream artifact - run: | - mkdir -p upstream - if git format-patch --stdout test_base > ci_fixes.patch; then - cp ci_fixes.patch upstream/ - fi - - uses: actions/upload-artifact@v3 - with: - path: upstream - name: upstream - - build-docs-pdf: - runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} - needs: [get_ci_fixes] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Update system packages - run: | - export PATH="build/bin:$PATH" - eval $(sage-print-system-package-command auto update) - eval $(sage-print-system-package-command auto --yes --no-install-recommends install zip) - eval $(sage-print-system-package-command auto --spkg --yes --no-install-recommends install git texlive) - - - - name: Add prebuilt tree as a worktree - id: worktree - run: | - git config --global --add safe.directory $(pwd) - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" - .ci/retrofit-worktree.sh worktree-image /sage - # Keep track of changes to built HTML - new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") - - - name: Download upstream artifact - uses: actions/download-artifact@v3 - with: - path: upstream - name: upstream - - - name: Apply CI fixes from sagemath/sage - # After applying the fixes, make sure all changes are marked as uncommitted changes. - run: | - if [ -r upstream/ci_fixes.patch ]; then - (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch - fi - - - name: Incremental build - id: incremental - run: | - # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 - SAGE_NUM_THREADS: 2 - - - name: Build (fallback to non-incremental) - id: build - if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' - run: | - set -ex - make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 - SAGE_NUM_THREADS: 2 - - - name: Build docs (PDF) - id: docbuild - if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') - run: make build V=0 && make doc-pdf - working-directory: ./worktree-image - env: - MAKE: make -j2 - SAGE_NUM_THREADS: 2 - - - name: Copy docs - if: always() && steps.docbuild.outcome == 'success' - run: | - # For some reason the deploy step below cannot find /sage/... - # So copy everything from there to local folder - mkdir -p ./docs - cp -r -L /sage/local/share/doc/sage/pdf/en/* ./docs - # Zip everything for increased performance - zip -r docs-pdf.zip docs - - - name: Upload docs - if: always() && steps.copy.outcome == 'success' - uses: actions/upload-artifact@v3 - with: - name: docs-pdf - path: docs-pdf.zip diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml deleted file mode 100644 index f8b338f50d6..00000000000 --- a/.github/workflows/doc-build.yml +++ /dev/null @@ -1,138 +0,0 @@ -name: Build documentation - -on: - pull_request: - merge_group: - push: - branches: - - master - - develop - workflow_dispatch: - # Allow to run manually - -concurrency: - # Cancel previous runs of this workflow for the same branch - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - get_ci_fixes: - runs-on: ubuntu-latest - steps: - - name: Checkout - id: checkout - uses: actions/checkout@v4 - - name: Merge CI fixes from sagemath/sage - run: | - .ci/merge-fixes.sh - env: - GH_TOKEN: ${{ github.token }} - - name: Store CI fixes in upstream artifact - run: | - mkdir -p upstream - if git format-patch --stdout test_base > ci_fixes.patch; then - cp ci_fixes.patch upstream/ - fi - - uses: actions/upload-artifact@v3 - with: - path: upstream - name: upstream - - build-docs: - runs-on: ubuntu-latest - container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev - needs: [get_ci_fixes] - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Update system packages - run: | - apt-get update && apt-get install -y git zip - - - name: Add prebuilt tree as a worktree - id: worktree - run: | - git config --global --add safe.directory $(pwd) - git config --global user.email "ci-sage@example.com" - git config --global user.name "Build & Test workflow" - .ci/retrofit-worktree.sh worktree-image /sage - # Keep track of changes to built HTML - new_version=$(cat src/VERSION.txt); (cd /sage/local/share/doc/sage/html/en && find . -name "*.html" | xargs sed -i '/class="sidebar-brand-text"/s/Sage [0-9a-z.]* /Sage '$new_version' /'; git init && (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; git add -A && git commit --quiet -m "old") - - - name: Download upstream artifact - uses: actions/download-artifact@v3 - with: - path: upstream - name: upstream - - - name: Apply CI fixes from sagemath/sage - # After applying the fixes, make sure all changes are marked as uncommitted changes. - run: | - if [ -r upstream/ci_fixes.patch ]; then - (cd worktree-image && git commit -q -m "current changes" --allow-empty -a && git am; git reset --quiet old; git add -N .) < upstream/ci_fixes.patch - fi - - - name: Incremental build - id: incremental - run: | - # Now re-bootstrap and build. The build is incremental because we were careful with the timestamps. - ./bootstrap && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 - - - name: Build (fallback to non-incremental) - id: build - if: always() && steps.worktree.outcome == 'success' && steps.incremental.outcome != 'success' - run: | - set -ex - make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage && ./config.status && make build - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 - - - name: Build docs - id: docbuild - if: always() && (steps.incremental.outcome == 'success' || steps.build.outcome == 'success') - # Always non-incremental because of the concern that - # incremental docbuild may introduce broken links (inter-file references) though build succeeds - run: | - set -ex - export SAGE_USE_CDNS=yes - mv /sage/local/share/doc/sage/html/en/.git /sage/.git-doc - make doc-clean doc-uninstall sagelib-clean && git clean -fx src/sage - mkdir -p /sage/local/share/doc/sage/html/en/ && mv /sage/.git-doc /sage/local/share/doc/sage/html/en/.git - ./config.status && make doc-html - working-directory: ./worktree-image - env: - MAKE: make -j2 --output-sync=recurse - SAGE_NUM_THREADS: 2 - - - name: Copy docs - id: copy - if: always() && steps.docbuild.outcome == 'success' - run: | - set -ex - mkdir -p ./docs - # Create changelog - echo '## Preview of CHANGES.html' - (cd /sage/local/share/doc/sage/html/en && git diff --name-only) | tee ./docs/CHANGES.txt - (cd /sage/local/share/doc/sage/html/en && git diff; rm -rf .git) > ./docs/html.diff - echo '## Preview of html.diff'; head -n 400 ./docs/html.diff - (echo '

HTML diff'; sed -E 's,(.*),

\1,' ./docs/CHANGES.txt) > ./docs/CHANGES.html - # For some reason the deploy step below cannot find /sage/... - # So copy everything from there to local folder - # We also need to replace the symlinks because netlify is not following them - cp -r -L /sage/local/share/doc/sage/html/en/* ./docs - # Zip everything for increased performance - zip -r docs.zip docs - - - name: Upload docs - if: always() && steps.copy.outcome == 'success' - uses: actions/upload-artifact@v3 - with: - name: docs - path: docs.zip From 459da442bd7207419bd431bb5e1022378bd0a35f Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 10 Oct 2023 20:46:56 -0700 Subject: [PATCH 32/32] .github/workflows/build.yml: Merge ci-linux-incremental.yml --- .github/workflows/build.yml | 92 ++++++++++++++ .github/workflows/ci-linux-incremental.yml | 132 --------------------- 2 files changed, 92 insertions(+), 132 deletions(-) delete mode 100644 .github/workflows/ci-linux-incremental.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e5a8c164ea8..4dedaff614f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -49,6 +49,31 @@ jobs: path: upstream name: upstream + changed_files: + runs-on: ubuntu-latest + name: List changed packages + outputs: + uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }} + build_targets: ${{ steps.build-targets.outputs.build_targets }} + steps: + - uses: actions/checkout@v4 + - name: Get all packages that have changed + id: changed-packages + uses: tj-actions/changed-files@v38 + with: + files_yaml: | + configures: + - 'build/pkgs/*/spkg-configure.m4' + pkgs: + - 'build/pkgs/**' + - 'pkgs/**' + - name: Determine targets to build + id: build-targets + run: | + echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT + echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT + cat $GITHUB_OUTPUT + build: runs-on: ubuntu-latest container: ghcr.io/sagemath/sage/sage-${{ github.event.inputs.platform || 'ubuntu-focal-standard' }}-with-targets:${{ github.event.inputs.docker_tag || 'dev'}} @@ -163,6 +188,73 @@ jobs: with: files: ./worktree-image/coverage.xml + linux-standard: + needs: [changed_files] + if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}" + uses: ./.github/workflows/docker.yml + with: + # Build incrementally from published Docker image + incremental: true + free_disk_space: true + from_docker_repository: ghcr.io/sagemath/sage/ + from_docker_target: "with-targets" + from_docker_tag: "dev" + docker_targets: "with-targets" + targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest" + tox_system_factors: >- + ["ubuntu-focal", "ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm", + "fedora-30", "fedora-38", "gentoo-python3.11", "debian-bullseye-i386"] + tox_packages_factors: >- + ["standard"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + permissions: + packages: write + + linux-standard-sitepackages: + needs: [changed_files] + if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}" + uses: ./.github/workflows/docker.yml + with: + # Build incrementally from published Docker image + incremental: true + free_disk_space: true + from_docker_repository: ghcr.io/sagemath/sage/ + from_docker_target: "with-targets" + from_docker_tag: "dev" + docker_targets: "with-targets" + targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest" + # Only test systems with a usable system python (>= 3.9) + tox_system_factors: >- + ["ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm", + "fedora-33", "fedora-38", "gentoo-python3.11", "archlinux", "debian-bullseye-i386"] + tox_packages_factors: >- + ["standard-sitepackages"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + permissions: + packages: write + + linux-minimal: + needs: [changed_files] + if: "${{needs.changed_files.outputs.uninstall_targets}}${{needs.changed_files.outputs.build_targets}}" + uses: ./.github/workflows/docker.yml + with: + # Build incrementally from published Docker image + incremental: true + free_disk_space: true + from_docker_repository: ghcr.io/sagemath/sage/ + from_docker_target: "with-targets" + from_docker_tag: "dev" + docker_targets: "with-targets" + targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest" + tox_system_factors: >- + ["ubuntu-focal", "ubuntu-jammy", "ubuntu-mantic", "debian-bullseye", "debian-bookworm", + "fedora-30", "fedora-38", "gentoo-python3.11", "debian-bullseye-i386"] + tox_packages_factors: >- + ["minimal"] + docker_push_repository: ghcr.io/${{ github.repository }}/ + permissions: + packages: write + build-docs: runs-on: ubuntu-latest container: ghcr.io/sagemath/sage/sage-ubuntu-focal-standard-with-targets:dev diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml deleted file mode 100644 index a4aa9ae99c7..00000000000 --- a/.github/workflows/ci-linux-incremental.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: CI Linux incremental - -## This GitHub Actions workflow runs SAGE_ROOT/tox.ini with select environments, -## whenever a GitHub pull request is opened or synchronized in a repository -## where GitHub Actions are enabled. -## -## It builds and checks some sage spkgs as defined in TARGETS. -## -## A job succeeds if there is no error. -## -## The build is run with "make V=0", so the build logs of individual packages are suppressed. -## -## At the end, all package build logs that contain an error are printed out. -## -## After all jobs have finished (or are canceled) and a short delay, -## tar files of all logs are made available as "build artifacts". - -on: - pull_request: - types: - # Defaults - - opened - - synchronize - - reopened - # When a CI label is added - - labeled - workflow_dispatch: - -concurrency: - # Cancel previous runs of this workflow for the same branch - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -permissions: - packages: write - -jobs: - - changed_files: - runs-on: ubuntu-latest - name: List changed packages - outputs: - uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }} - build_targets: ${{ steps.build-targets.outputs.build_targets }} - steps: - - uses: actions/checkout@v4 - - name: Get all packages that have changed - id: changed-packages - uses: tj-actions/changed-files@v38 - with: - files_yaml: | - configures: - - 'build/pkgs/*/spkg-configure.m4' - pkgs: - - 'build/pkgs/**' - - 'pkgs/**' - - name: Determine targets to build - id: build-targets - run: | - echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT - echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([-_.a-z0-9]*)/[^ ]* *,\2-ensure,;'; done | sort -u))" >> $GITHUB_OUTPUT - cat $GITHUB_OUTPUT - - test: - needs: [changed_files] - if: | - github.event_name != 'pull_request' || - ((github.event.action != 'labeled' && - (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') || - contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) || - (github.event.action == 'labeled' && - (github.event.label.name == 'c: packages: optional' || - github.event.label.name == 'c: packages: standard'))) - uses: ./.github/workflows/docker.yml - with: - # Build incrementally from published Docker image - incremental: true - free_disk_space: true - from_docker_repository: ghcr.io/sagemath/sage/ - from_docker_target: "with-targets" - from_docker_tag: "dev" - docker_targets: "with-targets" - targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest" - tox_system_factors: >- - ["ubuntu-focal", - "ubuntu-jammy", - "ubuntu-mantic", - "debian-bullseye", - "debian-bookworm", - "fedora-30", - "fedora-38", - "gentoo-python3.11", - "debian-bullseye-i386"] - tox_packages_factors: >- - ["standard", - "minimal"] - docker_push_repository: ghcr.io/${{ github.repository }}/ - - site: - needs: [changed_files] - if: | - github.event_name != 'pull_request' || - ((github.event.action != 'labeled' && - (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') || - contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) || - (github.event.action == 'labeled' && - (github.event.label.name == 'c: packages: optional' || - github.event.label.name == 'c: packages: standard'))) - uses: ./.github/workflows/docker.yml - with: - # Build incrementally from published Docker image - incremental: true - free_disk_space: true - from_docker_repository: ghcr.io/sagemath/sage/ - from_docker_target: "with-targets" - from_docker_tag: "dev" - docker_targets: "with-targets" - targets: "${{needs.changed_files.outputs.uninstall_targets}} ${{needs.changed_files.outputs.build_targets}} build doc-html ptest" - # Only test systems with a usable system python (>= 3.9) - tox_system_factors: >- - ["ubuntu-jammy", - "ubuntu-mantic", - "debian-bullseye", - "debian-bookworm", - "fedora-33", - "fedora-38", - "gentoo-python3.11", - "archlinux", - "debian-bullseye-i386"] - tox_packages_factors: >- - ["standard-sitepackages"] - docker_push_repository: ghcr.io/${{ github.repository }}/