diff --git a/.devcontainer/onCreate-conda.sh b/.devcontainer/onCreate-conda.sh index eaa045cd27b..2cbb94e3492 100755 --- a/.devcontainer/onCreate-conda.sh +++ b/.devcontainer/onCreate-conda.sh @@ -4,7 +4,7 @@ set -e # Create conda environment ./bootstrap-conda conda install mamba -n base -c conda-forge -y -mamba env create --file src/environment-dev.yml || mamba env update --file src/environment-dev.yml +mamba env create --file src/environment-dev-3.11.yml || mamba env update --file src/environment-dev-3.11.yml conda init bash # Build sage diff --git a/.github/workflows/ci-conda.yml b/.github/workflows/ci-conda.yml index 75babf3ab8c..e520fa17104 100644 --- a/.github/workflows/ci-conda.yml +++ b/.github/workflows/ci-conda.yml @@ -59,7 +59,7 @@ jobs: with: path: ~/conda_pkgs_dir key: - ${{ runner.os }}-conda-${{ hashFiles('src/environment.yml') }} + ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }} - name: Setup Conda uses: conda-incubator/setup-miniconda@v2 @@ -68,8 +68,8 @@ jobs: mamba-version: "*" channels: conda-forge,defaults channel-priority: true - activate-environment: sage-build - environment-file: src/${{ matrix.conda-env }}.yml + activate-environment: sage + environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}.yml - name: Print Conda environment shell: bash -l {0} @@ -83,7 +83,7 @@ jobs: run: | ./bootstrap echo "::add-matcher::.github/workflows/configure-systempackage-problem-matcher.json" - ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) + ./configure --enable-build-as-root --with-python=$CONDA_PREFIX/bin/python --prefix=$CONDA_PREFIX --enable-system-site-packages $(for pkg in $(./sage -package list :standard: --has-file spkg-configure.m4 --has-file distros/conda.txt --exclude rpy2); do echo --with-system-$pkg=force; done) echo "::remove-matcher owner=configure-system-package-warning::" echo "::remove-matcher owner=configure-system-package-error::" diff --git a/.gitignore b/.gitignore index 2cec8a0cf62..43f58abcafe 100644 --- a/.gitignore +++ b/.gitignore @@ -27,11 +27,29 @@ # no longer generated, but may still be in user worktrees /src/lib/pkgconfig +# Environment files generated by bootstrap-conda. +# The files without Python version are no longer generated +# but may still be in users' directories. /environment.yml +/environment-3.9.yml +/environment-3.10.yml +/environment-3.11.yml /environment-optional.yml +/environment-optional-3.9.yml +/environment-optional-3.10.yml +/environment-optional-3.11.yml /src/environment.yml +/src/environment-3.9.yml +/src/environment-3.10.yml +/src/environment-3.11.yml /src/environment-dev.yml +/src/environment-dev-3.9.yml +/src/environment-dev-3.10.yml +/src/environment-dev-3.11.yml /src/environment-optional.yml +/src/environment-optional-3.9.yml +/src/environment-optional-3.10.yml +/src/environment-optional-3.11.yml /src/setup.cfg /src/requirements.txt diff --git a/.gitpod.yml b/.gitpod.yml index 52ac8d7184c..51c2687c5d3 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -8,7 +8,7 @@ tasks: # Create conda environment, then configure and build sage init: >- ./bootstrap-conda - && mamba env create --file src/environment-dev.yml --prefix venv + && mamba env create --file src/environment-dev-3.11.yml --prefix venv && conda config --append envs_dirs $(pwd) && conda activate $(pwd)/venv && ./bootstrap diff --git a/Makefile b/Makefile index 11a7b77665a..ae90db4e9e2 100644 --- a/Makefile +++ b/Makefile @@ -158,11 +158,7 @@ bootstrap-clean: rm -rf config/install-sh config/compile config/config.guess config/config.sub config/missing configure build/make/Makefile-auto.in rm -f src/doc/en/installation/*.txt rm -rf src/doc/en/reference/spkg/*.rst - rm -f environment.yml - rm -f src/environment.yml - rm -f src/environment-dev.yml - rm -f environment-optional.yml - rm -f src/environment-optional.yml + for a in environment environment-optional src/environment src/environment-dev src/environment-optional; do rm -f $$a.yml $$a-3.[89].yml $$a-3.1[0-9].yml; done rm -f src/Pipfile rm -f src/pyproject.toml rm -f src/requirements.txt diff --git a/bootstrap b/bootstrap index 54d0a156239..2ae99949025 100755 --- a/bootstrap +++ b/bootstrap @@ -226,10 +226,10 @@ save () { build/make/Makefile-auto.in \ src/doc/en/installation/*.txt \ src/doc/en/reference/spkg/*.rst \ - environment.yml \ - src/environment.yml \ - environment-optional.yml \ - src/environment-optional.yml \ + environment-3.[89].yml environment-3.1[0-9].yml \ + src/environment-3.[89].yml src/environment-3.1[0-9].yml \ + environment-optional-3.[89].yml environment-optional-3.1[0-9].yml \ + src/environment-optional-3.[89].yml src/environment-optional-3.1[0-9].yml \ src/Pipfile \ src/pyproject.toml \ src/requirements.txt \ diff --git a/bootstrap-conda b/bootstrap-conda index ed4bb9e0d08..faa29513db4 100755 --- a/bootstrap-conda +++ b/bootstrap-conda @@ -11,112 +11,151 @@ STRIP_COMMENTS="sed s/#.*//;" shopt -s extglob DEVELOP_SPKG_PATTERN="@(_develop$(for a in $(head -n 1 build/pkgs/_develop/dependencies); do echo -n "|"$a; done))" -BOOTSTRAP_PACKAGES=$(echo $(${STRIP_COMMENTS} build/pkgs/_bootstrap/distros/conda.txt)) -SYSTEM_PACKAGES= -OPTIONAL_SYSTEM_PACKAGES= -SAGELIB_SYSTEM_PACKAGES= -SAGELIB_OPTIONAL_SYSTEM_PACKAGES= -DEVELOP_SYSTEM_PACKAGES= +BOOTSTRAP_PACKAGES=_bootstrap +PACKAGES= +OPTIONAL_PACKAGES= +SAGELIB_PACKAGES= +SAGELIB_OPTIONAL_PACKAGES= +DEVELOP_PACKAGES= + for PKG_BASE in $(sage-package list --has-file distros/conda.txt --exclude _sagemath); do PKG_SCRIPTS=build/pkgs/$PKG_BASE SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/distros/conda.txt PKG_TYPE=$(cat $PKG_SCRIPTS/type) PKG_SYSTEM_PACKAGES=$(echo $(${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE)) - if [ -n "PKG_SYSTEM_PACKAGES" ]; then + if [ -n "$PKG_SYSTEM_PACKAGES" ]; then if [ -f $PKG_SCRIPTS/spkg-configure.m4 ]; then + if grep -q SAGE_PYTHON_PACKAGE_CHECK $PKG_SCRIPTS/spkg-configure.m4; then + # Python package that would need --enable-system-site-packages to be used + # with the Sage distribution, but we do not recommend that for conda. + PKG_SAGELIB_ONLY=yes + else + PKG_SAGELIB_ONLY=no + fi + else + # No spkg-configure, so the Sage distribution is not able to make use of this package. + PKG_SAGELIB_ONLY=yes + fi + [ -n "$BOOTSTRAP_VERBOSE" ] && echo "$PKG_BASE:$PKG_TYPE:$PKG_SAGELIB_ONLY" + if [ $PKG_SAGELIB_ONLY = no ]; then case "$PKG_BASE:$PKG_TYPE" in *:standard) - SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac else case "$PKG_BASE:$PKG_TYPE" in *:standard) - SAGELIB_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_PACKAGES+=" $PKG_BASE" ;; $DEVELOP_SPKG_PATTERN:*) - DEVELOP_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + DEVELOP_PACKAGES+=" $PKG_BASE" ;; *) - SAGELIB_OPTIONAL_SYSTEM_PACKAGES+=" $PKG_SYSTEM_PACKAGES" + SAGELIB_OPTIONAL_PACKAGES+=" $PKG_BASE" ;; esac fi fi done +unset PKG_SYSTEM_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Collected:" && set | grep PACKAGES= + +# Translate to system packages +export ENABLE_SYSTEM_SITE_PACKAGES=yes # Disable filtering in sage-get-system-packages +SYSTEM_PACKAGES=$(sage-get-system-packages conda $PACKAGES) +BOOTSTRAP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $BOOTSTRAP_PACKAGES) +OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $OPTIONAL_PACKAGES) +SAGELIB_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_PACKAGES) +SAGELIB_OPTIONAL_SYSTEM_PACKAGES=$(sage-get-system-packages conda $SAGELIB_OPTIONAL_PACKAGES) +DEVELOP_SYSTEM_PACKAGES=$(sage-get-system-packages conda $DEVELOP_PACKAGES) +unset ENABLE_SYSTEM_SITE_PACKAGES + +[ -n "$BOOTSTRAP_VERBOSE" ] && echo "## Translated to system:" && set | grep SYSTEM_PACKAGES= + echo >&2 $0:$LINENO: generate conda environment files -( - echo "name: sage-build" - echo "channels:" - echo " - conda-forge" - echo " - nodefaults" - echo "dependencies:" - for pkg in $SYSTEM_PACKAGES; do - echo " - $pkg" - done - echo " # Packages needed for ./bootstrap" - for pkg in $BOOTSTRAP_PACKAGES; do - echo " - $pkg" - done -) > environment.yml -( - sed 's/name: sage-build/name: sage/' environment.yml - echo " # Additional packages providing all dependencies for the Sage library" - for pkg in $SAGELIB_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment.yml + ( + echo "name: sage-build" + echo "channels:" + echo " - conda-forge" + echo " - nodefaults" + echo "dependencies:" + for pkg in $SYSTEM_PACKAGES; do + echo " - $pkg" + done + echo " # Packages needed for ./bootstrap" + for pkg in $BOOTSTRAP_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > environment-template.yml + ( + sed 's/name: sage-build/name: sage/' environment-template.yml + echo " # Additional packages providing all dependencies for the Sage library" + for pkg in $SAGELIB_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-template.yml -( - sed 's/name: sage/name: sage-dev/' src/environment.yml - echo " # Additional dev tools" - for pkg in $DEVELOP_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-dev.yml + ( + cat environment-template.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > environment-optional-template.yml -( - cat environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > environment-optional.yml + ( + sed 's/name: sage/name: sage-dev/' src/environment-template.yml + echo " # Additional dev tools" + for pkg in $DEVELOP_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-dev-template.yml -( - cat src/environment.yml - echo " # optional packages" - for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do - echo " - $pkg" - done -) > src/environment-optional.yml -( - echo >&4 " - pip:" - echo >&5 " - pip:" - for PKG_BASE in $((sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src) | sort); do - PKG_SCRIPTS=build/pkgs/$PKG_BASE - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt - if [ ! -f $SYSTEM_PACKAGES_FILE ]; then - SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt - fi - PKG_TYPE=$(cat $PKG_SCRIPTS/type) - if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then - : # cannot install packages that depend on the Sage library - else - case "$PKG_BASE:$PKG_TYPE" in - $DEVELOP_SPKG_PATTERN:*) FD=4;; - *) FD=5;; - esac - ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do - [ -n "$line" ] && echo >&$FD " - $line" - done - fi + ( + cat src/environment-template.yml + echo " # optional packages" + for pkg in $OPTIONAL_SYSTEM_PACKAGES $SAGELIB_OPTIONAL_SYSTEM_PACKAGES; do + echo " - $pkg" + done + ) > src/environment-optional-template.yml + + ( + echo >&4 " - pip:" + echo >&5 " - pip:" + for PKG_BASE in $(sage-package list :standard: :optional: --has-file requirements.txt --no-file distros/conda.txt --no-file src; sage-package list :standard: :optional: --has-file install-requires.txt --no-file requirements.txt --no-file distros/conda.txt --no-file src); do + PKG_SCRIPTS=build/pkgs/$PKG_BASE + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/requirements.txt + if [ ! -f $SYSTEM_PACKAGES_FILE ]; then + SYSTEM_PACKAGES_FILE=$PKG_SCRIPTS/install-requires.txt + fi + PKG_TYPE=$(cat $PKG_SCRIPTS/type) + if grep -q SAGERUNTIME $PKG_SCRIPTS/dependencies $PKG_SCRIPTS/dependencies_order_only 2>/dev/null; then + : # cannot install packages that depend on the Sage library + else + case "$PKG_BASE:$PKG_TYPE" in + $DEVELOP_SPKG_PATTERN:*) FD=4;; + *:standard) FD="4 5";; + *) FD=5;; + esac + ${STRIP_COMMENTS} $SYSTEM_PACKAGES_FILE | while read -r line; do + [ -n "$line" ] && for fd in $FD; do echo >&$fd " - $line"; done + done + fi + done + ) 4>> /dev/null 5>> src/environment-optional-template.yml + +for f in environment environment-optional src/environment src/environment-optional src/environment-dev; do + for python_version in 3.9 3.10 3.11; do + sed -E 's/^( *- *)python *$/\1python='$python_version'/' $f-template.yml > $f-$python_version.yml done -) 4>> src/environment-dev.yml 5>> src/environment-optional.yml + rm -f $f-template.yml +done diff --git a/build/pkgs/matplotlib/distros/conda.txt b/build/pkgs/matplotlib/distros/conda.txt index 8b3901c7c9a..9fdcdfeb47f 100644 --- a/build/pkgs/matplotlib/distros/conda.txt +++ b/build/pkgs/matplotlib/distros/conda.txt @@ -1,2 +1,2 @@ # Trac #33642: For unknown reasons, without the version constraint, conda installs only 3.3.2 -"matplotlib>=3.5.1" +matplotlib>=3.5.1 diff --git a/src/doc/en/installation/conda.rst b/src/doc/en/installation/conda.rst index 6b4cb871dd7..14fe52ad2e2 100644 --- a/src/doc/en/installation/conda.rst +++ b/src/doc/en/installation/conda.rst @@ -75,13 +75,16 @@ from source as follows: - Create a new conda environment including all standard packages recognized by sage, and activate it:: - $ conda env create --file environment.yml --name sage-build + $ conda env create --file environment-3.11.yml --name sage-build $ conda activate sage-build - Alternatively, use ``environment-optional.yml`` in place of + Alternatively, use ``environment-optional-3.11.yml`` in place of ``environment.yml`` to create an environment with all standard and optional packages recognized by sage. + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. + - Then the SageMath distribution will be built using the compilers provided by Conda and using many packages installed by Conda:: @@ -123,16 +126,15 @@ Here we assume that you are using a git checkout. - Create and activate a new conda environment with the dependencies of Sage and a few additional developer tools:: - $ mamba env create --file src/environment-dev.yml --name sage-dev + $ mamba env create --file src/environment-dev-3.11.yml --name sage-dev $ conda activate sage-dev - Alternatively, you can use ``src/environment.yml`` or - ``src/environment-optional.yml``, which will only install standard + Alternatively, you can use ``src/environment-3.11.yml`` or + ``src/environment-optional-3.11.yml``, which will only install standard (and optional) packages without any additional developer tools. - By default, the most recent version of Python supported by Sage is - installed. You can use the additional option ``python=3.9`` in the above - ``env create`` command to select another Python version (here 3.9). + A different Python version can be selected by replacing ``3.11`` by ``3.9`` + or ``3.10`` in these commands. - Install the build prerequisites and the Sage library:: @@ -142,7 +144,7 @@ Here we assume that you are using a git checkout. - Verify that Sage has been installed:: $ sage -c 'print(version())' - SageMath version 9.6.beta5, Release Date: 2022-03-12 + SageMath version 10.2.beta4, Release Date: 2023-09-24 Note that ``make`` is not used at all. All dependencies (including all Python packages) are provided by conda. @@ -162,7 +164,7 @@ After editing any Cython files, rebuild the Sage library using:: In order to update the conda environment later, you can run:: - $ mamba env update --file src/environment-dev.yml --name sage-dev + $ mamba env update --file src/environment-dev-3.11.yml --name sage-dev To build the documentation, use:: diff --git a/tox.ini b/tox.ini index c084e4c821e..0f4540605e2 100644 --- a/tox.ini +++ b/tox.ini @@ -191,6 +191,7 @@ setenv = sitepackages: ENABLE_SYSTEM_SITE_PACKAGES=yes sitepackages: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages conda-environment: SAGE_PACKAGE_LIST_ARGS=_prereq + conda-environment-{src,dev}: CONFIG_CONFIGURE_ARGS_SITEPACKAGES=--enable-system-site-packages # Whether to add the system packages needed for bootstrapping EXTRA_SAGE_PACKAGES_0=_bootstrap nobootstrap: EXTRA_SAGE_PACKAGES_0= @@ -527,9 +528,14 @@ setenv = local-conda-miniconda: CONDA_INSTALLER_FILE=Miniconda3-latest-{env:CONDA_OS}-x86_64.sh local-conda: SETENV=. {env:CONDA_PREFIX}/bin/activate base local-conda-environment: CONDA_SAGE_ENVIRONMENT=sage-build - local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment.yml - local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional.yml + local-conda-environment: CONDA_SAGE_ENVIRONMENT_DIR= + local-conda-environment-src: CONDA_SAGE_ENVIRONMENT=sage local-conda-environment-src: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment: CONDA_SAGE_ENVIRONMENT_FILE=environment-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-optional: CONDA_SAGE_ENVIRONMENT_FILE=environment-optional-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT=sage-dev + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_DIR=src/ + local-conda-environment-dev: CONDA_SAGE_ENVIRONMENT_FILE=environment-dev-{env:PYTHON_MAJOR}.{env:PYTHON_MINOR}.yml local-conda-environment: SETENV_CONFIGURE=( {env:CONDA_PREFIX}/bin/conda env create -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} || {env:CONDA_PREFIX}/bin/conda env update -n {env:CONDA_SAGE_ENVIRONMENT} --file {env:CONDA_SAGE_ENVIRONMENT_DIR:}{env:CONDA_SAGE_ENVIRONMENT_FILE} ) && . {env:CONDA_PREFIX}/bin/activate {env:CONDA_SAGE_ENVIRONMENT} # # Configuration factors