From e526f04e5cd4e6305a7ee9a6eeaabeec857de695 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 24 Jun 2024 18:08:14 -0700 Subject: [PATCH 1/8] .github/workflows/dist.yml: Build manylinux aarch64 wheels via QEMU --- .github/workflows/dist.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 3d4028dfe78..ef27da4c052 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -166,6 +166,8 @@ jobs: arch: x86_64 - os: ubuntu-latest arch: i686 + - os: ubuntu-latest + arch: aarch64 - os: macos-13 arch: x86_64 - os: macos-14 @@ -190,6 +192,12 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up QEMU + if: runner.os == 'Linux' && matrix.arch != 'x86_64' && matrix.arch != 'i686' + uses: docker/setup-qemu-action@v3 + with: + platforms: all + - uses: actions/download-artifact@v4 with: name: dist From 3dd5ae9c43951ba8d99b6d733ba8b833177ee01c Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 24 Jun 2024 18:26:56 -0700 Subject: [PATCH 2/8] .github/workflows/dist.yml: Build musllinux wheels --- .github/workflows/dist.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index ef27da4c052..beef6b2ea52 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -179,8 +179,7 @@ jobs: # Non-Python packages to install as spkgs TARGETS_PRE: gmp mpfr mpc bliss coxeter3 mcqd meataxe sirocco boost_cropped tdlib # Disable building PyPy wheels on all platforms - # Disable musllinux until #33083 provides alpine package information - CIBW_SKIP: "pp* *-musllinux*" + CIBW_SKIP: "pp*" # CIBW_ARCHS: ${{ matrix.arch }} # https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python @@ -233,7 +232,7 @@ jobs: run: | "${{ steps.python.outputs.python-path }}" -m pip install pipx export PATH=build/bin:$PATH - export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt" + export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || $(sage-print-system-package-command alpine --yes --no-install-recommends install $(sage-get-system-packages alpine $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt" mkdir -p unpacked for pkg in sagemath*objects sagemath*categories sagemath*bliss sagemath*coxeter3 sagemath*mcqd sagemath*tdlib; do case "$pkg:${{ matrix.arch }}" in From 995f4f9bb75bedcce4e547110962b622c3340452 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 24 Jun 2024 20:48:09 -0700 Subject: [PATCH 3/8] .github/workflows/dist.yml: Set SAGE_NUM_THREADS --- .github/workflows/dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index beef6b2ea52..2c46aa2a9b8 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -185,7 +185,7 @@ jobs: # https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9, <3.13" # Environment during wheel build - CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt" + CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt SAGE_NUM_THREADS=6" # Use 'build', not 'pip wheel' CIBW_BUILD_FRONTEND: build steps: From 67d7ce370d42f39afb4b13d5998c1296b78a9a51 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Mon, 24 Jun 2024 23:07:36 -0700 Subject: [PATCH 4/8] .github/workflows/dist.yml: Split aarch64 job into manylinux, musllinux --- .github/workflows/dist.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index 2c46aa2a9b8..b45144cd28d 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -155,7 +155,7 @@ jobs: if: env.CAN_DEPLOY == 'true' build_wheels: - name: Build wheels on ${{ matrix.os }}, arch ${{ matrix.arch }} + name: Build wheels on ${{ matrix.os }}, arch ${{ matrix.arch }} ${{ matrix.build }} runs-on: ${{ matrix.os }} needs: sdists_for_pypi strategy: @@ -168,6 +168,10 @@ jobs: arch: i686 - os: ubuntu-latest arch: aarch64 + build: manylinux + - os: ubuntu-latest + arch: aarch64 + build: musllinux - os: macos-13 arch: x86_64 - os: macos-14 @@ -178,6 +182,7 @@ jobs: SPKGS: _bootstrap _prereq # Non-Python packages to install as spkgs TARGETS_PRE: gmp mpfr mpc bliss coxeter3 mcqd meataxe sirocco boost_cropped tdlib + CIBW_BUILD: "*${{ matrix.build }}*" # Disable building PyPy wheels on all platforms CIBW_SKIP: "pp*" # @@ -244,7 +249,7 @@ jobs: - uses: actions/upload-artifact@v4 with: - name: ${{ matrix.os }}-${{ matrix.arch }}-wheels + name: ${{ matrix.os }}-${{ matrix.build }}-${{ matrix.arch }}-wheels path: ./wheelhouse/*.whl upload_wheels: @@ -257,7 +262,7 @@ jobs: - uses: actions/download-artifact@v4 with: - pattern: "*-*-wheels" + pattern: "*-*-*-wheels" path: wheelhouse merge-multiple: true From bffc1d3f282fd46cbd22097a453fe2298aae9c53 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 25 Jun 2024 09:44:27 -0700 Subject: [PATCH 5/8] .github/workflows/dist.yml: Use PIP_FIND_LINKS --- .github/workflows/dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index b45144cd28d..a23c709de0c 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -190,7 +190,7 @@ jobs: # https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9, <3.13" # Environment during wheel build - CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt SAGE_NUM_THREADS=6" + CIBW_ENVIRONMENT: "PATH=$(pwd)/prefix/bin:$PATH CPATH=$(pwd)/prefix/include:$CPATH LIBRARY_PATH=$(pwd)/prefix/lib:$LIBRARY_PATH LD_LIBRARY_PATH=$(pwd)/prefix/lib:$LD_LIBRARY_PATH PKG_CONFIG_PATH=$(pwd)/prefix/share/pkgconfig:$PKG_CONFIG_PATH ACLOCAL_PATH=/usr/share/aclocal PIP_CONSTRAINT=$(pwd)/constraints.txt PIP_FIND_LINKS=file://$(pwd)/wheelhouse SAGE_NUM_THREADS=6" # Use 'build', not 'pip wheel' CIBW_BUILD_FRONTEND: build steps: From 74e2966506c2f4ec72fb0cdd8725b4c3b459a2ff Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 25 Jun 2024 09:51:16 -0700 Subject: [PATCH 6/8] .github/workflows/dist.yml: More matrix jobs --- .github/workflows/dist.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index a23c709de0c..cecbb02188e 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -161,17 +161,10 @@ jobs: strategy: fail-fast: false matrix: + os: [ubuntu-latest] + arch: [x86_64, i686, aarch64] + build: [manylinux, musllinux] include: - - os: ubuntu-latest - arch: x86_64 - - os: ubuntu-latest - arch: i686 - - os: ubuntu-latest - arch: aarch64 - build: manylinux - - os: ubuntu-latest - arch: aarch64 - build: musllinux - os: macos-13 arch: x86_64 - os: macos-14 From 9f73f5db55f22b54809e53be3132b639875be635 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 25 Jun 2024 11:50:51 -0700 Subject: [PATCH 7/8] .github/workflows/dist.yml: Separate step for each distribution --- .github/workflows/dist.yml | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index cecbb02188e..b5d93ca40c7 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -155,7 +155,7 @@ jobs: if: env.CAN_DEPLOY == 'true' build_wheels: - name: Build wheels on ${{ matrix.os }}, arch ${{ matrix.arch }} ${{ matrix.build }} + name: wheels ${{ matrix.arch }} ${{ matrix.build }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} needs: sdists_for_pypi strategy: @@ -214,7 +214,7 @@ jobs: eval $(sage-print-system-package-command auto --sudo --yes --no-install-recommends --spkg install _bootstrap) ./bootstrap - - name: Build platform wheels + - name: Unpack and prepare # We build the wheels from the sdists so that MANIFEST filtering becomes effective. # But we must run cibuildwheel with the unpacked source directory, not a tarball, # so that SAGE_ROOT is copied into the build containers. @@ -228,18 +228,34 @@ jobs: # # omit sagemath-{meataxe,sirocco} for now -- needs sagemath-modules run: | - "${{ steps.python.outputs.python-path }}" -m pip install pipx + "${{ steps.python.outputs.python-path }}" -m pip install cibuildwheel==2.18.0 export PATH=build/bin:$PATH - export CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || $(sage-print-system-package-command alpine --yes --no-install-recommends install $(sage-get-system-packages alpine $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt" + echo CIBW_BEFORE_ALL="( $(sage-print-system-package-command debian --yes --no-install-recommends install $(sage-get-system-packages debian $SPKGS)) || $(sage-print-system-package-command fedora --yes --no-install-recommends install $(sage-get-system-packages fedora $SPKGS | sed s/pkg-config/pkgconfig/)) || ( $(sage-print-system-package-command homebrew --yes --no-install-recommends install $(sage-get-system-packages homebrew $SPKGS)) || $(sage-print-system-package-command alpine --yes --no-install-recommends install $(sage-get-system-packages alpine $SPKGS)) || echo error ignored) ) && if cp /host/sage-\$AUDITWHEEL_PLAT/config.status . 2>/dev/null; then chmod +x config.status; fi && if [ -x ./config.status ]; then ./config.status; else ./configure --enable-build-as-root ${{ startsWith(matrix.os, 'ubuntu') && '--prefix=/host/sage-\$AUDITWHEEL_PLAT' || '' }} && cp config.status prefix/; fi && MAKE=\"make -j6\" make V=0 $TARGETS_PRE && (echo \"sage_conf @ file://\$(pwd)/pkgs/sage-conf\" && echo \"sage_setup @ file://\$(pwd)/pkgs/sage-setup\") > constraints.txt" >> "$GITHUB_ENV" mkdir -p unpacked - for pkg in sagemath*objects sagemath*categories sagemath*bliss sagemath*coxeter3 sagemath*mcqd sagemath*tdlib; do - case "$pkg:${{ matrix.arch }}" in - sagemath*tdlib:i686) continue;; # broken - boost-related - esac - (cd unpacked && tar xfz - ) < dist/$pkg*.tar.gz - "${{ steps.python.outputs.python-path }}" -m pipx run cibuildwheel==2.18.0 unpacked/$pkg* + for sdist in dist/$pkg*.tar.gz; do + (cd unpacked && tar xfz - ) < $sdist done + - name: sagemath-objects + run: | + "${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*objects* + + - name: sagemath-categories + run: | + "${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*categories* + + - name: sagemath-bliss + run: | + "${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*bliss* + + - name: sagemath-coxeter3 + run: | + "${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*coxeter3* + + - name: sagemath-mcqd + run: | + "${{ steps.python.outputs.python-path }}" -m cibuildwheel unpacked/sagemath*mcqd* + - uses: actions/upload-artifact@v4 with: name: ${{ matrix.os }}-${{ matrix.build }}-${{ matrix.arch }}-wheels From 4e538125893af3e587cfd4fa7d2bfba3fd01b0a3 Mon Sep 17 00:00:00 2001 From: Matthias Koeppe Date: Tue, 25 Jun 2024 15:29:26 -0700 Subject: [PATCH 8/8] .github/workflows/dist.yml: Cosmetic change --- .github/workflows/dist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dist.yml b/.github/workflows/dist.yml index b5d93ca40c7..8cc8249e3c6 100644 --- a/.github/workflows/dist.yml +++ b/.github/workflows/dist.yml @@ -155,7 +155,7 @@ jobs: if: env.CAN_DEPLOY == 'true' build_wheels: - name: wheels ${{ matrix.arch }} ${{ matrix.build }} on ${{ matrix.os }} + name: wheels ${{ matrix.build }}*_${{ matrix.arch }} runs-on: ${{ matrix.os }} needs: sdists_for_pypi strategy: @@ -167,8 +167,10 @@ jobs: include: - os: macos-13 arch: x86_64 + build: macosx - os: macos-14 arch: arm64 + build: macosx env: CAN_DEPLOY: ${{ secrets.SAGEMATH_PYPI_API_TOKEN != '' }} # SPKGs to install as system packages