From 76a4649761506eab69ad383e70a627a743f24610 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Mon, 8 Feb 2021 14:59:04 -0800 Subject: [PATCH] Add initial jq-based templating engine --- .architectures-lib | 65 ------- .gitattributes | 2 + .github/workflows/verify-templating.yml | 22 +++ .gitignore | 1 + 2.7/{ => buster}/Dockerfile | 55 +++--- 2.7/{slim => slim-buster}/Dockerfile | 61 +++---- {3.7 => 3.6/buster}/Dockerfile | 51 ++++-- 3.6/{slim => slim-buster}/Dockerfile | 55 +++--- {3.6 => 3.7/buster}/Dockerfile | 51 ++++-- 3.7/{slim => slim-buster}/Dockerfile | 55 +++--- ...slim.template => Dockerfile-linux.template | 107 +++++++++--- Dockerfile.template | 108 ------------ apply-templates.sh | 66 +++++++ generate-stackbrew-library.sh | 101 +++++++---- release-architectures | 13 -- update.sh | 124 +------------- versions.json | 97 +++++++++++ versions.sh | 162 ++++++++++++++++++ 18 files changed, 688 insertions(+), 508 deletions(-) delete mode 100644 .architectures-lib create mode 100644 .gitattributes create mode 100644 .github/workflows/verify-templating.yml create mode 100644 .gitignore rename 2.7/{ => buster}/Dockerfile (71%) rename 2.7/{slim => slim-buster}/Dockerfile (68%) rename {3.7 => 3.6/buster}/Dockerfile (74%) rename 3.6/{slim => slim-buster}/Dockerfile (74%) rename {3.6 => 3.7/buster}/Dockerfile (74%) rename 3.7/{slim => slim-buster}/Dockerfile (74%) rename Dockerfile-slim.template => Dockerfile-linux.template (56%) delete mode 100644 Dockerfile.template create mode 100755 apply-templates.sh delete mode 100644 release-architectures create mode 100644 versions.json create mode 100755 versions.sh diff --git a/.architectures-lib b/.architectures-lib deleted file mode 100644 index d9f98a7..0000000 --- a/.architectures-lib +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env bash - -_awkArch() { - local awkExpr="$1"; shift - awk "$@" "/^#|^\$/ { next } $awkExpr" release-architectures -} - -dpkgArches() { - _awkArch '{ print $2 }' -} - -dpkgToBashbrewArch() { - local dpkgArch="$1"; shift - _awkArch '$2 == dpkgArch { print $1; exit }' -v dpkgArch="$dpkgArch" -} - -dpkgToPyPyArch() { - local dpkgArch="$1"; shift - _awkArch '$2 == dpkgArch { print $3; exit }' -v dpkgArch="$dpkgArch" -} - -_generateParentRepoToArches() { - local repo="$1"; shift - local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' - - eval "declare -g -A parentRepoToArches=( $( - find -name 'Dockerfile' -exec awk ' - toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { - print "'"$officialImagesUrl"'" $2 - } - ' '{}' + \ - | sort -u \ - | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' - ) )" -} -_generateParentRepoToArches 'pypy' - -hasBashbrewArch() { - local dir="$1"; shift - local bashbrewArch="$1"; shift - grep -qE "^# ${bashbrewArch}\$" "$dir/Dockerfile" -} - -parent() { - local dir="$1"; shift # "2", "3/slim", etc - - local parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile")" - - echo "$parent" -} - -parentArches() { - local dir="$1"; shift # "2", "3/slim", etc - local parent="$1"; shift # "buildpack-deps:jessie", etc - - local parentArches="${parentRepoToArches[$parent]:-}" - - local arches=() - for arch in $parentArches; do - if hasBashbrewArch "$dir" "$arch"; then - arches+=( "$arch" ) - fi - done - echo "${arches[*]:-}" -} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..90ec81c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +/*/**/Dockerfile linguist-generated +/Dockerfile*.template linguist-language=Dockerfile diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml new file mode 100644 index 0000000..7e833f1 --- /dev/null +++ b/.github/workflows/verify-templating.yml @@ -0,0 +1,22 @@ +name: Verify Templating + +on: + pull_request: + push: + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + apply-templates: + name: Check For Uncomitted Changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Apply Templates + run: ./apply-templates.sh + - name: Check Git Status + run: | + status="$(git status --short)" + [ -z "$status" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d548f66 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.jq-template.awk diff --git a/2.7/Dockerfile b/2.7/buster/Dockerfile similarity index 71% rename from 2.7/Dockerfile rename to 2.7/buster/Dockerfile index 25a7b17..fbfe279 100644 --- a/2.7/Dockerfile +++ b/2.7/buster/Dockerfile @@ -1,8 +1,10 @@ -FROM buildpack-deps:buster +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM buildpack-deps:buster # runtime dependencies RUN set -eux; \ @@ -13,22 +15,31 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + # ensure local pypy is preferred over distribution pypy ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='f412b602ccd6912ddee0e7523e0e38f4b2c7a144449c2cad078cffbdb66fd7b1' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='23b145b7cfbaeefb6ee76fc8216c83b652ab1daffac490558718edbbd60082d8' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='bfbc81874b137837a8ba8c517b97de29f5a336f7ec500c52f2bfdbd3580d1703' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux64.tar.bz2'; \ + sha256='f412b602ccd6912ddee0e7523e0e38f4b2c7a144449c2cad078cffbdb66fd7b1'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-aarch64.tar.bz2'; \ + sha256='23b145b7cfbaeefb6ee76fc8216c83b652ab1daffac490558718edbbd60082d8'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux32.tar.bz2'; \ + sha256='bfbc81874b137837a8ba8c517b97de29f5a336f7ec500c52f2bfdbd3580d1703'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -41,30 +52,18 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy2.7-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy' '/usr/local/bin/pypy'; \ + ln -sv '/opt/pypy/bin/pypy' /usr/local/bin/; \ \ # smoke test pypy --version; \ \ -# on pypy3, rebuild ffi bits for compatibility with Debian Stretch+ - cd /opt/pypy/lib_pypy; \ -# https://github.com/docker-library/pypy/issues/24#issuecomment-409408657 - if [ -f _ssl_build.py ]; then \ - pypy _ssl_build.py; \ - fi; \ -# https://github.com/docker-library/pypy/issues/42 - if [ -f _lzma_build.py ]; then \ - pypy _lzma_build.py; \ - fi; \ -# TODO rebuild other cffi modules here too? (other _*_build.py files) - \ apt-mark auto '.*' > /dev/null; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ find /opt/pypy -type f -executable -exec ldd '{}' ';' \ @@ -101,7 +100,7 @@ RUN set -ex; \ pypy get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ # smoke test pip --version; \ diff --git a/2.7/slim/Dockerfile b/2.7/slim-buster/Dockerfile similarity index 68% rename from 2.7/slim/Dockerfile rename to 2.7/slim-buster/Dockerfile index 0a606e1..f17df02 100644 --- a/2.7/slim/Dockerfile +++ b/2.7/slim-buster/Dockerfile @@ -1,30 +1,41 @@ -FROM debian:buster-slim +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM debian:buster-slim RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates; \ rm -rf /var/lib/apt/lists/* +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + # ensure local pypy is preferred over distribution pypy ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='f412b602ccd6912ddee0e7523e0e38f4b2c7a144449c2cad078cffbdb66fd7b1' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='23b145b7cfbaeefb6ee76fc8216c83b652ab1daffac490558718edbbd60082d8' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='bfbc81874b137837a8ba8c517b97de29f5a336f7ec500c52f2bfdbd3580d1703' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux64.tar.bz2'; \ + sha256='f412b602ccd6912ddee0e7523e0e38f4b2c7a144449c2cad078cffbdb66fd7b1'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-aarch64.tar.bz2'; \ + sha256='23b145b7cfbaeefb6ee76fc8216c83b652ab1daffac490558718edbbd60082d8'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux32.tar.bz2'; \ + sha256='bfbc81874b137837a8ba8c517b97de29f5a336f7ec500c52f2bfdbd3580d1703'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -39,32 +50,18 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy2.7-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy' '/usr/local/bin/pypy'; \ + ln -sv '/opt/pypy/bin/pypy' /usr/local/bin/; \ \ # smoke test pypy --version; \ \ -# on pypy3, rebuild ffi bits for compatibility with Debian Stretch+ - cd /opt/pypy/lib_pypy; \ -# https://github.com/docker-library/pypy/issues/24#issuecomment-409408657 - if [ -f _ssl_build.py ]; then \ - apt-get install -y --no-install-recommends gcc libc6-dev libssl-dev; \ - pypy _ssl_build.py; \ - fi; \ -# https://github.com/docker-library/pypy/issues/42 - if [ -f _lzma_build.py ]; then \ - apt-get install -y --no-install-recommends gcc libc6-dev liblzma-dev; \ - pypy _lzma_build.py; \ - fi; \ -# TODO rebuild other cffi modules here too? (other _*_build.py files) - \ apt-mark auto '.*' > /dev/null; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ find /opt/pypy -type f -executable -exec ldd '{}' ';' \ @@ -95,9 +92,7 @@ ENV PYTHON_GET_PIP_SHA256 95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends \ - wget \ - ; \ + apt-get install -y --no-install-recommends wget; \ rm -rf /var/lib/apt/lists/*; \ \ wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ @@ -106,7 +101,7 @@ RUN set -ex; \ pypy get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ apt-get purge -y --auto-remove wget; \ # smoke test diff --git a/3.7/Dockerfile b/3.6/buster/Dockerfile similarity index 74% rename from 3.7/Dockerfile rename to 3.6/buster/Dockerfile index f0173b4..4076c69 100644 --- a/3.7/Dockerfile +++ b/3.6/buster/Dockerfile @@ -1,8 +1,10 @@ -FROM buildpack-deps:buster +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM buildpack-deps:buster # runtime dependencies RUN set -eux; \ @@ -13,24 +15,35 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -# ensure local pypy is preferred over distribution pypy +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# ensure local pypy3 is preferred over distribution pypy3 ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='37e2804c4661c86c857d709d28c7de716b000d31e89766599fdf5a98928b7096' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='ee4aa041558b58de6063dd6df93b3def221c4ca4c900d6a9db5b1b52135703a8' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='7d81b8e9fcd07c067cfe2f519ab770ec62928ee8787f952cadf2d2786246efc8' ;; \ -# s390x - s390x) pypyArch='s390x'; sha256='92000d90b9a37f2e9cb7885f2a872adfa9e48e74bf7f84a8b8185c8181f0502d' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux64.tar.bz2'; \ + sha256='4fb85fdd516482cab727bb9473b066ff8fb672940dedf7ccc32bf92957d29e0a'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-aarch64.tar.bz2'; \ + sha256='bc82cf7f0182b942a2cfad4a0d167f364bfbf18f434e100a2fe62bc88547ac9b'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux32.tar.bz2'; \ + sha256='f183c61e66fd2c536a65695bd7ff770748c2884c235a589b9c6ac63690770c69'; \ + ;; \ + 's390x') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-s390x.tar.bz2'; \ + sha256='0de9c33ff3500c6e7fd273d0a6d341bc839b0298f697c4d6fe141f2b54c5c3e2'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -43,14 +56,14 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy3.7-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy3' '/usr/local/bin/pypy3'; \ + ln -sv '/opt/pypy/bin/pypy3' /usr/local/bin/; \ \ # smoke test pypy3 --version; \ @@ -103,7 +116,7 @@ RUN set -ex; \ pypy3 get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ # smoke test pip --version; \ diff --git a/3.6/slim/Dockerfile b/3.6/slim-buster/Dockerfile similarity index 74% rename from 3.6/slim/Dockerfile rename to 3.6/slim-buster/Dockerfile index c1a08fc..2d41537 100644 --- a/3.6/slim/Dockerfile +++ b/3.6/slim-buster/Dockerfile @@ -1,32 +1,45 @@ -FROM debian:buster-slim +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM debian:buster-slim RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates; \ rm -rf /var/lib/apt/lists/* -# ensure local pypy is preferred over distribution pypy +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# ensure local pypy3 is preferred over distribution pypy3 ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='4fb85fdd516482cab727bb9473b066ff8fb672940dedf7ccc32bf92957d29e0a' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='bc82cf7f0182b942a2cfad4a0d167f364bfbf18f434e100a2fe62bc88547ac9b' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='f183c61e66fd2c536a65695bd7ff770748c2884c235a589b9c6ac63690770c69' ;; \ -# s390x - s390x) pypyArch='s390x'; sha256='0de9c33ff3500c6e7fd273d0a6d341bc839b0298f697c4d6fe141f2b54c5c3e2' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux64.tar.bz2'; \ + sha256='4fb85fdd516482cab727bb9473b066ff8fb672940dedf7ccc32bf92957d29e0a'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-aarch64.tar.bz2'; \ + sha256='bc82cf7f0182b942a2cfad4a0d167f364bfbf18f434e100a2fe62bc88547ac9b'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux32.tar.bz2'; \ + sha256='f183c61e66fd2c536a65695bd7ff770748c2884c235a589b9c6ac63690770c69'; \ + ;; \ + 's390x') \ + url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-s390x.tar.bz2'; \ + sha256='0de9c33ff3500c6e7fd273d0a6d341bc839b0298f697c4d6fe141f2b54c5c3e2'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -41,14 +54,14 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy3.6-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy3' '/usr/local/bin/pypy3'; \ + ln -sv '/opt/pypy/bin/pypy3' /usr/local/bin/; \ \ # smoke test pypy3 --version; \ @@ -97,9 +110,7 @@ ENV PYTHON_GET_PIP_SHA256 95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends \ - wget \ - ; \ + apt-get install -y --no-install-recommends wget; \ rm -rf /var/lib/apt/lists/*; \ \ wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ @@ -108,7 +119,7 @@ RUN set -ex; \ pypy3 get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ apt-get purge -y --auto-remove wget; \ # smoke test diff --git a/3.6/Dockerfile b/3.7/buster/Dockerfile similarity index 74% rename from 3.6/Dockerfile rename to 3.7/buster/Dockerfile index a27a912..f7946a2 100644 --- a/3.6/Dockerfile +++ b/3.7/buster/Dockerfile @@ -1,8 +1,10 @@ -FROM buildpack-deps:buster +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM buildpack-deps:buster # runtime dependencies RUN set -eux; \ @@ -13,24 +15,35 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -# ensure local pypy is preferred over distribution pypy +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# ensure local pypy3 is preferred over distribution pypy3 ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='4fb85fdd516482cab727bb9473b066ff8fb672940dedf7ccc32bf92957d29e0a' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='bc82cf7f0182b942a2cfad4a0d167f364bfbf18f434e100a2fe62bc88547ac9b' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='f183c61e66fd2c536a65695bd7ff770748c2884c235a589b9c6ac63690770c69' ;; \ -# s390x - s390x) pypyArch='s390x'; sha256='0de9c33ff3500c6e7fd273d0a6d341bc839b0298f697c4d6fe141f2b54c5c3e2' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux64.tar.bz2'; \ + sha256='37e2804c4661c86c857d709d28c7de716b000d31e89766599fdf5a98928b7096'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-aarch64.tar.bz2'; \ + sha256='ee4aa041558b58de6063dd6df93b3def221c4ca4c900d6a9db5b1b52135703a8'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux32.tar.bz2'; \ + sha256='7d81b8e9fcd07c067cfe2f519ab770ec62928ee8787f952cadf2d2786246efc8'; \ + ;; \ + 's390x') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-s390x.tar.bz2'; \ + sha256='92000d90b9a37f2e9cb7885f2a872adfa9e48e74bf7f84a8b8185c8181f0502d'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -43,14 +56,14 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy3.6-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy3' '/usr/local/bin/pypy3'; \ + ln -sv '/opt/pypy/bin/pypy3' /usr/local/bin/; \ \ # smoke test pypy3 --version; \ @@ -103,7 +116,7 @@ RUN set -ex; \ pypy3 get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ # smoke test pip --version; \ diff --git a/3.7/slim/Dockerfile b/3.7/slim-buster/Dockerfile similarity index 74% rename from 3.7/slim/Dockerfile rename to 3.7/slim-buster/Dockerfile index 7ea07a8..7e9d167 100644 --- a/3.7/slim/Dockerfile +++ b/3.7/slim-buster/Dockerfile @@ -1,32 +1,45 @@ -FROM debian:buster-slim +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +FROM debian:buster-slim RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates; \ rm -rf /var/lib/apt/lists/* -# ensure local pypy is preferred over distribution pypy +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# ensure local pypy3 is preferred over distribution pypy3 ENV PATH /opt/pypy/bin:$PATH ENV PYPY_VERSION 7.3.3 -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ -# amd64 - amd64) pypyArch='linux64'; sha256='37e2804c4661c86c857d709d28c7de716b000d31e89766599fdf5a98928b7096' ;; \ -# arm64v8 - arm64) pypyArch='aarch64'; sha256='ee4aa041558b58de6063dd6df93b3def221c4ca4c900d6a9db5b1b52135703a8' ;; \ -# i386 - i386) pypyArch='linux32'; sha256='7d81b8e9fcd07c067cfe2f519ab770ec62928ee8787f952cadf2d2786246efc8' ;; \ -# s390x - s390x) pypyArch='s390x'; sha256='92000d90b9a37f2e9cb7885f2a872adfa9e48e74bf7f84a8b8185c8181f0502d' ;; \ + 'amd64') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux64.tar.bz2'; \ + sha256='37e2804c4661c86c857d709d28c7de716b000d31e89766599fdf5a98928b7096'; \ + ;; \ + 'arm64') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-aarch64.tar.bz2'; \ + sha256='ee4aa041558b58de6063dd6df93b3def221c4ca4c900d6a9db5b1b52135703a8'; \ + ;; \ + 'i386') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux32.tar.bz2'; \ + sha256='7d81b8e9fcd07c067cfe2f519ab770ec62928ee8787f952cadf2d2786246efc8'; \ + ;; \ + 's390x') \ + url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-s390x.tar.bz2'; \ + sha256='92000d90b9a37f2e9cb7885f2a872adfa9e48e74bf7f84a8b8185c8181f0502d'; \ + ;; \ *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ esac; \ \ @@ -41,14 +54,14 @@ RUN set -ex; \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/pypy3.7-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/pypy3' '/usr/local/bin/pypy3'; \ + ln -sv '/opt/pypy/bin/pypy3' /usr/local/bin/; \ \ # smoke test pypy3 --version; \ @@ -97,9 +110,7 @@ ENV PYTHON_GET_PIP_SHA256 95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d RUN set -ex; \ apt-get update; \ - apt-get install -y --no-install-recommends \ - wget \ - ; \ + apt-get install -y --no-install-recommends wget; \ rm -rf /var/lib/apt/lists/*; \ \ wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ @@ -108,7 +119,7 @@ RUN set -ex; \ pypy3 get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ apt-get purge -y --auto-remove wget; \ # smoke test diff --git a/Dockerfile-slim.template b/Dockerfile-linux.template similarity index 56% rename from Dockerfile-slim.template rename to Dockerfile-linux.template index 05f6bbd..6a86a28 100644 --- a/Dockerfile-slim.template +++ b/Dockerfile-linux.template @@ -1,61 +1,112 @@ -FROM debian:%%BASE%%-slim - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 +{{ def is_slim: env.variant | startswith("slim-") -}} +{{ def is_3: env.version | startswith("3") -}} +{{ def cmd: if is_3 then "pypy3" else "pypy" end -}} +{{ if is_slim then ( -}} +FROM debian:{{ env.variant | ltrimstr("slim-") }}-slim RUN set -eux; \ apt-get update; \ apt-get install -y --no-install-recommends ca-certificates; \ rm -rf /var/lib/apt/lists/* +{{ ) else ( -}} +FROM buildpack-deps:{{ env.variant }} + +# runtime dependencies +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ + tcl \ + tk \ + ; \ + rm -rf /var/lib/apt/lists/* +{{ ) end -}} -# ensure local pypy is preferred over distribution pypy +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +# ensure local {{ cmd }} is preferred over distribution {{ cmd }} ENV PATH /opt/pypy/bin:$PATH -ENV PYPY_VERSION %%PYPY_VERSION%% +ENV PYPY_VERSION {{ .version }} -RUN set -ex; \ +RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" - %%ARCH-CASE%%; \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +{{ + [ + .arches | to_entries[] + | .key as $bashbrewArch + | ( + { + arm32v5: "armel", + arm32v7: "armhf", + arm64v8: "arm64", + mips64le: "mips64el", + ppc64le: "ppc64el", + } + | .[$bashbrewArch] // $bashbrewArch + ) as $dpkgArch + | .value + | ( +-}} + {{ $dpkgArch | @sh }}) \ + url={{ .url | @sh }}; \ + sha256={{ .sha256 | @sh }}; \ + ;; \ +{{ + ) + ] | add +-}} + *) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; \ + esac; \ \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends \ +{{ if is_slim then ( -}} bzip2 \ wget \ -# sometimes "%%CMD%%" itself is linked against libexpat1 / libncurses5, sometimes they're ".so" files in "/opt/pypy/lib_pypy" +{{ ) else "" end -}} +# sometimes "{{ cmd }}" itself is linked against libexpat1 / libncurses5, sometimes they're ".so" files in "/opt/pypy/lib_pypy" libexpat1 \ libncurses5 \ # (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) ; \ \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/%%TAR%%-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ + wget -O pypy.tar.bz2 "$url" --progress=dot:giga; \ + echo "$sha256 *pypy.tar.bz2" | sha256sum --check --strict -; \ mkdir /opt/pypy; \ tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ rm pypy.tar.bz2; \ \ - ln -svT '/opt/pypy/bin/%%CMD%%' '/usr/local/bin/%%CMD%%'; \ + ln -sv {{ "/opt/pypy/bin/" + cmd | @sh }} /usr/local/bin/; \ \ # smoke test - %%CMD%% --version; \ + {{ cmd }} --version; \ \ +{{ if is_3 then ( -}} # on pypy3, rebuild ffi bits for compatibility with Debian Stretch+ cd /opt/pypy/lib_pypy; \ # https://github.com/docker-library/pypy/issues/24#issuecomment-409408657 if [ -f _ssl_build.py ]; then \ +{{ if is_slim then ( -}} apt-get install -y --no-install-recommends gcc libc6-dev libssl-dev; \ - %%CMD%% _ssl_build.py; \ +{{ ) else "" end -}} + {{ cmd }} _ssl_build.py; \ fi; \ # https://github.com/docker-library/pypy/issues/42 if [ -f _lzma_build.py ]; then \ +{{ if is_slim then ( -}} apt-get install -y --no-install-recommends gcc libc6-dev liblzma-dev; \ - %%CMD%% _lzma_build.py; \ +{{ ) else "" end -}} + {{ cmd }} _lzma_build.py; \ fi; \ # TODO rebuild other cffi modules here too? (other _*_build.py files) \ +{{ ) else "" end -}} apt-mark auto '.*' > /dev/null; \ [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ find /opt/pypy -type f -executable -exec ldd '{}' ';' \ @@ -69,7 +120,7 @@ RUN set -ex; \ apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ rm -rf /var/lib/apt/lists/*; \ # smoke test again, to be sure - %%CMD%% --version; \ + {{ cmd }} --version; \ \ find /opt/pypy -depth \ \( \ @@ -79,27 +130,29 @@ RUN set -ex; \ \) -exec rm -rf '{}' + # if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION %%PIP_VERSION%% +ENV PYTHON_PIP_VERSION {{ .pip.version }} # https://github.com/pypa/get-pip -ENV PYTHON_GET_PIP_URL %%PYTHON_GET_PIP_URL%% -ENV PYTHON_GET_PIP_SHA256 %%PYTHON_GET_PIP_SHA256%% +ENV PYTHON_GET_PIP_URL {{ ."get-pip".url }} +ENV PYTHON_GET_PIP_SHA256 {{ ."get-pip".sha256 }} RUN set -ex; \ +{{ if is_slim then ( -}} apt-get update; \ - apt-get install -y --no-install-recommends \ - wget \ - ; \ + apt-get install -y --no-install-recommends wget; \ rm -rf /var/lib/apt/lists/*; \ +{{ ) else "" end -}} \ wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \ \ - %%CMD%% get-pip.py \ + {{ cmd }} get-pip.py \ --disable-pip-version-check \ --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ + "pip == $PYTHON_PIP_VERSION" \ ; \ +{{ if is_slim then ( -}} apt-get purge -y --auto-remove wget; \ +{{ ) else "" end -}} # smoke test pip --version; \ \ @@ -111,4 +164,4 @@ RUN set -ex; \ \) -exec rm -rf '{}' +; \ rm -f get-pip.py -CMD ["%%CMD%%"] +CMD {{ [ cmd ] | @json }} diff --git a/Dockerfile.template b/Dockerfile.template deleted file mode 100644 index a808632..0000000 --- a/Dockerfile.template +++ /dev/null @@ -1,108 +0,0 @@ -FROM buildpack-deps:%%BASE%% - -# http://bugs.python.org/issue19846 -# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. -ENV LANG C.UTF-8 - -# runtime dependencies -RUN set -eux; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ - tcl \ - tk \ - ; \ - rm -rf /var/lib/apt/lists/* - -# ensure local pypy is preferred over distribution pypy -ENV PATH /opt/pypy/bin:$PATH - -ENV PYPY_VERSION %%PYPY_VERSION%% - -RUN set -ex; \ - \ -# this "case" statement is generated via "update.sh" - %%ARCH-CASE%%; \ - \ - savedAptMark="$(apt-mark showmanual)"; \ - apt-get update; \ - apt-get install -y --no-install-recommends \ -# sometimes "%%CMD%%" itself is linked against libexpat1 / libncurses5, sometimes they're ".so" files in "/opt/pypy/lib_pypy" - libexpat1 \ - libncurses5 \ -# (so we'll add them temporarily, then use "ldd" later to determine which to keep based on usage per architecture) - ; \ - \ - wget -O pypy.tar.bz2 "https://downloads.python.org/pypy/%%TAR%%-v${PYPY_VERSION}-${pypyArch}.tar.bz2" --progress=dot:giga; \ - echo "$sha256 *pypy.tar.bz2" | sha256sum -c; \ - mkdir /opt/pypy; \ - tar -xjC /opt/pypy --strip-components=1 -f pypy.tar.bz2; \ - find /opt/pypy/lib-python -depth -type d -a \( -name test -o -name tests \) -exec rm -rf '{}' +; \ - rm pypy.tar.bz2; \ - \ - ln -svT '/opt/pypy/bin/%%CMD%%' '/usr/local/bin/%%CMD%%'; \ - \ -# smoke test - %%CMD%% --version; \ - \ -# on pypy3, rebuild ffi bits for compatibility with Debian Stretch+ - cd /opt/pypy/lib_pypy; \ -# https://github.com/docker-library/pypy/issues/24#issuecomment-409408657 - if [ -f _ssl_build.py ]; then \ - %%CMD%% _ssl_build.py; \ - fi; \ -# https://github.com/docker-library/pypy/issues/42 - if [ -f _lzma_build.py ]; then \ - %%CMD%% _lzma_build.py; \ - fi; \ -# TODO rebuild other cffi modules here too? (other _*_build.py files) - \ - apt-mark auto '.*' > /dev/null; \ - [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \ - find /opt/pypy -type f -executable -exec ldd '{}' ';' \ - | awk '/=>/ { print $(NF-1) }' \ - | sort -u \ - | xargs -r dpkg-query --search \ - | cut -d: -f1 \ - | sort -u \ - | xargs -r apt-mark manual \ - ; \ - apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \ - rm -rf /var/lib/apt/lists/*; \ -# smoke test again, to be sure - %%CMD%% --version; \ - \ - find /opt/pypy -depth \ - \( \ - \( -type d -a \( -name test -o -name tests \) \) \ - -o \ - \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ - \) -exec rm -rf '{}' + - -# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" -ENV PYTHON_PIP_VERSION %%PIP_VERSION%% -# https://github.com/pypa/get-pip -ENV PYTHON_GET_PIP_URL %%PYTHON_GET_PIP_URL%% -ENV PYTHON_GET_PIP_SHA256 %%PYTHON_GET_PIP_SHA256%% - -RUN set -ex; \ - \ - wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \ - echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum --check --strict -; \ - \ - %%CMD%% get-pip.py \ - --disable-pip-version-check \ - --no-cache-dir \ - "pip==$PYTHON_PIP_VERSION" \ - ; \ -# smoke test - pip --version; \ - \ - find /opt/pypy -depth \ - \( \ - \( -type d -a \( -name test -o -name tests \) \) \ - -o \ - \( -type f -a \( -name '*.pyc' -o -name '*.pyo' \) \) \ - \) -exec rm -rf '{}' +; \ - rm -f get-pip.py - -CMD ["%%CMD%%"] diff --git a/apply-templates.sh b/apply-templates.sh new file mode 100755 index 0000000..a448d8f --- /dev/null +++ b/apply-templates.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[ -f versions.json ] # run "versions.sh" first + +jqt='.jq-template.awk' +if [ -n "${BASHBREW_SCRIPTS:-}" ]; then + jqt="$BASHBREW_SCRIPTS/jq-template.awk" +elif [ "$BASH_SOURCE" -nt "$jqt" ]; then + wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/00e281f36edd19f52541a6ba2f215cc3c4645128/scripts/jq-template.awk' +fi + +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi + +generated_warning() { + cat <<-EOH + # + # NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" + # + # PLEASE DO NOT EDIT IT DIRECTLY. + # + + EOH +} + +for version; do + export version + + rm -rf "$version/" + + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + for variant in "${variants[@]}"; do + export variant + + dir="$version/$variant" + mkdir -p "$dir" + + case "$variant" in + windows/*) + # TODO someday? :) + variant="$(basename "$dir")" # "buster", "windowsservercore-1809", etc + windowsVariant="${variant%%-*}" # "windowsservercore", "nanoserver" + windowsRelease="${variant#$windowsVariant-}" # "1809", "ltsc2016", etc + windowsVariant="${windowsVariant#windows}" # "servercore", "nanoserver" + export windowsVariant windowsRelease + template='Dockerfile-windows.template' + ;; + + *) + template='Dockerfile-linux.template' + ;; + esac + + echo "processing $dir ..." + + { + generated_warning + gawk -f "$jqt" "$template" + } > "$dir/Dockerfile" + done +done diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 4c4d439..4c6e0a3 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -2,17 +2,22 @@ set -Eeuo pipefail declare -A aliases=( - [3.6]='3' - [2.7]='2' + ['3.6']='3' + ['2.7']='2' ) +defaultDebianSuite='buster' + self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi -versions=( */ ) -versions=( "${versions[@]%/}" ) +# sort version numbers with highest first +IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS # get the most recent commit which modified any of "$@" fileCommit() { @@ -24,18 +29,38 @@ dirCommit() { local dir="$1"; shift ( cd "$dir" - fileCommit \ - Dockerfile \ - $(git show HEAD:./Dockerfile | awk ' + files="$( + git show HEAD:./Dockerfile | awk ' toupper($1) == "COPY" { for (i = 2; i < NF; i++) { + if ($i ~ /^--from=/) { + next + } print $i } } - ') + ' + )" + fileCommit Dockerfile $files ) } +getArches() { + local repo="$1"; shift + local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' + + eval "declare -g -A parentRepoToArches=( $( + find -name 'Dockerfile' -exec awk ' + toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { + print "'"$officialImagesUrl"'" $2 + } + ' '{}' + \ + | sort -u \ + | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' + ) )" +} +getArches 'pypy' + cat <<-EOH # this file is generated via https://github.com/docker-library/pypy/blob/$(fileCommit "$self")/$self @@ -51,11 +76,12 @@ join() { echo "${out#$sep}" } -for version in "${versions[@]}"; do - commit="$(dirCommit "$version")" +for version; do + export version + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" - fullVersion="$(git show "$commit":"$version/Dockerfile" | awk '$1 == "ENV" && $2 == "PYPY_VERSION" { print $3; exit }')" - #fullVersion="$version-$fullVersion" + fullVersion="$(jq -r '.[env.version].version' versions.json)" pypyVersionAliases=() while [ "${fullVersion%[.-]*}" != "$fullVersion" ]; do @@ -75,28 +101,41 @@ for version in "${versions[@]}"; do fi done - for variant in '' slim; do - dir="$version${variant:+/$variant}" + for variant in "${variants[@]}"; do + dir="$version/$variant" [ -f "$dir/Dockerfile" ] || continue commit="$(dirCommit "$dir")" - variantAliases=( "${versionAliases[@]}" ) - if [ -n "$variant" ]; then - variantAliases=( "${variantAliases[@]/%/-$variant}" ) - variantAliases=( "${variantAliases[@]//latest-/}" ) - fi - - variantParent="$(parent "$dir")" - - suite="${variantParent#*:}" # "jessie-slim", "stretch" - suite="${suite%-slim}" # "jessie", "stretch" - - suiteAliases=( "${variantAliases[@]/%/-$suite}" ) - suiteAliases=( "${suiteAliases[@]//latest-/}" ) - variantAliases+=( "${suiteAliases[@]}" ) - - variantArches="$(parentArches "$dir" "$variantParent")" + variantAliases=( "${versionAliases[@]/%/-$variant}" ) + case "$variant" in + "$defaultDebianSuite") + variantAliases=( + "${versionAliases[@]}" + "${variantAliases[@]}" + ) + ;; + slim-"$defaultDebianSuite") + variantAliases=( + "${versionAliases[@]/%/-slim}" + "${variantAliases[@]}" + ) + ;; + esac + variantAliases=( "${variantAliases[@]//latest-/}" ) + + variantParent="$(awk 'toupper($1) == "FROM" { print $2; exit }' "$dir/Dockerfile")" + variantArches="${parentRepoToArches[$variantParent]:-}" + variantArches="$( + comm -12 \ + <( + jq -r ' + .[env.version].arches + | keys[] + ' versions.json | sort + ) \ + <(xargs -n1 <<<"$variantArches" | sort) + )" echo cat <<-EOE diff --git a/release-architectures b/release-architectures deleted file mode 100644 index 0106210..0000000 --- a/release-architectures +++ /dev/null @@ -1,13 +0,0 @@ -# see https://bitbucket.org/pypy/pypy/downloads/ -# and https://pypy.org/download.html#checksums - -# bashbrew-arch dpkg-arch pypy-release-arch -amd64 amd64 linux64 -arm32v5 armel linux-armel -arm32v7 armhf linux-armhf-raring -arm64v8 arm64 aarch64 -i386 i386 linux32 - -# see https://bitbucket.org/pypy/pypy/issues/2646 for some s390x/ppc64le caveats (mitigated in 3.x via https://github.com/docker-library/pypy/issues/24#issuecomment-476873691) -ppc64le ppc64el ppc64le -s390x s390x s390x diff --git a/update.sh b/update.sh index b6f9db3..bac2d75 100755 --- a/update.sh +++ b/update.sh @@ -1,125 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' - -versions=( "$@" ) -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi -versions=( "${versions[@]%/}" ) - -pipVersion="$(curl -fsSL 'https://pypi.org/pypi/pip/json' | jq '.releases | keys[] | select(startswith("20."))' -r | sort -rV | head -1)" # version 20.x is the last to support Python 2 -getPipCommit="$(curl -fsSL "https://github.com/pypa/get-pip/commits/$pipVersion/get-pip.py.atom" | tac|tac | awk -F '[[:space:]]*[<>/]+' '$2 == "id" && $3 ~ /Commit/ { print $4; exit }')" -getPipUrl="https://github.com/pypa/get-pip/raw/$getPipCommit/get-pip.py" -getPipSha256="$(curl -fsSL "$getPipUrl" | sha256sum | cut -d' ' -f1)" - -sha256s="$(curl -fsSL 'https://pypy.org/download.html')" -scrapeSha256() { - local pypy="$1"; shift - local fullVersion="$1"; shift - local arch="$1"; shift - - #

pypy2.7-5.4.0 sha256:

- #
-	# ...
-	# bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
-	# ...
-	# 
- grep -om1 -E '[a-f0-9]{64} '"$pypy-v$tryVersion"'-'"$arch"'.tar.bz2' <<<"$sha256s" \ - | cut -d' ' -f1 -} - -# see http://stackoverflow.com/a/2705678/433558 -sed_escape_rhs() { - echo "$@" | sed -e 's/[\/&]/\\&/g' | sed -e ':a;N;$!ba;s/\n/\\n/g' -} - -for version in "${versions[@]}"; do - case "$version" in - 3 | 3.*) cmd='pypy3'; base='buster' ;; - 2 | 2.*) cmd='pypy'; base='buster' ;; - *) echo >&2 "error: unknown pypy variant $version"; exit 1 ;; - esac - pypy="pypy$version" - - # pypy3.6-v7.3.1-aarch64.tar.bz2 - # pypy2.7-v7.3.1-aarch64.tar.bz2 - IFS=$'\n' - tryVersions=( $( - curl -fsSL --compressed 'https://downloads.python.org/pypy/' \ - | sed -rn 's/^.*'"$pypy"'-v([0-9.]+(-alpha[0-9]*)?)-linux64.tar.bz2.*$/\1/gp' \ - | sort -rV - ) ) - unset IFS - - fullVersion= - sha256sum= - for tryVersion in "${tryVersions[@]}"; do - if \ - sha256sum="$(scrapeSha256 "$pypy" "$tryVersion" 'linux64')" \ - && [ -n "$sha256sum" ] \ - ; then - fullVersion="$tryVersion" - break - fi - done - if [ -z "$fullVersion" ]; then - echo >&2 "error: cannot find suitable release for '$version'" - exit 1 - fi - - # if our current version is newer than the version we just scraped, this must be a fluke/flake (https://github.com/docker-library/official-images/pull/6163) - if currentVersion="$(awk '$1 == "ENV" && $2 == "PYPY_VERSION" { print $3; exit }' "$version/Dockerfile" 2>/dev/null)" && [ -n "$currentVersion" ] && [ "$currentVersion" != "$fullVersion" ]; then - newVersion="$( - { - echo "$fullVersion" - echo "$currentVersion" - } | sort -rV | head -1 - )" - if [ "$newVersion" = "$currentVersion" ]; then - echo >&2 "error: scraped version ($fullVersion) is older than our current version ($currentVersion)!" - echo >&2 " cowardly bailing to avoid unnecessary churn" - exit 1 - fi - fi - - echo "$version: $fullVersion" - - linuxArchCase='dpkgArch="$(dpkg --print-architecture)"; '$'\\\n' - linuxArchCase+=$'\t''case "${dpkgArch##*-}" in '$'\\\n' - for dpkgArch in $(dpkgArches); do - bashbrewArch="$(dpkgToBashbrewArch "$dpkgArch")" - case "$version/$bashbrewArch" in - 2.7/s390x | 2.7/ppc64le) - echo >&2 "warning: skipping $pypy on $bashbrewArch; https://bitbucket.org/pypy/pypy/issues/2646" - continue - ;; - esac - pypyArch="$(dpkgToPyPyArch "$dpkgArch")" - sha256="$(scrapeSha256 "$pypy" "$fullVersion" "$pypyArch")" || : - if [ -z "$sha256" ]; then - echo >&2 "warning: cannot find sha256 for $pypy-$fullVersion on arch $pypyArch ($bashbrewArch); skipping it" - continue - fi - linuxArchCase+="# $bashbrewArch"$'\n' - linuxArchCase+=$'\t\t'"$dpkgArch) pypyArch='$pypyArch'; sha256='$sha256' ;; "$'\\\n' - done - linuxArchCase+=$'\t\t''*) echo >&2 "error: current architecture ($dpkgArch) does not have a corresponding PyPy $PYPY_VERSION binary release"; exit 1 ;; '$'\\\n' - linuxArchCase+=$'\t''esac' - - for variant in slim ''; do - sed -r \ - -e 's!%%PYPY_VERSION%%!'"$fullVersion"'!g' \ - -e 's!%%PIP_VERSION%%!'"$pipVersion"'!g' \ - -e 's!%%PYTHON_GET_PIP_URL%%!'"$getPipUrl"'!' \ - -e 's!%%PYTHON_GET_PIP_SHA256%%!'"$getPipSha256"'!' \ - -e 's!%%TAR%%!'"$pypy"'!g' \ - -e 's!%%CMD%%!'"$cmd"'!g' \ - -e 's!%%BASE%%!'"$base"'!g' \ - -e 's!%%ARCH-CASE%%!'"$(sed_escape_rhs "$linuxArchCase")"'!g' \ - "Dockerfile${variant:+-$variant}.template" > "$version/$variant/Dockerfile" - done -done +./versions.sh "$@" +./apply-templates.sh "$@" diff --git a/versions.json b/versions.json new file mode 100644 index 0000000..91b59d9 --- /dev/null +++ b/versions.json @@ -0,0 +1,97 @@ +{ + "2.7": { + "arches": { + "amd64": { + "sha256": "f412b602ccd6912ddee0e7523e0e38f4b2c7a144449c2cad078cffbdb66fd7b1", + "url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux64.tar.bz2" + }, + "arm64v8": { + "sha256": "23b145b7cfbaeefb6ee76fc8216c83b652ab1daffac490558718edbbd60082d8", + "url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-aarch64.tar.bz2" + }, + "i386": { + "sha256": "bfbc81874b137837a8ba8c517b97de29f5a336f7ec500c52f2bfdbd3580d1703", + "url": "https://downloads.python.org/pypy/pypy2.7-v7.3.3-linux32.tar.bz2" + } + }, + "get-pip": { + "sha256": "95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d399b2d5218", + "url": "https://github.com/pypa/get-pip/raw/3843bff3a0a61da5b63ea0b7d34794c5c51a2f11/get-pip.py", + "version": "3843bff3a0a61da5b63ea0b7d34794c5c51a2f11" + }, + "pip": { + "version": "20.3.4" + }, + "variants": [ + "buster", + "slim-buster" + ], + "version": "7.3.3" + }, + "3.6": { + "arches": { + "amd64": { + "sha256": "4fb85fdd516482cab727bb9473b066ff8fb672940dedf7ccc32bf92957d29e0a", + "url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux64.tar.bz2" + }, + "arm64v8": { + "sha256": "bc82cf7f0182b942a2cfad4a0d167f364bfbf18f434e100a2fe62bc88547ac9b", + "url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-aarch64.tar.bz2" + }, + "i386": { + "sha256": "f183c61e66fd2c536a65695bd7ff770748c2884c235a589b9c6ac63690770c69", + "url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-linux32.tar.bz2" + }, + "s390x": { + "sha256": "0de9c33ff3500c6e7fd273d0a6d341bc839b0298f697c4d6fe141f2b54c5c3e2", + "url": "https://downloads.python.org/pypy/pypy3.6-v7.3.3-s390x.tar.bz2" + } + }, + "get-pip": { + "sha256": "95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d399b2d5218", + "url": "https://github.com/pypa/get-pip/raw/3843bff3a0a61da5b63ea0b7d34794c5c51a2f11/get-pip.py", + "version": "3843bff3a0a61da5b63ea0b7d34794c5c51a2f11" + }, + "pip": { + "version": "20.3.4" + }, + "variants": [ + "buster", + "slim-buster" + ], + "version": "7.3.3" + }, + "3.7": { + "arches": { + "amd64": { + "sha256": "37e2804c4661c86c857d709d28c7de716b000d31e89766599fdf5a98928b7096", + "url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux64.tar.bz2" + }, + "arm64v8": { + "sha256": "ee4aa041558b58de6063dd6df93b3def221c4ca4c900d6a9db5b1b52135703a8", + "url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-aarch64.tar.bz2" + }, + "i386": { + "sha256": "7d81b8e9fcd07c067cfe2f519ab770ec62928ee8787f952cadf2d2786246efc8", + "url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-linux32.tar.bz2" + }, + "s390x": { + "sha256": "92000d90b9a37f2e9cb7885f2a872adfa9e48e74bf7f84a8b8185c8181f0502d", + "url": "https://downloads.python.org/pypy/pypy3.7-v7.3.3-s390x.tar.bz2" + } + }, + "get-pip": { + "sha256": "95c5ee602b2f3cc50ae053d716c3c89bea62c58568f64d7d25924d399b2d5218", + "url": "https://github.com/pypa/get-pip/raw/3843bff3a0a61da5b63ea0b7d34794c5c51a2f11/get-pip.py", + "version": "3843bff3a0a61da5b63ea0b7d34794c5c51a2f11" + }, + "pip": { + "version": "20.3.4" + }, + "variants": [ + "buster", + "slim-buster" + ], + "version": "7.3.3" + } +} diff --git a/versions.sh b/versions.sh new file mode 100755 index 0000000..6cb5a4d --- /dev/null +++ b/versions.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# see https://downloads.python.org/pypy/ +declare -A pypyArches=( + ['amd64']='linux64' + ['arm32v5']='linux-armel' + ['arm32v7']='linux-armhf-raring' + ['arm64v8']='aarch64' + ['i386']='linux32' + + # see https://foss.heptapod.net/pypy/pypy/-/issues/2646 for some s390x/ppc64le caveats (mitigated in 3.x via https://github.com/docker-library/pypy/issues/24#issuecomment-476873691) + ['ppc64le']='ppc64le' + ['s390x']='s390x' +) + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +versions=( "$@" ) +if [ ${#versions[@]} -eq 0 ]; then + versions=( */ ) + json='{}' +else + json="$(< versions.json)" +fi +versions=( "${versions[@]%/}" ) + +pipVersion="$( + curl -fsSL 'https://pypi.org/pypi/pip/json' \ + | jq -r ' + .releases + | keys[] + # version 20.x is the last to support Python 2 + | select(startswith("20.")) + ' \ + | sort -rV \ + | head -1 +)" +getPipCommit="$(curl -fsSL "https://github.com/pypa/get-pip/commits/$pipVersion/get-pip.py.atom" | tac|tac | awk -F '[[:space:]]*[<>/]+' '$2 == "id" && $3 ~ /Commit/ { print $4; exit }')" +getPipUrl="https://github.com/pypa/get-pip/raw/$getPipCommit/get-pip.py" +getPipSha256="$(curl -fsSL "$getPipUrl" | sha256sum | cut -d' ' -f1)" +export pipVersion getPipCommit getPipUrl getPipSha256 + +sha256s="$(curl -fsSL --compressed 'https://pypy.org/download.html')" +pypy_tarball() { + local pypy="$1"; shift + local fullVersion="$1"; shift + local arch="$1"; shift + + echo "pypy$pypy-v$fullVersion-$arch.tar.bz2" +} +scrape_sha256() { + local tarball="$1"; shift + + #

pypy2.7-5.4.0 sha256:

+ #
+	# ...
+	# bdfea513d59dcd580970cb6f79f3a250d00191fd46b68133d5327e924ca845f8  pypy2-v5.4.0-linux64.tar.bz2
+	# ...
+	# 
+ grep -om1 -E "[a-f0-9]{64} $tarball" <<<"$sha256s" \ + | cut -d' ' -f1 +} + +downloads="$(curl -fsSL --compressed 'https://downloads.python.org/pypy/')" + +for version in "${versions[@]}"; do + export version + + IFS=$'\n' + tryVersions=( $( + sed -rn 's/^.*pypy'"$version"'-v([0-9.]+(-alpha[0-9]*)?)-'"${pypyArches['amd64']}"'[.]tar[.]bz2.*$/\1/gp' <<<"$downloads" \ + | sort -rV + ) ) + unset IFS + + fullVersion= + pypyArch="${pypyArches['amd64']}" + tarball= + sha256= + for tryVersion in "${tryVersions[@]}"; do + if tarball="$(pypy_tarball "$version" "$tryVersion" "$pypyArch")" && sha256="$(scrape_sha256 "$tarball")" && [ -n "$sha256" ]; then + fullVersion="$tryVersion" + break + fi + done + if [ -z "$fullVersion" ]; then + echo >&2 "error: cannot find suitable release for '$version'" + exit 1 + fi + + export fullVersion tarball sha256 + doc="$(jq -nc ' + { + version: env.fullVersion, + arches: { + amd64: { + sha256: env.sha256, + url: ("https://downloads.python.org/pypy/" + env.tarball), + }, + }, + pip: { version: env.pipVersion }, + "get-pip": { + version: env.getPipCommit, + sha256: env.getPipSha256, + url: env.getPipUrl, + }, + variants: [ "buster", "slim-buster" ], + } + ')" + + # if our current version is newer than the version we just scraped, this must be a fluke/flake (https://github.com/docker-library/official-images/pull/6163) + if \ + currentVersion="$( + jq -r '.[env.version].version // ""' versions.json 2>/dev/null + )" \ + && [ -n "$currentVersion" ] \ + && [ "$currentVersion" != "$fullVersion" ] \ + && newVersion="$( + { + echo "$fullVersion" + echo "$currentVersion" + } | sort -rV | head -1 + )" \ + && [ "$newVersion" = "$currentVersion" ] \ + ; then + echo >&2 "error: scraped version ($fullVersion) is older than our current version ($currentVersion)!" + echo >&2 " cowardly bailing to avoid unnecessary churn" + exit 1 + fi + + echo "$version: $fullVersion" + + for bashbrewArch in "${!pypyArches[@]}"; do + case "$version/$bashbrewArch" in + */amd64) + # we already collected the amd64 sha256 above + continue + ;; + + 2.7/s390x | 2.7/ppc64le) + echo >&2 "warning: skipping $version on $bashbrewArch; https://foss.heptapod.net/pypy/pypy/-/issues/2646" + continue + ;; + esac + + pypyArch="${pypyArches["$bashbrewArch"]}" + if tarball="$(pypy_tarball "$version" "$fullVersion" "$pypyArch")" && sha256="$(scrape_sha256 "$tarball")" && [ -n "$sha256" ]; then + export bashbrewArch tarball sha256 + doc="$(jq <<<"$doc" -c ' + .arches[env.bashbrewArch] = { + sha256: env.sha256, + url: ("https://downloads.python.org/pypy/" + env.tarball), + } + ')" + fi + done + + json="$(jq <<<"$json" -c --argjson doc "$doc" '.[env.version] = $doc')" +done + +jq <<<"$json" -S . > versions.json