From 5e970060c3d8916095bc1dadacdd76754305c3c2 Mon Sep 17 00:00:00 2001 From: Tianon Gravi Date: Wed, 19 Aug 2020 22:19:44 -0700 Subject: [PATCH] Add initial jq-based templating engine --- .architectures-lib | 62 ------- .gitattributes | 2 + .github/workflows/verify-templating.yml | 22 +++ .gitignore | 1 + 1.14/alpine3.11/Dockerfile | 58 +++++-- 1.14/alpine3.12/Dockerfile | 58 +++++-- 1.14/buster/Dockerfile | 69 ++++++-- 1.14/stretch/Dockerfile | 69 ++++++-- 1.14/windows/nanoserver-1809/Dockerfile | 6 + .../windows/windowsservercore-1809/Dockerfile | 8 +- .../windowsservercore-ltsc2016/Dockerfile | 8 +- 1.15/alpine3.12/Dockerfile | 58 +++++-- 1.15/buster/Dockerfile | 69 ++++++-- 1.15/windows/nanoserver-1809/Dockerfile | 6 + .../windows/windowsservercore-1809/Dockerfile | 8 +- .../windowsservercore-ltsc2016/Dockerfile | 8 +- Dockerfile-alpine.template | 56 +++++-- Dockerfile-debian.template | 60 ++++++- Dockerfile-windows-nanoserver.template | 6 +- ... => Dockerfile-windows-servercore.template | 8 +- apply-templates.sh | 66 ++++++++ generate-stackbrew-library.sh | 61 ++++--- release-architectures | 9 - update.sh | 115 +------------ versions.json | 108 ++++++++++++ versions.sh | 155 ++++++++++++++++++ 26 files changed, 843 insertions(+), 313 deletions(-) delete mode 100644 .architectures-lib create mode 100644 .gitattributes create mode 100644 .github/workflows/verify-templating.yml create mode 100644 .gitignore rename Dockerfile-windows-windowsservercore.template => Dockerfile-windows-servercore.template (92%) create mode 100755 apply-templates.sh delete mode 100644 release-architectures create mode 100644 versions.json create mode 100755 versions.sh diff --git a/.architectures-lib b/.architectures-lib deleted file mode 100644 index d3efd55c..00000000 --- a/.architectures-lib +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash - -_awkArch() { - local version="$1"; shift - local awkExpr="$1"; shift - awk "$@" "/^#|^\$/ { next } $awkExpr" release-architectures -} - -dpkgArches() { - local version="$1"; shift - _awkArch "$version" '{ print $2 }' -} - -hasBashbrewArch() { - local version="$1"; shift - local bashbrewArch="$1"; shift - _awkArch "$version" 'BEGIN { exitCode = 1 } $1 == bashbrewArch { exitCode = 0 } END { exit exitCode }' -v bashbrewArch="$bashbrewArch" -} - -dpkgToGoArch() { - local version="$1"; shift - local dpkgArch="$1"; shift - _awkArch "$version" '$2 == dpkgArch { print $3; exit }' -v dpkgArch="$dpkgArch" -} - -_generateParentRepoToArches() { - local repo="$1"; shift - local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' - - eval "declare -g -A parentRepoToArches=( $( - find -name 'Dockerfile' -exec awk ' - toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { - print "'"$officialImagesUrl"'" $2 - } - ' '{}' + \ - | sort -u \ - | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' - ) )" -} -_generateParentRepoToArches 'golang' - -parentArches() { - local version="$1"; shift # "1.8", etc - local variant="$1"; shift # "", "stretch", etc - - local parent="$(awk 'toupper($1) == "FROM" { print $2 }' "$version/$variant/Dockerfile")" - echo "${parentRepoToArches[$parent]:-}" -} -variantArches() { - local version="$1"; shift # "1.8", etc - local variant="$1"; shift # "", "stretch", etc - - local parentArches="$(parentArches "$version" "$variant")" - - local variantArches=() - for arch in $parentArches; do - if hasBashbrewArch "$version" "$arch"; then - variantArches+=( "$arch" ) - fi - done - echo "${variantArches[*]}" -} diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..de47a6e3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +**/Dockerfile linguist-generated +Dockerfile*.template linguist-language=Dockerfile diff --git a/.github/workflows/verify-templating.yml b/.github/workflows/verify-templating.yml new file mode 100644 index 00000000..7e833f1c --- /dev/null +++ b/.github/workflows/verify-templating.yml @@ -0,0 +1,22 @@ +name: Verify Templating + +on: + pull_request: + push: + +defaults: + run: + shell: 'bash -Eeuo pipefail -x {0}' + +jobs: + apply-templates: + name: Check For Uncomitted Changes + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Apply Templates + run: ./apply-templates.sh + - name: Check Git Status + run: | + status="$(git status --short)" + [ -z "$status" ] diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..d548f66d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.jq-template.awk diff --git a/1.14/alpine3.11/Dockerfile b/1.14/alpine3.11/Dockerfile index 4f4cddd7..f46d661b 100644 --- a/1.14/alpine3.11/Dockerfile +++ b/1.14/alpine3.11/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.11 RUN apk add --no-cache \ @@ -8,15 +14,18 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf +ENV PATH /usr/local/go/bin:$PATH + ENV GOLANG_VERSION 1.14.7 RUN set -eux; \ apk add --no-cache --virtual .build-deps \ bash \ gcc \ + gnupg \ + go \ musl-dev \ openssl \ - go \ ; \ export \ # set GOROOT_BOOTSTRAP such that we can actually build Go @@ -37,28 +46,53 @@ RUN set -eux; \ x86) export GO386='387' ;; \ esac; \ \ - wget -O go.tgz "https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz"; \ - echo '064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3 *go.tgz' | sha256sum -c -; \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + url='https://storage.googleapis.com/golang/go1.14.7.src.tar.gz'; \ + sha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ + \ + wget -O go.tgz.asc "$url.asc"; \ + wget -O go.tgz "$url"; \ + echo "$sha256 *go.tgz" | sha256sum -c -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ - cd /usr/local/go/src; \ - ./make.bash; \ + goEnv="$(go env | sed -rn -e '/^GO(OS|ARCH|ARM|386)=/s//export \0/p')"; \ + eval "$goEnv"; \ + [ -n "$GOOS" ]; \ + [ -n "$GOARCH" ]; \ + ( \ + cd /usr/local/go/src; \ + ./make.bash; \ + ); \ + \ + apk del --no-network .build-deps; \ \ +# pre-compile the standard library, just like the official binary release tarballs do + go install std; \ +# go install: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 +# go install -race std; \ + \ +# remove a few intermediate / bootstrapping files the official binary release tarballs do not contain rm -rf \ -# https://github.com/golang/go/blob/0b30cf534a03618162d3015c8705dd2231e34703/src/cmd/dist/buildtool.go#L121-L125 + /usr/local/go/pkg/*/cmd \ /usr/local/go/pkg/bootstrap \ -# https://golang.org/cl/82095 -# https://github.com/golang/build/blob/e3fe1605c30f6a3fd136b561569933312ede8782/cmd/release/releaselet.go#L56 /usr/local/go/pkg/obj \ + /usr/local/go/pkg/tool/*/api \ + /usr/local/go/pkg/tool/*/go_bootstrap \ + /usr/local/go/src/cmd/dist/dist \ ; \ - apk del .build-deps; \ \ - export PATH="/usr/local/go/bin:$PATH"; \ go version ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - +ENV PATH $GOPATH/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH diff --git a/1.14/alpine3.12/Dockerfile b/1.14/alpine3.12/Dockerfile index e1317341..f513d787 100644 --- a/1.14/alpine3.12/Dockerfile +++ b/1.14/alpine3.12/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.12 RUN apk add --no-cache \ @@ -8,15 +14,18 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf +ENV PATH /usr/local/go/bin:$PATH + ENV GOLANG_VERSION 1.14.7 RUN set -eux; \ apk add --no-cache --virtual .build-deps \ bash \ gcc \ + gnupg \ + go \ musl-dev \ openssl \ - go \ ; \ export \ # set GOROOT_BOOTSTRAP such that we can actually build Go @@ -37,28 +46,53 @@ RUN set -eux; \ x86) export GO386='387' ;; \ esac; \ \ - wget -O go.tgz "https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz"; \ - echo '064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3 *go.tgz' | sha256sum -c -; \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + url='https://storage.googleapis.com/golang/go1.14.7.src.tar.gz'; \ + sha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ + \ + wget -O go.tgz.asc "$url.asc"; \ + wget -O go.tgz "$url"; \ + echo "$sha256 *go.tgz" | sha256sum -c -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ - cd /usr/local/go/src; \ - ./make.bash; \ + goEnv="$(go env | sed -rn -e '/^GO(OS|ARCH|ARM|386)=/s//export \0/p')"; \ + eval "$goEnv"; \ + [ -n "$GOOS" ]; \ + [ -n "$GOARCH" ]; \ + ( \ + cd /usr/local/go/src; \ + ./make.bash; \ + ); \ + \ + apk del --no-network .build-deps; \ \ +# pre-compile the standard library, just like the official binary release tarballs do + go install std; \ +# go install: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 +# go install -race std; \ + \ +# remove a few intermediate / bootstrapping files the official binary release tarballs do not contain rm -rf \ -# https://github.com/golang/go/blob/0b30cf534a03618162d3015c8705dd2231e34703/src/cmd/dist/buildtool.go#L121-L125 + /usr/local/go/pkg/*/cmd \ /usr/local/go/pkg/bootstrap \ -# https://golang.org/cl/82095 -# https://github.com/golang/build/blob/e3fe1605c30f6a3fd136b561569933312ede8782/cmd/release/releaselet.go#L56 /usr/local/go/pkg/obj \ + /usr/local/go/pkg/tool/*/api \ + /usr/local/go/pkg/tool/*/go_bootstrap \ + /usr/local/go/src/cmd/dist/dist \ ; \ - apk del .build-deps; \ \ - export PATH="/usr/local/go/bin:$PATH"; \ go version ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - +ENV PATH $GOPATH/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH diff --git a/1.14/buster/Dockerfile b/1.14/buster/Dockerfile index d6273825..e07277f6 100644 --- a/1.14/buster/Dockerfile +++ b/1.14/buster/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM buildpack-deps:buster-scm # gcc for cgo @@ -15,27 +21,64 @@ ENV GOLANG_VERSION 1.14.7 RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ - amd64) goRelArch='linux-amd64'; goRelSha256='4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5' ;; \ - armhf) goRelArch='linux-armv6l'; goRelSha256='6079eb82bcf24b33dda0e32777c7fdddcc3b1ec70e374308cc8311562449b107' ;; \ - arm64) goRelArch='linux-arm64'; goRelSha256='fe5b6f6e441f3cb7b53ebf1a010bbebcb720ac98124984cfe2e51d72b8a58c71' ;; \ - i386) goRelArch='linux-386'; goRelSha256='2f5793f10bb6b08eedecd376aa3d594e10193c6b5cf198ada46200259ff76547' ;; \ - ppc64el) goRelArch='linux-ppc64le'; goRelSha256='bd1f12a2271a6d1689bcf3ec01d123c81cbaca5d16c3f7df294a2d725ac4d3d1' ;; \ - s390x) goRelArch='linux-s390x'; goRelSha256='ea570b3caa0c271da440f568ab646cfea80d712c51fb4d08189bb66bd5eb949c' ;; \ - *) goRelArch='src'; goRelSha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ - echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \ + 'amd64') \ + arch='linux-amd64'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz'; \ + sha256='4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5'; \ + ;; \ + 'armhf') \ + arch='linux-armv6l'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-armv6l.tar.gz'; \ + sha256='6079eb82bcf24b33dda0e32777c7fdddcc3b1ec70e374308cc8311562449b107'; \ + ;; \ + 'arm64') \ + arch='linux-arm64'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-arm64.tar.gz'; \ + sha256='fe5b6f6e441f3cb7b53ebf1a010bbebcb720ac98124984cfe2e51d72b8a58c71'; \ + ;; \ + 'i386') \ + arch='linux-386'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-386.tar.gz'; \ + sha256='2f5793f10bb6b08eedecd376aa3d594e10193c6b5cf198ada46200259ff76547'; \ + ;; \ + 'ppc64el') \ + arch='linux-ppc64le'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-ppc64le.tar.gz'; \ + sha256='bd1f12a2271a6d1689bcf3ec01d123c81cbaca5d16c3f7df294a2d725ac4d3d1'; \ + ;; \ + 's390x') \ + arch='linux-s390x'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-s390x.tar.gz'; \ + sha256='ea570b3caa0c271da440f568ab646cfea80d712c51fb4d08189bb66bd5eb949c'; \ + ;; \ + *) \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + arch='src'; \ + url='https://storage.googleapis.com/golang/go1.14.7.src.tar.gz'; \ + sha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ + echo >&2; \ + echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; \ + echo >&2; \ + ;; \ esac; \ \ - url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \ + wget -O go.tgz.asc "$url.asc" --progress=dot:giga; \ wget -O go.tgz "$url" --progress=dot:giga; \ - echo "${goRelSha256} *go.tgz" | sha256sum -c -; \ + echo "$sha256 *go.tgz" | sha256sum --strict --check -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ -# https://github.com/golang/go/issues/38536#issuecomment-616897960 - if [ "$goRelArch" = 'src' ]; then \ + if [ "$arch" = 'src' ]; then \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends golang-go; \ diff --git a/1.14/stretch/Dockerfile b/1.14/stretch/Dockerfile index aae7651b..9260c561 100644 --- a/1.14/stretch/Dockerfile +++ b/1.14/stretch/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM buildpack-deps:stretch-scm # gcc for cgo @@ -15,27 +21,64 @@ ENV GOLANG_VERSION 1.14.7 RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ - amd64) goRelArch='linux-amd64'; goRelSha256='4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5' ;; \ - armhf) goRelArch='linux-armv6l'; goRelSha256='6079eb82bcf24b33dda0e32777c7fdddcc3b1ec70e374308cc8311562449b107' ;; \ - arm64) goRelArch='linux-arm64'; goRelSha256='fe5b6f6e441f3cb7b53ebf1a010bbebcb720ac98124984cfe2e51d72b8a58c71' ;; \ - i386) goRelArch='linux-386'; goRelSha256='2f5793f10bb6b08eedecd376aa3d594e10193c6b5cf198ada46200259ff76547' ;; \ - ppc64el) goRelArch='linux-ppc64le'; goRelSha256='bd1f12a2271a6d1689bcf3ec01d123c81cbaca5d16c3f7df294a2d725ac4d3d1' ;; \ - s390x) goRelArch='linux-s390x'; goRelSha256='ea570b3caa0c271da440f568ab646cfea80d712c51fb4d08189bb66bd5eb949c' ;; \ - *) goRelArch='src'; goRelSha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ - echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \ + 'amd64') \ + arch='linux-amd64'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz'; \ + sha256='4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5'; \ + ;; \ + 'armhf') \ + arch='linux-armv6l'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-armv6l.tar.gz'; \ + sha256='6079eb82bcf24b33dda0e32777c7fdddcc3b1ec70e374308cc8311562449b107'; \ + ;; \ + 'arm64') \ + arch='linux-arm64'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-arm64.tar.gz'; \ + sha256='fe5b6f6e441f3cb7b53ebf1a010bbebcb720ac98124984cfe2e51d72b8a58c71'; \ + ;; \ + 'i386') \ + arch='linux-386'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-386.tar.gz'; \ + sha256='2f5793f10bb6b08eedecd376aa3d594e10193c6b5cf198ada46200259ff76547'; \ + ;; \ + 'ppc64el') \ + arch='linux-ppc64le'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-ppc64le.tar.gz'; \ + sha256='bd1f12a2271a6d1689bcf3ec01d123c81cbaca5d16c3f7df294a2d725ac4d3d1'; \ + ;; \ + 's390x') \ + arch='linux-s390x'; \ + url='https://storage.googleapis.com/golang/go1.14.7.linux-s390x.tar.gz'; \ + sha256='ea570b3caa0c271da440f568ab646cfea80d712c51fb4d08189bb66bd5eb949c'; \ + ;; \ + *) \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + arch='src'; \ + url='https://storage.googleapis.com/golang/go1.14.7.src.tar.gz'; \ + sha256='064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3'; \ + echo >&2; \ + echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; \ + echo >&2; \ + ;; \ esac; \ \ - url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \ + wget -O go.tgz.asc "$url.asc" --progress=dot:giga; \ wget -O go.tgz "$url" --progress=dot:giga; \ - echo "${goRelSha256} *go.tgz" | sha256sum -c -; \ + echo "$sha256 *go.tgz" | sha256sum --strict --check -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ -# https://github.com/golang/go/issues/38536#issuecomment-616897960 - if [ "$goRelArch" = 'src' ]; then \ + if [ "$arch" = 'src' ]; then \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends golang-go; \ diff --git a/1.14/windows/nanoserver-1809/Dockerfile b/1.14/windows/nanoserver-1809/Dockerfile index 49f40b51..05137cfa 100644 --- a/1.14/windows/nanoserver-1809/Dockerfile +++ b/1.14/windows/nanoserver-1809/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/nanoserver:1809 SHELL ["cmd", "/S", "/C"] diff --git a/1.14/windows/windowsservercore-1809/Dockerfile b/1.14/windows/windowsservercore-1809/Dockerfile index 87cf720f..05e43bf8 100644 --- a/1.14/windows/windowsservercore-1809/Dockerfile +++ b/1.14/windows/windowsservercore-1809/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/servercore:1809 # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 @@ -48,7 +54,7 @@ RUN $newPath = ('{0}\bin;C:\go\bin;{1}' -f $env:GOPATH, $env:PATH); \ ENV GOLANG_VERSION 1.14.7 -RUN $url = ('https://golang.org/dl/go{0}.windows-amd64.zip' -f $env:GOLANG_VERSION); \ +RUN $url = 'https://storage.googleapis.com/golang/go1.14.7.windows-amd64.zip'; \ Write-Host ('Downloading {0} ...' -f $url); \ Invoke-WebRequest -Uri $url -OutFile 'go.zip'; \ \ diff --git a/1.14/windows/windowsservercore-ltsc2016/Dockerfile b/1.14/windows/windowsservercore-ltsc2016/Dockerfile index 5e651d1d..7711c147 100644 --- a/1.14/windows/windowsservercore-ltsc2016/Dockerfile +++ b/1.14/windows/windowsservercore-ltsc2016/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/servercore:ltsc2016 # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 @@ -48,7 +54,7 @@ RUN $newPath = ('{0}\bin;C:\go\bin;{1}' -f $env:GOPATH, $env:PATH); \ ENV GOLANG_VERSION 1.14.7 -RUN $url = ('https://golang.org/dl/go{0}.windows-amd64.zip' -f $env:GOLANG_VERSION); \ +RUN $url = 'https://storage.googleapis.com/golang/go1.14.7.windows-amd64.zip'; \ Write-Host ('Downloading {0} ...' -f $url); \ Invoke-WebRequest -Uri $url -OutFile 'go.zip'; \ \ diff --git a/1.15/alpine3.12/Dockerfile b/1.15/alpine3.12/Dockerfile index 4580739b..b9e71cd7 100644 --- a/1.15/alpine3.12/Dockerfile +++ b/1.15/alpine3.12/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM alpine:3.12 RUN apk add --no-cache \ @@ -8,15 +14,18 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf +ENV PATH /usr/local/go/bin:$PATH + ENV GOLANG_VERSION 1.15 RUN set -eux; \ apk add --no-cache --virtual .build-deps \ bash \ gcc \ + gnupg \ + go \ musl-dev \ openssl \ - go \ ; \ export \ # set GOROOT_BOOTSTRAP such that we can actually build Go @@ -37,28 +46,53 @@ RUN set -eux; \ x86) export GO386='387' ;; \ esac; \ \ - wget -O go.tgz "https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz"; \ - echo '69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a *go.tgz' | sha256sum -c -; \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + url='https://storage.googleapis.com/golang/go1.15.src.tar.gz'; \ + sha256='69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a'; \ + \ + wget -O go.tgz.asc "$url.asc"; \ + wget -O go.tgz "$url"; \ + echo "$sha256 *go.tgz" | sha256sum -c -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ - cd /usr/local/go/src; \ - ./make.bash; \ + goEnv="$(go env | sed -rn -e '/^GO(OS|ARCH|ARM|386)=/s//export \0/p')"; \ + eval "$goEnv"; \ + [ -n "$GOOS" ]; \ + [ -n "$GOARCH" ]; \ + ( \ + cd /usr/local/go/src; \ + ./make.bash; \ + ); \ + \ + apk del --no-network .build-deps; \ \ +# pre-compile the standard library, just like the official binary release tarballs do + go install std; \ +# go install: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 +# go install -race std; \ + \ +# remove a few intermediate / bootstrapping files the official binary release tarballs do not contain rm -rf \ -# https://github.com/golang/go/blob/0b30cf534a03618162d3015c8705dd2231e34703/src/cmd/dist/buildtool.go#L121-L125 + /usr/local/go/pkg/*/cmd \ /usr/local/go/pkg/bootstrap \ -# https://golang.org/cl/82095 -# https://github.com/golang/build/blob/e3fe1605c30f6a3fd136b561569933312ede8782/cmd/release/releaselet.go#L56 /usr/local/go/pkg/obj \ + /usr/local/go/pkg/tool/*/api \ + /usr/local/go/pkg/tool/*/go_bootstrap \ + /usr/local/go/src/cmd/dist/dist \ ; \ - apk del .build-deps; \ \ - export PATH="/usr/local/go/bin:$PATH"; \ go version ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - +ENV PATH $GOPATH/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH diff --git a/1.15/buster/Dockerfile b/1.15/buster/Dockerfile index 7d84c47c..211b352d 100644 --- a/1.15/buster/Dockerfile +++ b/1.15/buster/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM buildpack-deps:buster-scm # gcc for cgo @@ -15,27 +21,64 @@ ENV GOLANG_VERSION 1.15 RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" dpkgArch="$(dpkg --print-architecture)"; \ case "${dpkgArch##*-}" in \ - amd64) goRelArch='linux-amd64'; goRelSha256='2d75848ac606061efe52a8068d0e647b35ce487a15bb52272c427df485193602' ;; \ - armhf) goRelArch='linux-armv6l'; goRelSha256='6d8914ddd25f85f2377c269ccfb359acf53adf71a42cdbf53434a7c76fa7a9bd' ;; \ - arm64) goRelArch='linux-arm64'; goRelSha256='7e18d92f61ddf480a4f9a57db09389ae7b9dadf68470d0cb9c00d734a0c57f8d' ;; \ - i386) goRelArch='linux-386'; goRelSha256='68ce979083126694ceef60233f69efe870f54af24d81a120f76265107a9e9aab' ;; \ - ppc64el) goRelArch='linux-ppc64le'; goRelSha256='4603736a158b3d8ac52b9245f39bf715936c801e05bb5ad7c44b1edd6d5ef6a2' ;; \ - s390x) goRelArch='linux-s390x'; goRelSha256='8825f93caaf87465e32f298408c48b98d4180f3ddb885bd027f2926e711d23e8' ;; \ - *) goRelArch='src'; goRelSha256='69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a'; \ - echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; \ + 'amd64') \ + arch='linux-amd64'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-amd64.tar.gz'; \ + sha256='2d75848ac606061efe52a8068d0e647b35ce487a15bb52272c427df485193602'; \ + ;; \ + 'armhf') \ + arch='linux-armv6l'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-armv6l.tar.gz'; \ + sha256='6d8914ddd25f85f2377c269ccfb359acf53adf71a42cdbf53434a7c76fa7a9bd'; \ + ;; \ + 'arm64') \ + arch='linux-arm64'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-arm64.tar.gz'; \ + sha256='7e18d92f61ddf480a4f9a57db09389ae7b9dadf68470d0cb9c00d734a0c57f8d'; \ + ;; \ + 'i386') \ + arch='linux-386'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-386.tar.gz'; \ + sha256='68ce979083126694ceef60233f69efe870f54af24d81a120f76265107a9e9aab'; \ + ;; \ + 'ppc64el') \ + arch='linux-ppc64le'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-ppc64le.tar.gz'; \ + sha256='4603736a158b3d8ac52b9245f39bf715936c801e05bb5ad7c44b1edd6d5ef6a2'; \ + ;; \ + 's390x') \ + arch='linux-s390x'; \ + url='https://storage.googleapis.com/golang/go1.15.linux-s390x.tar.gz'; \ + sha256='8825f93caaf87465e32f298408c48b98d4180f3ddb885bd027f2926e711d23e8'; \ + ;; \ + *) \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + arch='src'; \ + url='https://storage.googleapis.com/golang/go1.15.src.tar.gz'; \ + sha256='69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a'; \ + echo >&2; \ + echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; \ + echo >&2; \ + ;; \ esac; \ \ - url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \ + wget -O go.tgz.asc "$url.asc" --progress=dot:giga; \ wget -O go.tgz "$url" --progress=dot:giga; \ - echo "${goRelSha256} *go.tgz" | sha256sum -c -; \ + echo "$sha256 *go.tgz" | sha256sum --strict --check -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ -# https://github.com/golang/go/issues/38536#issuecomment-616897960 - if [ "$goRelArch" = 'src' ]; then \ + if [ "$arch" = 'src' ]; then \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends golang-go; \ diff --git a/1.15/windows/nanoserver-1809/Dockerfile b/1.15/windows/nanoserver-1809/Dockerfile index 7bb75ee2..c73b54a0 100644 --- a/1.15/windows/nanoserver-1809/Dockerfile +++ b/1.15/windows/nanoserver-1809/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/nanoserver:1809 SHELL ["cmd", "/S", "/C"] diff --git a/1.15/windows/windowsservercore-1809/Dockerfile b/1.15/windows/windowsservercore-1809/Dockerfile index cc681803..c415d2fc 100644 --- a/1.15/windows/windowsservercore-1809/Dockerfile +++ b/1.15/windows/windowsservercore-1809/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/servercore:1809 # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 @@ -48,7 +54,7 @@ RUN $newPath = ('{0}\bin;C:\go\bin;{1}' -f $env:GOPATH, $env:PATH); \ ENV GOLANG_VERSION 1.15 -RUN $url = ('https://golang.org/dl/go{0}.windows-amd64.zip' -f $env:GOLANG_VERSION); \ +RUN $url = 'https://storage.googleapis.com/golang/go1.15.windows-amd64.zip'; \ Write-Host ('Downloading {0} ...' -f $url); \ Invoke-WebRequest -Uri $url -OutFile 'go.zip'; \ \ diff --git a/1.15/windows/windowsservercore-ltsc2016/Dockerfile b/1.15/windows/windowsservercore-ltsc2016/Dockerfile index 061dfe06..5ae70dc9 100644 --- a/1.15/windows/windowsservercore-ltsc2016/Dockerfile +++ b/1.15/windows/windowsservercore-ltsc2016/Dockerfile @@ -1,3 +1,9 @@ +# +# NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" +# +# PLEASE DO NOT EDIT IT DIRECTLY. +# + FROM mcr.microsoft.com/windows/servercore:ltsc2016 # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 @@ -48,7 +54,7 @@ RUN $newPath = ('{0}\bin;C:\go\bin;{1}' -f $env:GOPATH, $env:PATH); \ ENV GOLANG_VERSION 1.15 -RUN $url = ('https://golang.org/dl/go{0}.windows-amd64.zip' -f $env:GOLANG_VERSION); \ +RUN $url = 'https://storage.googleapis.com/golang/go1.15.windows-amd64.zip'; \ Write-Host ('Downloading {0} ...' -f $url); \ Invoke-WebRequest -Uri $url -OutFile 'go.zip'; \ \ diff --git a/Dockerfile-alpine.template b/Dockerfile-alpine.template index 86975ad2..7d67b4b2 100644 --- a/Dockerfile-alpine.template +++ b/Dockerfile-alpine.template @@ -1,4 +1,4 @@ -FROM alpine:%%TAG%% +FROM alpine:{{ env.variant | ltrimstr("alpine") }} RUN apk add --no-cache \ ca-certificates @@ -8,15 +8,18 @@ RUN apk add --no-cache \ # - docker run --rm debian:stretch grep '^hosts:' /etc/nsswitch.conf RUN [ ! -e /etc/nsswitch.conf ] && echo 'hosts: files dns' > /etc/nsswitch.conf -ENV GOLANG_VERSION %%VERSION%% +ENV PATH /usr/local/go/bin:$PATH + +ENV GOLANG_VERSION {{ .version }} RUN set -eux; \ apk add --no-cache --virtual .build-deps \ bash \ gcc \ + gnupg \ + go \ musl-dev \ openssl \ - go \ ; \ export \ # set GOROOT_BOOTSTRAP such that we can actually build Go @@ -37,28 +40,53 @@ RUN set -eux; \ x86) export GO386='387' ;; \ esac; \ \ - wget -O go.tgz "https://golang.org/dl/go$GOLANG_VERSION.src.tar.gz"; \ - echo '%%SRC-SHA256%% *go.tgz' | sha256sum -c -; \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + url={{ .arches.src.url | @sh }}; \ + sha256={{ .arches.src.sha256 | @sh }}; \ + \ + wget -O go.tgz.asc "$url.asc"; \ + wget -O go.tgz "$url"; \ + echo "$sha256 *go.tgz" | sha256sum -c -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ - cd /usr/local/go/src; \ - ./make.bash; \ + goEnv="$(go env | sed -rn -e '/^GO(OS|ARCH|ARM|386)=/s//export \0/p')"; \ + eval "$goEnv"; \ + [ -n "$GOOS" ]; \ + [ -n "$GOARCH" ]; \ + ( \ + cd /usr/local/go/src; \ + ./make.bash; \ + ); \ + \ + apk del --no-network .build-deps; \ \ +# pre-compile the standard library, just like the official binary release tarballs do + go install std; \ +# go install: -race is only supported on linux/amd64, linux/ppc64le, linux/arm64, freebsd/amd64, netbsd/amd64, darwin/amd64 and windows/amd64 +# go install -race std; \ + \ +# remove a few intermediate / bootstrapping files the official binary release tarballs do not contain rm -rf \ -# https://github.com/golang/go/blob/0b30cf534a03618162d3015c8705dd2231e34703/src/cmd/dist/buildtool.go#L121-L125 + /usr/local/go/pkg/*/cmd \ /usr/local/go/pkg/bootstrap \ -# https://golang.org/cl/82095 -# https://github.com/golang/build/blob/e3fe1605c30f6a3fd136b561569933312ede8782/cmd/release/releaselet.go#L56 /usr/local/go/pkg/obj \ + /usr/local/go/pkg/tool/*/api \ + /usr/local/go/pkg/tool/*/go_bootstrap \ + /usr/local/go/src/cmd/dist/dist \ ; \ - apk del .build-deps; \ \ - export PATH="/usr/local/go/bin:$PATH"; \ go version ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - +ENV PATH $GOPATH/bin:$PATH RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" WORKDIR $GOPATH diff --git a/Dockerfile-debian.template b/Dockerfile-debian.template index 397c5ba6..f6bbf743 100644 --- a/Dockerfile-debian.template +++ b/Dockerfile-debian.template @@ -1,4 +1,4 @@ -FROM buildpack-deps:%%TAG%%-scm +FROM buildpack-deps:{{ env.variant }}-scm # gcc for cgo RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -11,21 +11,65 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV PATH /usr/local/go/bin:$PATH -ENV GOLANG_VERSION %%VERSION%% +ENV GOLANG_VERSION {{ .version }} RUN set -eux; \ \ -# this "case" statement is generated via "update.sh" - %%ARCH-CASE%%; \ + dpkgArch="$(dpkg --print-architecture)"; \ + case "${dpkgArch##*-}" in \ +{{ + [ + .arches | to_entries[] + | select(.value.arch | startswith("linux-")) + | .key as $bashbrewArch + | ( + { + arm32v5: "armel", + arm32v7: "armhf", + arm64v8: "arm64", + mips64le: "mips64el", + ppc64le: "ppc64el", + } + | .[$bashbrewArch] // $bashbrewArch + ) as $dpkgArch + | .value + | ( +-}} + {{ $dpkgArch | @sh }}) \ + arch={{ .arch | @sh }}; \ + url={{ .url | @sh }}; \ + sha256={{ .sha256 | @sh }}; \ + ;; \ +{{ + ) + ] | add +-}} + *) \ +# https://github.com/golang/go/issues/38536#issuecomment-616897960 + arch='src'; \ + url={{ .arches.src.url | @sh }}; \ + sha256={{ .arches.src.sha256 | @sh }}; \ + echo >&2; \ + echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; \ + echo >&2; \ + ;; \ + esac; \ \ - url="https://golang.org/dl/go${GOLANG_VERSION}.${goRelArch}.tar.gz"; \ + wget -O go.tgz.asc "$url.asc" --progress=dot:giga; \ wget -O go.tgz "$url" --progress=dot:giga; \ - echo "${goRelSha256} *go.tgz" | sha256sum -c -; \ + echo "$sha256 *go.tgz" | sha256sum --strict --check -; \ + \ +# https://github.com/golang/go/issues/14739#issuecomment-324767697 + export GNUPGHOME="$(mktemp -d)"; \ +# https://www.google.com/linuxrepositories/ + gpg --batch --keyserver ha.pool.sks-keyservers.net --recv-keys 'EB4C 1BFD 4F04 2F6D DDCC EC91 7721 F63B D38B 4796'; \ + gpg --batch --verify go.tgz.asc go.tgz; \ + rm -rf "$GNUPGHOME" go.tgz.asc; \ + \ tar -C /usr/local -xzf go.tgz; \ rm go.tgz; \ \ -# https://github.com/golang/go/issues/38536#issuecomment-616897960 - if [ "$goRelArch" = 'src' ]; then \ + if [ "$arch" = 'src' ]; then \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends golang-go; \ diff --git a/Dockerfile-windows-nanoserver.template b/Dockerfile-windows-nanoserver.template index a0759c3c..a455c358 100644 --- a/Dockerfile-windows-nanoserver.template +++ b/Dockerfile-windows-nanoserver.template @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/nanoserver:%%MICROSOFT-TAG%% +FROM mcr.microsoft.com/windows/{{ env.windowsVariant }}:{{ env.windowsRelease }} SHELL ["cmd", "/S", "/C"] @@ -14,9 +14,9 @@ RUN setx /m PATH "%GOPATH%\bin;C:\go\bin;%PATH%" USER ContainerUser # doing this first to share cache across versions more aggressively -ENV GOLANG_VERSION %%VERSION%% +ENV GOLANG_VERSION {{ .version }} -COPY --from=golang:%%VERSION%%-windowsservercore-%%MICROSOFT-TAG%% C:\\go C:\\go +COPY --from=golang:{{ .version }}-windowsservercore-{{ env.windowsRelease }} C:\\go C:\\go RUN go version WORKDIR $GOPATH diff --git a/Dockerfile-windows-windowsservercore.template b/Dockerfile-windows-servercore.template similarity index 92% rename from Dockerfile-windows-windowsservercore.template rename to Dockerfile-windows-servercore.template index 8e4ca4b1..b285834c 100644 --- a/Dockerfile-windows-windowsservercore.template +++ b/Dockerfile-windows-servercore.template @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/windows/servercore:%%MICROSOFT-TAG%% +FROM mcr.microsoft.com/windows/{{ env.windowsVariant }}:{{ env.windowsRelease }} # $ProgressPreference: https://github.com/PowerShell/PowerShell/issues/2138#issuecomment-251261324 SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"] @@ -46,13 +46,13 @@ RUN $newPath = ('{0}\bin;C:\go\bin;{1}' -f $env:GOPATH, $env:PATH); \ [Environment]::SetEnvironmentVariable('PATH', $newPath, [EnvironmentVariableTarget]::Machine); # doing this first to share cache across versions more aggressively -ENV GOLANG_VERSION %%VERSION%% +ENV GOLANG_VERSION {{ .version }} -RUN $url = ('https://golang.org/dl/go{0}.windows-amd64.zip' -f $env:GOLANG_VERSION); \ +RUN $url = '{{ .arches["windows-amd64"].url }}'; \ Write-Host ('Downloading {0} ...' -f $url); \ Invoke-WebRequest -Uri $url -OutFile 'go.zip'; \ \ - $sha256 = '%%WIN-SHA256%%'; \ + $sha256 = '{{ .arches["windows-amd64"].sha256 }}'; \ Write-Host ('Verifying sha256 ({0}) ...' -f $sha256); \ if ((Get-FileHash go.zip -Algorithm sha256).Hash -ne $sha256) { \ Write-Host 'FAILED!'; \ diff --git a/apply-templates.sh b/apply-templates.sh new file mode 100755 index 00000000..72e7c750 --- /dev/null +++ b/apply-templates.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +[ -f versions.json ] # run "versions.sh" first + +jqt='.jq-template.awk' +if [ -n "${BASHBREW_SCRIPTS:-}" ]; then + jqt="$BASHBREW_SCRIPTS/jq-template.awk" +elif [ "$BASH_SOURCE" -nt "$jqt" ]; then + wget -qO "$jqt" 'https://github.com/docker-library/bashbrew/raw/5f0c26381fb7cc78b2d217d58007800bdcfbcfa1/scripts/jq-template.awk' +fi + +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi + +generated_warning() { + cat <<-EOH + # + # NOTE: THIS DOCKERFILE IS GENERATED VIA "apply-templates.sh" + # + # PLEASE DO NOT EDIT IT DIRECTLY. + # + + EOH +} + +for version; do + export version + + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" + + for dir in "${variants[@]}"; do + mkdir -p "$version/$dir" + + variant="$(basename "$dir")" # "buster", "windowsservercore-1809", etc + export variant + + case "$dir" in + alpine*) + template='Dockerfile-alpine.template' + ;; + + windows/*) + windowsVariant="${variant%%-*}" # "windowsservercore", "nanoserver" + windowsRelease="${variant#$windowsVariant-}" # "1809", "ltsc2016", etc + windowsVariant="${windowsVariant#windows}" # "servercore", "nanoserver" + export windowsVariant windowsRelease + template="Dockerfile-windows-$windowsVariant.template" + ;; + + *) + template='Dockerfile-debian.template' + ;; + esac + + echo "processing $version/$dir ..." + + { + generated_warning + gawk -f "$jqt" "$template" + } > "$version/$dir/Dockerfile" + done +done diff --git a/generate-stackbrew-library.sh b/generate-stackbrew-library.sh index 09675780..f7ab8ee5 100755 --- a/generate-stackbrew-library.sh +++ b/generate-stackbrew-library.sh @@ -18,13 +18,13 @@ declare -A alpineVersion=( self="$(basename "$BASH_SOURCE")" cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' - -versions=( */ ) -versions=( "${versions[@]%/}" ) +if [ "$#" -eq 0 ]; then + versions="$(jq -r 'keys | map(@sh) | join(" ")' versions.json)" + eval "set -- $versions" +fi # sort version numbers with highest first -IFS=$'\n'; versions=( $(echo "${versions[*]}" | sort -rV) ); unset IFS +IFS=$'\n'; set -- $(sort -rV <<<"$*"); unset IFS # get the most recent commit which modified any of "$@" fileCommit() { @@ -52,6 +52,22 @@ dirCommit() { ) } +getArches() { + local repo="$1"; shift + local officialImagesUrl='https://github.com/docker-library/official-images/raw/master/library/' + + eval "declare -g -A parentRepoToArches=( $( + find -name 'Dockerfile' -exec awk ' + toupper($1) == "FROM" && $2 !~ /^('"$repo"'|scratch|.*\/.*)(:|$)/ { + print "'"$officialImagesUrl"'" $2 + } + ' '{}' + \ + | sort -u \ + | xargs bashbrew cat --format '[{{ .RepoName }}:{{ .TagName }}]="{{ join " " .TagEntry.Architectures }}"' + ) )" +} +getArches 'golang' + cat <<-EOH # this file is generated via https://github.com/docker-library/golang/blob/$(fileCommit "$self")/$self @@ -68,28 +84,24 @@ join() { echo "${out#$sep}" } -for version in "${versions[@]}"; do - rcVersion="${version%-rc}" +for version; do + export version + variants="$(jq -r '.[env.version].variants | map(@sh) | join(" ")' versions.json)" + eval "variants=( $variants )" versionAliases=( $version ${aliases[$version]:-} ) - for v in \ - buster stretch alpine{3.12,3.11} \ - windows/windowsservercore-{ltsc2016,1809} \ - windows/nanoserver-1809 \ - ; do + for v in "${variants[@]}"; do dir="$version/$v" - [ -f "$dir/Dockerfile" ] || continue variant="$(basename "$v")" versionSuite="${debianSuite[$version]:-$defaultDebianSuite}" - commit="$(dirCommit "$dir")" - fullVersion="$(git show "$commit":"$dir/Dockerfile" | awk '$1 == "ENV" && $2 == "GOLANG_VERSION" { print $3; exit }')" + fullVersion="$(jq -r '.[env.version].version' versions.json)" [[ "$fullVersion" == *.*[^0-9]* ]] || fullVersion+='.0' @@ -107,15 +119,20 @@ for version in "${versions[@]}"; do fi case "$v" in - windows/*) variantArches='windows-amd64' ;; - - # stretch's "golang-go" package doesn't include GOARM for arm32v5 and "gccgo" in stretch can't build mips64le - stretch) - variantArches="$(variantArches "$version" "$v")" + windows/*) + variantArches='windows-amd64' ;; *) - variantArches="$(parentArches "$version" "$v")" + variantParent="$(awk 'toupper($1) == "FROM" { print $2 }' "$dir/Dockerfile")" + variantArches="${parentRepoToArches[$variantParent]}" + + if [ "$variant" = 'stretch' ]; then + # stretch's "golang-go" package doesn't include GOARM for arm32v5 + variantArches="$(sed <<<" $variantArches " -e 's/ arm32v5 / /g')" + # ... and "gccgo" in stretch can't build mips64le + variantArches="$(sed <<<" $variantArches " -e 's/ mips64le / /g')" + fi ;; esac @@ -140,6 +157,8 @@ for version in "${versions[@]}"; do fi fi + commit="$(dirCommit "$dir")" + echo echo "Tags: $(join ', ' "${variantAliases[@]}")" if [ "${#sharedTags[@]}" -gt 0 ]; then diff --git a/release-architectures b/release-architectures deleted file mode 100644 index 6b7fa2c8..00000000 --- a/release-architectures +++ /dev/null @@ -1,9 +0,0 @@ -# see https://golang.org/dl/ - -# bashbrew-arch dpkg-arch golang-release-arch -amd64 amd64 amd64 -arm32v7 armhf armv6l -arm64v8 arm64 arm64 -i386 i386 386 -ppc64le ppc64el ppc64le -s390x s390x s390x diff --git a/update.sh b/update.sh index 01867ae8..bac2d758 100755 --- a/update.sh +++ b/update.sh @@ -3,116 +3,5 @@ set -Eeuo pipefail cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" -source '.architectures-lib' - -versions=( "$@" ) -if [ ${#versions[@]} -eq 0 ]; then - versions=( */ ) -fi -versions=( "${versions[@]%/}" ) - -# see http://stackoverflow.com/a/2705678/433558 -sed_escape_lhs() { - echo "$@" | sed -e 's/[]\/$*.^|[]/\\&/g' -} -sed_escape_rhs() { - echo "$@" | sed -e 's/[\/&]/\\&/g' | sed -e ':a;N;$!ba;s/\n/\\n/g' -} - -# https://github.com/golang/go/issues/13220 -allGoVersions=() -apiBaseUrl='https://www.googleapis.com/storage/v1/b/golang/o?fields=nextPageToken,items%2Fname' -pageToken= -while [ "$pageToken" != 'null' ]; do - page="$(curl -fsSL "$apiBaseUrl&pageToken=$pageToken")" - allGoVersions+=( $( - echo "$page" \ - | jq -r '.items[].name' \ - | grep -E '^go[0-9].*[.]src[.]tar[.]gz$' \ - | sed -r -e 's!^go!!' -e 's![.]src[.]tar[.]gz$!!' - ) ) - # TODO extract per-version "available binary tarballs" information while we've got it handy here? - pageToken="$(echo "$page" | jq -r '.nextPageToken')" -done - -for version in "${versions[@]}"; do - rcVersion="${version%-rc}" - rcGrepV='-v' - if [ "$rcVersion" != "$version" ]; then - rcGrepV= - fi - rcGrepV+=' -E' - rcGrepExpr='beta|rc' - - fullVersion="$( - echo "${allGoVersions[@]}" | xargs -n1 \ - | grep $rcGrepV -- "$rcGrepExpr" \ - | grep -E "^${rcVersion}([.a-z]|$)" \ - | sort -V \ - | tail -1 - )" || true - if [ -z "$fullVersion" ]; then - echo >&2 "warning: cannot find full version for $version" - continue - fi - fullVersion="${fullVersion#go}" # strip "go" off "go1.4.2" - - # https://github.com/golang/build/commit/24f7399f96feb8dd2fc54f064e47a886c2f8bb4a - srcSha256="$(curl -fsSL "https://storage.googleapis.com/golang/go${fullVersion}.src.tar.gz.sha256")" - if [ -z "$srcSha256" ]; then - echo >&2 "warning: cannot find sha256 for $fullVersion src tarball" - continue - fi - - linuxArchCase='dpkgArch="$(dpkg --print-architecture)"; '$'\\\n' - linuxArchCase+=$'\t''case "${dpkgArch##*-}" in '$'\\\n' - for dpkgArch in $(dpkgArches "$version"); do - goArch="$(dpkgToGoArch "$version" "$dpkgArch")" - sha256="$(curl -fsSL "https://storage.googleapis.com/golang/go${fullVersion}.linux-${goArch}.tar.gz.sha256")" - if [ -z "$sha256" ]; then - echo >&2 "warning: cannot find sha256 for $fullVersion on arch $goArch" - continue 2 - fi - linuxArchCase+=$'\t\t'"$dpkgArch) goRelArch='linux-$goArch'; goRelSha256='$sha256' ;; "$'\\\n' - done - linuxArchCase+=$'\t\t'"*) goRelArch='src'; goRelSha256='$srcSha256'; "$'\\\n' - linuxArchCase+=$'\t\t\t''echo >&2; echo >&2 "warning: current architecture ($dpkgArch) does not have a corresponding Go binary release; will be building from source"; echo >&2 ;; '$'\\\n' - linuxArchCase+=$'\t''esac' - - windowsSha256="$(curl -fsSL "https://storage.googleapis.com/golang/go${fullVersion}.windows-amd64.zip.sha256")" - - for variant in \ - alpine{3.11,3.12} \ - stretch buster \ - ; do - if [ -d "$version/$variant" ]; then - tag="$variant" - template='debian' - case "$variant" in - alpine*) tag="${variant#alpine}"; template='alpine' ;; - esac - - sed -r \ - -e 's!%%VERSION%%!'"$fullVersion"'!g' \ - -e 's!%%TAG%%!'"$tag"'!g' \ - -e 's!%%SRC-SHA256%%!'"$srcSha256"'!g' \ - -e 's!%%ARCH-CASE%%!'"$(sed_escape_rhs "$linuxArchCase")"'!g' \ - "Dockerfile-${template}.template" > "$version/$variant/Dockerfile" - fi - done - - for winVariant in \ - nanoserver-1809 \ - windowsservercore-{ltsc2016,1809} \ - ; do - if [ -d "$version/windows/$winVariant" ]; then - sed -r \ - -e 's!%%VERSION%%!'"$fullVersion"'!g' \ - -e 's!%%WIN-SHA256%%!'"$windowsSha256"'!g' \ - -e 's!%%MICROSOFT-TAG%%!'"${winVariant#*-}"'!g' \ - "Dockerfile-windows-${winVariant%%-*}.template" > "$version/windows/$winVariant/Dockerfile" - fi - done - - echo "$version: $fullVersion ($srcSha256)" -done +./versions.sh "$@" +./apply-templates.sh "$@" diff --git a/versions.json b/versions.json new file mode 100644 index 00000000..85ac20a7 --- /dev/null +++ b/versions.json @@ -0,0 +1,108 @@ +{ + "1.14": { + "arches": { + "amd64": { + "arch": "linux-amd64", + "sha256": "4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-amd64.tar.gz" + }, + "arm32v7": { + "arch": "linux-armv6l", + "sha256": "6079eb82bcf24b33dda0e32777c7fdddcc3b1ec70e374308cc8311562449b107", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-armv6l.tar.gz" + }, + "arm64v8": { + "arch": "linux-arm64", + "sha256": "fe5b6f6e441f3cb7b53ebf1a010bbebcb720ac98124984cfe2e51d72b8a58c71", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-arm64.tar.gz" + }, + "i386": { + "arch": "linux-386", + "sha256": "2f5793f10bb6b08eedecd376aa3d594e10193c6b5cf198ada46200259ff76547", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-386.tar.gz" + }, + "ppc64le": { + "arch": "linux-ppc64le", + "sha256": "bd1f12a2271a6d1689bcf3ec01d123c81cbaca5d16c3f7df294a2d725ac4d3d1", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-ppc64le.tar.gz" + }, + "s390x": { + "arch": "linux-s390x", + "sha256": "ea570b3caa0c271da440f568ab646cfea80d712c51fb4d08189bb66bd5eb949c", + "url": "https://storage.googleapis.com/golang/go1.14.7.linux-s390x.tar.gz" + }, + "src": { + "arch": "src", + "sha256": "064392433563660c73186991c0a315787688e7c38a561e26647686f89b6c30e3", + "url": "https://storage.googleapis.com/golang/go1.14.7.src.tar.gz" + }, + "windows-amd64": { + "arch": "windows-amd64", + "sha256": "ddf35c01f444f5698dee7b265c172abe29dd7138bef4415e6b60bf322ddcffa5", + "url": "https://storage.googleapis.com/golang/go1.14.7.windows-amd64.zip" + } + }, + "variants": [ + "buster", + "stretch", + "alpine3.12", + "alpine3.11", + "windows/windowsservercore-1809", + "windows/windowsservercore-ltsc2016", + "windows/nanoserver-1809" + ], + "version": "1.14.7" + }, + "1.15": { + "arches": { + "amd64": { + "arch": "linux-amd64", + "sha256": "2d75848ac606061efe52a8068d0e647b35ce487a15bb52272c427df485193602", + "url": "https://storage.googleapis.com/golang/go1.15.linux-amd64.tar.gz" + }, + "arm32v7": { + "arch": "linux-armv6l", + "sha256": "6d8914ddd25f85f2377c269ccfb359acf53adf71a42cdbf53434a7c76fa7a9bd", + "url": "https://storage.googleapis.com/golang/go1.15.linux-armv6l.tar.gz" + }, + "arm64v8": { + "arch": "linux-arm64", + "sha256": "7e18d92f61ddf480a4f9a57db09389ae7b9dadf68470d0cb9c00d734a0c57f8d", + "url": "https://storage.googleapis.com/golang/go1.15.linux-arm64.tar.gz" + }, + "i386": { + "arch": "linux-386", + "sha256": "68ce979083126694ceef60233f69efe870f54af24d81a120f76265107a9e9aab", + "url": "https://storage.googleapis.com/golang/go1.15.linux-386.tar.gz" + }, + "ppc64le": { + "arch": "linux-ppc64le", + "sha256": "4603736a158b3d8ac52b9245f39bf715936c801e05bb5ad7c44b1edd6d5ef6a2", + "url": "https://storage.googleapis.com/golang/go1.15.linux-ppc64le.tar.gz" + }, + "s390x": { + "arch": "linux-s390x", + "sha256": "8825f93caaf87465e32f298408c48b98d4180f3ddb885bd027f2926e711d23e8", + "url": "https://storage.googleapis.com/golang/go1.15.linux-s390x.tar.gz" + }, + "src": { + "arch": "src", + "sha256": "69438f7ed4f532154ffaf878f3dfd83747e7a00b70b3556eddabf7aaee28ac3a", + "url": "https://storage.googleapis.com/golang/go1.15.src.tar.gz" + }, + "windows-amd64": { + "arch": "windows-amd64", + "sha256": "dc491314dff5b87ad50bf1cf56715de8f8c54489be30f3e19239bc2ad1af25e3", + "url": "https://storage.googleapis.com/golang/go1.15.windows-amd64.zip" + } + }, + "variants": [ + "buster", + "alpine3.12", + "windows/windowsservercore-1809", + "windows/windowsservercore-ltsc2016", + "windows/nanoserver-1809" + ], + "version": "1.15" + } +} diff --git a/versions.sh b/versions.sh new file mode 100755 index 00000000..03241d9c --- /dev/null +++ b/versions.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# see https://golang.org/dl/ +declare -A golangArches=( + ['amd64']='linux-amd64' + ['arm32v7']='linux-armv6l' + ['arm64v8']='linux-arm64' + ['i386']='linux-386' + ['ppc64le']='linux-ppc64le' + ['s390x']='linux-s390x' + ['windows-amd64']='windows-amd64' + + # special case (fallback) + ['src']='src' +) + +cd "$(dirname "$(readlink -f "$BASH_SOURCE")")" + +versions=( "$@" ) +if [ ${#versions[@]} -eq 0 ]; then + versions=( */ ) + json='{}' +else + json="$(< versions.json)" +fi +versions=( "${versions[@]%/}" ) + +# https://github.com/golang/go/issues/13220 +allGoVersions='{}' +apiBaseUrl='https://www.googleapis.com/storage/v1/b/golang/o?fields=nextPageToken,items%2Fname' +pageToken= +while [ "$pageToken" != 'null' ]; do + page="$(curl -fsSL "$apiBaseUrl&pageToken=$pageToken")" + # now that we have this page's data, get ready for the next request + pageToken="$(jq <<<"$page" -r '.nextPageToken')" + + # for each API page, collect the "version => arches" pairs we find + goVersions="$( + jq <<<"$page" -r ' + [ + .items as $items + | $items[].name + | match("^go([0-9].*)[.](src|(linux|windows)-[^.]+)[.](tar[.]gz|zip)$") + | .captures[0].string as $version + | .captures[1].string as $arch + | { version: $version, arch: $arch } + ] | reduce .[] as $o ( + {}; + .[$o.version] += [ $o.arch ] + ) + ' + )" + + # ... and aggregate them together into a single object of "version => arches" pairs + allGoVersions="$( + jq <<<"$allGoVersions"$'\n'"$goVersions" -cs ' + map(to_entries) | add + | reduce .[] as $o ( + {}; + .[$o.key] = ( + $o.value + .[$o.key] + | unique + ) + ) + ' + )" +done + +for version in "${versions[@]}"; do + rcVersion="${version%-rc}" + rcRegex='^[^a-z]*$' + if [ "$rcVersion" != "$version" ]; then + # beta, rc, etc + rcRegex='[a-z]+[0-9]*$' + fi + + export rcVersion rcRegex + fullVersion="$( + jq <<<"$allGoVersions" -r ' + . as $map + | keys[] | select( + startswith(env.rcVersion) + and ( + ltrimstr(env.rcVersion) + | test(env.rcRegex) + ) + and ($map[.] | index("src")) + ) + ' | sort -rV | head -1 + )" + if [ -z "$fullVersion" ]; then + echo >&2 "warning: cannot find full version for $version" + continue + fi + + echo "$version: $fullVersion" + + export fullVersion + doc="$( + jq -nc '{ + version: env.fullVersion, + arches: {}, + variants: [], + }' + )" + + arches="$(jq <<<"$allGoVersions" -c '.[env.fullVersion]')" + + # loop over bashbrew arches, get sha256 for each one supported + for bashbrewArch in "${!golangArches[@]}"; do + arch="${golangArches[$bashbrewArch]}" + export arch + if jq <<<"$arches" -e 'index(env.arch) != null' > /dev/null; then + file="go${fullVersion}.$arch.$([[ "$arch" == windows-* ]] && echo 'zip' || echo 'tar.gz')" + url="https://storage.googleapis.com/golang/$file" + # https://github.com/golang/build/commit/24f7399f96feb8dd2fc54f064e47a886c2f8bb4a + if sha256="$(curl -fsSL "$url.sha256")"; then + export bashbrewArch arch url sha256 + doc="$( + jq <<<"$doc" -c '.arches[env.bashbrewArch] = { + arch: env.arch, + url: env.url, + sha256: env.sha256, + }' + )" + fi + fi + done + + # order here controls the order of the library/ file + for variant in \ + buster \ + stretch \ + \ + alpine3.12 \ + alpine3.11 \ + \ + windows/windowsservercore-{1809,ltsc2016} \ + windows/nanoserver-1809 \ + ; do + base="${variant%%/*}" # "buster", "windows", etc. + [ -d "$version/$base" ] || continue + if [ "$base" = 'windows' ] && ! jq <<<"$arches" -e 'index("windows-amd64")' > /dev/null; then + continue + fi + export variant + doc="$(jq <<<"$doc" -c '.variants += [ env.variant ]')" + done + + export version + json="$(jq <<<"$json" -c --argjson doc "$doc" '.[env.version] = $doc')" +done + +jq <<<"$json" -S . > versions.json