From cdb98b7fd972edb2803701103dee11a157a0e2ee Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 18 Feb 2018 16:57:34 +0100 Subject: [PATCH 1/4] Extracts files from ConfigMap syntax and adds kubectl --- kafka/Dockerfile | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 kafka/Dockerfile diff --git a/kafka/Dockerfile b/kafka/Dockerfile new file mode 100644 index 00000000..b1b73c13 --- /dev/null +++ b/kafka/Dockerfile @@ -0,0 +1,40 @@ +FROM solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + +# Verifies compatibility with Yolean/kubernetes-kafka, if you use your own FROM and/or install Kafka above +RUN set -ex; \ + java -version; \ + stat ./bin/kafka-server-start.sh + +ENV \ + CONFIG_MOUNT_PATH=/etc/kafka \ + KUBERNETES_VERSION=1.9.3 \ + KUBERNETES_CLIENTS_SHA256=2f509c05f0c4e1c1ac9e98879a1924f24546905349457904344d79dc639217be + +COPY 10broker-config.yml $CONFIG_MOUNT_PATH/10broker-config.yml + +# Extracts individual files as if config was a mounted volume +# Installs runtime dependencies for the init script +RUN set -ex; \ + export DEBIAN_FRONTEND=noninteractive; \ + runDeps=''; \ + buildDeps='curl ca-certificates jq'; \ + apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \ + \ + YQ_VERSION=1.14.0; \ + YQ_SHA256=0b0b79959b24eebdfeacfe634ce906674e16e48160222a8bb91904142b9166ed; \ + curl -sLS -o /usr/local/bin/yq -k https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64; \ + echo "$YQ_SHA256 /usr/local/bin/yq" | sha256sum -c; \ + chmod u+x /usr/local/bin/yq; \ + \ + for F in init.sh server.properties log4j.properties; do yq r -j $CONFIG_MOUNT_PATH/10broker-config.yml "data[$F]" | jq -r . | tee $CONFIG_MOUNT_PATH/$F; done; \ + chmod u+x $CONFIG_MOUNT_PATH/init.sh; \ + rm $CONFIG_MOUNT_PATH/10broker-config.yml; \ + \ + curl -sLS -o k.tar.gz -k https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-client-linux-amd64.tar.gz; \ + echo "$KUBERNETES_CLIENTS_SHA256 k.tar.gz" | sha256sum -c; \ + tar -xvzf k.tar.gz -C /usr/local/bin/ --strip-components=3 kubernetes/client/bin/kubectl; \ + rm k.tar.gz; \ + \ + apt-get purge -y --auto-remove $buildDeps; \ + rm -rf /var/lib/apt/lists/*; \ + rm -rf /var/log/dpkg.log /var/log/alternatives.log /var/log/apt From 5ead2879e00be63a698fc95319b04c51d3ac4a33 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 18 Feb 2018 17:28:31 +0100 Subject: [PATCH 2/4] Cleans up after yaml value extraction --- kafka/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index b1b73c13..892fadbd 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -30,6 +30,8 @@ RUN set -ex; \ chmod u+x $CONFIG_MOUNT_PATH/init.sh; \ rm $CONFIG_MOUNT_PATH/10broker-config.yml; \ \ + rm /usr/local/bin/yq; \ + \ curl -sLS -o k.tar.gz -k https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-client-linux-amd64.tar.gz; \ echo "$KUBERNETES_CLIENTS_SHA256 k.tar.gz" | sha256sum -c; \ tar -xvzf k.tar.gz -C /usr/local/bin/ --strip-components=3 kubernetes/client/bin/kubectl; \ From de06b6669f28894d6631610055c24693dbdac58f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 19 Feb 2018 07:54:11 +0100 Subject: [PATCH 3/4] There's only kubectl in the archive nowadays --- kafka/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 892fadbd..8ad07317 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -8,7 +8,7 @@ RUN set -ex; \ ENV \ CONFIG_MOUNT_PATH=/etc/kafka \ KUBERNETES_VERSION=1.9.3 \ - KUBERNETES_CLIENTS_SHA256=2f509c05f0c4e1c1ac9e98879a1924f24546905349457904344d79dc639217be + KUBERNETES_CLIENT_SHA256=2f509c05f0c4e1c1ac9e98879a1924f24546905349457904344d79dc639217be COPY 10broker-config.yml $CONFIG_MOUNT_PATH/10broker-config.yml @@ -33,7 +33,7 @@ RUN set -ex; \ rm /usr/local/bin/yq; \ \ curl -sLS -o k.tar.gz -k https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-client-linux-amd64.tar.gz; \ - echo "$KUBERNETES_CLIENTS_SHA256 k.tar.gz" | sha256sum -c; \ + echo "$KUBERNETES_CLIENT_SHA256 k.tar.gz" | sha256sum -c; \ tar -xvzf k.tar.gz -C /usr/local/bin/ --strip-components=3 kubernetes/client/bin/kubectl; \ rm k.tar.gz; \ \ From 81e97da8eb052cd8fe63bfbafbe19ee6f04ec333 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 19 Feb 2018 12:37:33 +0100 Subject: [PATCH 4/4] A minor detail: the actual invocation :) An alternative would be to instead build a dedicated init container image. Now we have to make sure the environment matches from init, where we tweaked the script, to the broker pod. --- kafka/50kafka.yml | 12 ++++++++++++ kafka/Dockerfile | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 229bc4dc..5ac512e5 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,6 +42,18 @@ spec: - name: broker image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - name: JMX_PORT diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 8ad07317..4c530343 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -27,9 +27,11 @@ RUN set -ex; \ chmod u+x /usr/local/bin/yq; \ \ for F in init.sh server.properties log4j.properties; do yq r -j $CONFIG_MOUNT_PATH/10broker-config.yml "data[$F]" | jq -r . | tee $CONFIG_MOUNT_PATH/$F; done; \ - chmod u+x $CONFIG_MOUNT_PATH/init.sh; \ rm $CONFIG_MOUNT_PATH/10broker-config.yml; \ \ + chmod u+x $CONFIG_MOUNT_PATH/init.sh; \ + sed -i "2i$CONFIG_MOUNT_PATH/init.sh" ./bin/kafka-server-start.sh; \ + \ rm /usr/local/bin/yq; \ \ curl -sLS -o k.tar.gz -k https://dl.k8s.io/v${KUBERNETES_VERSION}/kubernetes-client-linux-amd64.tar.gz; \