From c60c28df526defb55b80d4f1c372658dbea128ba Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 13:51:34 +0200 Subject: [PATCH 1/5] Adds the ps command as a layer atop the existing kafka image --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 7a42219b..f6ee66d6 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 659c9d4d..b6abd87f 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f551fbdd..c934fdf0 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index d5aaf8b6..570c471c 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index f3cfa83e..5e9acc2c 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index 317d9110..5bdb6101 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 00a13ae8..31fb0a7b 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index d923ee97..b5a27ea6 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 88b781af..54e44120 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties From 8fb8dfab08fb958dafdacacef609e8c1193450a1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:04:05 +0200 Subject: [PATCH 2/5] But the script doesn't wait for termination --- kafka/50kafka.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index b6abd87f..ee2d8be4 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -58,6 +58,10 @@ spec: command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties + lifecycle: + preStop: + exec: + command: ["./bin/kafka-server-stop.sh"] resources: requests: cpu: 100m From af5a8bf690876d282f0ef5910b03e1433140e126 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:14:32 +0200 Subject: [PATCH 3/5] Waits for termination before exiting hook --- kafka/50kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index ee2d8be4..ba14d01e 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -61,7 +61,7 @@ spec: lifecycle: preStop: exec: - command: ["./bin/kafka-server-stop.sh"] + command: ["sh", "-ce", "kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done"] resources: requests: cpu: 100m From e282aecd652eb0c7a8240ddb256f8b5f8be98abe Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:26:54 +0200 Subject: [PATCH 4/5] Zookeeper's shutdown is identical in the kafka dist too --- zookeeper/50pzoo.yml | 4 ++++ zookeeper/51zoo.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index b5a27ea6..b6b0e2a4 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -40,6 +40,10 @@ spec: command: - ./bin/zookeeper-server-start.sh - /etc/kafka/zookeeper.properties + lifecycle: + preStop: + exec: + command: ["sh", "-ce", "kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done"] ports: - containerPort: 2181 name: client diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 54e44120..f423896f 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -43,6 +43,10 @@ spec: command: - ./bin/zookeeper-server-start.sh - /etc/kafka/zookeeper.properties + lifecycle: + preStop: + exec: + command: ["sh", "-ce", "kill -s TERM 1; while $(kill -0 1 2>/dev/null); do sleep 1; done"] ports: - containerPort: 2181 name: client From 72ca76dbd2409c3bf0083cd89aa382ff40705f13 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Sep 2018 14:27:42 +0200 Subject: [PATCH 5/5] Turns out we didn't need the ps command anyway This reverts commit c60c28df526defb55b80d4f1c372658dbea128ba. --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index f6ee66d6..7a42219b 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index ba14d01e..376d063a 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index c934fdf0..f551fbdd 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 570c471c..d5aaf8b6 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index 5e9acc2c..f3cfa83e 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index 5bdb6101..317d9110 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 31fb0a7b..00a13ae8 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index b6b0e2a4..68e1dbdd 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index f423896f..42c12128 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2-ps@sha256:632da34b98ec5b1161086d3c30eb062a319d6658db1732deeeede0593dd2b1a1 + image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties