From 80f29039ef0cb17000841b54a3c7d165581ffba3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 14 Aug 2018 14:50:15 +0200 Subject: [PATCH 1/3] Kafka and importantly Zookeeper upgraded to 2.0.0 --- avro-tools/test/rest-curl.yml | 2 +- kafka/50kafka.yml | 2 +- kafka/test/kafkacat.yml | 2 +- kafka/test/produce-consume.yml | 6 +++--- maintenance/preferred-replica-election-job.yml | 2 +- maintenance/reassign-paritions-job.yml | 2 +- maintenance/replication-factor-increase-job.yml | 2 +- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml index 7a42219b..0b491cff 100644 --- a/avro-tools/test/rest-curl.yml +++ b/avro-tools/test/rest-curl.yml @@ -114,7 +114,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 659c9d4d..fad17e6e 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -42,7 +42,7 @@ spec: mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f551fbdd..6f1aa5b5 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index d5aaf8b6..7634ac13 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-topics.sh - --zookeeper @@ -95,7 +95,7 @@ spec: spec: containers: - name: producer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 @@ -114,7 +114,7 @@ spec: - name: shared mountPath: /shared - name: consumer - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: BOOTSTRAP value: bootstrap.kafka:9092 diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml index f3cfa83e..16060d82 100644 --- a/maintenance/preferred-replica-election-job.yml +++ b/maintenance/preferred-replica-election-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 command: - ./bin/kafka-preferred-replica-election.sh - --zookeeper diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml index 317d9110..d7eab61d 100644 --- a/maintenance/reassign-paritions-job.yml +++ b/maintenance/reassign-paritions-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml index 00a13ae8..40781af5 100644 --- a/maintenance/replication-factor-increase-job.yml +++ b/maintenance/replication-factor-increase-job.yml @@ -10,7 +10,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: ZOOKEEPER value: zookeeper.kafka:2181 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index d923ee97..9132f9e3 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -33,7 +33,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 88b781af..46f67976 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -36,7 +36,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:1.0.2@sha256:7fdb326994bcde133c777d888d06863b7c1a0e80f043582816715d76643ab789 + image: solsson/kafka:2.0.0@sha256:8bc5ccb5a63fdfb977c1e207292b72b34370d2c9fe023bdc0f8ce0d8e0da1670 env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties From 44da767ddffeeaef35a5f84bf219870c4a719a51 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 16:29:10 +0100 Subject: [PATCH 2/3] Scales to 2 brokers + 3 zookeeper instances --- kafka/10broker-config.yml | 4 ++-- kafka/50kafka.yml | 2 +- zookeeper/10zookeeper-config.yml | 2 -- zookeeper/50pzoo.yml | 2 +- zookeeper/51zoo.yml | 2 +- 5 files changed, 5 insertions(+), 7 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index c26ef450..5971a79e 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -58,9 +58,9 @@ data: # the brokers. num.partitions=1 - default.replication.factor=3 + default.replication.factor=2 - min.insync.replicas=2 + min.insync.replicas=1 auto.create.topics.enable=true diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index fad17e6e..17ce1b25 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -8,7 +8,7 @@ spec: matchLabels: app: kafka serviceName: "broker" - replicas: 3 + replicas: 2 updateStrategy: type: OnDelete template: diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index 06943914..42dc6fd8 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -23,9 +23,7 @@ data: syncLimit=2 server.1=pzoo-0.pzoo:2888:3888:participant server.2=pzoo-1.pzoo:2888:3888:participant - server.3=pzoo-2.pzoo:2888:3888:participant server.4=zoo-0.zoo:2888:3888:participant - server.5=zoo-1.zoo:2888:3888:participant log4j.properties: |- log4j.rootLogger=INFO, stdout diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 9132f9e3..6819461f 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -9,7 +9,7 @@ spec: app: zookeeper storage: persistent serviceName: "pzoo" - replicas: 3 + replicas: 2 updateStrategy: type: OnDelete template: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 46f67976..15c01aa8 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -9,7 +9,7 @@ spec: app: zookeeper storage: ephemeral serviceName: "zoo" - replicas: 2 + replicas: 1 updateStrategy: type: OnDelete template: From 0018eff3da875f8c69e535fe546e524be028cc51 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 3 Feb 2018 21:25:58 +0100 Subject: [PATCH 3/3] Same as default.replication.factor/min.insync.replicas --- kafka/10broker-config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 5971a79e..f672fb49 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -115,9 +115,9 @@ data: ############################# Internal Topic Settings ############################# # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. - #offsets.topic.replication.factor=1 - #transaction.state.log.replication.factor=1 - #transaction.state.log.min.isr=1 + offsets.topic.replication.factor=2 + transaction.state.log.replication.factor=2 + transaction.state.log.min.isr=1 ############################# Log Flush Policy #############################