From c5b818b702efeb88f27123cf9986a6a766d8ab65 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 09:08:29 +0100 Subject: [PATCH 1/3] Auto create topics, in our experience, creates more pains than it solves so we want to undo https://github.com/Yolean/kubernetes-kafka/pull/107. It was partially based on a false assumption, as pointed out in https://github.com/Yolean/kubernetes-kafka/issues/101#issuecomment-365600781 Topics are created not only at produce but also at for example kafkacat -C. Typos cost us more time than it would take to automate topic creation and run ./bin/kafka-topics.sh in a temporary pod when we haven't automated. --- kafka/10broker-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index c26ef450..16e8ce4a 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -62,7 +62,7 @@ data: min.insync.replicas=2 - auto.create.topics.enable=true + auto.create.topics.enable=false # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. # This value is recommended to be increased for installations with data dirs located in RAID array. From 7f992c30b2c389e0ca19861186d5bafd0830360c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 13:14:26 +0100 Subject: [PATCH 2/3] Adds topic creation for eventrouter --- events-kube/topic-create.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 events-kube/topic-create.yaml diff --git a/events-kube/topic-create.yaml b/events-kube/topic-create.yaml new file mode 100644 index 00000000..086aaad9 --- /dev/null +++ b/events-kube/topic-create.yaml @@ -0,0 +1,24 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-create-event-kube + namespace: kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:2.1.0@sha256:ac3f06d87d45c7be727863f31e79fbfdcb9c610b51ba9cf03c75a95d602f15e1 + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - ops.kube-events.stream.json + - --partitions + - "12" + - --replication-factor + - "2" + restartPolicy: Never From 3b8ac9872eacbc9f2180a3619c8581f6e1916af7 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 28 Nov 2018 13:26:59 +0100 Subject: [PATCH 3/3] Quite minimal resource limits on the topic-admin script now that we're running Java 11. Chosen based on trial and error: 80Mi got OOMKilled, 100m cpu took >twice the time to complete. --- events-kube/topic-create.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/events-kube/topic-create.yaml b/events-kube/topic-create.yaml index 086aaad9..ee0d592d 100644 --- a/events-kube/topic-create.yaml +++ b/events-kube/topic-create.yaml @@ -21,4 +21,8 @@ spec: - "12" - --replication-factor - "2" + resources: + limits: + cpu: 200m + memory: 100Mi restartPolicy: Never