From 7ddf8a1a8619bd181aa54c17d1fda05b626b20b0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 26 Mar 2019 08:40:28 +0100 Subject: [PATCH 01/10] Celebrates the k8s 1.14 release: a kustomization --- README.md | 6 ++++++ kafka/kustomization.yaml | 5 +++++ variants/scale-1/kafka.yaml | 7 +++++++ variants/scale-1/kustomization.yaml | 4 ++++ 4 files changed, 22 insertions(+) create mode 100644 kafka/kustomization.yaml create mode 100644 variants/scale-1/kafka.yaml create mode 100644 variants/scale-1/kustomization.yaml diff --git a/README.md b/README.md index 827859a0..e6022b3c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,12 @@ That'll give you client "bootstrap" `bootstrap.kafka.svc.cluster.local:9092`. Our only dependency is `kubectl`. Not because we dislike Helm or Operators, but because we think plain manifests make it easier to collaborate. If you begin to rely on this kafka setup we recommend you fork, for example to edit [broker config](https://github.com/Yolean/kubernetes-kafka/blob/master/kafka/10broker-config.yml#L47). +## Kustomize + +With the introduction of [app customization](https://kubectl.docs.kubernetes.io/pages/app_customization/introduction.html) in `kubectl` 1.14 there's an alternative to forks. We as a community can maintain a set of overlays. + +See the [variants](./variants) folder for different overlays. For example to scale to 1 kafka broker try `kubectl apply -k variants/scale-1/`. + ## Version history | tag | k8s ≥ | highlights | diff --git a/kafka/kustomization.yaml b/kafka/kustomization.yaml new file mode 100644 index 00000000..0590b6b5 --- /dev/null +++ b/kafka/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- 10broker-config.yml +- 20dns.yml +- 30bootstrap-service.yml +- 50kafka.yml diff --git a/variants/scale-1/kafka.yaml b/variants/scale-1/kafka.yaml new file mode 100644 index 00000000..18d01c80 --- /dev/null +++ b/variants/scale-1/kafka.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + namespace: kafka +spec: + replicas: 1 diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml new file mode 100644 index 00000000..f9da309c --- /dev/null +++ b/variants/scale-1/kustomization.yaml @@ -0,0 +1,4 @@ +bases: +- ../../kafka +patchesStrategicMerge: +- kafka.yaml From e7cf27e5af7a92aa1a798b97be7f2eb9102efbf8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 08:09:53 +0100 Subject: [PATCH 02/10] Fails because of type difference and achor definition order --- variants/scale-1/kustomization.yaml | 2 ++ variants/scale-1/zookeeper.yaml | 44 +++++++++++++++++++++++++++++ zookeeper/kustomization.yaml | 7 +++++ 3 files changed, 53 insertions(+) create mode 100644 variants/scale-1/zookeeper.yaml create mode 100644 zookeeper/kustomization.yaml diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index f9da309c..2cc065d5 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -1,4 +1,6 @@ bases: - ../../kafka +- ../../zookeeper patchesStrategicMerge: - kafka.yaml +- zookeeper.yaml diff --git a/variants/scale-1/zookeeper.yaml b/variants/scale-1/zookeeper.yaml new file mode 100644 index 00000000..d2f232e8 --- /dev/null +++ b/variants/scale-1/zookeeper.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: pzoo + namespace: kafka +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pzoo + namespace: kafka +spec: + replicas: &replicasPzoo 1 + template: + spec: + initContainers: + - name: init-config + env: + - name: PZOO_REPLICAS + value: *replicasPzoo + #- name: ZOO_REPLICAS + # value: *replicasZoo +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zoo + namespace: kafka +spec: + replicas: &replicasZoo 1 + template: + spec: + initContainers: + - name: init-config + env: + - name: PZOO_REPLICAS + value: > + "" + *replicasPzoo + - name: ZOO_REPLICAS + value: > + "" + *replicasZoo diff --git a/zookeeper/kustomization.yaml b/zookeeper/kustomization.yaml new file mode 100644 index 00000000..978b228a --- /dev/null +++ b/zookeeper/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- 10zookeeper-config.yml +- 20pzoo-service.yml +- 21zoo-service.yml +- 30service.yml +- 50pzoo.yml +- 51zoo.yml From 7b9e3b6e0a7517eddd25eb47eb94e2ae7fa5f594 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 11:45:42 +0100 Subject: [PATCH 03/10] Idea, non-operational, on scripted servers config --- variants/scale-1/zookeeper.yaml | 18 +++++++----------- zookeeper/10zookeeper-config.yml | 14 +++++++++++++- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/variants/scale-1/zookeeper.yaml b/variants/scale-1/zookeeper.yaml index d2f232e8..d46723b3 100644 --- a/variants/scale-1/zookeeper.yaml +++ b/variants/scale-1/zookeeper.yaml @@ -11,16 +11,16 @@ metadata: name: pzoo namespace: kafka spec: - replicas: &replicasPzoo 1 + replicas: 2 template: spec: initContainers: - name: init-config env: - name: PZOO_REPLICAS - value: *replicasPzoo - #- name: ZOO_REPLICAS - # value: *replicasZoo + value: '2' + - name: ZOO_REPLICAS + value: '1' --- apiVersion: apps/v1 kind: StatefulSet @@ -28,17 +28,13 @@ metadata: name: zoo namespace: kafka spec: - replicas: &replicasZoo 1 + replicas: 1 template: spec: initContainers: - name: init-config env: - name: PZOO_REPLICAS - value: > - "" - *replicasPzoo + value: '2' - name: ZOO_REPLICAS - value: > - "" - *replicasZoo + value: '1' diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index faa8c3db..52454194 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -14,9 +14,21 @@ data: export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid cp -Lur /etc/kafka-configmap/* /etc/kafka/ + [ ! -z "$PZOO_REPLICAS" ] && { + sed -i "s/^server\\.\([0-9]\+\)=pzoo-/#server.\1=pzoo-/" /etc/kafka/zookeeper.properties + for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done + } + cat /etc/kafka/zookeeper.properties + [ ! -z "$ZOO_REPLICAS" ] && { + OFF=$PZOO_REPLICAS # TODO require both, or count pzoo lines? + sed -i "s/^server\\.\([0-9]\+\)=zoo-/#server.\1=zoo-/" /etc/kafka/zookeeper.properties + for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $OFF + $N - 1 ))=zoo-$(( $OFF + $N - 2 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done + } + cat /etc/kafka/zookeeper.properties sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties + cat /etc/kafka/zookeeper.properties - zookeeper.properties: |- + zookeeper.properties: | tickTime=2000 dataDir=/var/lib/zookeeper/data dataLogDir=/var/lib/zookeeper/log From c100e3351f66f60d2a80bdc3761ac1bffb54cc84 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:08:08 +0100 Subject: [PATCH 04/10] Genereates zookeeper servers config from replica counts Caveats now include: - Duplication withint the zookeeper.yaml file (have tried but failed to use yaml anchors) - Numbers need to add up like the inif-copnfig script assumes - Pods pass readiness probe even if for example they fail to recognize their own ID in the list (i.e. the 0.0.0.0 sed fails) --- variants/scale-1/zookeeper.yaml | 3 +++ zookeeper/10zookeeper-config.yml | 11 +++-------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/variants/scale-1/zookeeper.yaml b/variants/scale-1/zookeeper.yaml index d46723b3..0144c595 100644 --- a/variants/scale-1/zookeeper.yaml +++ b/variants/scale-1/zookeeper.yaml @@ -34,7 +34,10 @@ spec: initContainers: - name: init-config env: + # There's no validation on these numbers adding up to a coherent zk config, so watch out - name: PZOO_REPLICAS value: '2' - name: ZOO_REPLICAS value: '1' + - name: ID_OFFSET + value: '3' diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index 52454194..219838a9 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -14,15 +14,10 @@ data: export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid cp -Lur /etc/kafka-configmap/* /etc/kafka/ - [ ! -z "$PZOO_REPLICAS" ] && { - sed -i "s/^server\\.\([0-9]\+\)=pzoo-/#server.\1=pzoo-/" /etc/kafka/zookeeper.properties + [ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$ZOO_REPLICAS" ] && { + sed -i "s/^server\\./#server./" /etc/kafka/zookeeper.properties for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done - } - cat /etc/kafka/zookeeper.properties - [ ! -z "$ZOO_REPLICAS" ] && { - OFF=$PZOO_REPLICAS # TODO require both, or count pzoo lines? - sed -i "s/^server\\.\([0-9]\+\)=zoo-/#server.\1=zoo-/" /etc/kafka/zookeeper.properties - for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $OFF + $N - 1 ))=zoo-$(( $OFF + $N - 2 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done + for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done } cat /etc/kafka/zookeeper.properties sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties From 70334f4ac69ce4501fc793f2b3d3cccef7827ac1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:18:16 +0100 Subject: [PATCH 05/10] On how to use kustomize instead of a fork --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index e6022b3c..925ba9b9 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,16 @@ With the introduction of [app customization](https://kubectl.docs.kubernetes.io/ See the [variants](./variants) folder for different overlays. For example to scale to 1 kafka broker try `kubectl apply -k variants/scale-1/`. +Currently `apply -k` replaces `apply -f ./zookeeper; apply -f ./kafka`. +The original applies result in `error: unable to decode "zookeeper/kustomization.yaml": Object 'Kind' is missing in ...` +and though they still seem to work you can get around that with a v1.14+ kubectl using: `k apply -k variants/as-is/`. + +### Maintaining your own kustomization + +`kubectl apply -k` takes a siungle overlay, meaning that you can't compose different overlays from this repo. +You'll probably want to maintain your own variant. +One option is to keep kubernets-kafka as a git submodule and edit the relative path from an example variant. + ## Version history | tag | k8s ≥ | highlights | From 84a8b2e7f21248d889eec2b80003417896138deb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:22:02 +0100 Subject: [PATCH 06/10] A variant with 2 brokers and 3 zk --- variants/scale-2/kafka.yaml | 7 +++++ variants/scale-2/kustomization.yaml | 6 ++++ variants/scale-2/zookeeper.yaml | 43 +++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 variants/scale-2/kafka.yaml create mode 100644 variants/scale-2/kustomization.yaml create mode 100644 variants/scale-2/zookeeper.yaml diff --git a/variants/scale-2/kafka.yaml b/variants/scale-2/kafka.yaml new file mode 100644 index 00000000..c80e25a9 --- /dev/null +++ b/variants/scale-2/kafka.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka + namespace: kafka +spec: + replicas: 2 diff --git a/variants/scale-2/kustomization.yaml b/variants/scale-2/kustomization.yaml new file mode 100644 index 00000000..2cc065d5 --- /dev/null +++ b/variants/scale-2/kustomization.yaml @@ -0,0 +1,6 @@ +bases: +- ../../kafka +- ../../zookeeper +patchesStrategicMerge: +- kafka.yaml +- zookeeper.yaml diff --git a/variants/scale-2/zookeeper.yaml b/variants/scale-2/zookeeper.yaml new file mode 100644 index 00000000..0144c595 --- /dev/null +++ b/variants/scale-2/zookeeper.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: pzoo + namespace: kafka +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pzoo + namespace: kafka +spec: + replicas: 2 + template: + spec: + initContainers: + - name: init-config + env: + - name: PZOO_REPLICAS + value: '2' + - name: ZOO_REPLICAS + value: '1' +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zoo + namespace: kafka +spec: + replicas: 1 + template: + spec: + initContainers: + - name: init-config + env: + # There's no validation on these numbers adding up to a coherent zk config, so watch out + - name: PZOO_REPLICAS + value: '2' + - name: ZOO_REPLICAS + value: '1' + - name: ID_OFFSET + value: '3' From d8a4fe2c5ba6ae42beccdaba7cbfdc98ddd554b0 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:23:05 +0100 Subject: [PATCH 07/10] How to change the number of zk replicas --- variants/scale-1/zookeeper.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/variants/scale-1/zookeeper.yaml b/variants/scale-1/zookeeper.yaml index 0144c595..8779b468 100644 --- a/variants/scale-1/zookeeper.yaml +++ b/variants/scale-1/zookeeper.yaml @@ -11,16 +11,16 @@ metadata: name: pzoo namespace: kafka spec: - replicas: 2 + replicas: 1 template: spec: initContainers: - name: init-config env: - name: PZOO_REPLICAS - value: '2' - - name: ZOO_REPLICAS value: '1' + - name: ZOO_REPLICAS + value: '0' --- apiVersion: apps/v1 kind: StatefulSet @@ -28,7 +28,7 @@ metadata: name: zoo namespace: kafka spec: - replicas: 1 + replicas: 0 template: spec: initContainers: @@ -36,8 +36,8 @@ spec: env: # There's no validation on these numbers adding up to a coherent zk config, so watch out - name: PZOO_REPLICAS - value: '2' - - name: ZOO_REPLICAS value: '1' + - name: ZOO_REPLICAS + value: '0' - name: ID_OFFSET - value: '3' + value: '2' From e2bcc7007fb0844ff908529467c66fdefba8f020 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:36:49 +0100 Subject: [PATCH 08/10] Removes debug output --- zookeeper/10zookeeper-config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index 219838a9..54c7c8ab 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -19,9 +19,7 @@ data: for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done } - cat /etc/kafka/zookeeper.properties sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties - cat /etc/kafka/zookeeper.properties zookeeper.properties: | tickTime=2000 From 3ca5c8108c5c3af5309d7f64dee0eee99ecc9988 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:39:04 +0100 Subject: [PATCH 09/10] Adds the as-is variant, i.e. the workaround for the error message from plain apply due to kustomization.yaml --- variants/as-is/kustomization.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 variants/as-is/kustomization.yaml diff --git a/variants/as-is/kustomization.yaml b/variants/as-is/kustomization.yaml new file mode 100644 index 00000000..f54376f4 --- /dev/null +++ b/variants/as-is/kustomization.yaml @@ -0,0 +1,3 @@ +bases: +- ../../kafka +- ../../zookeeper From 83b88ad630aa6dc90ac09fd0d8b775258f63ad05 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 28 Mar 2019 12:40:59 +0100 Subject: [PATCH 10/10] Clarifies the as-is situation a bit --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 925ba9b9..ce2009dc 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,8 @@ With the introduction of [app customization](https://kubectl.docs.kubernetes.io/ See the [variants](./variants) folder for different overlays. For example to scale to 1 kafka broker try `kubectl apply -k variants/scale-1/`. Currently `apply -k` replaces `apply -f ./zookeeper; apply -f ./kafka`. -The original applies result in `error: unable to decode "zookeeper/kustomization.yaml": Object 'Kind' is missing in ...` -and though they still seem to work you can get around that with a v1.14+ kubectl using: `k apply -k variants/as-is/`. +The original commands now result in `error: unable to decode "zookeeper/kustomization.yaml": Object 'Kind' is missing in ...` +and though they still seem to work you can get around that with a v1.14+ kubectl using: `kubectl apply -k variants/as-is/`. ### Maintaining your own kustomization