diff --git a/README.md b/README.md index 552d8fa5..9079cf46 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,8 @@ Then pick and chose from patches our [example variants](https://github.com/Yolea | tag | k8s ≥ | highlights | | ------ | ----- | ----------- | -| TBD | 1.13+ | Kafka [2.4.0](https://github.com/Yolean/kubernetes-kafka/pull/297) + [standard storage class](https://github.com/Yolean/kubernetes-kafka/pull/294) | +| v7.0.0 | 1.15+ | [Breaking](https://github.com/Yolean/kubernetes-kafka/pull/311#issuecomment-657181714) with [nonroot](./nonroot/) and [native](./native/) bases | +| v6.0.x | 1.13+ | Kafka [2.4.0](https://github.com/Yolean/kubernetes-kafka/pull/297) + [standard storage class](https://github.com/Yolean/kubernetes-kafka/pull/294) | | v6.0.0 | 1.11+ | Kafka 2.2.0 + `apply -k` (kubectl 1.14+) + [#270](https://github.com/Yolean/kubernetes-kafka/pull/270) | | v5.1.0 | 1.11+ | Kafka 2.1.1 | | v5.0.3 | 1.11+ | Zookeeper fix [#227](https://github.com/Yolean/kubernetes-kafka/pull/227) + [maxClientCnxns=1](https://github.com/Yolean/kubernetes-kafka/pull/230#issuecomment-445953857) | diff --git a/cruise-control/topic-create.yml b/cruise-control/topic-create.yml index 49eef860..c8730c15 100644 --- a/cruise-control/topic-create.yml +++ b/cruise-control/topic-create.yml @@ -8,7 +8,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:native-cli@sha256:7a4cc4ef875aea50b8d4dbb525cbac0ef3a5c6e611fcafc2c415ca432ebe5601 + image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/events-kube/topic-create.yaml b/events-kube/topic-create.yaml index 5021a394..2cf4f062 100644 --- a/events-kube/topic-create.yaml +++ b/events-kube/topic-create.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:native-cli@sha256:7a4cc4ef875aea50b8d4dbb525cbac0ef3a5c6e611fcafc2c415ca432ebe5601 + image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 857e1f13..49fc74c4 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -41,6 +41,7 @@ data: } printf '%s\n' "${SEDS[@]}" | sed -f - /etc/kafka-configmap/server.properties > /etc/kafka/server.properties.tmp [ $? -eq 0 ] && mv /etc/kafka/server.properties.tmp /etc/kafka/server.properties + ln -s /etc/kafka/server.properties /etc/kafka/server.properties.$POD_NAME server.properties: |- ############################# Log Basics ############################# @@ -166,7 +167,7 @@ data: zookeeper.connect=zookeeper:2181 # Timeout in ms for connecting to zookeeper - #zookeeper.connection.timeout.ms=6000 + #zookeeper.connection.timeout.ms=18000 ############################# Group Coordinator Settings ############################# diff --git a/kafka/20dns.yml b/kafka/20dns.yml index 4088c311..3a74b708 100644 --- a/kafka/20dns.yml +++ b/kafka/20dns.yml @@ -3,9 +3,12 @@ apiVersion: v1 kind: Service metadata: - name: broker + name: kafka namespace: kafka + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" spec: + publishNotReadyAddresses: true ports: - port: 9092 # [podname].broker.kafka.svc.cluster.local diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index 3ac02b2c..452509f8 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -7,7 +7,7 @@ spec: selector: matchLabels: app: kafka - serviceName: "broker" + serviceName: "kafka" replicas: 3 updateStrategy: type: RollingUpdate @@ -47,6 +47,14 @@ spec: - name: broker image: solsson/kafka:2.4.1@sha256:79761e15919b4fe9857ec00313c9df799918ad0340b684c0163ab7035907bb5a env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: CLASSPATH value: /opt/kafka/libs/extensions/* - name: KAFKA_LOG4J_OPTS @@ -62,7 +70,7 @@ spec: containerPort: 5555 command: - ./bin/kafka-server-start.sh - - /etc/kafka/server.properties + - /etc/kafka/server.properties.$(POD_NAME) lifecycle: preStop: exec: diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 45bf923e..b0f169ea 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -72,7 +72,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:native-cli@sha256:7a4cc4ef875aea50b8d4dbb525cbac0ef3a5c6e611fcafc2c415ca432ebe5601 + image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index fa23c771..02cbad0e 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -55,7 +55,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:native-cli@sha256:7a4cc4ef875aea50b8d4dbb525cbac0ef3a5c6e611fcafc2c415ca432ebe5601 + image: solsson/kafka:native-cli@sha256:fbf29c59182fb87921c5199783d2d5796856ecbfe34a9c03eca658b3cf50f3c4 command: - ./bin/kafka-topics.sh - --zookeeper diff --git a/native/distroless.yaml b/native/distroless.yaml new file mode 100644 index 00000000..e00b5205 --- /dev/null +++ b/native/distroless.yaml @@ -0,0 +1,9 @@ +# The more specific removes are to make sure that there was a shell that we're removing +- op: remove + path: /spec/template/spec/containers/0/readinessProbe/exec +- op: remove + path: /spec/template/spec/containers/0/readinessProbe +- op: remove + path: /spec/template/spec/containers/0/lifecycle/preStop/exec +- op: remove + path: /spec/template/spec/containers/0/lifecycle/preStop diff --git a/native/kustomization.yaml b/native/kustomization.yaml new file mode 100644 index 00000000..6a489db7 --- /dev/null +++ b/native/kustomization.yaml @@ -0,0 +1,17 @@ +bases: +- ../nonroot +patchesStrategicMerge: +- native-image-zookeeper.yaml +patchesJson6902: +- target: + group: apps + version: v1 + kind: StatefulSet + name: pzoo + path: distroless.yaml +- target: + group: apps + version: v1 + kind: StatefulSet + name: zoo + path: distroless.yaml \ No newline at end of file diff --git a/native/native-image-zookeeper.yaml b/native/native-image-zookeeper.yaml new file mode 100644 index 00000000..5d38e65d --- /dev/null +++ b/native/native-image-zookeeper.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pzoo +spec: + template: + spec: + containers: + - name: zookeeper + image: solsson/kafka:native-zookeeper-server-start@sha256:ba3a0632240b8906a3b5bb6441e98ad9d9de73cb716b156ca68f1b435c819e8b + resources: + requests: + cpu: 10m + memory: 25Mi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zoo +spec: + template: + spec: + containers: + - name: zookeeper + image: solsson/kafka:native-zookeeper-server-start@sha256:ba3a0632240b8906a3b5bb6441e98ad9d9de73cb716b156ca68f1b435c819e8b + resources: + requests: + cpu: 10m + memory: 25Mi diff --git a/nonroot/entrypoint-from-image.yaml b/nonroot/entrypoint-from-image.yaml new file mode 100644 index 00000000..fe110c35 --- /dev/null +++ b/nonroot/entrypoint-from-image.yaml @@ -0,0 +1,2 @@ +- op: remove + path: /spec/template/spec/containers/0/command diff --git a/nonroot/kustomization.yaml b/nonroot/kustomization.yaml index d23bcfa1..a3526dfb 100644 --- a/nonroot/kustomization.yaml +++ b/nonroot/kustomization.yaml @@ -2,6 +2,9 @@ bases: - ../rbac-namespace-default - ../kafka - ../zookeeper +patchesStrategicMerge: +- nonroot-image-kafka.yaml +- nonroot-image-zookeeper.yaml patchesJson6902: - target: group: apps @@ -21,7 +24,21 @@ patchesJson6902: kind: StatefulSet name: zoo path: fsgroup-65534.yaml -# https://github.com/kubernetes-sigs/kustomize/issues/915#issuecomment-477808963 -patchesStrategicMerge: -- nonroot-image-kafka.yaml -- nonroot-image-zookeeper.yaml +- target: + group: apps + version: v1 + kind: StatefulSet + name: kafka + path: entrypoint-from-image.yaml +- target: + group: apps + version: v1 + kind: StatefulSet + name: pzoo + path: entrypoint-from-image.yaml +- target: + group: apps + version: v1 + kind: StatefulSet + name: zoo + path: entrypoint-from-image.yaml diff --git a/nonroot/nonroot-image-kafka.yaml b/nonroot/nonroot-image-kafka.yaml index c5136489..9bfb41f3 100644 --- a/nonroot/nonroot-image-kafka.yaml +++ b/nonroot/nonroot-image-kafka.yaml @@ -5,6 +5,11 @@ metadata: spec: template: spec: + initContainers: + - name: init-config + image: solsson/kafka:initutils-nonroot@sha256:87f6bb39fd47a6f382018a2dc55a484d1b71eee48a58019c7e65a9bc53e8dca2 containers: - name: broker - image: solsson/kafka:2.4.1-nonroot@sha256:ddfdd23081c075b86132f3e5d8e15389c78c4334dbb22166d9ad1d57f362007f + image: solsson/kafka:2.5.0-kafka-server-start@sha256:c6d43d1240c358d9ab8bb60de61a0b9578f04fd727ee4016f7706800998e5351 + args: + - /etc/kafka/server.properties.$(POD_NAME) diff --git a/nonroot/nonroot-image-zookeeper.yaml b/nonroot/nonroot-image-zookeeper.yaml index 51ba96ba..8db695a9 100644 --- a/nonroot/nonroot-image-zookeeper.yaml +++ b/nonroot/nonroot-image-zookeeper.yaml @@ -5,9 +5,14 @@ metadata: spec: template: spec: + initContainers: + - name: init-config + image: solsson/kafka:initutils-nonroot@sha256:87f6bb39fd47a6f382018a2dc55a484d1b71eee48a58019c7e65a9bc53e8dca2 containers: - name: zookeeper - image: solsson/kafka:2.4.1-nonroot@sha256:ddfdd23081c075b86132f3e5d8e15389c78c4334dbb22166d9ad1d57f362007f + image: solsson/kafka:zookeeper-server-start@sha256:c34a9928fcc5ac15b2243ee2994cf09f5a3608b6090de378f39f89ad12320276 + args: + - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) --- apiVersion: apps/v1 kind: StatefulSet @@ -16,6 +21,11 @@ metadata: spec: template: spec: + initContainers: + - name: init-config + image: solsson/kafka:initutils-nonroot@sha256:87f6bb39fd47a6f382018a2dc55a484d1b71eee48a58019c7e65a9bc53e8dca2 containers: - name: zookeeper - image: solsson/kafka:2.4.1-nonroot@sha256:ddfdd23081c075b86132f3e5d8e15389c78c4334dbb22166d9ad1d57f362007f + image: solsson/kafka:zookeeper-server-start@sha256:c34a9928fcc5ac15b2243ee2994cf09f5a3608b6090de378f39f89ad12320276 + args: + - /etc/kafka/zookeeper.properties.scale-$(REPLICAS).$(POD_NAME) diff --git a/variants/dev-small/listener-localhost.json b/variants/dev-small/listener-localhost.json index 7b9e683f..496b1664 100644 --- a/variants/dev-small/listener-localhost.json +++ b/variants/dev-small/listener-localhost.json @@ -1,4 +1,4 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/command/2", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/3", "value": "advertised.listeners=PLAINTEXT://:9092,OUTSIDE://localhost:9094"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/1", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/2", "value": "advertised.listeners=PLAINTEXT://:9092,OUTSIDE://localhost:9094"} ] diff --git a/variants/dev-small/num-partitions-1.json b/variants/dev-small/num-partitions-1.json index 3cfa8e0c..b8211f7f 100644 --- a/variants/dev-small/num-partitions-1.json +++ b/variants/dev-small/num-partitions-1.json @@ -1,4 +1,4 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "num.partitions=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "num.partitions=1"} ] diff --git a/variants/scale-1-ephemeral/kafka-scale1-overrides.json b/variants/scale-1-ephemeral/kafka-scale1-overrides.json index d9d03749..13187671 100644 --- a/variants/scale-1-ephemeral/kafka-scale1-overrides.json +++ b/variants/scale-1-ephemeral/kafka-scale1-overrides.json @@ -1,10 +1,10 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "default.replication.factor=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.num.partitions=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "default.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "min.insync.replicas=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.num.partitions=1"} ] \ No newline at end of file diff --git a/variants/scale-1/kafka-scale1-overrides.json b/variants/scale-1/kafka-scale1-overrides.json index d9d03749..b4dcceca 100644 --- a/variants/scale-1/kafka-scale1-overrides.json +++ b/variants/scale-1/kafka-scale1-overrides.json @@ -1,10 +1,12 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "default.replication.factor=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=1"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.num.partitions=1"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "zookeeper.connect=zoo-0.zoo.$(POD_NAMESPACE).svc.cluster.local:2181" }, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "default.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "min.insync.replicas=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.replication.factor=1"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.num.partitions=1"} ] \ No newline at end of file diff --git a/variants/scale-1/kustomization.yaml b/variants/scale-1/kustomization.yaml index 6cd41f97..ab741e0b 100644 --- a/variants/scale-1/kustomization.yaml +++ b/variants/scale-1/kustomization.yaml @@ -1,5 +1,5 @@ bases: -- ../../nonroot +- ../../native patchesStrategicMerge: - kafka.yaml - zookeeper.yaml diff --git a/variants/scale-1/zookeeper.yaml b/variants/scale-1/zookeeper.yaml index 8779b468..bc23fc67 100644 --- a/variants/scale-1/zookeeper.yaml +++ b/variants/scale-1/zookeeper.yaml @@ -19,8 +19,13 @@ spec: env: - name: PZOO_REPLICAS value: '1' - - name: ZOO_REPLICAS - value: '0' + - name: REPLICAS + value: '1' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '1' --- apiVersion: apps/v1 kind: StatefulSet @@ -37,7 +42,12 @@ spec: # There's no validation on these numbers adding up to a coherent zk config, so watch out - name: PZOO_REPLICAS value: '1' - - name: ZOO_REPLICAS - value: '0' + - name: REPLICAS + value: '1' - name: ID_OFFSET value: '2' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '1' diff --git a/variants/scale-2/kafka-scale2-overrides.json b/variants/scale-2/kafka-scale2-overrides.json index 72d2c344..a18753cb 100644 --- a/variants/scale-2/kafka-scale2-overrides.json +++ b/variants/scale-2/kafka-scale2-overrides.json @@ -1,10 +1,10 @@ [ - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "default.replication.factor=2"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "min.insync.replicas=2"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.replication.factor=2"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "--override"}, - {"op": "add", "path": "/spec/template/spec/containers/0/command/-", "value": "offsets.topic.num.partitions=2"} + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "default.replication.factor=2"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "min.insync.replicas=2"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.replication.factor=2"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "offsets.topic.num.partitions=2"} ] diff --git a/variants/scale-2/zookeeper.yaml b/variants/scale-2/zookeeper.yaml index fa62b228..96c8b189 100644 --- a/variants/scale-2/zookeeper.yaml +++ b/variants/scale-2/zookeeper.yaml @@ -22,7 +22,12 @@ spec: # There's no validation on these numbers adding up to a coherent zk config, so watch out - name: PZOO_REPLICAS value: '0' - - name: ZOO_REPLICAS + - name: REPLICAS value: '2' - name: ID_OFFSET value: '1' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '2' diff --git a/variants/scale-3-3/kafka-zookeeper-connect-only-zoo.json b/variants/scale-3-3/kafka-zookeeper-connect-only-zoo.json new file mode 100644 index 00000000..605bf743 --- /dev/null +++ b/variants/scale-3-3/kafka-zookeeper-connect-only-zoo.json @@ -0,0 +1,6 @@ +[ + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": + "zookeeper.connect=zoo-0.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-1.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-2.zoo.$(POD_NAMESPACE).svc.cluster.local:2181" + } +] \ No newline at end of file diff --git a/variants/scale-3-3/kustomization.yaml b/variants/scale-3-3/kustomization.yaml new file mode 100644 index 00000000..11b82724 --- /dev/null +++ b/variants/scale-3-3/kustomization.yaml @@ -0,0 +1,12 @@ +bases: +- ../scale-3-5 +patchesStrategicMerge: +- ./only-zoo-3.yaml +patchesJson6902: +- target: + group: apps + version: v1 + kind: StatefulSet + name: kafka + namespace: kafka + path: kafka-zookeeper-connect-only-zoo.json diff --git a/variants/scale-3-3/only-zoo-3.yaml b/variants/scale-3-3/only-zoo-3.yaml new file mode 100644 index 00000000..bced2320 --- /dev/null +++ b/variants/scale-3-3/only-zoo-3.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pzoo + namespace: kafka +spec: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zoo + namespace: kafka +spec: + replicas: 3 + template: + spec: + initContainers: + - name: init-config + env: + # There's no validation on these numbers adding up to a coherent zk config, so watch out + - name: PZOO_REPLICAS + value: '0' + - name: REPLICAS + value: '3' + - name: ID_OFFSET + value: '1' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '3' diff --git a/variants/scale-3-5-nopzoo/kafka-zookeeper-connect-only-zoo.json b/variants/scale-3-5-nopzoo/kafka-zookeeper-connect-only-zoo.json new file mode 100644 index 00000000..37c9f1fc --- /dev/null +++ b/variants/scale-3-5-nopzoo/kafka-zookeeper-connect-only-zoo.json @@ -0,0 +1,4 @@ +[ + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "zookeeper.connect=zoo:2181"} +] \ No newline at end of file diff --git a/variants/scale-3-5-nopzoo/kustomization.yaml b/variants/scale-3-5-nopzoo/kustomization.yaml index 4254a479..5c7ec29d 100644 --- a/variants/scale-3-5-nopzoo/kustomization.yaml +++ b/variants/scale-3-5-nopzoo/kustomization.yaml @@ -2,3 +2,11 @@ bases: - ../scale-3-5 patchesStrategicMerge: - ./only-zoo-5.yaml +patchesJson6902: +- target: + group: apps + version: v1 + kind: StatefulSet + name: kafka + namespace: kafka + path: kafka-zookeeper-connect-only-zoo.json diff --git a/variants/scale-3-5-nopzoo/only-zoo-5.yaml b/variants/scale-3-5-nopzoo/only-zoo-5.yaml index 6d096373..ed59e8de 100644 --- a/variants/scale-3-5-nopzoo/only-zoo-5.yaml +++ b/variants/scale-3-5-nopzoo/only-zoo-5.yaml @@ -22,7 +22,12 @@ spec: # There's no validation on these numbers adding up to a coherent zk config, so watch out - name: PZOO_REPLICAS value: '0' - - name: ZOO_REPLICAS + - name: REPLICAS value: '5' - name: ID_OFFSET value: '1' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '5' diff --git a/variants/scale-6-9/kafka-6.yaml b/variants/scale-6-9/kafka-6.yaml new file mode 100644 index 00000000..a7e319a7 --- /dev/null +++ b/variants/scale-6-9/kafka-6.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: kafka +spec: + replicas: 6 diff --git a/variants/scale-6-9/kafka-zookeeper-connect-only-zoo.json b/variants/scale-6-9/kafka-zookeeper-connect-only-zoo.json new file mode 100644 index 00000000..06d3bb1a --- /dev/null +++ b/variants/scale-6-9/kafka-zookeeper-connect-only-zoo.json @@ -0,0 +1,6 @@ +[ + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--override"}, + {"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": + "zookeeper.connect=zoo-0.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-1.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-2.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-3.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-4.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-5.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-6.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-7.zoo.$(POD_NAMESPACE).svc.cluster.local:2181,zoo-8.zoo.$(POD_NAMESPACE).svc.cluster.local:2181" + } +] \ No newline at end of file diff --git a/variants/scale-6-9/kustomization.yaml b/variants/scale-6-9/kustomization.yaml new file mode 100644 index 00000000..bd1afdd4 --- /dev/null +++ b/variants/scale-6-9/kustomization.yaml @@ -0,0 +1,26 @@ +bases: +- ../../native +patchesStrategicMerge: +- zoo-9.yaml +- kafka-6.yaml +patchesJson6902: +- target: + group: apps + version: v1 + kind: StatefulSet + name: kafka + path: kafka-zookeeper-connect-only-zoo.json +# The nonroot image is distroless and doesn't support a shell that the prestop hook needs +- target: + group: apps + version: v1 + kind: StatefulSet + name: zoo + path: lifecycle-remove.json +# The nonroot image is distroless and has neither shell nor the nc command +- target: + group: apps + version: v1 + kind: StatefulSet + name: zoo + path: zoo-readiness-without-shell.yaml diff --git a/variants/scale-6-9/lifecycle-remove.json b/variants/scale-6-9/lifecycle-remove.json new file mode 100644 index 00000000..d6ed5370 --- /dev/null +++ b/variants/scale-6-9/lifecycle-remove.json @@ -0,0 +1,4 @@ +[ + {"op": "remove", "path": "/spec/template/spec/containers/0/lifecycle"} +] + \ No newline at end of file diff --git a/variants/scale-6-9/zoo-9.yaml b/variants/scale-6-9/zoo-9.yaml new file mode 100644 index 00000000..e2babff0 --- /dev/null +++ b/variants/scale-6-9/zoo-9.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: pzoo + namespace: kafka +spec: + replicas: 0 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zoo + namespace: kafka +spec: + replicas: 9 + template: + spec: + initContainers: + - name: init-config + env: + # There's no validation on these numbers adding up to a coherent zk config, so watch out + - name: PZOO_REPLICAS + value: '0' + - name: REPLICAS + value: '9' + - name: ID_OFFSET + value: '1' + containers: + - name: zookeeper + env: + - name: REPLICAS + value: '9' diff --git a/variants/scale-6-9/zoo-readiness-without-shell.yaml b/variants/scale-6-9/zoo-readiness-without-shell.yaml new file mode 100644 index 00000000..72fe2284 --- /dev/null +++ b/variants/scale-6-9/zoo-readiness-without-shell.yaml @@ -0,0 +1,6 @@ +- path: /spec/template/spec/containers/0/readinessProbe + # op: replace + # value: + # tcpSocket: + # port: 2181 + op: remove diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index e2bf8cba..57728474 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -9,17 +9,17 @@ data: set -e set -x - [ -d /var/lib/zookeeper/data ] || mkdir /var/lib/zookeeper/data + [ ! -d /var/lib/zookeeper/data ] && mkdir -m 770 /var/lib/zookeeper/data && chgrp $(stat -c '%g' /var/lib/zookeeper) /var/lib/zookeeper/data [ -z "$ID_OFFSET" ] && ID_OFFSET=1 export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid cp -Lur /etc/kafka-configmap/* /etc/kafka/ - [ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$ZOO_REPLICAS" ] && { + [ ! -z "$PZOO_REPLICAS" ] && [ ! -z "$REPLICAS" ] && { sed -i "s/^server\\./#server./" /etc/kafka/zookeeper.properties - for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done - for N in $(seq $ZOO_REPLICAS); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done + for N in $(seq $PZOO_REPLICAS); do echo "server.$N=pzoo-$(( $N - 1 )).pzoo.$POD_NAMESPACE.svc.cluster.local:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done + for N in $(seq $(( $REPLICAS - $PZOO_REPLICAS ))); do echo "server.$(( $PZOO_REPLICAS + $N ))=zoo-$(( $N - 1 )).zoo.$POD_NAMESPACE.svc.cluster.local:2888:3888:participant" >> /etc/kafka/zookeeper.properties; done } - sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties + ln -s /etc/kafka/zookeeper.properties /etc/kafka/zookeeper.properties.scale-$REPLICAS.$POD_NAME zookeeper.properties: | 4lw.commands.whitelist=ruok @@ -27,9 +27,11 @@ data: dataDir=/var/lib/zookeeper/data dataLogDir=/var/lib/zookeeper/log clientPort=2181 - maxClientCnxns=2 + maxClientCnxns=3 initLimit=5 syncLimit=2 + tcpKeepAlive=true + electionPortBindRetry=0 server.1=pzoo-0.pzoo:2888:3888:participant server.2=pzoo-1.pzoo:2888:3888:participant server.3=pzoo-2.pzoo:2888:3888:participant @@ -42,6 +44,5 @@ data: log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n - # Suppress connection log messages, three lines per livenessProbe execution - log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN + # Suppress connection log messages, one line per readiness probe log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 2bf8a4b5..c4dfdcc7 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -25,6 +25,15 @@ spec: - name: init-config image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: configmap mountPath: /etc/kafka-configmap @@ -36,11 +45,15 @@ spec: - name: zookeeper image: solsson/kafka:2.4.1@sha256:79761e15919b4fe9857ec00313c9df799918ad0340b684c0163ab7035907bb5a env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties command: - ./bin/zookeeper-server-start.sh - - /etc/kafka/zookeeper.properties + - /etc/kafka/zookeeper.properties.scale-5.$(POD_NAME) lifecycle: preStop: exec: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 20d271a3..8abe8b40 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -26,6 +26,14 @@ spec: image: solsson/kafka-initutils@sha256:f6d9850c6c3ad5ecc35e717308fddb47daffbde18eb93e98e031128fe8b899ef command: ['/bin/bash', '/etc/kafka-configmap/init.sh'] env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace - name: ID_OFFSET value: "4" volumeMounts: @@ -39,11 +47,15 @@ spec: - name: zookeeper image: solsson/kafka:2.4.1@sha256:79761e15919b4fe9857ec00313c9df799918ad0340b684c0163ab7035907bb5a env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties command: - ./bin/zookeeper-server-start.sh - - /etc/kafka/zookeeper.properties + - /etc/kafka/zookeeper.properties.scale-5.$(POD_NAME) lifecycle: preStop: exec: