Persistent volumes for Zookeeper

README.md

@@ -24,18 +24,13 @@ kubectl -n kafka get pvc
 
 ## Set up Zookeeper
 
-The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances,
-so we use the [official docker image](https://hub.docker.com/_/zookeeper/)
-but with a [startup script change to guess node id from hostname](https://github.com/solsson/zookeeper-docker/commit/df9474f858ad548be8a365cb000a4dd2d2e3a217).
+The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances.
+We use the zookeeper build that comes with the Kafka distribution, and tweak the startup command to support StatefulSet.
 
 ```
 kubectl create -f ./zookeeper/
 ```
 
-Despite being a StatefulSet, there is no persistent volume by default.
-If you lose your zookeeper cluster, kafka will be unaware that persisted topics exist.
-The data is still there, but you need to re-create topics.
-
 ## Start Kafka
 
 Assuming you have your PVCs `Bound`, or enabled automatic provisioning (see above), go ahead and:
@@ -85,4 +80,6 @@ Is the metrics system up and running?
 ```
 kubectl logs -c metrics kafka-0
 kubectl exec -c broker kafka-0 -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics'
+kubectl logs -c metrics zoo-0
+kubectl exec -c zookeeper zoo-0 -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics'
 ```

zookeeper/10zookeeper-config.yml

@@ -11,11 +11,11 @@ data:
     clientPort=2181
     initLimit=5
     syncLimit=2
-    server.1=zoo-0.zoo:2888:3888:participant
-    server.2=zoo-1.zoo:2888:3888:participant
-    server.3=zoo-2.zoo:2888:3888:participant
-    server.4=zoo-3.zoo:2888:3888:participant
-    server.5=zoo-4.zoo:2888:3888:participant
+    server.1=pzoo-0.pzoo:2888:3888:participant
+    server.2=pzoo-1.pzoo:2888:3888:participant
+    server.3=pzoo-2.pzoo:2888:3888:participant
+    server.4=zoo-0.zoo:2888:3888:participant
+    server.5=zoo-1.zoo:2888:3888:participant
 
   log4j.properties: |-
     log4j.rootLogger=INFO, stdout

zookeeper/20pzoo-service.yml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: pzoo
+  namespace: kafka
+spec:
+  ports:
+  - port: 2888
+    name: peer
+  - port: 3888
+    name: leader-election
+  clusterIP: None
+  selector:
+    app: zookeeper
+    storage: persistent

zookeeper/20zoo-service.yml → zookeeper/21zoo-service.yml

@@ -12,3 +12,4 @@ spec:
   clusterIP: None
   selector:
     app: zookeeper
+    storage: ephemeral

zookeeper/50pzoo.yml

@@ -0,0 +1,68 @@
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: pzoo
+  namespace: kafka
+spec:
+  serviceName: "pzoo"
+  replicas: 3
+  template:
+    metadata:
+      labels:
+        app: zookeeper
+        storage: persistent
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
+    spec:
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: metrics
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
+        command:
+        - "java"
+        - "-jar"
+        - "jmx_prometheus_httpserver.jar"
+        - "5556"
+        - example_configs/zookeeper.yaml
+        ports:
+        - containerPort: 5556
+      - name: zookeeper
+        image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1
+        env:
+        - name: JMX_PORT
+          value: "5555"
+        command:
+        - sh
+        - -c
+        - >
+          set -e;
+          export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 1));
+          echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid;
+          sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties;
+          cat config/zookeeper.properties;
+          ./bin/zookeeper-server-start.sh config/zookeeper.properties
+        ports:
+        - containerPort: 2181
+          name: client
+        - containerPort: 2888
+          name: peer
+        - containerPort: 3888
+          name: leader-election
+        volumeMounts:
+        - name: config
+          mountPath: /usr/local/kafka/config
+        - name: data
+          mountPath: /var/lib/zookeeper/data
+      volumes:
+      - name: config
+        configMap:
+          name: zookeeper-config
+  volumeClaimTemplates:
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+      resources:
+        requests:
+          storage: 10Gi

zookeeper/50zoo.yml → zookeeper/51zoo.yml

@@ -5,22 +5,39 @@ metadata:
   namespace: kafka
 spec:
   serviceName: "zoo"
-  replicas: 5
+  replicas: 2
   template:
     metadata:
       labels:
         app: zookeeper
+        storage: ephemeral
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
     spec:
       terminationGracePeriodSeconds: 10
       containers:
+      - name: metrics
+        image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d
+        command:
+        - "java"
+        - "-jar"
+        - "jmx_prometheus_httpserver.jar"
+        - "5556"
+        - example_configs/zookeeper.yaml
+        ports:
+        - containerPort: 5556
       - name: zookeeper
         image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1
+        env:
+        - name: JMX_PORT
+          value: "5555"
         command:
         - sh
         - -c
         - >
           set -e;
-          export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 1));
+          export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 4));
           echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid;
           sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties;
           cat config/zookeeper.properties;
@@ -35,11 +52,11 @@ spec:
         volumeMounts:
         - name: config
           mountPath: /usr/local/kafka/config
-        - name: datadir
+        - name: data
           mountPath: /var/lib/zookeeper/data
       volumes:
       - name: config
         configMap:
           name: zookeeper-config
-      - name: datadir
+      - name: data
         emptyDir: {}