From 30b888a01f5f2f02522fb376a4fc827ce3882405 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 09:42:11 +0200 Subject: [PATCH 01/17] Creates a pod for expermentation with kubectl and kafkacat --- addon-events/events-kube-kafka.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 addon-events/events-kube-kafka.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml new file mode 100644 index 00000000..9eeed58a --- /dev/null +++ b/addon-events/events-kube-kafka.yml @@ -0,0 +1,27 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: events-kube-kafka + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + from: kube + to: kafka + spec: + containers: + - name: kubectl-kafkacat + image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + command: + - sh + - -ec + - 'tail -f /dev/null' From 4bdfb2b606a9db2fc9d1bba07e29f482b28ae82d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 11:52:15 +0200 Subject: [PATCH 02/17] Produces and reads events --- addon-events/events-kube-kafka.yml | 12 +++++-- addon-events/test-event-consumer.yml | 30 +++++++++++++++++ .../topic-ops-kube-events-all-json.yml | 32 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 addon-events/test-event-consumer.yml create mode 100644 addon-events/topic-ops-kube-events-all-json.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 9eeed58a..95d0ad2f 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -22,6 +22,14 @@ spec: - name: kubectl-kafkacat image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b command: - - sh + - /bin/bash - -ec - - 'tail -f /dev/null' + - > + kubectl + get events + -w + | + kafkacat + -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + -t ops-kube-events-all-json-001 + -P diff --git a/addon-events/test-event-consumer.yml b/addon-events/test-event-consumer.yml new file mode 100644 index 00000000..d98f88ca --- /dev/null +++ b/addon-events/test-event-consumer.yml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-events-consumer + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + type: test + from: kafka-ops + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - kafkacat + - -b + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - -t + - ops-kube-events-all-json-001 + - -C diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml new file mode 100644 index 00000000..e6396a44 --- /dev/null +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-events-all-json + namespace: kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-events-all-json + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-events-all-json-001 + - --partitions + - "1" + - --replication-factor + - "1" + - --config + # this might be eight days + - retention.ms=69125000 + restartPolicy: Never From ccdd8bec306601a82b8faea056a82e50c4b4c1de Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:11:40 +0200 Subject: [PATCH 03/17] wip --- addon-events/events-kube-kafka.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 95d0ad2f..d45d480d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -19,15 +19,16 @@ spec: to: kafka spec: containers: - - name: kubectl-kafkacat - image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + - name: kafkacat-curl + image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b command: - /bin/bash - -ec - > - kubectl - get events - -w + curl + --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 From 5ee97a2ef2e7bb451db3d36c6767b5e6a4e49e89 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:48:51 +0200 Subject: [PATCH 04/17] Quite interesting way to test a command from yaml - see it in logs --- addon-events/events-kube-kafka.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index d45d480d..45297859 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,12 +25,14 @@ spec: - /bin/bash - -ec - > + echo ' curl --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt - --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + '; tail -f /dev/null From 4da60f8453f128c9b466a51f6a2cdbb9f9f32c92 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:50:24 +0200 Subject: [PATCH 05/17] Produces proper json. Let's see when curl gives up and the container restarts. --- addon-events/events-kube-kafka.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 45297859..ed2bde47 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,8 +25,8 @@ spec: - /bin/bash - -ec - > - echo ' curl + -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events @@ -35,4 +35,3 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - '; tail -f /dev/null From efacc180a893cd7ce83ecb99e2f9c0d70339d64b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:06:17 +0200 Subject: [PATCH 06/17] Testing compression but solsson/kafka:0.11.0.0 throws ... Caused by: java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy at org.xerial.snappy.SnappyInputStream.readFully(SnappyInputStream.java:145) at org.xerial.snappy.SnappyInputStream.readHeader(SnappyInputStream.java:99) at org.xerial.snappy.SnappyInputStream.(SnappyInputStream.java:59) at org.apache.kafka.common.record.CompressionType$3.wrapForInput(CompressionType.java:82) ... 40 more --- addon-events/events-kube-kafka.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index ed2bde47..e188a92d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,3 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + -z snappy From b6575a536fea27a36972da2d599c8d2f29f773b6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:08:57 +0200 Subject: [PATCH 07/17] Works with my current kafka image, and the test consumer still sees JSON --- addon-events/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index e188a92d..5cc7883c 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z snappy + -z gzip From a30f22e324594ab1c6d14db1bdfd4f2ff1f7698e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 09:17:27 +0200 Subject: [PATCH 08/17] Latest build suppors snappy compression --- addon-events/events-kube-kafka.yml | 2 +- addon-events/topic-ops-kube-events-all-json.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 5cc7883c..e188a92d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z gzip + -z snappy diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml index e6396a44..8eb3b660 100644 --- a/addon-events/topic-ops-kube-events-all-json.yml +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -13,7 +13,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce command: - ./bin/kafka-topics.sh - --zookeeper From f626a7764513fd85bdf721be766646312fd848b1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 06:24:38 +0200 Subject: [PATCH 09/17] Adds RBAC policy for curl, default service account --- rbac-namespace-default/events-watcher.yml | 30 +++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 rbac-namespace-default/events-watcher.yml diff --git a/rbac-namespace-default/events-watcher.yml b/rbac-namespace-default/events-watcher.yml new file mode 100644 index 00000000..c8384b67 --- /dev/null +++ b/rbac-namespace-default/events-watcher.yml @@ -0,0 +1,30 @@ +# If events-kube-kafka-* goes crashlooping you probably need this +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kafka-events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: events-watcher +subjects: +- kind: ServiceAccount + name: default + namespace: kafka From 4f13cf571b18c06074f5216852b722cdb3d5dd67 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 06:59:29 +0200 Subject: [PATCH 10/17] Moves the example comsumer to the test namespace --- .../test-event-consumer.yml => test/events-topic.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) rename addon-events/test-event-consumer.yml => test/events-topic.yml (85%) diff --git a/addon-events/test-event-consumer.yml b/test/events-topic.yml similarity index 85% rename from addon-events/test-event-consumer.yml rename to test/events-topic.yml index d98f88ca..6d11d775 100644 --- a/addon-events/test-event-consumer.yml +++ b/test/events-topic.yml @@ -1,8 +1,8 @@ apiVersion: extensions/v1beta1 kind: Deployment metadata: - name: test-events-consumer - namespace: kafka + name: events-topic + namespace: test-kafka spec: replicas: 1 strategy: @@ -14,9 +14,8 @@ spec: template: metadata: labels: - app: events - type: test - from: kafka-ops + test-target: events-topic + test-type: readiness spec: containers: - name: kafkacat From 8935cf549857b9b4f6dfb68209681789483b15d8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 07:01:09 +0200 Subject: [PATCH 11/17] Not a proper test yet, but the PR documents how to use it to see events --- test/events-topic.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test/events-topic.yml b/test/events-topic.yml index 6d11d775..68ca9198 100644 --- a/test/events-topic.yml +++ b/test/events-topic.yml @@ -5,17 +5,12 @@ metadata: namespace: test-kafka spec: replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - # prefer duplicate events over missed - maxUnavailable: 0 - maxSurge: 1 template: metadata: labels: test-target: events-topic - test-type: readiness + # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced + #test-type: readiness spec: containers: - name: kafkacat From 8d0d99302b80e960b592d1d8779b3c502815fc19 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 21:34:19 +0200 Subject: [PATCH 12/17] Use an image with new kafkacat and curl versions --- addon-events/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index e188a92d..2d6913a9 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -20,7 +20,7 @@ spec: spec: containers: - name: kafkacat-curl - image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b + image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 command: - /bin/bash - -ec From b9e7b8a02e328309907bd024619008d8d7cdfc07 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 21:56:03 +0200 Subject: [PATCH 13/17] Makes things a bit less obscure if for example RBAC fails --- addon-events/events-kube-kafka.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 2d6913a9..062d097c 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -21,10 +21,18 @@ spec: containers: - name: kafkacat-curl image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-events-all-json-001 command: - /bin/bash - -ec + # curl errors will go to kafka, kafkacat errors will got to log - > + echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" + ; curl -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -32,7 +40,9 @@ spec: https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat - -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - -t ops-kube-events-all-json-001 + -b $BOOTSTRAP + -t $TOPIC -P -z snappy + -v + -d broker,topic From e4c517984066f8e76909ec8a38fd414ada2c10c4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:02:53 +0200 Subject: [PATCH 14/17] Argh, curl will pretty print the json ... and kafkacat will produce each line separately --- addon-events/events-kube-kafka.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 062d097c..a707b1c5 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -28,7 +28,7 @@ spec: value: ops-kube-events-all-json-001 command: - /bin/bash - - -ec + - -c # curl errors will go to kafka, kafkacat errors will got to log - > echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" @@ -46,3 +46,13 @@ spec: -z snappy -v -d broker,topic + ; + echo "Last message on topic:" + ; + kafkacat + -b $BOOTSTRAP + -t $TOPIC + -C + -o -1 + -c 1 + -z snappy From d628275ff253366f93c2533c7bdd191c909ebd69 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:07:21 +0200 Subject: [PATCH 15/17] bash -e is important, in combination with curl -f, so ... we don't send error messages to the topic (even if it's json -- format is different and it's pretty printed) --- addon-events/events-kube-kafka.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index a707b1c5..92387aae 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -28,12 +28,12 @@ spec: value: ops-kube-events-all-json-001 command: - /bin/bash - - -c - # curl errors will go to kafka, kafkacat errors will got to log + - -ec - > echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" ; curl + -f -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" @@ -46,13 +46,3 @@ spec: -z snappy -v -d broker,topic - ; - echo "Last message on topic:" - ; - kafkacat - -b $BOOTSTRAP - -t $TOPIC - -C - -o -1 - -c 1 - -z snappy From fc5a8362abda5e5e92df00d7bce21b3f5a7ee280 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:34:30 +0200 Subject: [PATCH 16/17] kafkacat's debug output solves the silence, but ... to see the error message (like missing RBAC) you basically have to run the curl command. I'd like startup to be as fast as possible, to not miss events, so I hesitate to add checks. --- addon-events/events-kube-kafka.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 92387aae..2cd18912 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -30,8 +30,6 @@ spec: - /bin/bash - -ec - > - echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" - ; curl -f -s From 7d4d30d221fc7eb53890446dda6a976cc34c11f6 Mon Sep 17 00:00:00 2001 From: Anton Lindgren Date: Mon, 30 Oct 2017 08:58:43 +0100 Subject: [PATCH 17/17] Fix retention from 0.8 days to 8 days Silly change, but 0.8 days is very short. Google helped me verify the math. --- addon-events/topic-ops-kube-events-all-json.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml index 8eb3b660..62f2b807 100644 --- a/addon-events/topic-ops-kube-events-all-json.yml +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -27,6 +27,6 @@ spec: - --replication-factor - "1" - --config - # this might be eight days - - retention.ms=69125000 + # 8 days + - retention.ms=691200000 restartPolicy: Never