From c7c7e9cde4e2b41dad09536391254c0377c7c174 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 09:42:11 +0200 Subject: [PATCH 1/8] Creates a pod for expermentation with kubectl and kafkacat --- addon-events/events-kube-kafka.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 addon-events/events-kube-kafka.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml new file mode 100644 index 00000000..9eeed58a --- /dev/null +++ b/addon-events/events-kube-kafka.yml @@ -0,0 +1,27 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: events-kube-kafka + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + from: kube + to: kafka + spec: + containers: + - name: kubectl-kafkacat + image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + command: + - sh + - -ec + - 'tail -f /dev/null' From bbd8c478b6c3b0b6b28c7d35e45df0a133b3c634 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 11:52:15 +0200 Subject: [PATCH 2/8] Produces and reads events --- addon-events/events-kube-kafka.yml | 12 +++++-- addon-events/test-event-consumer.yml | 30 +++++++++++++++++ .../topic-ops-kube-events-all-json.yml | 32 +++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 addon-events/test-event-consumer.yml create mode 100644 addon-events/topic-ops-kube-events-all-json.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 9eeed58a..95d0ad2f 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -22,6 +22,14 @@ spec: - name: kubectl-kafkacat image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b command: - - sh + - /bin/bash - -ec - - 'tail -f /dev/null' + - > + kubectl + get events + -w + | + kafkacat + -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + -t ops-kube-events-all-json-001 + -P diff --git a/addon-events/test-event-consumer.yml b/addon-events/test-event-consumer.yml new file mode 100644 index 00000000..d98f88ca --- /dev/null +++ b/addon-events/test-event-consumer.yml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-events-consumer + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + type: test + from: kafka-ops + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - kafkacat + - -b + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - -t + - ops-kube-events-all-json-001 + - -C diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml new file mode 100644 index 00000000..e6396a44 --- /dev/null +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-events-all-json + namespace: kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-events-all-json + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-events-all-json-001 + - --partitions + - "1" + - --replication-factor + - "1" + - --config + # this might be eight days + - retention.ms=69125000 + restartPolicy: Never From b0a600a7cfd73c81bf539c01393e172af031acd8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:11:40 +0200 Subject: [PATCH 3/8] wip --- addon-events/events-kube-kafka.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 95d0ad2f..d45d480d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -19,15 +19,16 @@ spec: to: kafka spec: containers: - - name: kubectl-kafkacat - image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + - name: kafkacat-curl + image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b command: - /bin/bash - -ec - > - kubectl - get events - -w + curl + --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 From 8c99e813c2c0e42a6192e061d43723eb5dc5bfef Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:48:51 +0200 Subject: [PATCH 4/8] Quite interesting way to test a command from yaml - see it in logs --- addon-events/events-kube-kafka.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index d45d480d..45297859 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,12 +25,14 @@ spec: - /bin/bash - -ec - > + echo ' curl --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt - --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + '; tail -f /dev/null From 31b8eb24d8df7882bd9d105ba06ce928377ceb84 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:50:24 +0200 Subject: [PATCH 5/8] Produces proper json. Let's see when curl gives up and the container restarts. --- addon-events/events-kube-kafka.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 45297859..ed2bde47 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,8 +25,8 @@ spec: - /bin/bash - -ec - > - echo ' curl + -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events @@ -35,4 +35,3 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - '; tail -f /dev/null From 748d408fcc7ab851eb940693f2cfa3d00da6c1ac Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:06:17 +0200 Subject: [PATCH 6/8] Testing compression but solsson/kafka:0.11.0.0 throws ... Caused by: java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy at org.xerial.snappy.SnappyInputStream.readFully(SnappyInputStream.java:145) at org.xerial.snappy.SnappyInputStream.readHeader(SnappyInputStream.java:99) at org.xerial.snappy.SnappyInputStream.(SnappyInputStream.java:59) at org.apache.kafka.common.record.CompressionType$3.wrapForInput(CompressionType.java:82) ... 40 more --- addon-events/events-kube-kafka.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index ed2bde47..e188a92d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,3 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + -z snappy From b56a8ecd84640c1d6b691782e27c38d3b433fd03 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:08:57 +0200 Subject: [PATCH 7/8] Works with my current kafka image, and the test consumer still sees JSON --- addon-events/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index e188a92d..5cc7883c 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z snappy + -z gzip From a6bc27f74cbeb47f63f8500d625b8593a7d477b9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 09:17:27 +0200 Subject: [PATCH 8/8] Latest build suppors snappy compression --- addon-events/events-kube-kafka.yml | 2 +- addon-events/topic-ops-kube-events-all-json.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 5cc7883c..e188a92d 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z gzip + -z snappy diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml index e6396a44..8eb3b660 100644 --- a/addon-events/topic-ops-kube-events-all-json.yml +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -13,7 +13,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce command: - ./bin/kafka-topics.sh - --zookeeper