From 30b888a01f5f2f02522fb376a4fc827ce3882405 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 6 Jul 2017 09:42:11 +0200
Subject: [PATCH 01/25] Creates a pod for expermentation with kubectl and
 kafkacat

---
 addon-events/events-kube-kafka.yml | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 addon-events/events-kube-kafka.yml

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
new file mode 100644
index 00000000..9eeed58a
--- /dev/null
+++ b/addon-events/events-kube-kafka.yml
@@ -0,0 +1,27 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: events-kube-kafka
+  namespace: kafka
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      # prefer duplicate events over missed
+      maxUnavailable: 0
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: events
+        from: kube
+        to: kafka
+    spec:
+      containers:
+      - name: kubectl-kafkacat
+        image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b
+        command:
+        - sh
+        - -ec
+        - 'tail -f /dev/null'

From 4bdfb2b606a9db2fc9d1bba07e29f482b28ae82d Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 6 Jul 2017 11:52:15 +0200
Subject: [PATCH 02/25] Produces and reads events

---
 addon-events/events-kube-kafka.yml            | 12 +++++--
 addon-events/test-event-consumer.yml          | 30 +++++++++++++++++
 .../topic-ops-kube-events-all-json.yml        | 32 +++++++++++++++++++
 3 files changed, 72 insertions(+), 2 deletions(-)
 create mode 100644 addon-events/test-event-consumer.yml
 create mode 100644 addon-events/topic-ops-kube-events-all-json.yml

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 9eeed58a..95d0ad2f 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -22,6 +22,14 @@ spec:
       - name: kubectl-kafkacat
         image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b
         command:
-        - sh
+        - /bin/bash
         - -ec
-        - 'tail -f /dev/null'
+        - >
+          kubectl
+          get events
+          -w
+          |
+          kafkacat
+          -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+          -t ops-kube-events-all-json-001
+          -P
diff --git a/addon-events/test-event-consumer.yml b/addon-events/test-event-consumer.yml
new file mode 100644
index 00000000..d98f88ca
--- /dev/null
+++ b/addon-events/test-event-consumer.yml
@@ -0,0 +1,30 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: test-events-consumer
+  namespace: kafka
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      # prefer duplicate events over missed
+      maxUnavailable: 0
+      maxSurge: 1
+  template:
+    metadata:
+      labels:
+        app: events
+        type: test
+        from: kafka-ops
+    spec:
+      containers:
+      - name: kafkacat
+        image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8
+        command:
+        - kafkacat
+        - -b
+        - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - -t
+        - ops-kube-events-all-json-001
+        - -C
diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml
new file mode 100644
index 00000000..e6396a44
--- /dev/null
+++ b/addon-events/topic-ops-kube-events-all-json.yml
@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: topic-ops-kube-events-all-json
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      labels:
+        app: topic-create
+        topic-id: ops-kube-events-all-json
+        topic-gen: "001"
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756
+        command:
+        - ./bin/kafka-topics.sh
+        - --zookeeper
+        - zookeeper:2181
+        - --create
+        - --if-not-exists
+        - --topic
+        - ops-kube-events-all-json-001
+        - --partitions
+        - "1"
+        - --replication-factor
+        - "1"
+        - --config
+        # this might be eight days
+        - retention.ms=69125000
+      restartPolicy: Never

From ccdd8bec306601a82b8faea056a82e50c4b4c1de Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 7 Jul 2017 07:11:40 +0200
Subject: [PATCH 03/25] wip

---
 addon-events/events-kube-kafka.yml | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 95d0ad2f..d45d480d 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -19,15 +19,16 @@ spec:
         to: kafka
     spec:
       containers:
-      - name: kubectl-kafkacat
-        image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b
+      - name: kafkacat-curl
+        image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b
         command:
         - /bin/bash
         - -ec
         - >
-          kubectl
-          get events
-          -w
+          curl
+          --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt
+          --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)"
+          https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events
           |
           kafkacat
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092

From 5ee97a2ef2e7bb451db3d36c6767b5e6a4e49e89 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 7 Jul 2017 07:48:51 +0200
Subject: [PATCH 04/25] Quite interesting way to test a command from yaml - see
 it in logs

---
 addon-events/events-kube-kafka.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index d45d480d..45297859 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -25,12 +25,14 @@ spec:
         - /bin/bash
         - -ec
         - >
+          echo '
           curl
           --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt
-          --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)"
+          --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)"
           https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events
           |
           kafkacat
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-events-all-json-001
           -P
+          '; tail -f /dev/null

From 4da60f8453f128c9b466a51f6a2cdbb9f9f32c92 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 7 Jul 2017 07:50:24 +0200
Subject: [PATCH 05/25] Produces proper json. Let's see when curl gives up and
 the container restarts.

---
 addon-events/events-kube-kafka.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 45297859..ed2bde47 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -25,8 +25,8 @@ spec:
         - /bin/bash
         - -ec
         - >
-          echo '
           curl
+          -s
           --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt
           --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)"
           https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events
@@ -35,4 +35,3 @@ spec:
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-events-all-json-001
           -P
-          '; tail -f /dev/null

From efacc180a893cd7ce83ecb99e2f9c0d70339d64b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 7 Jul 2017 08:06:17 +0200
Subject: [PATCH 06/25] Testing compression but solsson/kafka:0.11.0.0 throws
 ...

Caused by: java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy
	at org.xerial.snappy.SnappyInputStream.readFully(SnappyInputStream.java:145)
	at org.xerial.snappy.SnappyInputStream.readHeader(SnappyInputStream.java:99)
	at org.xerial.snappy.SnappyInputStream.<init>(SnappyInputStream.java:59)
	at org.apache.kafka.common.record.CompressionType$3.wrapForInput(CompressionType.java:82)
	... 40 more
---
 addon-events/events-kube-kafka.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index ed2bde47..e188a92d 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -35,3 +35,4 @@ spec:
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-events-all-json-001
           -P
+          -z snappy

From b6575a536fea27a36972da2d599c8d2f29f773b6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 7 Jul 2017 08:08:57 +0200
Subject: [PATCH 07/25] Works with my current kafka image, and the test
 consumer still sees JSON

---
 addon-events/events-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index e188a92d..5cc7883c 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -35,4 +35,4 @@ spec:
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-events-all-json-001
           -P
-          -z snappy
+          -z gzip

From a30f22e324594ab1c6d14db1bdfd4f2ff1f7698e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 09:17:27 +0200
Subject: [PATCH 08/25] Latest build suppors snappy compression

---
 addon-events/events-kube-kafka.yml              | 2 +-
 addon-events/topic-ops-kube-events-all-json.yml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 5cc7883c..e188a92d 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -35,4 +35,4 @@ spec:
           -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
           -t ops-kube-events-all-json-001
           -P
-          -z gzip
+          -z snappy
diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml
index e6396a44..8eb3b660 100644
--- a/addon-events/topic-ops-kube-events-all-json.yml
+++ b/addon-events/topic-ops-kube-events-all-json.yml
@@ -13,7 +13,7 @@ spec:
     spec:
       containers:
       - name: kafka
-        image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756
+        image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
         command:
         - ./bin/kafka-topics.sh
         - --zookeeper

From f626a7764513fd85bdf721be766646312fd848b1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 5 Aug 2017 06:24:38 +0200
Subject: [PATCH 09/25] Adds RBAC policy for curl, default service account

---
 rbac-namespace-default/events-watcher.yml | 30 +++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 rbac-namespace-default/events-watcher.yml

diff --git a/rbac-namespace-default/events-watcher.yml b/rbac-namespace-default/events-watcher.yml
new file mode 100644
index 00000000..c8384b67
--- /dev/null
+++ b/rbac-namespace-default/events-watcher.yml
@@ -0,0 +1,30 @@
+# If events-kube-kafka-* goes crashlooping you probably need this
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: events-watcher
+  labels:
+    origin: github.com_Yolean_kubernetes-kafka
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - watch
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  name: kafka-events-watcher
+  labels:
+    origin: github.com_Yolean_kubernetes-kafka
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: events-watcher
+subjects:
+- kind: ServiceAccount
+  name: default
+  namespace: kafka

From 4f13cf571b18c06074f5216852b722cdb3d5dd67 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 5 Aug 2017 06:59:29 +0200
Subject: [PATCH 10/25] Moves the example comsumer to the test namespace

---
 .../test-event-consumer.yml => test/events-topic.yml     | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)
 rename addon-events/test-event-consumer.yml => test/events-topic.yml (85%)

diff --git a/addon-events/test-event-consumer.yml b/test/events-topic.yml
similarity index 85%
rename from addon-events/test-event-consumer.yml
rename to test/events-topic.yml
index d98f88ca..6d11d775 100644
--- a/addon-events/test-event-consumer.yml
+++ b/test/events-topic.yml
@@ -1,8 +1,8 @@
 apiVersion: extensions/v1beta1
 kind: Deployment
 metadata:
-  name: test-events-consumer
-  namespace: kafka
+  name: events-topic
+  namespace: test-kafka
 spec:
   replicas: 1
   strategy:
@@ -14,9 +14,8 @@ spec:
   template:
     metadata:
       labels:
-        app: events
-        type: test
-        from: kafka-ops
+        test-target: events-topic
+        test-type: readiness
     spec:
       containers:
       - name: kafkacat

From 8935cf549857b9b4f6dfb68209681789483b15d8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 5 Aug 2017 07:01:09 +0200
Subject: [PATCH 11/25] Not a proper test yet, but the PR documents how to use
 it to see events

---
 test/events-topic.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/test/events-topic.yml b/test/events-topic.yml
index 6d11d775..68ca9198 100644
--- a/test/events-topic.yml
+++ b/test/events-topic.yml
@@ -5,17 +5,12 @@ metadata:
   namespace: test-kafka
 spec:
   replicas: 1
-  strategy:
-    type: RollingUpdate
-    rollingUpdate:
-      # prefer duplicate events over missed
-      maxUnavailable: 0
-      maxSurge: 1
   template:
     metadata:
       labels:
         test-target: events-topic
-        test-type: readiness
+        # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced
+        #test-type: readiness
     spec:
       containers:
       - name: kafkacat

From 8d0d99302b80e960b592d1d8779b3c502815fc19 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Oct 2017 21:34:19 +0200
Subject: [PATCH 12/25] Use an image with new kafkacat and curl versions

---
 addon-events/events-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index e188a92d..2d6913a9 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -20,7 +20,7 @@ spec:
     spec:
       containers:
       - name: kafkacat-curl
-        image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b
+        image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0
         command:
         - /bin/bash
         - -ec

From b9e7b8a02e328309907bd024619008d8d7cdfc07 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Oct 2017 21:56:03 +0200
Subject: [PATCH 13/25] Makes things a bit less obscure if for example RBAC
 fails

---
 addon-events/events-kube-kafka.yml | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 2d6913a9..062d097c 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -21,10 +21,18 @@ spec:
       containers:
       - name: kafkacat-curl
         image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - name: TOPIC
+          value: ops-kube-events-all-json-001
         command:
         - /bin/bash
         - -ec
+          # curl errors will go to kafka, kafkacat errors will got to log
         - >
+          echo "Started at $(date -u +%FT%TZ) producing to $TOPIC"
+          ;
           curl
           -s
           --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -32,7 +40,9 @@ spec:
           https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events
           |
           kafkacat
-          -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
-          -t ops-kube-events-all-json-001
+          -b $BOOTSTRAP
+          -t $TOPIC
           -P
           -z snappy
+          -v
+          -d broker,topic

From e4c517984066f8e76909ec8a38fd414ada2c10c4 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Oct 2017 22:02:53 +0200
Subject: [PATCH 14/25] Argh, curl will pretty print the json ...

and kafkacat will produce each line separately
---
 addon-events/events-kube-kafka.yml | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 062d097c..a707b1c5 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -28,7 +28,7 @@ spec:
           value: ops-kube-events-all-json-001
         command:
         - /bin/bash
-        - -ec
+        - -c
           # curl errors will go to kafka, kafkacat errors will got to log
         - >
           echo "Started at $(date -u +%FT%TZ) producing to $TOPIC"
@@ -46,3 +46,13 @@ spec:
           -z snappy
           -v
           -d broker,topic
+          ;
+          echo "Last message on topic:"
+          ;
+          kafkacat
+          -b $BOOTSTRAP
+          -t $TOPIC
+          -C
+          -o -1
+          -c 1
+          -z snappy

From d628275ff253366f93c2533c7bdd191c909ebd69 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Oct 2017 22:07:21 +0200
Subject: [PATCH 15/25] bash -e is important, in combination with curl -f, so
 ...

we don't send error messages to the topic
(even if it's json -- format is different and it's pretty printed)
---
 addon-events/events-kube-kafka.yml | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index a707b1c5..92387aae 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -28,12 +28,12 @@ spec:
           value: ops-kube-events-all-json-001
         command:
         - /bin/bash
-        - -c
-          # curl errors will go to kafka, kafkacat errors will got to log
+        - -ec
         - >
           echo "Started at $(date -u +%FT%TZ) producing to $TOPIC"
           ;
           curl
+          -f
           -s
           --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt
           --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)"
@@ -46,13 +46,3 @@ spec:
           -z snappy
           -v
           -d broker,topic
-          ;
-          echo "Last message on topic:"
-          ;
-          kafkacat
-          -b $BOOTSTRAP
-          -t $TOPIC
-          -C
-          -o -1
-          -c 1
-          -z snappy

From fc5a8362abda5e5e92df00d7bce21b3f5a7ee280 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 26 Oct 2017 22:34:30 +0200
Subject: [PATCH 16/25] kafkacat's debug output solves the silence, but ...

to see the error message (like missing RBAC) you basically have to run the curl command.

I'd like startup to be as fast as possible, to not miss events, so I hesitate to add checks.
---
 addon-events/events-kube-kafka.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml
index 92387aae..2cd18912 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/addon-events/events-kube-kafka.yml
@@ -30,8 +30,6 @@ spec:
         - /bin/bash
         - -ec
         - >
-          echo "Started at $(date -u +%FT%TZ) producing to $TOPIC"
-          ;
           curl
           -f
           -s

From 071f973fd2210cb5dc3edca17682e4b520b68b71 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 27 Oct 2017 08:33:39 +0200
Subject: [PATCH 17/25] Manifests updated to 1.8 and v3.0.0 pre-merge

---
 {addon-events => events-kube}/events-kube-kafka.yml        | 7 ++++++-
 {test => events-kube/test}/events-topic.yml                | 5 ++++-
 .../topic-ops-kube-events-all-json.yml                     | 0
 3 files changed, 10 insertions(+), 2 deletions(-)
 rename {addon-events => events-kube}/events-kube-kafka.yml (91%)
 rename {test => events-kube/test}/events-topic.yml (89%)
 rename {addon-events => events-kube}/topic-ops-kube-events-all-json.yml (100%)

diff --git a/addon-events/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml
similarity index 91%
rename from addon-events/events-kube-kafka.yml
rename to events-kube/events-kube-kafka.yml
index 2cd18912..769fad1b 100644
--- a/addon-events/events-kube-kafka.yml
+++ b/events-kube/events-kube-kafka.yml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta1
+apiVersion: extensions/v1beta2
 kind: Deployment
 metadata:
   name: events-kube-kafka
@@ -11,6 +11,11 @@ spec:
       # prefer duplicate events over missed
       maxUnavailable: 0
       maxSurge: 1
+  selector:
+    matchLabels:
+      app: events
+      from: kube
+      to: kafka
   template:
     metadata:
       labels:
diff --git a/test/events-topic.yml b/events-kube/test/events-topic.yml
similarity index 89%
rename from test/events-topic.yml
rename to events-kube/test/events-topic.yml
index 68ca9198..bbed4fa0 100644
--- a/test/events-topic.yml
+++ b/events-kube/test/events-topic.yml
@@ -1,10 +1,13 @@
-apiVersion: extensions/v1beta1
+apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
   name: events-topic
   namespace: test-kafka
 spec:
   replicas: 1
+  selector:
+    matchLabels:
+      test-target: events-topic
   template:
     metadata:
       labels:
diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml
similarity index 100%
rename from addon-events/topic-ops-kube-events-all-json.yml
rename to events-kube/topic-ops-kube-events-all-json.yml

From 1de4d2cb4452b47f2755cbe94b655d3aad2ead13 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 27 Oct 2017 10:08:41 +0200
Subject: [PATCH 18/25] An actual assertion

---
 events-kube/test/events-topic.yml | 82 +++++++++++++++++++++++++++----
 1 file changed, 72 insertions(+), 10 deletions(-)

diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml
index bbed4fa0..126d4fb5 100644
--- a/events-kube/test/events-topic.yml
+++ b/events-kube/test/events-topic.yml
@@ -1,3 +1,34 @@
+---
+kind: ConfigMap
+metadata:
+  name: events-topic
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  setup.sh: |-
+    touch /tmp/testlog
+
+    tail -f /tmp/testlog
+
+  test.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    PREVIOUS=$(sha1sum /tmp/event || echo "")
+    kafkacat -b $BOOTSTRAP -t $TOPIC -C -o -1 -c 1 | tee /tmp/event
+    CURRENT=$(sha1sum /tmp/event)
+    [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-status\”: \"No new event in $TOPIC\"}" && exit 1
+
+    exit 0
+
+  quit-on-nonzero-exit.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    exit 0
+
+---
 apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
@@ -5,23 +36,54 @@ metadata:
   namespace: test-kafka
 spec:
   replicas: 1
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       test-target: events-topic
+      test-type: readiness
   template:
     metadata:
       labels:
         test-target: events-topic
-        # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced
-        #test-type: readiness
+        test-type: readiness
+        # for example:
+        # readonly - can be used in production
+        # isolated - read/write but in a manner that does not affect other services
+        # load - unsuitable for production because it uses significant resources
+        # chaos - unsuitable for production because it injects failure modes
+        #test-use:
     spec:
       containers:
-      - name: kafkacat
-        image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8
+      - name: testcase
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+        - name: TOPIC
+          value: ops-kube-events-all-json-001
         command:
-        - kafkacat
-        - -b
-        - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
-        - -t
-        - ops-kube-events-all-json-001
-        - -C
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/test.sh
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/quit-on-nonzero-exit.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+      volumes:
+      - name: config
+        configMap:
+          name: events-topic

From 8eb4a4d272a78a8f75e29a7ab4339b6bbc3787a7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 27 Oct 2017 11:04:29 +0200
Subject: [PATCH 19/25] Needs to be tweaked depending on the churn

---
 events-kube/test/events-topic.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml
index 126d4fb5..844c136f 100644
--- a/events-kube/test/events-topic.yml
+++ b/events-kube/test/events-topic.yml
@@ -73,7 +73,7 @@ spec:
             - -e
             - /test/test.sh
           initialDelaySeconds: 10
-          periodSeconds: 10
+          periodSeconds: 60
         livenessProbe:
           exec:
             command:

From 8ee3bebca5f979b4f99b213042181c1d9a1a3098 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 27 Oct 2017 13:04:52 +0200
Subject: [PATCH 20/25] Fixes JSON syntax, topic is jq:able now

---
 events-kube/test/events-topic.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml
index 844c136f..29891f72 100644
--- a/events-kube/test/events-topic.yml
+++ b/events-kube/test/events-topic.yml
@@ -15,10 +15,10 @@ data:
     exec >> /tmp/testlog
     exec 2>&1
 
-    PREVIOUS=$(sha1sum /tmp/event || echo "")
+    PREVIOUS=$(sha1sum /tmp/event 2>/dev/null || echo "")
     kafkacat -b $BOOTSTRAP -t $TOPIC -C -o -1 -c 1 | tee /tmp/event
     CURRENT=$(sha1sum /tmp/event)
-    [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-status\”: \"No new event in $TOPIC\"}" && exit 1
+    [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-result\": \"No new event in $TOPIC\"}" && exit 1
 
     exit 0
 

From 97f03a469bf4911da00256da880ec31779d85ae1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 27 Oct 2017 13:07:32 +0200
Subject: [PATCH 21/25] Aligns RBAC with addons->folders for v3.0.0

---
 .../rbac/cluster-events-watcher.yml                               | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename rbac-namespace-default/events-watcher.yml => events-kube/rbac/cluster-events-watcher.yml (100%)

diff --git a/rbac-namespace-default/events-watcher.yml b/events-kube/rbac/cluster-events-watcher.yml
similarity index 100%
rename from rbac-namespace-default/events-watcher.yml
rename to events-kube/rbac/cluster-events-watcher.yml

From d4a3bc6565e14b2f068a2e0a40733c319274bd95 Mon Sep 17 00:00:00 2001
From: Anton Lindgren <lindgren.anton@gmail.com>
Date: Mon, 30 Oct 2017 08:58:43 +0100
Subject: [PATCH 22/25] Fix retention from 0.8 days to 8 days

Silly change, but 0.8 days is very short.

Google helped me verify the math.
---
 events-kube/topic-ops-kube-events-all-json.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/events-kube/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml
index 8eb3b660..62f2b807 100644
--- a/events-kube/topic-ops-kube-events-all-json.yml
+++ b/events-kube/topic-ops-kube-events-all-json.yml
@@ -27,6 +27,6 @@ spec:
         - --replication-factor
         - "1"
         - --config
-        # this might be eight days
-        - retention.ms=69125000
+        # 8 days
+        - retention.ms=691200000  
       restartPolicy: Never

From 3fca1e7ee736d3e8d3647453de07e9986bcfe2fa Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 30 Oct 2017 22:21:37 +0100
Subject: [PATCH 23/25] Fixes manifest. I must have had a local edit last time.

---
 events-kube/events-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml
index 769fad1b..086137d3 100644
--- a/events-kube/events-kube-kafka.yml
+++ b/events-kube/events-kube-kafka.yml
@@ -1,4 +1,4 @@
-apiVersion: extensions/v1beta2
+apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
   name: events-kube-kafka

From 907467ca1d262581cf5754a3a6fd8cd3de1b4a1a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 9 Nov 2017 16:58:08 +0100
Subject: [PATCH 24/25] Uses kafkacat build as in tests and logs-streaming

---
 events-kube/events-kube-kafka.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml
index 086137d3..0b8bf109 100644
--- a/events-kube/events-kube-kafka.yml
+++ b/events-kube/events-kube-kafka.yml
@@ -25,7 +25,7 @@ spec:
     spec:
       containers:
       - name: kafkacat-curl
-        image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0
+        image: solsson/kafkacat-curl@sha256:65568eb5b0c22d360546c7591d1efdee7ce919e4b299be33c0e4b47ff997c1e8
         env:
         - name: BOOTSTRAP
           value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092

From a861237ed8b2ff6aaff46c9404f1bb5a05d82edb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Mon, 18 Dec 2017 08:29:58 +0100
Subject: [PATCH 25/25] Modernize based on

 * https://github.com/Yolean/kubernetes-kafka/pull/107
 * https://github.com/Yolean/kubernetes-kafka/issues/101
 * https://github.com/Yolean/fluent-bit-kubernetes-kafka/commit/e045ef1ef03f9fd299575a7aa9ca312e6687e441
 * https://github.com/Yolean/kubernetes-kafka/pull/52
---
 events-kube/events-kube-kafka.yml             |  6 ++--
 events-kube/test/events-topic.yml             |  6 ++--
 .../topic-ops-kube-events-all-json.yml        | 32 -------------------
 3 files changed, 6 insertions(+), 38 deletions(-)
 delete mode 100644 events-kube/topic-ops-kube-events-all-json.yml

diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml
index 0b8bf109..31703b49 100644
--- a/events-kube/events-kube-kafka.yml
+++ b/events-kube/events-kube-kafka.yml
@@ -25,12 +25,12 @@ spec:
     spec:
       containers:
       - name: kafkacat-curl
-        image: solsson/kafkacat-curl@sha256:65568eb5b0c22d360546c7591d1efdee7ce919e4b299be33c0e4b47ff997c1e8
+        image: solsson/kafkacat-curl@sha256:b5484379301937129e67550331782a0f7ac030a4b913a254d084faea4bcf44a2
         env:
         - name: BOOTSTRAP
-          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+          value: bootstrap.kafka:9092
         - name: TOPIC
-          value: ops-kube-events-all-json-001
+          value: ops.kube-events-all.stream.json.001
         command:
         - /bin/bash
         - -ec
diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml
index 29891f72..0c48c364 100644
--- a/events-kube/test/events-topic.yml
+++ b/events-kube/test/events-topic.yml
@@ -56,12 +56,12 @@ spec:
     spec:
       containers:
       - name: testcase
-        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        image: solsson/kafkacat@sha256:2c539e4f58960ab7872976ebc664dd92de18cf27e7cbbeb296d654a2351f6ca4
         env:
         - name: BOOTSTRAP
-          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+          value: bootstrap.kafka:9092
         - name: TOPIC
-          value: ops-kube-events-all-json-001
+          value: ops.kube-events-all.stream.json.001
         command:
         - /bin/bash
         - -e
diff --git a/events-kube/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml
deleted file mode 100644
index 62f2b807..00000000
--- a/events-kube/topic-ops-kube-events-all-json.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: topic-ops-kube-events-all-json
-  namespace: kafka
-spec:
-  template:
-    metadata:
-      labels:
-        app: topic-create
-        topic-id: ops-kube-events-all-json
-        topic-gen: "001"
-    spec:
-      containers:
-      - name: kafka
-        image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
-        command:
-        - ./bin/kafka-topics.sh
-        - --zookeeper
-        - zookeeper:2181
-        - --create
-        - --if-not-exists
-        - --topic
-        - ops-kube-events-all-json-001
-        - --partitions
-        - "1"
-        - --replication-factor
-        - "1"
-        - --config
-        # 8 days
-        - retention.ms=691200000  
-      restartPolicy: Never