From 7480401f202bb875c2ab1d3ee61e22866c639ad5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 19:12:36 +0200 Subject: [PATCH 1/8] just a container sitting there (on each node) doing nothing --- addon-logs/logs-kube-kafka.yml | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 addon-logs/logs-kube-kafka.yml diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml new file mode 100644 index 00000000..a541a735 --- /dev/null +++ b/addon-logs/logs-kube-kafka.yml @@ -0,0 +1,40 @@ +apiVersion: extensions/v1beta1 +kind: DaemonSet +metadata: + name: logs-kafka + namespace: kube-system +spec: + template: + metadata: + labels: + k8s-app: logs-kafka + version: v1 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - sh + - -ec + - 'tail -f /dev/null' + resources: + limits: + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers From d36551fe552d5079e22ad751c825338e3deba8bd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 19:15:50 +0200 Subject: [PATCH 2/8] Creates the topic --- addon-logs/topic-ops-kube-logs-raw.yml | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 addon-logs/topic-ops-kube-logs-raw.yml diff --git a/addon-logs/topic-ops-kube-logs-raw.yml b/addon-logs/topic-ops-kube-logs-raw.yml new file mode 100644 index 00000000..043b798d --- /dev/null +++ b/addon-logs/topic-ops-kube-logs-raw.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-logs-raw + namespace: kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-logs-raw + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-logs-raw-001 + - --partitions + - "1" + - --replication-factor + - "1" + - --config + # this might be eight days + - retention.ms=69125000 + restartPolicy: Never From 8c87999cacfcde7df78e42369a3f7ef0fb55c2f8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 19:55:57 +0200 Subject: [PATCH 3/8] Reading metadata of containers will have to be some other producer's responsibility --- addon-logs/logs-kube-kafka.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml index a541a735..ad7b9b03 100644 --- a/addon-logs/logs-kube-kafka.yml +++ b/addon-logs/logs-kube-kafka.yml @@ -27,14 +27,9 @@ spec: volumeMounts: - name: varlog mountPath: /var/log - - name: varlibdockercontainers - mountPath: /var/lib/docker/containers readOnly: true terminationGracePeriodSeconds: 10 volumes: - name: varlog hostPath: path: /var/log - - name: varlibdockercontainers - hostPath: - path: /var/lib/docker/containers From 30a1042b5c82275592cd3527c5449008b7e3ff3a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 19:56:48 +0200 Subject: [PATCH 4/8] Gets the logs into Kafka all right, but you can only delimit on single chars so records look ugly --- addon-logs/logs-kube-kafka.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml index ad7b9b03..f1b437a5 100644 --- a/addon-logs/logs-kube-kafka.yml +++ b/addon-logs/logs-kube-kafka.yml @@ -17,7 +17,19 @@ spec: command: - sh - -ec - - 'tail -f /dev/null' + - > + tail + -n 0 + -f + /var/log/containers/*.log + | + kafkacat + -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + -t ops-kube-logs-raw-001 + -P + -D '\n\n==> ' + -K ' <==\n' + #-z snappy resources: limits: memory: 100Mi From eb5319b1b936288263ee8c1e24a421e0e262fa9d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 20:29:46 +0200 Subject: [PATCH 5/8] I didn't understand how -z is supposed to work --- addon-logs/logs-kube-kafka.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml index f1b437a5..7991751e 100644 --- a/addon-logs/logs-kube-kafka.yml +++ b/addon-logs/logs-kube-kafka.yml @@ -18,17 +18,18 @@ spec: - sh - -ec - > + cd /var/log/containers/; tail -n 0 -f - /var/log/containers/*.log + --zero-terminated + *.log | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-logs-raw-001 -P - -D '\n\n==> ' - -K ' <==\n' + -D '\0' #-z snappy resources: limits: From 3551331731c64cdf3e12a8722dda349e0eb97be9 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 20:30:34 +0200 Subject: [PATCH 6/8] Won't discover new files anyway. Read the excellent manual ... https://www.gnu.org/software/coreutils/manual/html_node/tail-invocation.html --- addon-logs/logs-kube-kafka.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml index 7991751e..750778b0 100644 --- a/addon-logs/logs-kube-kafka.yml +++ b/addon-logs/logs-kube-kafka.yml @@ -21,15 +21,13 @@ spec: cd /var/log/containers/; tail -n 0 - -f - --zero-terminated + --follow=name *.log | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-logs-raw-001 -P - -D '\0' #-z snappy resources: limits: From c816c2617a40d7250ff99128e6b5345bab09b30e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 10:24:41 +0100 Subject: [PATCH 7/8] Latest kafkacat image --- addon-logs/logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-logs/logs-kube-kafka.yml b/addon-logs/logs-kube-kafka.yml index 750778b0..7b1f49ee 100644 --- a/addon-logs/logs-kube-kafka.yml +++ b/addon-logs/logs-kube-kafka.yml @@ -13,7 +13,7 @@ spec: spec: containers: - name: kafkacat - image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef command: - sh - -ec From 4e7a201dc50ff01094dbf3e3811a6da726e254f5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 10:26:03 +0100 Subject: [PATCH 8/8] Latest kafka image, for topic creation --- addon-logs/topic-ops-kube-logs-raw.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-logs/topic-ops-kube-logs-raw.yml b/addon-logs/topic-ops-kube-logs-raw.yml index 043b798d..9fd9418b 100644 --- a/addon-logs/topic-ops-kube-logs-raw.yml +++ b/addon-logs/topic-ops-kube-logs-raw.yml @@ -13,7 +13,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: - ./bin/kafka-topics.sh - --zookeeper