From 3a0b097a2d695f0aea26bc5bad245c3c74809714 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:18:51 +0100 Subject: [PATCH 01/13] Copies the result of the original filebeat branch ... but changes naming and namespace, to run alongside -raw. --- logs-streaming/00namespace.yml | 5 ++ logs-streaming/filebeat-config.yml | 33 ++++++++++ logs-streaming/filebeat-logs-kube-kafka.yml | 63 +++++++++++++++++++ .../topic-ops-kube-logs-filebeat.yml | 31 +++++++++ 4 files changed, 132 insertions(+) create mode 100644 logs-streaming/00namespace.yml create mode 100644 logs-streaming/filebeat-config.yml create mode 100644 logs-streaming/filebeat-logs-kube-kafka.yml create mode 100644 logs-streaming/topic-ops-kube-logs-filebeat.yml diff --git a/logs-streaming/00namespace.yml b/logs-streaming/00namespace.yml new file mode 100644 index 00000000..d4209416 --- /dev/null +++ b/logs-streaming/00namespace.yml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: logs-kafka diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml new file mode 100644 index 00000000..ba7a4b16 --- /dev/null +++ b/logs-streaming/filebeat-config.yml @@ -0,0 +1,33 @@ +kind: ConfigMap +metadata: + name: filebeat-config + namespace: logs-kafka +apiVersion: v1 +data: + filebeat.yml: |+ + + filebeat.prospectors: + - type: log + enabled: true + paths: + - /var/log/containers/*.log + symlinks: true + # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs + exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-'] + + processors: + - add_kubernetes_metadata: + in_cluster: true + + output.kafka: + hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"] + topic: ${TOPIC} + + partition.round_robin: + reachable_only: false + + client_id: filebeat-kubernetes + version: 0.11.0.0 + required_acks: 1 + compression: gzip + max_message_bytes: 1000000 \ No newline at end of file diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml new file mode 100644 index 00000000..e1b931da --- /dev/null +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -0,0 +1,63 @@ +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: filebeat-kube-kafka + namespace: logs-kafka +spec: + selector: + matchLabels: + k8s-app: filebeat-kube-kafka + version: v1 + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: filebeat-kube-kafka + version: v1 + kubernetes.io/cluster-service: "true" + spec: + containers: + - name: filebeat + image: solsson/kafka-filebeat-kubernetes@sha256:79b40d81b892ffb3b917eb249f4b3050badf57fadffbdc35272060c9b377209e + command: + - filebeat + - -e + - -c + - /etc/filebeat/filebeat.yml + - -d + - "service,beat" + env: + - name: TOPIC + value: ops-kube-logs-filebeat-001 + resources: + limits: + memory: 100Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat + readOnly: true + - name: data + mountPath: /data + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: config + configMap: + name: filebeat-config + - name: data + hostPath: + path: /kubernets-filebeat-state + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers \ No newline at end of file diff --git a/logs-streaming/topic-ops-kube-logs-filebeat.yml b/logs-streaming/topic-ops-kube-logs-filebeat.yml new file mode 100644 index 00000000..d4aaf584 --- /dev/null +++ b/logs-streaming/topic-ops-kube-logs-filebeat.yml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-logs-filebeat + namespace: logs-kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-logs-filebeat + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-logs-filebeat-001 + - --partitions + - "1" + - --replication-factor + - "2" + - --config + - retention.ms=6912500000 + restartPolicy: Never From 760cff5aa240954b568b3f4c957d029854030d63 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:22:09 +0100 Subject: [PATCH 02/13] filebeat 6.0.0-rc2 --- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index e1b931da..cbb52a1f 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -18,7 +18,7 @@ spec: spec: containers: - name: filebeat - image: solsson/kafka-filebeat-kubernetes@sha256:79b40d81b892ffb3b917eb249f4b3050badf57fadffbdc35272060c9b377209e + image: solsson/kafka-filebeat-kubernetes@sha256:c2a68766e718d354b565b20f92baf0f50047b4bdf90a5055a20107a1131bee80 command: - filebeat - -e From fcc184a7b482c1fa7dd4fb4ecee3d4a90bd2571b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:28:37 +0100 Subject: [PATCH 03/13] Fixes pod start in GKE, read-only file system Error: failed to start container "filebeat": Error response from daemon: error while creating mount source path '/kubernets-filebeat-state': mkdir /kubernets-filebeat-state: read-only file system --- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index cbb52a1f..8e3c36d0 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -54,7 +54,7 @@ spec: name: filebeat-config - name: data hostPath: - path: /kubernets-filebeat-state + path: /tmp/kubernets-filebeat-state - name: varlog hostPath: path: /var/log From 7f0e564c612d33e85acd4b5904a90e279fc20304 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:33:07 +0100 Subject: [PATCH 04/13] Adds test (without asserts) for filebeat topic, jq-compatible log --- logs-streaming/test/logs-filebeat.yml | 75 +++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 logs-streaming/test/logs-filebeat.yml diff --git a/logs-streaming/test/logs-filebeat.yml b/logs-streaming/test/logs-filebeat.yml new file mode 100644 index 00000000..1d96ecd5 --- /dev/null +++ b/logs-streaming/test/logs-filebeat.yml @@ -0,0 +1,75 @@ +--- +kind: ConfigMap +metadata: + name: logs-filebeat + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + echo "" + echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" + kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e + + exit 0 + + quit-on-nonzero-exit.sh: |- + exit 0 + +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: logs-filebeat + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: logs-streaming-filebeat + test-type: readiness + template: + metadata: + labels: + test-target: logs-streaming-filebeat + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-logs-filebeat-001 + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: logs-filebeat From a401bdafca9734c267536146a94d83cb74076ff6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:52:13 +0100 Subject: [PATCH 05/13] Tentative RBAC; kubectl describe's error messages are helpful --- logs-streaming/filebeat-logs-kube-kafka.yml | 1 + logs-streaming/rbac/filebeat.yml | 24 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 logs-streaming/rbac/filebeat.yml diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index 8e3c36d0..7e7fcd37 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -16,6 +16,7 @@ spec: version: v1 kubernetes.io/cluster-service: "true" spec: + serviceAccountName: filebeat containers: - name: filebeat image: solsson/kafka-filebeat-kubernetes@sha256:c2a68766e718d354b565b20f92baf0f50047b4bdf90a5055a20107a1131bee80 diff --git a/logs-streaming/rbac/filebeat.yml b/logs-streaming/rbac/filebeat.yml new file mode 100644 index 00000000..1e1dc0b0 --- /dev/null +++ b/logs-streaming/rbac/filebeat.yml @@ -0,0 +1,24 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: logs-kafka + annotations: + manifest-origin: 'github.com/Yolean/kubernetes-kafka' +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: filebeat-sa-view + namespace: logs-kafka + annotations: + manifest-origin: 'github.com/Yolean/kubernetes-kafka' +subjects: +- kind: ServiceAccount + name: filebeat + namespace: logs-kafka +roleRef: + kind: ClusterRole + name: view + apiGroup: rbac.authorization.k8s.io From f0b7fde8149cec677cf103d493955d4fbd3a155c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 16:52:46 +0100 Subject: [PATCH 06/13] ClusterRoleBinding fixes RBAC for filebeat --- logs-streaming/rbac/filebeat.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/logs-streaming/rbac/filebeat.yml b/logs-streaming/rbac/filebeat.yml index 1e1dc0b0..a2f21156 100644 --- a/logs-streaming/rbac/filebeat.yml +++ b/logs-streaming/rbac/filebeat.yml @@ -7,11 +7,10 @@ metadata: annotations: manifest-origin: 'github.com/Yolean/kubernetes-kafka' --- -kind: RoleBinding +kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: - name: filebeat-sa-view - namespace: logs-kafka + name: logs-kafka-filebeat-sa-view annotations: manifest-origin: 'github.com/Yolean/kubernetes-kafka' subjects: From 52c28e8057a28270c3197766654ea46a9225976c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:01:26 +0100 Subject: [PATCH 07/13] Fixes JSON syntax getting broken by "Reached end of topic" --- logs-streaming/filebeat-config.yml | 1 + logs-streaming/test/logs-filebeat.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml index ba7a4b16..2df5ac73 100644 --- a/logs-streaming/filebeat-config.yml +++ b/logs-streaming/filebeat-config.yml @@ -11,6 +11,7 @@ data: enabled: true paths: - /var/log/containers/*.log + # Logs say a lot of https://github.com/elastic/beats/blob/v6.0.0-rc2/filebeat/prospector/log/prospector.go#L250 symlinks: true # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-'] diff --git a/logs-streaming/test/logs-filebeat.yml b/logs-streaming/test/logs-filebeat.yml index 1d96ecd5..ef53c96e 100644 --- a/logs-streaming/test/logs-filebeat.yml +++ b/logs-streaming/test/logs-filebeat.yml @@ -17,7 +17,7 @@ data: echo "" echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" - kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e + kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e -q exit 0 From 504911a038f51c943a5bd335ae48235256d4eed6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:19:06 +0100 Subject: [PATCH 08/13] Fixes kubernetes metadata inclusion thanks to https://discuss.elastic.co/t/kubernetes-metadata/90865/15 --- logs-streaming/filebeat-config.yml | 4 ++-- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml index 2df5ac73..828bdc0c 100644 --- a/logs-streaming/filebeat-config.yml +++ b/logs-streaming/filebeat-config.yml @@ -10,8 +10,7 @@ data: - type: log enabled: true paths: - - /var/log/containers/*.log - # Logs say a lot of https://github.com/elastic/beats/blob/v6.0.0-rc2/filebeat/prospector/log/prospector.go#L250 + - /var/lib/docker/containers/*/*.log symlinks: true # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-'] @@ -19,6 +18,7 @@ data: processors: - add_kubernetes_metadata: in_cluster: true + namespace: logs-kafka output.kafka: hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"] diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index 7e7fcd37..52bc2458 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -26,7 +26,7 @@ spec: - -c - /etc/filebeat/filebeat.yml - -d - - "service,beat" + - "service,beat,kubernetes" env: - name: TOPIC value: ops-kube-logs-filebeat-001 From 8878b5cac56b9dd579ce28aa0749da04526e8471 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:21:33 +0100 Subject: [PATCH 09/13] Removes kubernetes debug as the result is ... lots and lots of messages like 2017/11/04 16:16:03.383326 indexing.go:58: DBG Using container id: %!(EXTRA string=509e0dd0dcfdf526779c116ca3634c31c1b58102e762671ee31ab41febe22e3e) 2017/11/04 16:16:03.386038 indexing.go:52: DBG Incoming source value: %!(EXTRA string=/var/lib/docker/containers/509e0dd0dcfdf526779c116ca3634c31c1b58102e762671ee31ab41febe22e3e/509e0dd0dcfdf526779c116ca3634c31c1b58102e762671ee31ab41febe22e3e-json.log) --- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index 52bc2458..7e7fcd37 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -26,7 +26,7 @@ spec: - -c - /etc/filebeat/filebeat.yml - -d - - "service,beat,kubernetes" + - "service,beat" env: - name: TOPIC value: ops-kube-logs-filebeat-001 From e04a40357d191675d5513702c67cdc44ebf8a64b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:23:52 +0100 Subject: [PATCH 10/13] Dropped the logs- prefix now in the logs-kafka namespace --- logs-streaming/filebeat-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml index 828bdc0c..e1414034 100644 --- a/logs-streaming/filebeat-config.yml +++ b/logs-streaming/filebeat-config.yml @@ -13,7 +13,7 @@ data: - /var/lib/docker/containers/*/*.log symlinks: true # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs - exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-'] + exclude_files: ['^(.*\/)?logs-','^(.*\/)?filebeat-','^(.*\/)?test-'] processors: - add_kubernetes_metadata: From d61b3419957975e937251a58177d7b950f152c20 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:42:15 +0100 Subject: [PATCH 11/13] Memory use little higher than fluent-bit and tail+kafkacat --- logs-streaming/filebeat-logs-kube-kafka.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index 7e7fcd37..d2645d22 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -31,11 +31,12 @@ spec: - name: TOPIC value: ops-kube-logs-filebeat-001 resources: - limits: - memory: 100Mi requests: - cpu: 100m - memory: 100Mi + cpu: 2m + memory: 10Mi + limits: + cpu: 10m + memory: 20Mi volumeMounts: - name: config mountPath: /etc/filebeat From a83c6ebb1cf767c929443ac65f9167bd11fb36ec Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 4 Nov 2017 17:45:46 +0100 Subject: [PATCH 12/13] Got OOMKilled --- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index d2645d22..1da22694 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -36,7 +36,7 @@ spec: memory: 10Mi limits: cpu: 10m - memory: 20Mi + memory: 30Mi volumeMounts: - name: config mountPath: /etc/filebeat From 65d902fd23794e4467a0e314bfd8cadaa9c936fc Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 7 Nov 2017 12:50:47 +0100 Subject: [PATCH 13/13] Had lots of restarts still. Memory is now 10x that of tail+kafkacat. --- logs-streaming/filebeat-logs-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml index 1da22694..1294d50b 100644 --- a/logs-streaming/filebeat-logs-kube-kafka.yml +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -36,7 +36,7 @@ spec: memory: 10Mi limits: cpu: 10m - memory: 30Mi + memory: 40Mi volumeMounts: - name: config mountPath: /etc/filebeat