From 5ec870976efcc7b494a0f95fc8d8bbf0c1ee2b0a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 11 Nov 2017 16:06:30 +0100 Subject: [PATCH 1/3] Adds topic and test for https://github.com/fluent/fluent-bit-kubernetes-logging/pull/11 but beware of the aggregation recursion with the test, you'll see escaped escaped ... escaped json. --- logs-streaming/test/logs-fluentbit.yml | 75 +++++++++++++++++++ .../topic-fluent-bit-kubernetes-logging.yml | 32 ++++++++ 2 files changed, 107 insertions(+) create mode 100644 logs-streaming/test/logs-fluentbit.yml create mode 100644 logs-streaming/topic-fluent-bit-kubernetes-logging.yml diff --git a/logs-streaming/test/logs-fluentbit.yml b/logs-streaming/test/logs-fluentbit.yml new file mode 100644 index 00000000..909833bb --- /dev/null +++ b/logs-streaming/test/logs-fluentbit.yml @@ -0,0 +1,75 @@ +--- +kind: ConfigMap +metadata: + name: logs-fluentbit + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + echo "" + echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" + kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e -q + + exit 0 + + quit-on-nonzero-exit.sh: |- + exit 0 + +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: logs-fluentbit + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: logs-streaming-fluentbit + test-type: readiness + template: + metadata: + labels: + test-target: logs-streaming-fluentbit + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-logs-fluentbit-001 + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: logs-fluentbit diff --git a/logs-streaming/topic-fluent-bit-kubernetes-logging.yml b/logs-streaming/topic-fluent-bit-kubernetes-logging.yml new file mode 100644 index 00000000..cb27e884 --- /dev/null +++ b/logs-streaming/topic-fluent-bit-kubernetes-logging.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-logs-fluentbit + # same namespace as https://github.com/fluent/fluent-bit-kubernetes-logging + namespace: logging +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-logs-fluentbit + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-logs-fluentbit-001 + - --partitions + - "1" + - --replication-factor + - "2" + - --config + - retention.ms=6912500000 + restartPolicy: Never From 9ad41343c276cbf5aeba5c0fde27bee9cc50084a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 12:52:21 +0100 Subject: [PATCH 2/3] Adapts topic to our naming convention (#101) --- logs-streaming/test/logs-fluentbit.yml | 4 +-- .../topic-fluent-bit-kubernetes-logging.yml | 32 ------------------- 2 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 logs-streaming/topic-fluent-bit-kubernetes-logging.yml diff --git a/logs-streaming/test/logs-fluentbit.yml b/logs-streaming/test/logs-fluentbit.yml index 909833bb..86af0e3b 100644 --- a/logs-streaming/test/logs-fluentbit.yml +++ b/logs-streaming/test/logs-fluentbit.yml @@ -47,9 +47,9 @@ spec: image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 - name: TOPIC - value: ops-kube-logs-fluentbit-001 + value: ops.kube-logs-fluentbit.stream.json.001 command: - /bin/bash - -e diff --git a/logs-streaming/topic-fluent-bit-kubernetes-logging.yml b/logs-streaming/topic-fluent-bit-kubernetes-logging.yml deleted file mode 100644 index cb27e884..00000000 --- a/logs-streaming/topic-fluent-bit-kubernetes-logging.yml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: topic-ops-kube-logs-fluentbit - # same namespace as https://github.com/fluent/fluent-bit-kubernetes-logging - namespace: logging -spec: - template: - metadata: - labels: - app: topic-create - topic-id: ops-kube-logs-fluentbit - topic-gen: "001" - spec: - containers: - - name: kafka - image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper.kafka:2181 - - --create - - --if-not-exists - - --topic - - ops-kube-logs-fluentbit-001 - - --partitions - - "1" - - --replication-factor - - "2" - - --config - - retention.ms=6912500000 - restartPolicy: Never From a6ebef191d98e853d1004c6ab439338878f43af3 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 8 Jan 2018 12:53:03 +0100 Subject: [PATCH 3/3] Prints log sample less frequently, as these entries get aggregated too --- logs-streaming/test/logs-fluentbit.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logs-streaming/test/logs-fluentbit.yml b/logs-streaming/test/logs-fluentbit.yml index 86af0e3b..4ecc3c00 100644 --- a/logs-streaming/test/logs-fluentbit.yml +++ b/logs-streaming/test/logs-fluentbit.yml @@ -60,6 +60,8 @@ spec: - /bin/bash - -e - /test/test.sh + initialDelaySeconds: 0 + periodSeconds: 60 livenessProbe: exec: command: