From f890bcbedf3692fc09524f076ffc413ceb5f1d04 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 06:59:57 +0200
Subject: [PATCH 1/8] Modifies the image to run kafka-rest ouput instead of es

---
 docker-image/Dockerfile      |  2 +-
 docker-image/fluent-bit.conf | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/docker-image/Dockerfile b/docker-image/Dockerfile
index b868939..b2affa9 100644
--- a/docker-image/Dockerfile
+++ b/docker-image/Dockerfile
@@ -1,4 +1,4 @@
-FROM fluent/fluent-bit:0.11.15
+FROM fluent/fluent-bit:0.12-dev
 MAINTAINER Eduardo Silva <eduardo@treasure-data.com>
 USER root
 
diff --git a/docker-image/fluent-bit.conf b/docker-image/fluent-bit.conf
index bb2853b..4452de1 100644
--- a/docker-image/fluent-bit.conf
+++ b/docker-image/fluent-bit.conf
@@ -19,9 +19,14 @@
     Merge_JSON_Log On
 
 [OUTPUT]
-    Name   es
-    Match  *
-    Host   ${FLUENT_ELASTICSEARCH_HOST}
-    Port   ${FLUENT_ELASTICSEARCH_PORT}
-    Logstash_Format On
-    Retry_Limit False
+    Name            kafka-rest
+    Match           *
+    Host            ${KAFKA_REST_HOST}
+    Port            ${KAFKA_REST_PORT}
+    Time_Key        @timestamp
+    Tag_Key         _fluent-tag
+    Include_Tag_Key On
+    Tag_Key         My_Tag_Key
+    Topic           ${KAFKA_TOPIC}
+    # Partition     0
+    # Message_Key abc

From b756454cffac8233756e42822c84a41fef51692c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:01:30 +0200
Subject: [PATCH 2/8] Adds manifest for kafka-rest output daemonset

---
 fluent-bit-daemonset-kafka.yml | 49 ++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 fluent-bit-daemonset-kafka.yml

diff --git a/fluent-bit-daemonset-kafka.yml b/fluent-bit-daemonset-kafka.yml
new file mode 100644
index 0000000..414a2e7
--- /dev/null
+++ b/fluent-bit-daemonset-kafka.yml
@@ -0,0 +1,49 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: fluent-bit
+  namespace: kube-system
+  labels:
+    k8s-app: fluent-bit-logging
+    version: v1
+    kubernetes.io/cluster-service: "true"
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: fluent-bit-logging
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: fluent-bit
+        image: fluent
+        # build using eval $(minikube docker-env) && docker build -t fluent docker-image/
+        imagePullPolicy: Never
+        env:
+          - name:  KAFKA_REST_HOST
+            value: rest.kafka.svc.cluster.local
+          - name:  KAFKA_REST_PORT
+            value: "80"
+          - name:  KAFKA_TOPIC
+            value: logs
+        resources:
+          limits:
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers

From d3c6f46c31304466cf11752f34c3b9d0a5fdeff1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:02:01 +0200
Subject: [PATCH 3/8] Adds utility for use with kubectl exec

---
 testpod.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 testpod.yml

diff --git a/testpod.yml b/testpod.yml
new file mode 100644
index 0000000..5950953
--- /dev/null
+++ b/testpod.yml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: testpod-fluentbit-kafka
+  namespace: default
+spec:
+  containers:
+  - name: kafkacat
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    command:
+      - sh
+      - -ec
+      - >
+        cd /usr/local/bin;
+        echo '#!/bin/sh' > logs-cat;
+        chmod +x logs-cat;
+        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -t logs -C" >> logs-cat;
+
+        echo "Won't cat logs to stdout because it would create an infinite loop of log aggregation.";
+        echo "Instead run something like:";
+        echo "kubectl -n default exec -ti testpod-fluentbit-kafka -- logs-cat";
+        tail -f /dev/null;

From 6ae99121c84ae3e044b64e9919a1e54809b15005 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:11:54 +0200
Subject: [PATCH 4/8] Simplifies customization by using ConfigMap instead of
 docker build

---
 fluent-bit-config-kafka-rest.yml | 86 ++++++++++++++++++++++++++++++++
 fluent-bit-daemonset-kafka.yml   | 15 ++++--
 2 files changed, 97 insertions(+), 4 deletions(-)
 create mode 100644 fluent-bit-config-kafka-rest.yml

diff --git a/fluent-bit-config-kafka-rest.yml b/fluent-bit-config-kafka-rest.yml
new file mode 100644
index 0000000..cfd3089
--- /dev/null
+++ b/fluent-bit-config-kafka-rest.yml
@@ -0,0 +1,86 @@
+kind: ConfigMap
+metadata:
+  name: fluent-bit-config-kafka-rest
+  namespace: kube-system
+apiVersion: v1
+data:
+  fluent-bit.conf: |-
+    [SERVICE]
+        Flush          1
+        Daemon         Off
+        Log_Level      info
+        Parsers_File   parsers.conf
+
+    [INPUT]
+        Name           tail
+        Tag            kube.*
+        Path           /var/log/containers/*.log
+        Parser         docker
+        DB             /var/log/flb_kube.db
+        Mem_Buf_Limit  5MB
+
+    [FILTER]
+        Name           kubernetes
+        Match          kube.*
+        Kube_URL       https://kubernetes.default.svc:443
+        Merge_JSON_Log On
+
+    [OUTPUT]
+        Name            kafka-rest
+        Match           *
+        Host            ${KAFKA_REST_HOST}
+        Port            ${KAFKA_REST_PORT}
+        Time_Key        @timestamp
+        Tag_Key         _fluent-tag
+        Include_Tag_Key On
+        Tag_Key         My_Tag_Key
+        Topic           ${KAFKA_TOPIC}
+        # Partition     0
+        # Message_Key abc
+
+  parsers.conf: |-
+    [PARSER]
+        Name   apache
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache2
+        Format regex
+        Regex  ^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   apache_error
+        Format regex
+        Regex  ^\[[^ ]* (?<time>[^\]]*)\] \[(?<level>[^\]]*)\](?: \[pid (?<pid>[^\]]*)\])?( \[client (?<client>[^\]]*)\])? (?<message>.*)$
+
+    [PARSER]
+        Name   nginx
+        Format regex
+        Regex ^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name   json-test
+        Format json
+        Time_Key time
+        Time_Format %d/%b/%Y:%H:%M:%S %z
+
+    [PARSER]
+        Name        docker
+        Format      json
+        Time_Key    time
+        Time_Format %Y-%m-%dT%H:%M:%S.%L
+        Time_Keep   On
+
+    [PARSER]
+        Name        syslog
+        Format      regex
+        Regex       ^\<(?<pri>[0-9]+)\>(?<time>[^ ]* {1,2}[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?(?:[^\:]*\:)? *(?<message>.*)$
+        Time_Key    time
+        Time_Format %b %d %H:%M:%S
diff --git a/fluent-bit-daemonset-kafka.yml b/fluent-bit-daemonset-kafka.yml
index 414a2e7..cc7c099 100644
--- a/fluent-bit-daemonset-kafka.yml
+++ b/fluent-bit-daemonset-kafka.yml
@@ -17,16 +17,18 @@ spec:
     spec:
       containers:
       - name: fluent-bit
-        image: fluent
-        # build using eval $(minikube docker-env) && docker build -t fluent docker-image/
-        imagePullPolicy: Never
+        image: fluent/fluent-bit:0.12-dev
         env:
           - name:  KAFKA_REST_HOST
             value: rest.kafka.svc.cluster.local
           - name:  KAFKA_REST_PORT
             value: "80"
           - name:  KAFKA_TOPIC
-            value: logs
+            value: logs-json
+        command:
+          - /fluent-bit/bin/fluent-bit
+          - -c
+          - /fluent-bit/etc/fluent-bit.conf
         resources:
           limits:
             memory: 100Mi
@@ -34,6 +36,8 @@ spec:
             cpu: 100m
             memory: 100Mi
         volumeMounts:
+        - name: config
+          mountPath: /fluent-bit/etc
         - name: varlog
           mountPath: /var/log
         - name: varlibdockercontainers
@@ -41,6 +45,9 @@ spec:
           readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
+      - name: config
+        configMap:
+          name: fluent-bit-config-kafka-rest
       - name: varlog
         hostPath:
           path: /var/log

From 5f58cd5b238fc2856d70dd0ff1b6b87ff12ab198 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:12:45 +0200
Subject: [PATCH 5/8] Restores the docker image for use with es

This reverts commit f890bcbedf3692fc09524f076ffc413ceb5f1d04.
---
 docker-image/Dockerfile      |  2 +-
 docker-image/fluent-bit.conf | 17 ++++++-----------
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/docker-image/Dockerfile b/docker-image/Dockerfile
index b2affa9..b868939 100644
--- a/docker-image/Dockerfile
+++ b/docker-image/Dockerfile
@@ -1,4 +1,4 @@
-FROM fluent/fluent-bit:0.12-dev
+FROM fluent/fluent-bit:0.11.15
 MAINTAINER Eduardo Silva <eduardo@treasure-data.com>
 USER root
 
diff --git a/docker-image/fluent-bit.conf b/docker-image/fluent-bit.conf
index 4452de1..bb2853b 100644
--- a/docker-image/fluent-bit.conf
+++ b/docker-image/fluent-bit.conf
@@ -19,14 +19,9 @@
     Merge_JSON_Log On
 
 [OUTPUT]
-    Name            kafka-rest
-    Match           *
-    Host            ${KAFKA_REST_HOST}
-    Port            ${KAFKA_REST_PORT}
-    Time_Key        @timestamp
-    Tag_Key         _fluent-tag
-    Include_Tag_Key On
-    Tag_Key         My_Tag_Key
-    Topic           ${KAFKA_TOPIC}
-    # Partition     0
-    # Message_Key abc
+    Name   es
+    Match  *
+    Host   ${FLUENT_ELASTICSEARCH_HOST}
+    Port   ${FLUENT_ELASTICSEARCH_PORT}
+    Logstash_Format On
+    Retry_Limit False

From b964745c776628f96771d7693774e64dcbf7e837 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:14:46 +0200
Subject: [PATCH 6/8] github.com/fluent/fluent-bit-kubernetes-daemonset/pull/5

---
 fluent-bit-daemonset-kafka.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/fluent-bit-daemonset-kafka.yml b/fluent-bit-daemonset-kafka.yml
index cc7c099..4e94555 100644
--- a/fluent-bit-daemonset-kafka.yml
+++ b/fluent-bit-daemonset-kafka.yml
@@ -43,6 +43,9 @@ spec:
         - name: varlibdockercontainers
           mountPath: /var/lib/docker/containers
           readOnly: true
+        - name: minikube
+          mountPath: /mnt/sda1/var/lib/docker/containers
+          readOnly: true
       terminationGracePeriodSeconds: 10
       volumes:
       - name: config
@@ -54,3 +57,6 @@ spec:
       - name: varlibdockercontainers
         hostPath:
           path: /var/lib/docker/containers
+      - name: minikube
+        hostPath:
+          path: /mnt/sda1/var/lib/docker/containers

From 9ab38be3635c26f93cfda5ade449e0907c2d40d3 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:22:07 +0200
Subject: [PATCH 7/8] Just naming

---
 ...daemonset-kafka.yml => fluent-bit-daemonset-kafka-rest.yml | 2 +-
 testpod.yml                                                   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename fluent-bit-daemonset-kafka.yml => fluent-bit-daemonset-kafka-rest.yml (98%)

diff --git a/fluent-bit-daemonset-kafka.yml b/fluent-bit-daemonset-kafka-rest.yml
similarity index 98%
rename from fluent-bit-daemonset-kafka.yml
rename to fluent-bit-daemonset-kafka-rest.yml
index 4e94555..ebb3f45 100644
--- a/fluent-bit-daemonset-kafka.yml
+++ b/fluent-bit-daemonset-kafka-rest.yml
@@ -24,7 +24,7 @@ spec:
           - name:  KAFKA_REST_PORT
             value: "80"
           - name:  KAFKA_TOPIC
-            value: logs-json
+            value: fluent-bit
         command:
           - /fluent-bit/bin/fluent-bit
           - -c
diff --git a/testpod.yml b/testpod.yml
index 5950953..79bea3a 100644
--- a/testpod.yml
+++ b/testpod.yml
@@ -8,13 +8,13 @@ spec:
   - name: kafkacat
     image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
     command:
-      - sh
+      - bash
       - -ec
       - >
         cd /usr/local/bin;
         echo '#!/bin/sh' > logs-cat;
         chmod +x logs-cat;
-        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -t logs -C" >> logs-cat;
+        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -t fluent-bit -C" >> logs-cat;
 
         echo "Won't cat logs to stdout because it would create an infinite loop of log aggregation.";
         echo "Instead run something like:";

From a32359a10a187202632356d44af7ba4c511f4dbd Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 27 Jul 2017 07:38:27 +0200
Subject: [PATCH 8/8] Makes the test behave better when there's lots of logs

---
 testpod.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/testpod.yml b/testpod.yml
index 79bea3a..9ee3cd0 100644
--- a/testpod.yml
+++ b/testpod.yml
@@ -12,9 +12,9 @@ spec:
       - -ec
       - >
         cd /usr/local/bin;
-        echo '#!/bin/sh' > logs-cat;
+        echo '#!/bin/bash' > logs-cat;
         chmod +x logs-cat;
-        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -t fluent-bit -C" >> logs-cat;
+        echo "kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092 -C -t fluent-bit -o end" >> logs-cat;
 
         echo "Won't cat logs to stdout because it would create an infinite loop of log aggregation.";
         echo "Instead run something like:";