diff --git a/logs-streaming/00namespace.yml b/logs-streaming/00namespace.yml new file mode 100644 index 00000000..d4209416 --- /dev/null +++ b/logs-streaming/00namespace.yml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: logs-kafka diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml new file mode 100644 index 00000000..e1414034 --- /dev/null +++ b/logs-streaming/filebeat-config.yml @@ -0,0 +1,34 @@ +kind: ConfigMap +metadata: + name: filebeat-config + namespace: logs-kafka +apiVersion: v1 +data: + filebeat.yml: |+ + + filebeat.prospectors: + - type: log + enabled: true + paths: + - /var/lib/docker/containers/*/*.log + symlinks: true + # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs + exclude_files: ['^(.*\/)?logs-','^(.*\/)?filebeat-','^(.*\/)?test-'] + + processors: + - add_kubernetes_metadata: + in_cluster: true + namespace: logs-kafka + + output.kafka: + hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"] + topic: ${TOPIC} + + partition.round_robin: + reachable_only: false + + client_id: filebeat-kubernetes + version: 0.11.0.0 + required_acks: 1 + compression: gzip + max_message_bytes: 1000000 \ No newline at end of file diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml new file mode 100644 index 00000000..1294d50b --- /dev/null +++ b/logs-streaming/filebeat-logs-kube-kafka.yml @@ -0,0 +1,65 @@ +apiVersion: apps/v1beta2 +kind: DaemonSet +metadata: + name: filebeat-kube-kafka + namespace: logs-kafka +spec: + selector: + matchLabels: + k8s-app: filebeat-kube-kafka + version: v1 + kubernetes.io/cluster-service: "true" + template: + metadata: + labels: + k8s-app: filebeat-kube-kafka + version: v1 + kubernetes.io/cluster-service: "true" + spec: + serviceAccountName: filebeat + containers: + - name: filebeat + image: solsson/kafka-filebeat-kubernetes@sha256:c2a68766e718d354b565b20f92baf0f50047b4bdf90a5055a20107a1131bee80 + command: + - filebeat + - -e + - -c + - /etc/filebeat/filebeat.yml + - -d + - "service,beat" + env: + - name: TOPIC + value: ops-kube-logs-filebeat-001 + resources: + requests: + cpu: 2m + memory: 10Mi + limits: + cpu: 10m + memory: 40Mi + volumeMounts: + - name: config + mountPath: /etc/filebeat + readOnly: true + - name: data + mountPath: /data + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + terminationGracePeriodSeconds: 10 + volumes: + - name: config + configMap: + name: filebeat-config + - name: data + hostPath: + path: /tmp/kubernets-filebeat-state + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers \ No newline at end of file diff --git a/logs-streaming/rbac/filebeat.yml b/logs-streaming/rbac/filebeat.yml new file mode 100644 index 00000000..a2f21156 --- /dev/null +++ b/logs-streaming/rbac/filebeat.yml @@ -0,0 +1,23 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: filebeat + namespace: logs-kafka + annotations: + manifest-origin: 'github.com/Yolean/kubernetes-kafka' +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: logs-kafka-filebeat-sa-view + annotations: + manifest-origin: 'github.com/Yolean/kubernetes-kafka' +subjects: +- kind: ServiceAccount + name: filebeat + namespace: logs-kafka +roleRef: + kind: ClusterRole + name: view + apiGroup: rbac.authorization.k8s.io diff --git a/logs-streaming/test/logs-filebeat.yml b/logs-streaming/test/logs-filebeat.yml new file mode 100644 index 00000000..ef53c96e --- /dev/null +++ b/logs-streaming/test/logs-filebeat.yml @@ -0,0 +1,75 @@ +--- +kind: ConfigMap +metadata: + name: logs-filebeat + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + echo "" + echo "{\"#---\":\"$(date -u --iso-8601='ns')\"}" + kafkacat -b $BOOTSTRAP -C -t $TOPIC -f '{"#topic":"%t","#partition":%p,"#offset":%o,"#key":"%k","=":%s}\n' -o -10 -e -q + + exit 0 + + quit-on-nonzero-exit.sh: |- + exit 0 + +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: logs-filebeat + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: logs-streaming-filebeat + test-type: readiness + template: + metadata: + labels: + test-target: logs-streaming-filebeat + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-logs-filebeat-001 + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: logs-filebeat diff --git a/logs-streaming/topic-ops-kube-logs-filebeat.yml b/logs-streaming/topic-ops-kube-logs-filebeat.yml new file mode 100644 index 00000000..d4aaf584 --- /dev/null +++ b/logs-streaming/topic-ops-kube-logs-filebeat.yml @@ -0,0 +1,31 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-logs-filebeat + namespace: logs-kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-logs-filebeat + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-logs-filebeat-001 + - --partitions + - "1" + - --replication-factor + - "2" + - --config + - retention.ms=6912500000 + restartPolicy: Never