From e6d70a3f1f9ec4744f846ec975d57ea7b256e7cb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 20:16:11 +0200 Subject: [PATCH 1/9] Draft rbac. I was allowed to do create -f after ... adding myself to cluster-admin clusterrole --- 01rbac.yml | 27 +++++++++++++++++++++++++++ 50kafka.yml | 1 + 2 files changed, 28 insertions(+) create mode 100644 01rbac.yml diff --git a/01rbac.yml b/01rbac.yml new file mode 100644 index 00000000..5cd0732d --- /dev/null +++ b/01rbac.yml @@ -0,0 +1,27 @@ +--- +# Sufficient rights to look up self's pod description and pod's node description +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: pod-node-reader +rules: + - apiGroups: + - "" + resources: + - pods + - nodes + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: pod-node-reader-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pod-node-reader +subjects: +- kind: ServiceAccount + name: kafka + namespace: kafka diff --git a/50kafka.yml b/50kafka.yml index 4404a6be..525c9eb8 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -13,6 +13,7 @@ spec: annotations: spec: terminationGracePeriodSeconds: 30 + serviceAccountName: system:serviceaccounts:kafka initContainers: - name: init-config image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce From 4d00ead79928b6b19631d62264af98691da534be Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 06:30:03 +0200 Subject: [PATCH 2/9] Creates the service account kafka --- 01rbac.yml | 6 ++++++ 50kafka.yml | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/01rbac.yml b/01rbac.yml index 5cd0732d..1e44cdd9 100644 --- a/01rbac.yml +++ b/01rbac.yml @@ -1,4 +1,10 @@ --- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kafka + namespace: kafka +--- # Sufficient rights to look up self's pod description and pod's node description kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/50kafka.yml b/50kafka.yml index 525c9eb8..b148d90c 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -13,7 +13,7 @@ spec: annotations: spec: terminationGracePeriodSeconds: 30 - serviceAccountName: system:serviceaccounts:kafka + serviceAccountName: kafka initContainers: - name: init-config image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce From f6c48e20cbfd825236e49c60114a228540c8d135 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 07:07:41 +0200 Subject: [PATCH 3/9] Are there conventions for naming RBAC stuff? --- 01rbac.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/01rbac.yml b/01rbac.yml index 1e44cdd9..afd5aebf 100644 --- a/01rbac.yml +++ b/01rbac.yml @@ -22,7 +22,7 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: pod-node-reader-kafka + name: kafka-pod-node-reader roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole From 56f0f9ce51c9119d29b4fd9b146fbe39f1d0e389 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 07:08:38 +0200 Subject: [PATCH 4/9] Oddly /run/secrets doesn't get mounted --- 01rbac.yml | 1 + test/rack-awareness.yml | 77 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 test/rack-awareness.yml diff --git a/01rbac.yml b/01rbac.yml index afd5aebf..1427d796 100644 --- a/01rbac.yml +++ b/01rbac.yml @@ -4,6 +4,7 @@ kind: ServiceAccount metadata: name: kafka namespace: kafka +automountServiceAccountToken: true --- # Sufficient rights to look up self's pod description and pod's node description kind: ClusterRole diff --git a/test/rack-awareness.yml b/test/rack-awareness.yml new file mode 100644 index 00000000..aa70003a --- /dev/null +++ b/test/rack-awareness.yml @@ -0,0 +1,77 @@ +--- +kind: ConfigMap +metadata: + name: test-rack-awareness + # To test using the service account we need to be in the kafka namespace for now + namespace: kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 + +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: test-rack-awareness + # To test using the service account we need to be in the kafka namespace for now + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: kafka + test-type: readiness + spec: + serviceAccountName: kafka + containers: + - name: testcase + image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: test-rack-awareness From b3a6bbce9752778ea5e93116f7a1110b488e3dd8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 07:11:10 +0200 Subject: [PATCH 5/9] Secrets do get mounted when using the kafka image ... instead of curl. Could it be that curl is based on Alpine? --- test/rack-awareness.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/rack-awareness.yml b/test/rack-awareness.yml index aa70003a..2a80a54a 100644 --- a/test/rack-awareness.yml +++ b/test/rack-awareness.yml @@ -39,7 +39,7 @@ spec: serviceAccountName: kafka containers: - name: testcase - image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce env: - name: MY_NODE_NAME valueFrom: From 7912b82077648edf337f4595c35aacda2934fa8b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 07:46:48 +0200 Subject: [PATCH 6/9] Using kubectl because curl would get 401 as system:anonymous, but be prepared for misleading error messages (for an RBAC noob like me) when your operation does not match the Role's rights: ``` root@test-rack-awareness-267009956-k0ffs:/opt/kafka# kubectl get pod $HOSTNAME NAME READY STATUS RESTARTS AGE test-rack-awareness-267009956-k0ffs 1/1 Running 0 14m root@test-rack-awareness-267009956-k0ffs:/opt/kafka# kubectl get pods Error from server (Forbidden): User "system:serviceaccount:kafka:kafka" cannot list pods in the namespace "kafka".: "Unknown user \"system:serviceaccount:kafka:kafka\"" (get pods) ``` --- 10broker-config.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/10broker-config.yml b/10broker-config.yml index 93bc8f0e..5bebdecd 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -17,6 +17,13 @@ data: # todo add curl to kafka image, switch to a curl image for init or write the whole lookup in java hash curl 2>/dev/null || { apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install curl -y --no-install-recommends; } + hash kubectl 2>/dev/null || { + curl -sLS -o k.tar.gz -k https://dl.k8s.io/v1.7.2/kubernetes-client-linux-amd64.tar.gz + echo "9c2363710d61a12a28df2d8a4688543b785156369973d33144ab1f2c1d5c7b53 k.tar.gz" | sha256sum -c + tar xvf k.tar.gz -C /usr/local/bin/ --strip-components=3 kubernetes/client/bin/kubectl + rm k.tar.gz + } + API=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api AUTH="--cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header \"Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)\"" From f26a13f602177ccec2db32a23fd1563a72ac8550 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 12:36:21 +0200 Subject: [PATCH 7/9] kubectl gets the designated role and kan read node label --- 01rbac.yml | 7 +++---- 10broker-config.yml | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/01rbac.yml b/01rbac.yml index 1427d796..252d0d20 100644 --- a/01rbac.yml +++ b/01rbac.yml @@ -10,12 +10,11 @@ automountServiceAccountToken: true kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: pod-node-reader + name: node-reader rules: - apiGroups: - "" resources: - - pods - nodes verbs: - get @@ -23,11 +22,11 @@ rules: kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 metadata: - name: kafka-pod-node-reader + name: kafka-node-reader roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: pod-node-reader + name: node-reader subjects: - kind: ServiceAccount name: kafka diff --git a/10broker-config.yml b/10broker-config.yml index 5bebdecd..fc23111e 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -11,12 +11,12 @@ data: export KAFKA_BROKER_ID=${HOSTNAME##*-} sed -i "s/\${KAFKA_BROKER_ID}/$KAFKA_BROKER_ID/" /etc/kafka/server.properties - PODNAME=$HOSTNAME - NAMESPACE=$(cat /var/run/secrets/kubernetes.io/serviceaccount/namespace) - # todo add curl to kafka image, switch to a curl image for init or write the whole lookup in java hash curl 2>/dev/null || { apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install curl -y --no-install-recommends; } + echo "Service Account is $MY_POD_SERVICE_ACCOUNT" + + # curl auth was with system:anonymous so trying kubectl instead hash kubectl 2>/dev/null || { curl -sLS -o k.tar.gz -k https://dl.k8s.io/v1.7.2/kubernetes-client-linux-amd64.tar.gz echo "9c2363710d61a12a28df2d8a4688543b785156369973d33144ab1f2c1d5c7b53 k.tar.gz" | sha256sum -c @@ -24,12 +24,13 @@ data: rm k.tar.gz } - API=https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api - AUTH="--cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header \"Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)\"" - - curl -s $AUTH $API/namespaces/kafka/pods/$PODNAME -I --fail-early || { - echo "Access problems. Could be RBAC." - } + ZONE=$(kubectl get node $MY_NODE_NAME -o=go-template='{{index .metadata.labels "failure-domain.beta.kubernetes.io/zone"}}') + [ -z "$ZONE" ] && + if [ ! -z "$ZONE" ]; then + sed -i "s/#broker.rack=$/broker.rack=$ZONE/" /etc/kafka/server.properties + else + sed -i "s/#broker.rack=$# No zone label found/" /etc/kafka/server.properties + fi server.properties: |- # Licensed to the Apache Software Foundation (ASF) under one or more @@ -54,7 +55,7 @@ data: # The id of the broker. This must be set to a unique integer for each broker. broker.id=${KAFKA_BROKER_ID} - #broker.rack=${KAFKA_BROKER_RACK} + #broker.rack= # Switch to enable topic deletion or not, default value is false delete.topic.enable=true From 54a95fc233fbab9f350e69dd4d65a7f2cb137a62 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 12:46:12 +0200 Subject: [PATCH 8/9] Adds nodeName downwardApi for zone lookup, and service account for debugging --- 50kafka.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/50kafka.yml b/50kafka.yml index b148d90c..1f379e0b 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -18,6 +18,15 @@ spec: - name: init-config image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce command: ['/bin/bash', '/etc/kafka/init.sh'] + env: + - name: MY_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName volumeMounts: - name: config mountPath: /etc/kafka From 9cdb4b9a49506d21056f9b26d84b978948108252 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 2 Aug 2017 12:55:10 +0200 Subject: [PATCH 9/9] Removing the test as I can't see how to assert rack awareness --- test/rack-awareness.yml | 77 ----------------------------------------- 1 file changed, 77 deletions(-) delete mode 100644 test/rack-awareness.yml diff --git a/test/rack-awareness.yml b/test/rack-awareness.yml deleted file mode 100644 index 2a80a54a..00000000 --- a/test/rack-awareness.yml +++ /dev/null @@ -1,77 +0,0 @@ ---- -kind: ConfigMap -metadata: - name: test-rack-awareness - # To test using the service account we need to be in the kafka namespace for now - namespace: kafka -apiVersion: v1 -data: - - setup.sh: |- - touch /tmp/testlog - - tail -f /tmp/testlog - - continue.sh: |- - exit 0 - - run.sh: |- - exec >> /tmp/testlog - exec 2>&1 - - exit 0 - ---- -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: test-rack-awareness - # To test using the service account we need to be in the kafka namespace for now - namespace: kafka -spec: - replicas: 1 - template: - metadata: - labels: - test-target: kafka - test-type: readiness - spec: - serviceAccountName: kafka - containers: - - name: testcase - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_POD_SERVICE_ACCOUNT - valueFrom: - fieldRef: - fieldPath: spec.serviceAccountName - # Test set up - command: - - /bin/bash - - -e - - /test/setup.sh - # Test run, again and again - readinessProbe: - exec: - command: - - /bin/bash - - -e - - /test/run.sh - # Test quit on nonzero exit - livenessProbe: - exec: - command: - - /bin/bash - - -e - - /test/continue.sh - volumeMounts: - - name: config - mountPath: /test - volumes: - - name: config - configMap: - name: test-rack-awareness