From d384382e3af8e60428d3c7b46b4d3b6b7d518ccc Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 12:43:46 +0200 Subject: [PATCH 1/5] Just a pod with the Kafka distribution --- mirrormaker/mirrormaker.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 mirrormaker/mirrormaker.yml diff --git a/mirrormaker/mirrormaker.yml b/mirrormaker/mirrormaker.yml new file mode 100644 index 00000000..7b2a2d1d --- /dev/null +++ b/mirrormaker/mirrormaker.yml @@ -0,0 +1,28 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mirror-a + namespace: kafka + labels: + app: mirrormaker +spec: + replicas: 1 + selector: + matchLabels: + app: mirrormaker + strategy: + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + app: mirrormaker + spec: + containers: + - name: kafka + image: solsson/kafka:1.1@sha256:ba863ca7dc28563930584e37f93d57c2cbf3f46b1c1fa104fe8af7bcc0c31df4 + command: + - tail + - -f + - /dev/null From 9b66ed077cbc0e8bc2cdfe67b49450beeaace000 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 17:31:51 +0200 Subject: [PATCH 2/5] Specific to my current test clusters --- mirrormaker/mirrormaker.yml | 30 +++++++++++++++++++++++++++--- mirrormaker/source-endpoints.yml | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 mirrormaker/source-endpoints.yml diff --git a/mirrormaker/mirrormaker.yml b/mirrormaker/mirrormaker.yml index 7b2a2d1d..fe6fd505 100644 --- a/mirrormaker/mirrormaker.yml +++ b/mirrormaker/mirrormaker.yml @@ -22,7 +22,31 @@ spec: containers: - name: kafka image: solsson/kafka:1.1@sha256:ba863ca7dc28563930584e37f93d57c2cbf3f46b1c1fa104fe8af7bcc0c31df4 + resources: + requests: + cpu: 0m + memory: 80Mi + limits: + cpu: 20m + memory: 200Mi + env: + - name: TARGET_BOOTSTRAP + value: bootstrap.kafka:9092 + - name: SOURCE_BOOTSTRAP + value: mirror-from.kafka:32400,mirror-from.kafka:32401,mirror-from.kafka:32402 + - name: WHITELIST + value: .*yolean.*yolean.*operations.* + - name: GROUP_ID + value: mirror-to-4 command: - - tail - - -f - - /dev/null + - bash + - -cex + - > + echo "bootstrap.servers=$SOURCE_BOOTSTRAP" > ./config/consumer.properties; + echo "group.id=$GROUP_ID" >> ./config/consumer.properties; + echo "auto.offset.reset=earliest" >> ./config/consumer.properties; + echo "bootstrap.servers=$TARGET_BOOTSTRAP" > ./config/producer.properties; + echo "log4j.logger.org.apache.kafka.clients.Metadata=DEBUG" >> ./config/tools-log4j.properties; + echo "log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=INFO" >> ./config/tools-log4j.properties; + echo "log4j.logger.org.apache.kafka.clients.NetworkClient=INFO" >> ./config/tools-log4j.properties; + ./bin/kafka-mirror-maker.sh --consumer.config ./config/consumer.properties --producer.config ./config/producer.properties --whitelist "$WHITELIST" diff --git a/mirrormaker/source-endpoints.yml b/mirrormaker/source-endpoints.yml new file mode 100644 index 00000000..352acaee --- /dev/null +++ b/mirrormaker/source-endpoints.yml @@ -0,0 +1,32 @@ +kind: Service +apiVersion: v1 +metadata: + name: mirror-from + namespace: kafka +spec: + ports: + - name: outside-0 + port: 32400 + - name: outside-1 + port: 32401 + - name: outside-2 + port: 32402 +--- +kind: Endpoints +apiVersion: v1 +metadata: + name: mirror-from + namespace: kafka +subsets: +- addresses: + # source cluster: kubectl -n kafka get pods -l app=kafka -o yaml | grep outside + - ip: 10.132.0.13 + - ip: 10.132.0.9 + - ip: 10.132.0.10 + ports: + - name: outside-0 + port: 32400 + - name: outside-1 + port: 32401 + - name: outside-2 + port: 32402 From abbb2c3084ff1b7bb590752becb1fb19eceb5f5c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 17:32:39 +0200 Subject: [PATCH 3/5] Challenge: auto.offset.reset earliest makes it easy to add a topic, but is quite dangerous for existing topics in case you get an offset reset for some reason, like expiry. --- mirrormaker/mirrormaker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrormaker/mirrormaker.yml b/mirrormaker/mirrormaker.yml index fe6fd505..e993c54d 100644 --- a/mirrormaker/mirrormaker.yml +++ b/mirrormaker/mirrormaker.yml @@ -44,7 +44,7 @@ spec: - > echo "bootstrap.servers=$SOURCE_BOOTSTRAP" > ./config/consumer.properties; echo "group.id=$GROUP_ID" >> ./config/consumer.properties; - echo "auto.offset.reset=earliest" >> ./config/consumer.properties; + echo "auto.offset.reset=none" >> ./config/consumer.properties; echo "bootstrap.servers=$TARGET_BOOTSTRAP" > ./config/producer.properties; echo "log4j.logger.org.apache.kafka.clients.Metadata=DEBUG" >> ./config/tools-log4j.properties; echo "log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=INFO" >> ./config/tools-log4j.properties; From 7b789be66bd8fe8ac6cb8cab93a592f2397f7bd8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 17:41:27 +0200 Subject: [PATCH 4/5] Adapts to WARNING: The default partition assignment strategy of the new-consumer-based mirror maker will change from 'range' to 'roundrobin' in an upcoming release --- mirrormaker/mirrormaker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/mirrormaker/mirrormaker.yml b/mirrormaker/mirrormaker.yml index e993c54d..2a8bbb7c 100644 --- a/mirrormaker/mirrormaker.yml +++ b/mirrormaker/mirrormaker.yml @@ -45,6 +45,7 @@ spec: echo "bootstrap.servers=$SOURCE_BOOTSTRAP" > ./config/consumer.properties; echo "group.id=$GROUP_ID" >> ./config/consumer.properties; echo "auto.offset.reset=none" >> ./config/consumer.properties; + echo "partition.assignment.strategy=org.apache.kafka.clients.consumer.RoundRobinAssignor" >> ./config/consumer.properties; echo "bootstrap.servers=$TARGET_BOOTSTRAP" > ./config/producer.properties; echo "log4j.logger.org.apache.kafka.clients.Metadata=DEBUG" >> ./config/tools-log4j.properties; echo "log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=INFO" >> ./config/tools-log4j.properties; From 247a1c97769326d3ee4e942f24db13d68b8e59e5 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 May 2018 17:42:20 +0200 Subject: [PATCH 5/5] More cluster specific stuff, but I'd recommend comma separated exact topic names --- mirrormaker/mirrormaker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirrormaker/mirrormaker.yml b/mirrormaker/mirrormaker.yml index 2a8bbb7c..295df097 100644 --- a/mirrormaker/mirrormaker.yml +++ b/mirrormaker/mirrormaker.yml @@ -35,7 +35,7 @@ spec: - name: SOURCE_BOOTSTRAP value: mirror-from.kafka:32400,mirror-from.kafka:32401,mirror-from.kafka:32402 - name: WHITELIST - value: .*yolean.*yolean.*operations.* + value: .*yolean.*operations.* - name: GROUP_ID value: mirror-to-4 command: