Skip to content

Commit f89bd74

Browse files
solssonsolsson
committed
Merge remote-tracking branch 'origin/metrics-improve-scrape-times' into kubernetes-1.8-test
2 parents d0a5088 + e4fadac commit f89bd74

File tree

7 files changed

+263
-0
lines changed

7 files changed

+263
-0
lines changed

‎10broker-config.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -273,3 +273,22 @@ data:
273273
# Change to DEBUG to enable audit log for the authorizer
274274
log4j.logger.kafka.authorizer.logger=WARN, authorizerAppender
275275
log4j.additivity.kafka.authorizer.logger=false
276+
277+
jmx-kafka-prometheus.yml: |+
278+
lowercaseOutputName: true
279+
jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi
280+
ssl: false
281+
whitelistObjectNames: ["kafka.server:*","java.lang:*"]
282+
rules:
283+
- pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value
284+
- pattern : kafka.server<type=BrokerTopicMetrics, name=(BytesInPerSec|BytesOutPerSec|MessagesInPerSec), topic=(.+)><>OneMinuteRate
285+
- pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate
286+
- pattern : kafka.server<type=Produce><>queue-size
287+
- pattern : kafka.server<type=ReplicaManager, name=(PartitionCount|UnderReplicatedPartitions)><>(Value|OneMinuteRate)
288+
- pattern : kafka.server<type=controller-channel-metrics, broker-id=(.+)><>(.*)
289+
- pattern : kafka.server<type=socket-server-metrics, networkProcessor=(.+)><>(.*)
290+
- pattern : kafka.server<type=Fetch><>queue-size
291+
- pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate
292+
- pattern : java.lang<type=OperatingSystem><>SystemCpuLoad
293+
- pattern : java.lang<type=Memory><HeapMemoryUsage>used
294+
- pattern : java.lang<type=OperatingSystem><>FreePhysicalMemorySize

‎50kafka.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ spec:
1515
labels:
1616
app: kafka
1717
annotations:
18+
prometheus.io/scrape: "true"
19+
prometheus.io/port: "5556"
1820
spec:
1921
terminationGracePeriodSeconds: 30
2022
initContainers:
@@ -43,6 +45,8 @@ spec:
4345
env:
4446
- name: KAFKA_LOG4J_OPTS
4547
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
48+
- name: JMX_PORT
49+
value: "5555"
4650
ports:
4751
- containerPort: 9092
4852
- containerPort: 9094
@@ -70,6 +74,32 @@ spec:
7074
mountPath: /etc/kafka
7175
- name: data
7276
mountPath: /var/lib/kafka/data
77+
- name: metrics
78+
image: solsson/kafka-prometheus-jmx-exporter@sha256:348b0f6510b08dff70ba468a16d25dc8def480fe79aca0e3c76f098d67b108a3
79+
command:
80+
- java
81+
- -Xmx64M
82+
- -XX:MaxMetaspaceSize=32m
83+
- -jar
84+
- jmx_prometheus_httpserver.jar
85+
- "5556"
86+
- /etc/kafka/jmx-kafka-prometheus.yml
87+
ports:
88+
- containerPort: 5556
89+
livenessProbe:
90+
httpGet:
91+
path: /liveness
92+
port: 5556
93+
periodSeconds: 60
94+
resources:
95+
requests:
96+
cpu: 0m
97+
memory: 100Mi
98+
limits:
99+
memory: 150Mi
100+
volumeMounts:
101+
- name: config
102+
mountPath: /etc/kafka
73103
volumes:
74104
- name: config
75105
configMap:

‎test/jmx-selftest.yml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Sets up a pod that monitors itself, to test resource usage etc.
2+
# kubectl exec -n test-kafka jmx-selftest-... -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics'
3+
apiVersion: extensions/v1beta1
4+
kind: Deployment
5+
metadata:
6+
name: jmx-selftest
7+
namespace: test-kafka
8+
spec:
9+
replicas: 1
10+
template:
11+
metadata:
12+
labels:
13+
test-target: jmx-exporter
14+
test-type: readiness
15+
# Uncomment to test with Prometheus
16+
#annotations:
17+
# prometheus.io/scrape: "true"
18+
# prometheus.io/port: "5556"
19+
spec:
20+
containers:
21+
- name: monitor
22+
image: solsson/kafka-prometheus-jmx-exporter@sha256:348b0f6510b08dff70ba468a16d25dc8def480fe79aca0e3c76f098d67b108a3
23+
command:
24+
- java
25+
- -Dcom.sun.management.jmxremote.ssl=false
26+
- -Dcom.sun.management.jmxremote.authenticate=false
27+
- -Dcom.sun.management.jmxremote.port=5555
28+
- -jar
29+
- jmx_prometheus_httpserver.jar
30+
- "5556"
31+
- example_configs/httpserver_sample_config.yml
32+
ports:
33+
- name: jmx
34+
containerPort: 5555
35+
- name: slashmetrics
36+
containerPort: 5556
37+
# Test run, again and again
38+
readinessProbe:
39+
httpGet:
40+
path: /metrics
41+
port: 5556

‎test/metrics.yml‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# kubectl apply -f test/metrics.yml && kubectl scale --replicas=0 deploy/metrics && kubectl scale --replicas=1 deploy/metrics
2+
# kubectl exec metrics-... -- tail -f /tmp/loglast | egrep 'time_total|^jmx_scrape_duration_seconds|^java_lang_memory_heapmemoryusage_used|^java_lang_memory_nonheapmemoryusage_used'
3+
---
4+
kind: ConfigMap
5+
metadata:
6+
name: metrics
7+
namespace: test-kafka
8+
apiVersion: v1
9+
data:
10+
11+
curl-format.txt: |-
12+
\n
13+
# ------ curl stats ------\n
14+
time_namelookup %{time_namelookup}\n
15+
time_connect %{time_connect}\n
16+
time_appconnect %{time_appconnect}\n
17+
time_pretransfer %{time_pretransfer}\n
18+
time_redirect %{time_redirect}\n
19+
time_starttransfer %{time_starttransfer}\n
20+
\n
21+
time_total{url="%{url_effective}"} %{time_total}\n
22+
\n
23+
http_code{url="%{url_effective}"} %{http_code}\n
24+
size_download{url="%{url_effective}"} %{size_download}\n
25+
content_type %{content_type}\n
26+
# ----- curl complete -----\n
27+
\n
28+
29+
setup.sh: |-
30+
touch /tmp/testlog
31+
tail -f /tmp/testlog
32+
33+
continue.sh: |-
34+
exit 0
35+
36+
run.sh: |-
37+
exec >> /tmp/testlog
38+
exec 2>&1
39+
40+
date -u -Iseconds | tee /tmp/loglast
41+
42+
curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
43+
http://kafka-0.broker.kafka.svc.cluster.local:5556/metrics \
44+
| tee -a /tmp/loglast \
45+
| grep http_code \
46+
| grep 200
47+
48+
curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
49+
http://zoo-0.zoo.kafka.svc.cluster.local:5556/metrics \
50+
| tee -a /tmp/loglast \
51+
| grep http_code \
52+
| grep 200
53+
54+
curl -w "@/test/curl-format.txt" -s --max-time $MAX_RESPONSE_TIME \
55+
http://pzoo-0.pzoo.kafka.svc.cluster.local:5556/metrics \
56+
| tee -a /tmp/loglast \
57+
| grep http_code \
58+
| grep 200
59+
60+
exit 0
61+
62+
---
63+
apiVersion: apps/v1beta1
64+
kind: Deployment
65+
metadata:
66+
name: metrics
67+
namespace: test-kafka
68+
spec:
69+
replicas: 1
70+
template:
71+
metadata:
72+
labels:
73+
test-target: kafka
74+
test-type: readiness
75+
spec:
76+
containers:
77+
- name: testcase
78+
image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf
79+
env:
80+
- name: MAX_RESPONSE_TIME
81+
value: "3"
82+
# Test set up
83+
command:
84+
- /bin/bash
85+
- -e
86+
- /test/setup.sh
87+
# Test run, again and again
88+
readinessProbe:
89+
exec:
90+
command:
91+
- /bin/bash
92+
- -e
93+
- /test/run.sh
94+
# Test quit on nonzero exit
95+
livenessProbe:
96+
exec:
97+
command:
98+
- /bin/bash
99+
- -e
100+
- /test/continue.sh
101+
volumeMounts:
102+
- name: config
103+
mountPath: /test
104+
volumes:
105+
- name: config
106+
configMap:
107+
name: metrics

‎zookeeper/10zookeeper-config.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,3 +35,22 @@ data:
3535
# Suppress connection log messages, three lines per livenessProbe execution
3636
log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN
3737
log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN
38+
39+
jmx-zookeeper-prometheus.yaml: |+
40+
rules:
41+
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d)><>(\\w+)"
42+
name: "zookeeper_$2"
43+
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d)><>(\\w+)"
44+
name: "zookeeper_$3"
45+
labels:
46+
replicaId: "$2"
47+
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+)><>(\\w+)"
48+
name: "zookeeper_$4"
49+
labels:
50+
replicaId: "$2"
51+
memberType: "$3"
52+
- pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+), name3=(\\w+)><>(\\w+)"
53+
name: "zookeeper_$4_$5"
54+
labels:
55+
replicaId: "$2"
56+
memberType: "$3"

‎zookeeper/50pzoo.yml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ spec:
1717
app: zookeeper
1818
storage: persistent
1919
annotations:
20+
prometheus.io/scrape: "true"
21+
prometheus.io/port: "5556"
2022
spec:
2123
terminationGracePeriodSeconds: 10
2224
initContainers:
@@ -34,6 +36,8 @@ spec:
3436
env:
3537
- name: KAFKA_LOG4J_OPTS
3638
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
39+
- name: JMX_PORT
40+
value: "5555"
3741
command:
3842
- ./bin/zookeeper-server-start.sh
3943
- /etc/kafka/zookeeper.properties
@@ -65,6 +69,24 @@ spec:
6569
mountPath: /etc/kafka
6670
- name: data
6771
mountPath: /var/lib/zookeeper/data
72+
- name: metrics
73+
image: solsson/kafka-prometheus-jmx-exporter@sha256:348b0f6510b08dff70ba468a16d25dc8def480fe79aca0e3c76f098d67b108a3
74+
command:
75+
- java
76+
- -Xmx64M
77+
- -XX:MaxMetaspaceSize=32m
78+
- -jar
79+
- jmx_prometheus_httpserver.jar
80+
- "5556"
81+
- example_configs/zookeeper.yaml
82+
ports:
83+
- containerPort: 5556
84+
resources:
85+
requests:
86+
cpu: 0m
87+
memory: 100Mi
88+
limits:
89+
memory: 150Mi
6890
volumes:
6991
- name: config
7092
configMap:

‎zookeeper/51zoo.yml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@ spec:
1616
app: zookeeper
1717
storage: ephemeral
1818
annotations:
19+
prometheus.io/scrape: "true"
20+
prometheus.io/port: "5556"
1921
spec:
2022
terminationGracePeriodSeconds: 10
2123
initContainers:
@@ -36,6 +38,8 @@ spec:
3638
env:
3739
- name: KAFKA_LOG4J_OPTS
3840
value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties
41+
- name: JMX_PORT
42+
value: "5555"
3943
command:
4044
- ./bin/zookeeper-server-start.sh
4145
- /etc/kafka/zookeeper.properties
@@ -67,6 +71,27 @@ spec:
6771
mountPath: /etc/kafka
6872
- name: data
6973
mountPath: /var/lib/zookeeper/data
74+
- name: metrics
75+
image: solsson/kafka-prometheus-jmx-exporter@sha256:348b0f6510b08dff70ba468a16d25dc8def480fe79aca0e3c76f098d67b108a3
76+
command:
77+
- java
78+
- -Xmx64M
79+
- -XX:MaxMetaspaceSize=32m
80+
- -jar
81+
- jmx_prometheus_httpserver.jar
82+
- "5556"
83+
- /etc/kafka/jmx-zookeeper-prometheus.yaml
84+
ports:
85+
- containerPort: 5556
86+
resources:
87+
requests:
88+
cpu: 0m
89+
memory: 100Mi
90+
limits:
91+
memory: 150Mi
92+
volumeMounts:
93+
- name: config
94+
mountPath: /etc/kafka
7095
volumes:
7196
- name: config
7297
configMap:

0 commit comments

Comments
 (0)