From 1be8ffaa616de51bf5bc62405caee670b6beff49 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 12 Aug 2024 17:06:29 -0700 Subject: [PATCH 1/3] migrate to kraft --- _unit-test/create-docker-volumes-test.sh | 1 - docker-compose.yml | 40 +++++++----------------- install/create-docker-volumes.sh | 1 - install/turn-things-off.sh | 4 +++ 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh index 6b1176c5c73..a2b17bf5731 100755 --- a/_unit-test/create-docker-volumes-test.sh +++ b/_unit-test/create-docker-volumes-test.sh @@ -15,7 +15,6 @@ sentry-kafka sentry-postgres sentry-redis sentry-symbolicator -sentry-zookeeper" before=$(get_volumes) diff --git a/docker-compose.yml b/docker-compose.yml index af7aab1d2de..f26c86447d1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,42 +143,27 @@ services: read_only: true source: ./postgres/ target: /opt/sentry/ - zookeeper: - <<: *restart_policy - image: "confluentinc/cp-zookeeper:7.6.1" - environment: - ZOOKEEPER_CLIENT_PORT: "2181" - CONFLUENT_SUPPORT_METRICS_ENABLE: "false" - ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" - ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" - KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" - ulimits: - nofile: - soft: 4096 - hard: 4096 - volumes: - - "sentry-zookeeper:/var/lib/zookeeper/data" - - "sentry-zookeeper-log:/var/lib/zookeeper/log" - - "sentry-secrets:/etc/zookeeper/secrets" - healthcheck: - <<: *healthcheck_defaults - test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 localhost 2181 | grep imok'] kafka: <<: *restart_policy - depends_on: - zookeeper: - <<: *depends_on-healthy image: "confluentinc/cp-kafka:7.6.1" environment: - KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" - KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" + # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example + KAFKA_PROCESS_ROLES: "broker,controller" + KAFKA_CONTROLLER_QUORUM_VOTERS: "1001@127.0.0.1:29093" + KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" + KAFKA_NODE_ID: "1001" + CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" + KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092" + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT" + KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" KAFKA_LOG_RETENTION_HOURS: "24" KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: "false" - KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" + KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN" KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" ulimits: @@ -544,8 +529,6 @@ volumes: external: true sentry-redis: external: true - sentry-zookeeper: - external: true sentry-kafka: external: true sentry-clickhouse: @@ -561,7 +544,6 @@ volumes: sentry-secrets: sentry-smtp: sentry-nginx-cache: - sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: sentry-clickhouse-log: diff --git a/install/create-docker-volumes.sh b/install/create-docker-volumes.sh index ca3ef0b23ed..15f20d54409 100644 --- a/install/create-docker-volumes.sh +++ b/install/create-docker-volumes.sh @@ -6,6 +6,5 @@ echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-postgres)." echo "Created $(docker volume create --name=sentry-redis)." echo "Created $(docker volume create --name=sentry-symbolicator)." -echo "Created $(docker volume create --name=sentry-zookeeper)." echo "${_endgroup}" diff --git a/install/turn-things-off.sh b/install/turn-things-off.sh index 270dc4e78c2..aa616d38d47 100644 --- a/install/turn-things-off.sh +++ b/install/turn-things-off.sh @@ -6,6 +6,10 @@ if [[ -n "$MINIMIZE_DOWNTIME" ]]; then else # Clean up old stuff and ensure nothing is working while we install/update $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans + # TODO(getsentry/self-hosted#2489) + if docker volume ls | grep -qw sentry-zookeeper; then + docker volume rm sentry-zookeeper + fi fi echo "${_endgroup}" From 052811ce1cb221cc9e861ab4d3c66759336d5739 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Mon, 12 Aug 2024 17:11:54 -0700 Subject: [PATCH 2/3] fix syntax error --- _unit-test/create-docker-volumes-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_unit-test/create-docker-volumes-test.sh b/_unit-test/create-docker-volumes-test.sh index a2b17bf5731..2cb9b962a8b 100755 --- a/_unit-test/create-docker-volumes-test.sh +++ b/_unit-test/create-docker-volumes-test.sh @@ -14,7 +14,7 @@ sentry-data sentry-kafka sentry-postgres sentry-redis -sentry-symbolicator +sentry-symbolicator" before=$(get_volumes) From a18bd046b9b3962ed4169e655e7d4054eddc1729 Mon Sep 17 00:00:00 2001 From: hubertdeng123 Date: Tue, 13 Aug 2024 09:45:44 -0700 Subject: [PATCH 3/3] move zookeeper volume removal to end of installation --- install/turn-things-off.sh | 4 ---- install/wrap-up.sh | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/install/turn-things-off.sh b/install/turn-things-off.sh index aa616d38d47..270dc4e78c2 100644 --- a/install/turn-things-off.sh +++ b/install/turn-things-off.sh @@ -6,10 +6,6 @@ if [[ -n "$MINIMIZE_DOWNTIME" ]]; then else # Clean up old stuff and ensure nothing is working while we install/update $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans - # TODO(getsentry/self-hosted#2489) - if docker volume ls | grep -qw sentry-zookeeper; then - docker volume rm sentry-zookeeper - fi fi echo "${_endgroup}" diff --git a/install/wrap-up.sh b/install/wrap-up.sh index a811f81a618..8840262f25d 100644 --- a/install/wrap-up.sh +++ b/install/wrap-up.sh @@ -28,3 +28,8 @@ else echo "-----------------------------------------------------------------" echo "" fi + +# TODO(getsentry/self-hosted#2489) +if docker volume ls | grep -qw sentry-zookeeper; then + docker volume rm sentry-zookeeper +fi