feat: no more zookeeper

.github/actions/setup-sentry/action.yml

@@ -172,7 +172,7 @@ runs:
         fi
 
         if [ "$NEED_KAFKA" = "true" ]; then
-          services+=(zookeeper kafka)
+          services+=(kafka)
         fi
 
         sentry devservices up "${services[@]}"

Makefile

@@ -163,7 +163,7 @@ test-tools:
 	@echo ""
 
 # JavaScript relay tests are meant to be run within Symbolicator test suite, as they are parametrized to verify both processing pipelines during migration process.
-# Running Locally: Run `sentry devservices up kafka zookeeper` before starting these tests
+# Running Locally: Run `sentry devservices up kafka` before starting these tests
 test-symbolicator: create-db
 	@echo "--> Running symbolicator tests"
 	pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml"

config/clickhouse/dist_config.xml

@@ -1,7 +1,6 @@
 <yandex>
     <max_server_memory_usage_to_ram_ratio>0.3</max_server_memory_usage_to_ram_ratio>
 
-
     <remote_servers>
         <cluster_one_sh>
             <shard>
@@ -13,13 +12,6 @@
         </cluster_one_sh>
     </remote_servers>
 
-    <zookeeper>
-        <node>
-            <host>sentry_zookeeper</host>
-            <port>2181</port>
-        </node>
-    </zookeeper>
-
     <macros>
         <shard>1</shard>
         <replica>1</replica>

src/sentry/conf/server.py

@@ -2676,37 +2676,28 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str]
             "entrypoint": "/cdc/postgres-entrypoint.sh" if settings.SENTRY_USE_CDC_DEV else None,
         }
     ),
-    "zookeeper": lambda settings, options: (
-        {
-            # On Apple arm64, we upgrade to version 6.x to allow zookeeper to run properly on Apple's arm64
-            # See details https://github.com/confluentinc/kafka-images/issues/80#issuecomment-855511438
-            "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0",
-            "environment": {
-                "ZOOKEEPER_CLIENT_PORT": "2181",
-                "KAFKA_OPTS": "-Dzookeeper.4lw.commands.whitelist=ruok",
-            },
-            "volumes": {"zookeeper_6": {"bind": "/var/lib/zookeeper/data"}},
-            "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY,
-        }
-    ),
     "kafka": lambda settings, options: (
         {
-            "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0",
+            "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:7.5.0",
             "ports": {"9092/tcp": 9092},
+            # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
             "environment": {
-                "KAFKA_ZOOKEEPER_CONNECT": "{containers[zookeeper][name]}:2181",
-                "KAFKA_LISTENERS": "INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092",
-                "KAFKA_ADVERTISED_LISTENERS": "INTERNAL://{containers[kafka][name]}:9093,EXTERNAL://{containers[kafka]"
-                "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}",
-                "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT",
-                "KAFKA_INTER_BROKER_LISTENER_NAME": "INTERNAL",
+                "KAFKA_PROCESS_ROLES": "broker,controller",
+                "KAFKA_CONTROLLER_QUORUM_VOTERS": "[email protected]:29093",
+                "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER",
+                "KAFKA_NODE_ID": "1",
+                "CLUSTER_ID": "MkU3OEVBNTcwNTJENDM2Qk",
+                "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093",
+                "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://127.0.0.1:29092,INTERNAL://sentry_kafka:9093,EXTERNAL://127.0.0.1:9092",
+                "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT",
+                "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT",
                 "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1",
                 "KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS": "1",
                 "KAFKA_LOG_RETENTION_HOURS": "24",
                 "KAFKA_MESSAGE_MAX_BYTES": "50000000",
                 "KAFKA_MAX_REQUEST_SIZE": "50000000",
             },
-            "volumes": {"kafka_6": {"bind": "/var/lib/kafka/data"}},
+            "volumes": {"kafka": {"bind": "/var/lib/kafka/data"}},
             "only_if": "kafka" in settings.SENTRY_EVENTSTREAM
             or settings.SENTRY_USE_RELAY
             or settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS

src/sentry/runner/commands/devserver.py

@@ -329,10 +329,10 @@ def devserver(
     if kafka_consumers:
         with get_docker_client() as docker:
             containers = {c.name for c in docker.containers.list(filters={"status": "running"})}
-        if "sentry_zookeeper" not in containers or "sentry_kafka" not in containers:
+        if "sentry_kafka" not in containers:
             raise click.ClickException(
                 f"""
-Devserver is configured to start some kafka consumers, but Kafka + Zookeeper
+Devserver is configured to start some kafka consumers, but Kafka
 don't seem to be running.
 
 The following consumers were intended to be started: {kafka_consumers}
@@ -345,7 +345,7 @@ def devserver(
 
     SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream"
 
-and run `sentry devservices up kafka zookeeper`.
+and run `sentry devservices up kafka`.
 
 Alternatively, run without --workers.
 """

src/sentry/runner/commands/devservices.py

@@ -235,6 +235,8 @@ def up(
     if services:
         for service in services:
             if service not in containers:
+                if service == "zookeeper":
+                    continue
                 click.secho(
                     f"Service `{service}` is not known or not enabled.\n",
                     err=True,

tests/sentry/post_process_forwarder/test_post_process_forwarder.py

@@ -19,7 +19,6 @@
 pytestmark = [requires_kafka]
 
 SENTRY_KAFKA_HOSTS = os.environ.get("SENTRY_KAFKA_HOSTS", "127.0.0.1:9092")
-SENTRY_ZOOKEEPER_HOSTS = os.environ.get("SENTRY_ZOOKEEPER_HOSTS", "127.0.0.1:2181")
 settings.KAFKA_CLUSTERS["default"] = {"common": {"bootstrap.servers": SENTRY_KAFKA_HOSTS}}
 
 

tests/tools/test_devservices_healthcheck.py

@@ -73,7 +73,7 @@ def test_postgres_running(mock_subprocess_run: mock.MagicMock) -> None:
     assert mock_subprocess_run.call_count == 2
 
 
-def test_kafka_zookeper_running(mock_subprocess_run: mock.MagicMock) -> None:
+def test_kafka_running(mock_subprocess_run: mock.MagicMock) -> None:
     running = mock.Mock()
     running.stdout = "running\n"
     running.code = 0
@@ -84,15 +84,6 @@ def run(
         cmd_args: List[str], capture_output: bool = False, text: bool = False, check: bool = False
     ) -> mock.Mock:
         if cmd_args == (
-            "docker",
-            "container",
-            "inspect",
-            "-f",
-            "{{.State.Status}}",
-            "sentry_zookeeper",
-        ):
-            return running
-        elif cmd_args == (
             "docker",
             "container",
             "inspect",
@@ -106,16 +97,8 @@ def run(
             "exec",
             "sentry_kafka",
             "kafka-topics",
-            "--zookeeper",
-            "sentry_zookeeper:2181",
-            "--list",
-        ) or (
-            "docker",
-            "exec",
-            "sentry_kafka",
-            "kafka-topics",
-            "--zookeeper",
-            "127.0.0.1:2181",
+            "--bootstrap-server",
+            "127.0.0.1:9092",
             "--list",
         ):
             return healthcheck
@@ -124,4 +107,4 @@ def run(
     mock_subprocess_run.side_effect = run
 
     check_health(["kafka"])
-    assert mock_subprocess_run.call_count == 3
+    assert mock_subprocess_run.call_count == 2

tools/devservices_healthcheck.py

@@ -48,8 +48,8 @@ def check_kafka():
             "exec",
             "sentry_kafka",
             "kafka-topics",
-            "--zookeeper",
-            "sentry_zookeeper:2181",
+            "--bootstrap-server",
+            "127.0.0.1:9092",
             "--list",
         ),
         check=True,
@@ -75,12 +75,6 @@ def check_postgres() -> None:
         "sentry_kafka",
         os.getenv("NEED_KAFKA") == "true",
         check_kafka,
-        deps=["zookeeper"],
-    ),
-    "zookeeper": HealthCheck(
-        "zookeeper",
-        "sentry_zookeeper",
-        os.getenv("NEED_KAFKA") == "true",
     ),
 }