diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml index a81b770dc8354d..911f62da15620e 100644 --- a/.github/actions/setup-sentry/action.yml +++ b/.github/actions/setup-sentry/action.yml @@ -172,7 +172,7 @@ runs: fi if [ "$NEED_KAFKA" = "true" ]; then - services+=(zookeeper kafka) + services+=(kafka) fi sentry devservices up "${services[@]}" diff --git a/Makefile b/Makefile index e9d81281c4e8f7..92d72e2db433e8 100644 --- a/Makefile +++ b/Makefile @@ -163,7 +163,7 @@ test-tools: @echo "" # JavaScript relay tests are meant to be run within Symbolicator test suite, as they are parametrized to verify both processing pipelines during migration process. -# Running Locally: Run `sentry devservices up kafka zookeeper` before starting these tests +# Running Locally: Run `sentry devservices up kafka` before starting these tests test-symbolicator: create-db @echo "--> Running symbolicator tests" pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml" diff --git a/config/clickhouse/dist_config.xml b/config/clickhouse/dist_config.xml index bfb90d93f6224d..a2f21d89412a7f 100644 --- a/config/clickhouse/dist_config.xml +++ b/config/clickhouse/dist_config.xml @@ -1,7 +1,6 @@ 0.3 - @@ -13,13 +12,6 @@ - - - sentry_zookeeper - 2181 - - - 1 1 diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 1d00113d433d10..391321943b651a 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2676,37 +2676,28 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "entrypoint": "/cdc/postgres-entrypoint.sh" if settings.SENTRY_USE_CDC_DEV else None, } ), - "zookeeper": lambda settings, options: ( - { - # On Apple arm64, we upgrade to version 6.x to allow zookeeper to run properly on Apple's arm64 - # See details https://github.com/confluentinc/kafka-images/issues/80#issuecomment-855511438 - "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0", - "environment": { - "ZOOKEEPER_CLIENT_PORT": "2181", - "KAFKA_OPTS": "-Dzookeeper.4lw.commands.whitelist=ruok", - }, - "volumes": {"zookeeper_6": {"bind": "/var/lib/zookeeper/data"}}, - "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY, - } - ), "kafka": lambda settings, options: ( { - "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0", + "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:7.5.0", "ports": {"9092/tcp": 9092}, + # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example "environment": { - "KAFKA_ZOOKEEPER_CONNECT": "{containers[zookeeper][name]}:2181", - "KAFKA_LISTENERS": "INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092", - "KAFKA_ADVERTISED_LISTENERS": "INTERNAL://{containers[kafka][name]}:9093,EXTERNAL://{containers[kafka]" - "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}", - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT", - "KAFKA_INTER_BROKER_LISTENER_NAME": "INTERNAL", + "KAFKA_PROCESS_ROLES": "broker,controller", + "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@127.0.0.1:29093", + "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", + "KAFKA_NODE_ID": "1", + "CLUSTER_ID": "MkU3OEVBNTcwNTJENDM2Qk", + "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093", + "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://127.0.0.1:29092,INTERNAL://sentry_kafka:9093,EXTERNAL://127.0.0.1:9092", + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT", + "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT", "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", "KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS": "1", "KAFKA_LOG_RETENTION_HOURS": "24", "KAFKA_MESSAGE_MAX_BYTES": "50000000", "KAFKA_MAX_REQUEST_SIZE": "50000000", }, - "volumes": {"kafka_6": {"bind": "/var/lib/kafka/data"}}, + "volumes": {"kafka": {"bind": "/var/lib/kafka/data"}}, "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY or settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS diff --git a/src/sentry/runner/commands/devserver.py b/src/sentry/runner/commands/devserver.py index 44e2a0e146e125..16b4a02bfe8009 100644 --- a/src/sentry/runner/commands/devserver.py +++ b/src/sentry/runner/commands/devserver.py @@ -329,10 +329,10 @@ def devserver( if kafka_consumers: with get_docker_client() as docker: containers = {c.name for c in docker.containers.list(filters={"status": "running"})} - if "sentry_zookeeper" not in containers or "sentry_kafka" not in containers: + if "sentry_kafka" not in containers: raise click.ClickException( f""" -Devserver is configured to start some kafka consumers, but Kafka + Zookeeper +Devserver is configured to start some kafka consumers, but Kafka don't seem to be running. The following consumers were intended to be started: {kafka_consumers} @@ -345,7 +345,7 @@ def devserver( SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" -and run `sentry devservices up kafka zookeeper`. +and run `sentry devservices up kafka`. Alternatively, run without --workers. """ diff --git a/src/sentry/runner/commands/devservices.py b/src/sentry/runner/commands/devservices.py index 6a263474ceec53..e884685ee3c242 100644 --- a/src/sentry/runner/commands/devservices.py +++ b/src/sentry/runner/commands/devservices.py @@ -235,6 +235,8 @@ def up( if services: for service in services: if service not in containers: + if service == "zookeeper": + continue click.secho( f"Service `{service}` is not known or not enabled.\n", err=True, diff --git a/tests/sentry/post_process_forwarder/test_post_process_forwarder.py b/tests/sentry/post_process_forwarder/test_post_process_forwarder.py index 436d714e15581c..6e08c2399cc15f 100644 --- a/tests/sentry/post_process_forwarder/test_post_process_forwarder.py +++ b/tests/sentry/post_process_forwarder/test_post_process_forwarder.py @@ -19,7 +19,6 @@ pytestmark = [requires_kafka] SENTRY_KAFKA_HOSTS = os.environ.get("SENTRY_KAFKA_HOSTS", "127.0.0.1:9092") -SENTRY_ZOOKEEPER_HOSTS = os.environ.get("SENTRY_ZOOKEEPER_HOSTS", "127.0.0.1:2181") settings.KAFKA_CLUSTERS["default"] = {"common": {"bootstrap.servers": SENTRY_KAFKA_HOSTS}} diff --git a/tests/tools/test_devservices_healthcheck.py b/tests/tools/test_devservices_healthcheck.py index 6e6ff4dd2ba348..1eb0039defe078 100644 --- a/tests/tools/test_devservices_healthcheck.py +++ b/tests/tools/test_devservices_healthcheck.py @@ -73,7 +73,7 @@ def test_postgres_running(mock_subprocess_run: mock.MagicMock) -> None: assert mock_subprocess_run.call_count == 2 -def test_kafka_zookeper_running(mock_subprocess_run: mock.MagicMock) -> None: +def test_kafka_running(mock_subprocess_run: mock.MagicMock) -> None: running = mock.Mock() running.stdout = "running\n" running.code = 0 @@ -84,15 +84,6 @@ def run( cmd_args: List[str], capture_output: bool = False, text: bool = False, check: bool = False ) -> mock.Mock: if cmd_args == ( - "docker", - "container", - "inspect", - "-f", - "{{.State.Status}}", - "sentry_zookeeper", - ): - return running - elif cmd_args == ( "docker", "container", "inspect", @@ -106,16 +97,8 @@ def run( "exec", "sentry_kafka", "kafka-topics", - "--zookeeper", - "sentry_zookeeper:2181", - "--list", - ) or ( - "docker", - "exec", - "sentry_kafka", - "kafka-topics", - "--zookeeper", - "127.0.0.1:2181", + "--bootstrap-server", + "127.0.0.1:9092", "--list", ): return healthcheck @@ -124,4 +107,4 @@ def run( mock_subprocess_run.side_effect = run check_health(["kafka"]) - assert mock_subprocess_run.call_count == 3 + assert mock_subprocess_run.call_count == 2 diff --git a/tools/devservices_healthcheck.py b/tools/devservices_healthcheck.py index 9898acf70dcf96..1337531e22c566 100644 --- a/tools/devservices_healthcheck.py +++ b/tools/devservices_healthcheck.py @@ -48,8 +48,8 @@ def check_kafka(): "exec", "sentry_kafka", "kafka-topics", - "--zookeeper", - "sentry_zookeeper:2181", + "--bootstrap-server", + "127.0.0.1:9092", "--list", ), check=True, @@ -75,12 +75,6 @@ def check_postgres() -> None: "sentry_kafka", os.getenv("NEED_KAFKA") == "true", check_kafka, - deps=["zookeeper"], - ), - "zookeeper": HealthCheck( - "zookeeper", - "sentry_zookeeper", - os.getenv("NEED_KAFKA") == "true", ), }