Introduce patches with external kafka

.pre-commit-config.yaml

@@ -1,3 +1,4 @@
+exclude: '\.patch$'
 repos:
 - repo: local
   hooks:
@@ -11,7 +12,6 @@ repos:
       args: [-w, -d]
       files: .*\.sh
       stages: [commit, merge-commit, push, manual]
-
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.3.0
   hooks:

optional-modifications/README.md

@@ -0,0 +1,41 @@
+# Optional Modifications
+
+Other than the default self-hosted Sentry installation, sometimes users
+can leverage their existing infrastructure to help them with limited
+resources. "Patches", or you might call this like a "plugin system", is
+a collection of patch files (see [man patch(1)](https://man7.org/linux/man-pages/man1/patch.1.html))
+that can be used with to modify the existing configuration to achieve
+the desired goal.
+
+> [!WARNING]
+> Beware that this is very experimental and might not work as expected.
+>
+> **Use it at your own risk!**
+
+## How to use patches
+
+The patches are designed mostly to help modify the existing
+configuration files. You will need to run the `install.sh` script
+afterwards.
+
+They should be run from the root directory. For example, the
+`external-kafka` patches should be run as:
+
+```bash
+patch < optional-modifications/patches/external-kafka/.env.patch
+patch < optional-modifications/patches/external-kafka/config.example.yml.patch
+patch < optional-modifications/patches/external-kafka/sentry.conf.example.py.patch
+patch < optional-modifications/patches/external-kafka/docker-compose.yml.patch
+```
+
+Some patches might require additional steps to be taken, like providing
+credentials or additional TLS certificates.
+
+## Official support
+
+Sentry employees are not obliged to provide dedicated support for
+patches, but they can help by providing information to move us forward.
+We encourage the community to contribute for any bug fixes or
+improvements.
+
+See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information.

optional-modifications/_lib.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+test "${DEBUG:-}" && set -x
+
+function patch_file() {
+  target="$1"
+  content="$2"
+  if [[ -f "$target" ]]; then
+    echo "🙈 Patching $target ..."
+    patch -p1 <"$content"
+  else
+    echo "🙊 Skipping $target ..."
+  fi
+}

optional-modifications/patches/external-kafka/.env.patch

@@ -0,0 +1,22 @@
+--- .env	2025-02-04 07:31:54.868049984 +0700
++++ .env.external-kafka	2025-05-15 08:33:15.442361105 +0700
+@@ -22,3 +22,19 @@
+ POSTGRES_MAX_CONNECTIONS=100
+ # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets
+ # SETUP_JS_SDK_ASSETS=1
++
++################################################################################
++## Additional External Kafka options
++################################################################################
++KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092
++# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
++KAFKA_SECURITY_PROTOCOL=PLAINTEXT
++# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable.
++# KAFKA_SASL_MECHANISM=PLAIN
++# KAFKA_SASL_USERNAME=username
++# KAFKA_SASL_PASSWORD=password
++# Put your certificates on the \`certificates/kafka\` directory.
++# The certificates will be mounted as read-only volumes.
++# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem
++# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem
++# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key

optional-modifications/patches/external-kafka/config.example.yml.patch

@@ -0,0 +1,19 @@
+--- relay/config.example.yml	2025-05-15 08:27:40.426876887 +0700
++++ relay/config.example.external-kafka.yml	2025-05-15 08:34:21.113311217 +0700
+@@ -7,8 +7,15 @@
+ processing:
+   enabled: true
+   kafka_config:
+-    - {name: "bootstrap.servers", value: "kafka:9092"}
++    - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"}
+     - {name: "message.max.bytes", value: 50000000} # 50MB
++    - {name: "security.protocol", value: "PLAINTEXT"}
++    - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used.
++    - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used.
++    - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used.
++    - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used.
++    - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used.
++    - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used.
+   redis: redis://redis:6379
+   geoip_path: "/geoip/GeoLite2-City.mmdb"
+

optional-modifications/patches/external-kafka/docker-compose.yml.patch

@@ -0,0 +1,142 @@
+--- docker-compose.yml	2025-03-17 13:32:15.120328412 +0700
++++ docker-compose.external-kafka.yml	2025-05-15 08:39:05.509951068 +0700
+@@ -26,8 +26,6 @@
+   depends_on:
+     redis:
+       <<: *depends_on-healthy
+-    kafka:
+-      <<: *depends_on-healthy
+     postgres:
+       <<: *depends_on-healthy
+     memcached:
+@@ -59,6 +57,14 @@
+     SENTRY_EVENT_RETENTION_DAYS:
+     SENTRY_MAIL_HOST:
+     SENTRY_MAX_EXTERNAL_SOURCEMAP_SIZE:
++    KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
++    KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
++    KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-}
++    KAFKA_SSL_CERTIFICATE_LOCATION: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
++    KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-}
++    KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
++    KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
++    KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
+   volumes:
+     - "sentry-data:/data"
+     - "./sentry:/etc/sentry"
+@@ -69,15 +75,20 @@
+   depends_on:
+     clickhouse:
+       <<: *depends_on-healthy
+-    kafka:
+-      <<: *depends_on-healthy
+     redis:
+       <<: *depends_on-healthy
+   image: "$SNUBA_IMAGE"
+   environment:
+     SNUBA_SETTINGS: self_hosted
+     CLICKHOUSE_HOST: clickhouse
+-    DEFAULT_BROKERS: "kafka:9092"
++    DEFAULT_BROKERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
++    KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
++    KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-}
++    KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
++    KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-}
++    KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
++    KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
++    KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
+     REDIS_HOST: redis
+     UWSGI_MAX_REQUESTS: "10000"
+     UWSGI_DISABLE_LOGGING: "true"
+@@ -140,43 +151,7 @@
+       POSTGRES_HOST_AUTH_METHOD: "trust"
+     volumes:
+       - "sentry-postgres:/var/lib/postgresql/data"
+-  kafka:
+-    <<: *restart_policy
+-    image: "confluentinc/cp-kafka:7.6.1"
+-    environment:
+-      # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example
+-      KAFKA_PROCESS_ROLES: "broker,controller"
+-      KAFKA_CONTROLLER_QUORUM_VOTERS: "[email protected]:29093"
+-      KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER"
+-      KAFKA_NODE_ID: "1001"
+-      CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk"
+-      KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093"
+-      KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092"
+-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT"
+-      KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT"
+-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1"
+-      KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1"
+-      KAFKA_LOG_RETENTION_HOURS: "24"
+-      KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust
+-      KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too
+-      CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
+-      KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN"
+-      KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN"
+-      KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN"
+-    ulimits:
+-      nofile:
+-        soft: 4096
+-        hard: 4096
+-    volumes:
+-      - "sentry-kafka:/var/lib/kafka/data"
+-      - "sentry-kafka-log:/var/lib/kafka/log"
+-      - "sentry-secrets:/etc/kafka/secrets"
+-    healthcheck:
+-      <<: *healthcheck_defaults
+-      test: ["CMD-SHELL", "nc -z localhost 9092"]
+-      interval: 10s
+-      timeout: 10s
+-      retries: 30
++  kafka: !reset null
+   clickhouse:
+     <<: *restart_policy
+     image: clickhouse-self-hosted-local
+@@ -475,9 +450,8 @@
+         read_only: true
+         source: ./geoip
+         target: /geoip
++      - ./certificates/kafka:/kafka-certificates:ro
+     depends_on:
+-      kafka:
+-        <<: *depends_on-healthy
+       redis:
+         <<: *depends_on-healthy
+       web:
+@@ -486,15 +460,21 @@
+     <<: *restart_policy
+     image: "$VROOM_IMAGE"
+     environment:
+-      SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092"
+-      SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092"
++      SENTRY_KAFKA_BROKERS_PROFILING: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
++      SENTRY_KAFKA_BROKERS_OCCURRENCES: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
++      SENTRY_KAFKA_BROKERS_SPANS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
++      SENTRY_KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT}
++      SENTRY_KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-}
++      SENTRY_KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-}
++      SENTRY_KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-}
++      SENTRY_KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-}
++      SENTRY_KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-}
++      SENTRY_KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-}
+       SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles
+       SENTRY_SNUBA_HOST: "http://snuba-api:1218"
+     volumes:
+       - sentry-vroom:/var/lib/sentry-profiles
+-    depends_on:
+-      kafka:
+-        <<: *depends_on-healthy
++      - ./certificates/kafka:/kafka-certificates:ro
+     profiles:
+       - feature-complete
+   vroom-cleanup:
+@@ -523,8 +503,6 @@
+     external: true
+   sentry-redis:
+     external: true
+-  sentry-kafka:
+-    external: true
+   sentry-clickhouse:
+     external: true
+   sentry-symbolicator:

optional-modifications/patches/external-kafka/sentry.conf.example.py.patch

@@ -0,0 +1,21 @@
+--- sentry/sentry.conf.example.py	2025-05-15 08:27:40.427876868 +0700
++++ sentry/sentry.conf.example.external-kafka.py	2025-05-15 08:32:44.845127931 +0700
+@@ -132,9 +132,17 @@
+ SENTRY_CACHE = "sentry.cache.redis.RedisCache"
+
+ DEFAULT_KAFKA_OPTIONS = {
+-    "bootstrap.servers": "kafka:9092",
++    "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"),
+     "message.max.bytes": 50000000,
+     "socket.timeout.ms": 1000,
++    "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL
++    # If you don't use any of these options below, you can remove them or set them to `None`.
++    "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable.
++    "sasl.username": env("KAFKA_SASL_USERNAME", None),
++    "sasl.password": env("KAFKA_SASL_PASSWORD", None),
++    "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used.
++    "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used.
++    "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used.
+ }
+
+ SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream"