From 019cdcf25e4d826d3053acc240b4353ef8489c2e Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 21 Aug 2023 09:38:17 -0700 Subject: [PATCH 01/13] redpanda --- .github/actions/setup-sentry/action.yml | 29 +++++---------- Makefile | 2 +- config/clickhouse/dist_config.xml | 8 ----- src/sentry/conf/server.py | 36 ++++++------------- src/sentry/runner/commands/devserver.py | 6 ++-- .../test_post_process_forwarder.py | 1 - tools/devservices_healthcheck.py | 13 +++---- 7 files changed, 27 insertions(+), 68 deletions(-) diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml index e34f7790c71af7..e2ec11766b2398 100644 --- a/.github/actions/setup-sentry/action.yml +++ b/.github/actions/setup-sentry/action.yml @@ -167,34 +167,21 @@ runs: services="${services} chartcuterie" fi - sentry devservices up $services & - - # TODO: Use devservices kafka. See https://github.com/getsentry/sentry/pull/20986#issuecomment-704510570 + # TODO: reuse devservices if [ "$NEED_KAFKA" = "true" ]; then - # This is *not* the production version. Unclear reason as to why this was chosen - # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L631 - docker run \ - --name sentry_zookeeper \ - -d --network host \ - -e ZOOKEEPER_CLIENT_PORT=2181 \ - confluentinc/cp-zookeeper:4.1.0 \ - & - - # This is the production version; do not change w/o changing it there as well - # https://github.com/getsentry/ops/blob/c823e62f930ecc6c97bb08898c71e49edc7232f6/cookbooks/getsentry/attributes/default.rb#L643 docker run \ --name sentry_kafka \ -d --network host \ - -e KAFKA_ZOOKEEPER_CONNECT=127.0.0.1:2181 \ - -e KAFKA_LISTENERS=INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092 \ - -e KAFKA_ADVERTISED_LISTENERS=INTERNAL://127.0.0.1:9093,EXTERNAL://127.0.0.1:9092 \ - -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT \ - -e KAFKA_INTER_BROKER_LISTENER_NAME=INTERNAL \ - -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ - confluentinc/cp-kafka:5.1.2 \ + redpandadata/redpanda:v22.3.23 \ + redpanda start \ + --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:9092 \ + --advertise-kafka-addr internal://127.0.0.1:9093,external://127.0.0.1:9092 \ + --mode dev-container \ & fi + sentry devservices up $services & + wait docker ps -a diff --git a/Makefile b/Makefile index 5b9a08f5e4471d..e94f10fe94dff8 100644 --- a/Makefile +++ b/Makefile @@ -159,7 +159,7 @@ backend-typing: @echo "" # JavaScript relay tests are meant to be run within Symbolicator test suite, as they are parametrized to verify both processing pipelines during migration process. -# Running Locally: Run `sentry devservices up kafka zookeeper` before starting these tests +# Running Locally: Run `sentry devservices up kafka` before starting these tests test-symbolicator: create-db @echo "--> Running symbolicator tests" pytest tests/symbolicator -vv --cov . --cov-report="xml:.artifacts/symbolicator.coverage.xml" diff --git a/config/clickhouse/dist_config.xml b/config/clickhouse/dist_config.xml index bfb90d93f6224d..a2f21d89412a7f 100644 --- a/config/clickhouse/dist_config.xml +++ b/config/clickhouse/dist_config.xml @@ -1,7 +1,6 @@ 0.3 - @@ -13,13 +12,6 @@ - - - sentry_zookeeper - 2181 - - - 1 1 diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 2a20e775fa3fe8..80963cf7cd5c07 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2560,34 +2560,20 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "entrypoint": "/cdc/postgres-entrypoint.sh" if settings.SENTRY_USE_CDC_DEV else None, } ), - "zookeeper": lambda settings, options: ( - { - # On Apple arm64, we upgrade to version 6.x to allow zookeeper to run properly on Apple's arm64 - # See details https://github.com/confluentinc/kafka-images/issues/80#issuecomment-855511438 - "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-zookeeper:6.2.0", - "environment": {"ZOOKEEPER_CLIENT_PORT": "2181"}, - "volumes": {"zookeeper_6": {"bind": "/var/lib/zookeeper/data"}}, - "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY, - } - ), "kafka": lambda settings, options: ( { - "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:6.2.0", + "image": "redpandadata/redpanda:v22.3.23", "ports": {"9092/tcp": 9092}, - "environment": { - "KAFKA_ZOOKEEPER_CONNECT": "{containers[zookeeper][name]}:2181", - "KAFKA_LISTENERS": "INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092", - "KAFKA_ADVERTISED_LISTENERS": "INTERNAL://{containers[kafka][name]}:9093,EXTERNAL://{containers[kafka]" - "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}", - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT", - "KAFKA_INTER_BROKER_LISTENER_NAME": "INTERNAL", - "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", - "KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS": "1", - "KAFKA_LOG_RETENTION_HOURS": "24", - "KAFKA_MESSAGE_MAX_BYTES": "50000000", - "KAFKA_MAX_REQUEST_SIZE": "50000000", - }, - "volumes": {"kafka_6": {"bind": "/var/lib/kafka/data"}}, + "command": [ + "redpanda", + "start", + "--kafka-addr", + "internal://0.0.0.0:9093,external://0.0.0.0:9092", + "--advertise-kafka-addr", + "internal://sentry_kafka:9093,external://127.0.0.1:9092", + "--mode", + "dev-container", + ], "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY or settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS diff --git a/src/sentry/runner/commands/devserver.py b/src/sentry/runner/commands/devserver.py index f87d8f213d9b80..1a5ad46b16ad62 100644 --- a/src/sentry/runner/commands/devserver.py +++ b/src/sentry/runner/commands/devserver.py @@ -329,10 +329,10 @@ def devserver( if kafka_consumers: with get_docker_client() as docker: containers = {c.name for c in docker.containers.list(filters={"status": "running"})} - if "sentry_zookeeper" not in containers or "sentry_kafka" not in containers: + if "sentry_kafka" not in containers: raise click.ClickException( f""" -Devserver is configured to start some kafka consumers, but Kafka + Zookeeper +Devserver is configured to start some kafka consumers, but Kafka don't seem to be running. The following consumers were intended to be started: {kafka_consumers} @@ -345,7 +345,7 @@ def devserver( SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" -and run `sentry devservices up kafka zookeeper`. +and run `sentry devservices up kafka`. Alternatively, run without --workers. """ diff --git a/tests/sentry/post_process_forwarder/test_post_process_forwarder.py b/tests/sentry/post_process_forwarder/test_post_process_forwarder.py index 213eb970b6acdc..3e5bef42450ec8 100644 --- a/tests/sentry/post_process_forwarder/test_post_process_forwarder.py +++ b/tests/sentry/post_process_forwarder/test_post_process_forwarder.py @@ -16,7 +16,6 @@ from sentry.utils.batching_kafka_consumer import wait_for_topics SENTRY_KAFKA_HOSTS = os.environ.get("SENTRY_KAFKA_HOSTS", "127.0.0.1:9092") -SENTRY_ZOOKEEPER_HOSTS = os.environ.get("SENTRY_ZOOKEEPER_HOSTS", "127.0.0.1:2181") settings.KAFKA_CLUSTERS["default"] = {"common": {"bootstrap.servers": SENTRY_KAFKA_HOSTS}} diff --git a/tools/devservices_healthcheck.py b/tools/devservices_healthcheck.py index 0874947d44f2a3..01616d0e9c747b 100644 --- a/tools/devservices_healthcheck.py +++ b/tools/devservices_healthcheck.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import subprocess import time @@ -29,16 +28,12 @@ def main() -> None: "docker", "exec", "sentry_kafka", - "kafka-topics", - "--zookeeper", - # TODO: sentry_zookeeper:2181 doesn't work in CI, but 127.0.0.1 doesn't work locally - "127.0.0.1:2181", - "--list", + "rpk", + "topic", + "list", ] - healthchecks = [postgres_healthcheck] - if os.getenv("NEED_KAFKA") == "true": - healthchecks.append(kafka_healthcheck) + healthchecks = [postgres_healthcheck, kafka_healthcheck] for check in healthchecks: run_cmd(check) From 6be6472cd7a05db36bd02093d5ba5eb6897479d8 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 21 Aug 2023 09:53:32 -0700 Subject: [PATCH 02/13] fix --- tools/devservices_healthcheck.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/devservices_healthcheck.py b/tools/devservices_healthcheck.py index 01616d0e9c747b..f599603301ae33 100644 --- a/tools/devservices_healthcheck.py +++ b/tools/devservices_healthcheck.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import subprocess import time @@ -33,7 +34,9 @@ def main() -> None: "list", ] - healthchecks = [postgres_healthcheck, kafka_healthcheck] + healthchecks = [postgres_healthcheck] + if os.getenv("NEED_KAFKA") == "true": + healthchecks.append(kafka_healthcheck) for check in healthchecks: run_cmd(check) From 7126921ab7896371b9543c463c592cb238691286 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 5 Sep 2023 17:14:36 -0700 Subject: [PATCH 03/13] try it out --- .github/actions/setup-sentry/action.yml | 17 +- .github/workflows/acceptance.yml | 147 -------- .github/workflows/bootstrap.yml | 17 - .../workflows/bump-sentry-in-getsentry.yml | 56 --- .github/workflows/bump-version.yml | 74 ---- .github/workflows/codeql.yml | 80 ----- .github/workflows/dependency-review.yml | 19 - .github/workflows/development-environment.yml | 139 -------- .../workflows/enforce-license-compliance.yml | 14 - .github/workflows/fast-revert.yml | 40 --- .github/workflows/frontend.yml | 225 ------------ .github/workflows/getsentry-dispatch.yml | 81 ----- .github/workflows/jest-balance.yml | 42 --- .github/workflows/label-pullrequest.yml | 69 ---- .github/workflows/lock.yml | 16 - .../meta-deploys-detect-change-type.yml | 33 -- .github/workflows/migrations.yml | 93 ----- .github/workflows/openapi-diff.yml | 60 ---- .github/workflows/openapi.yml | 73 ---- .github/workflows/pre-commit.yml | 83 ----- .../react-to-product-owners-yml-changes.yml | 29 -- .github/workflows/release.yml | 31 -- .github/workflows/self-hosted-e2e-tests.yml | 38 -- .github/workflows/sentry-pull-request-bot.yml | 74 ---- .github/workflows/shuffle-tests.yml | 55 --- .github/workflows/sync-labels.yml | 72 ---- .github/workflows/visual-snapshots.yml | 335 ------------------ src/sentry/conf/server.py | 35 +- tools/devservices_healthcheck.py | 7 +- 29 files changed, 38 insertions(+), 2016 deletions(-) delete mode 100644 .github/workflows/acceptance.yml delete mode 100644 .github/workflows/bootstrap.yml delete mode 100644 .github/workflows/bump-sentry-in-getsentry.yml delete mode 100644 .github/workflows/bump-version.yml delete mode 100644 .github/workflows/codeql.yml delete mode 100644 .github/workflows/dependency-review.yml delete mode 100644 .github/workflows/development-environment.yml delete mode 100644 .github/workflows/enforce-license-compliance.yml delete mode 100644 .github/workflows/fast-revert.yml delete mode 100644 .github/workflows/frontend.yml delete mode 100644 .github/workflows/getsentry-dispatch.yml delete mode 100644 .github/workflows/jest-balance.yml delete mode 100644 .github/workflows/label-pullrequest.yml delete mode 100644 .github/workflows/lock.yml delete mode 100644 .github/workflows/meta-deploys-detect-change-type.yml delete mode 100644 .github/workflows/migrations.yml delete mode 100644 .github/workflows/openapi-diff.yml delete mode 100644 .github/workflows/openapi.yml delete mode 100644 .github/workflows/pre-commit.yml delete mode 100644 .github/workflows/react-to-product-owners-yml-changes.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/self-hosted-e2e-tests.yml delete mode 100644 .github/workflows/sentry-pull-request-bot.yml delete mode 100644 .github/workflows/shuffle-tests.yml delete mode 100644 .github/workflows/sync-labels.yml delete mode 100644 .github/workflows/visual-snapshots.yml diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml index e2ec11766b2398..ac7603debabf3f 100644 --- a/.github/actions/setup-sentry/action.yml +++ b/.github/actions/setup-sentry/action.yml @@ -171,12 +171,17 @@ runs: if [ "$NEED_KAFKA" = "true" ]; then docker run \ --name sentry_kafka \ - -d --network host \ - redpandadata/redpanda:v22.3.23 \ - redpanda start \ - --kafka-addr internal://0.0.0.0:9093,external://0.0.0.0:9092 \ - --advertise-kafka-addr internal://127.0.0.1:9093,external://127.0.0.1:9092 \ - --mode dev-container \ + -e KAFKA_PROCESS_ROLES=broker,controller \ + -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:29093 \ + -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \ + -e KAFKA_NODE_ID=1 \ + -e CLUSTER_ID=MkU3OEVBNTcwNTJENDM2Qk \ + -e KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093 \ + -e KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://127.0.0.1:9093,EXTERNAL://127.0.0.1:9092 \ + -e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT \ + -e KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT \ + -e KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1 \ + confluentinc/cp-kafka:7.5.0 \ & fi diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml deleted file mode 100644 index 1c742ef58a558e..00000000000000 --- a/.github/workflows/acceptance.yml +++ /dev/null @@ -1,147 +0,0 @@ -# TODO(billy): this workflow has not been re-named from `acceptance` because -# Visual Snapshots compares against artifacts from the same workflow name (on main branch) -# We should rename this when we have a more finalized naming scheme. -# -# Also note that this name *MUST* match the filename because GHA -# only provides the workflow name (https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables#default-environment-variables) -# and GH APIs only support querying by workflow *FILENAME* (https://developer.github.com/v3/actions/workflows/#get-a-workflow) -name: acceptance -on: - push: - branches: - - master - - releases/** - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - files-changed: - name: detect what files changed - runs-on: ubuntu-20.04 - timeout-minutes: 3 - # Map a step output to a job output - outputs: - acceptance: ${{ steps.changes.outputs.acceptance }} - backend_all: ${{ steps.changes.outputs.backend_all }} - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Check for backend file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - acceptance: - if: needs.files-changed.outputs.acceptance == 'true' - needs: files-changed - name: acceptance - runs-on: ubuntu-20.04 - timeout-minutes: 30 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2, 3, 4] - pg-version: ['14'] - env: - # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. - MATRIX_INSTANCE_TOTAL: 5 - VISUAL_SNAPSHOT_ENABLE: 1 - TEST_GROUP_STRATEGY: roundrobin - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Step configurations - id: config - run: echo "webpack-path=.webpack_cache" >> "$GITHUB_OUTPUT" - - - name: webpack cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - with: - path: ${{ steps.config.outputs.webpack-path }} - key: ${{ runner.os }}-v2-webpack-cache-${{ hashFiles('webpack.config.ts') }} - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: webpack - env: - WEBPACK_CACHE_PATH: ${{ steps.config.outputs.webpack-path }} - SENTRY_INSTRUMENTATION: 1 - # this is fine to not have for forks, it shouldn't fail - SENTRY_WEBPACK_WEBHOOK_SECRET: ${{ secrets.SENTRY_WEBPACK_WEBHOOK_SECRET }} - run: | - yarn build-acceptance - - - name: Build chartcuterie configuration module - run: | - make build-chartcuterie-config - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - chartcuterie: true - pg-version: ${{ matrix.pg-version }} - - - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: | - mkdir -p ${{ steps.setup.outputs.acceptance-dir }} - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-mobile - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-tooltips - make run-acceptance - env: - PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }} - - # This job runs when FE or BE changes happen, however, we only upload coverage data for - # BE changes since it conflicts with codecov's carry forward functionality - # Upload coverage data even if running the tests step fails since - # it reduces large coverage fluctuations - - name: Handle artifacts - uses: ./.github/actions/artifacts - if: ${{ always() && needs.files-changed.outputs.backend_all == 'true' }} - with: - token: ${{ secrets.CODECOV_TOKEN }} - - acceptance-required-checks: - # this is a required check so we need this job to always run and report a status. - if: always() - name: Acceptance - needs: [acceptance, files-changed] - runs-on: ubuntu-20.04 - timeout-minutes: 3 - steps: - # If any jobs we depend on fail, we will fail since this check calls the - # visual snapshot service which creates the "Visual Snapshot" status, - # which is a required check. - # NOTE: A timeout is considered a failure - - name: Check for failures - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml deleted file mode 100644 index aaa1096d95747a..00000000000000 --- a/.github/workflows/bootstrap.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: bootstrap -on: - schedule: - # Run on Mondays in hopes to fix things before new engineers run the script - # It's far more likely that somethings regresses because there's new software released than - # by our code changes regressing it - - cron: '0 3 * * 1' - -jobs: - bootstrap-script: - name: bootstrap - runs-on: macos-13 - timeout-minutes: 30 - steps: - - name: Run bootstrap scripts - run: | - bash <(curl -s https://raw.githubusercontent.com/getsentry/bootstrap-sentry/master/bootstrap.sh) diff --git a/.github/workflows/bump-sentry-in-getsentry.yml b/.github/workflows/bump-sentry-in-getsentry.yml deleted file mode 100644 index 51643299296e44..00000000000000 --- a/.github/workflows/bump-sentry-in-getsentry.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: bump sentry in getsentry - -# Serializes the execution of this workflow. -concurrency: - group: ${{ github.workflow }} - -on: - push: - branches: - - master - -defaults: - run: - # the default default is: - # bash --noprofile --norc -eo pipefail {0} - shell: bash --noprofile --norc -eo pipefail -ux {0} - -jobs: - bump-sentry: - runs-on: ubuntu-20.04 - steps: - - name: checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # For getsentry/bin/bump-sentry, sentry needs to be at ../sentry relative to getsentry. - path: sentry - - - name: checkout getsentry - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - repository: 'getsentry/getsentry' - path: getsentry - # This PAT (Personal Access Token) belongs to getsentry-bot, - # who can write to getsentry and is SAML+SSO ready. - token: ${{ secrets.BUMP_SENTRY_TOKEN }} - - - name: bump-sentry ${{ github.sha }} - run: | - cd getsentry - - python -S -m bin.bump_sentry ${{ github.sha }} - - # If getsentry is pushed to by any other means while we were here, - # we won't be able to push. - for i in 1 2 3 4 5; do - git push origin master && exit 0 - # There's a little bit of network delay here that suffices - # as a small sleep. - git \ - -c user.name=getsentry-bot \ - -c user.email=bot@sentry.io \ - pull --rebase origin master - done - - # 5th and final attempt. - git push origin master diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml deleted file mode 100644 index 5c44b8ab4c1178..00000000000000 --- a/.github/workflows/bump-version.yml +++ /dev/null @@ -1,74 +0,0 @@ -on: - workflow_dispatch: - inputs: - package: - required: true - type: string - description: package name such as `sentry-arroyo` - version: - required: true - type: string - description: desired version such as `1.2.3`, or `latest` to pull the latest version from PyPI - pr_options: - type: string - default: '' - description: additional options for gh pr create, such as for asking for specific reviewers - - # for use in other (cron/scheduled) workflows to bump specific - # company-internal dependencies on a more aggressive schedule - workflow_call: - inputs: - package: - required: true - type: string - version: - required: true - type: string - pr_options: - type: string - default: '' - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - bump-version: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - token: ${{ secrets.BUMP_SENTRY_TOKEN }} - - run: | - set -euxo pipefail - - if [ "$VERSION" = latest ]; then - VERSION="$(curl -sL https://pypi.org/pypi/$PACKAGE/json | jq -r .info.version)" - fi - - git checkout -b "bot/bump-version/$PACKAGE/$VERSION" - - re="$(sed 's/[_-]/[_-]/g' <<< "$PACKAGE")" - sed -i "s/^$re==.*/$PACKAGE==$VERSION/g" -- requirements*.txt - sed -i "s/^$re>=.*/$PACKAGE>=$VERSION/g" -- requirements*.txt - - if git diff --exit-code; then - exit 0 - fi - - git \ - -c user.name=getsentry-bot \ - -c user.email='10587625+getsentry-bot@users.noreply.github.com' \ - commit \ - --all \ - --message "ref: bump $PACKAGE to $VERSION" \ - --message "Co-Authored-By: $SENDER <$SENDER_ID+$SENDER@users.noreply.github.com>" - - git push origin HEAD --quiet - - gh pr create --fill ${{ inputs.pr_options }} - env: - GH_TOKEN: ${{ secrets.BUMP_SENTRY_TOKEN }} - PACKAGE: ${{ inputs.package }} - VERSION: ${{ inputs.version }} - SENDER: ${{ github.event.sender.login }} - SENDER_ID: ${{ github.event.sender.id }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 8145557b79a644..00000000000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,80 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: 'CodeQL' - -on: - push: - branches: ['master'] - pull_request: - # The branches below must be a subset of the branches above - branches: ['master'] - paths: - - '**.js' - - '**.ejs' - - '**.tsx' - - '**.ts' - - '**.py' - - '!tests/**' - schedule: - - cron: '44 12 * * 1' - -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: ['javascript', 'python'] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 - with: - config-file: ./.github/codeql/codeql-config.yml - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: security-extended - - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language - - #- run: | - # make bootstrap - # make release - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 - with: - category: '/language:${{matrix.language}}' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml deleted file mode 100644 index 2f1c34516a4c74..00000000000000 --- a/.github/workflows/dependency-review.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: 'Dependency Review' -on: - pull_request: - branches: ['master'] - -permissions: - contents: read - -jobs: - dependency-review: - runs-on: ubuntu-latest - steps: - - name: 'Checkout Repository' - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 - - name: Dependency Review - uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94 # v3.0.3 - with: - # Possible values: "critical", "high", "moderate", "low" - fail-on-severity: high diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml deleted file mode 100644 index 3689938c729dc0..00000000000000 --- a/.github/workflows/development-environment.yml +++ /dev/null @@ -1,139 +0,0 @@ -name: dev env -on: - pull_request: - paths: - - '.pre-commit-config.yaml' - - 'Makefile' - - '.github/workflows/development-environment.yml' - - 'requirements-*.txt' - - '.python-version' - - '.envrc' - - 'Brewfile' - - 'scripts/**' - - 'tools/**' - - 'src/sentry/runner/commands/devserver.py' - - 'src/sentry/runner/commands/devservices.py' - - 'bin/load-mocks' - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - # This workflow is optimized to test the dev env with dev services as fast as possible - # The bootstrap workflow (see last workflow) tests the experience of first time engineers - docker-setup: - name: Docker set up - runs-on: macos-13 - timeout-minutes: 40 - env: - # Make the environment more similar to what Mac defaults to - SHELL: /bin/zsh - - steps: - - name: Checkout sentry - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Set up - id: info - run: | - echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" - - # Trick for unattended Docker installations - # https://github.com/docker/for-mac/issues/2359#issuecomment-943131345 - # NOTE: This can sometimes take up to 10 minutes - - name: Install Docker - run: | - # d4m 4.11+ never starts: https://github.com/docker/for-mac/issues/6450 - curl -o /tmp/docker.rb https://raw.githubusercontent.com/Homebrew/homebrew-cask/fe866ec0765de141599745f03e215452db7f511b/Casks/docker.rb - HOMEBREW_NO_AUTO_UPDATE=1 brew install -v --HEAD -s /tmp/docker.rb - sudo /Applications/Docker.app/Contents/MacOS/Docker --unattended --install-privileged-components - open -a /Applications/Docker.app --args --unattended --accept-license - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - # This handles Python's cache - - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 - with: - python-version: 3.8.16 - cache-dependency-path: requirements-dev-frozen.txt - - # This tests starting up the dev services, loading mocks and pre-commit installation - # This can take over 15 minutes - - name: make bootstrap - # GHA pythons are miscompiled and report macos 10.16 - env: - SYSTEM_VERSION_COMPAT: 0 - run: make bootstrap - - # The pyenv set up takes long, thus, separating it into its own - pyenv-setup: - name: pyenv set up - runs-on: macos-13 - timeout-minutes: 25 - env: - # This is to support this code https://github.com/getsentry/sentry/blob/47b837a69c38e190a7555de81e6d7d216498b854/scripts/pyenv_setup.sh#L17-L40 - SHELL: /bin/zsh - - steps: - - name: Checkout sentry - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Install pyenv - run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv - - # Only after we source ~/.zprofile that the right Python will be selected - - name: Set up pyenv - run: | - make setup-pyenv - [[ $(which python) != "${HOME}/.pyenv/shims/python" ]] - source ~/.zprofile - [[ $(which python) == "${HOME}/.pyenv/shims/python" ]] - [[ $(python -V) == "Python $(cat .python-version)" ]] - python -m venv .venv - source .venv/bin/activate - [[ $(python -V) == "Python $(cat .python-version)" ]] - - tools-tests: - runs-on: ubuntu-20.04 - timeout-minutes: 5 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 - with: - python-version: 3.8.16 - cache-dependency-path: | - requirements-dev.txt - requirements-dev-frozen.txt - install-cmd: pip install -r requirements-dev.txt -c requirements-dev-frozen.txt - - name: run tests - run: make test-tools - - name: Handle artifacts - uses: ./.github/actions/artifacts - with: - token: ${{ secrets.CODECOV_TOKEN }} - - # We don't yet test the bootstrap step - # https://github.com/getsentry/bootstrap-sentry/blob/7af557be84920dd587e48613dbc308c937bc0e08/bootstrap.sh#L618-L619 - bootstrap-script: - runs-on: macos-13 - timeout-minutes: 40 - env: - # Make the environment more similar to what Mac defaults to - SHELL: /bin/zsh - - steps: - - name: Run bootstrap code - env: - STRAP_DEBUG: 1 - # This ensures that the bootstrap code will test against this branch - CI_CHECKOUT_BRANCH: ${{ github.head_ref || github.sha }} - run: | - bash <(curl -s https://raw.githubusercontent.com/getsentry/bootstrap-sentry/main/bootstrap.sh) diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml deleted file mode 100644 index 3c7ec30fa0aa98..00000000000000 --- a/.github/workflows/enforce-license-compliance.yml +++ /dev/null @@ -1,14 +0,0 @@ -name: Enforce License Compliance - -on: - pull_request: - branches: [master, main] - -jobs: - enforce-license-compliance: - runs-on: ubuntu-latest - steps: - - name: 'Enforce License Compliance' - uses: getsentry/action-enforce-license-compliance@520fb640b532c27b4da9644116d102b579ef84f5 # main - with: - fossa_api_key: ${{ secrets.FOSSA_API_KEY }} diff --git a/.github/workflows/fast-revert.yml b/.github/workflows/fast-revert.yml deleted file mode 100644 index c87ecee323a63b..00000000000000 --- a/.github/workflows/fast-revert.yml +++ /dev/null @@ -1,40 +0,0 @@ -on: - pull_request_target: - types: [labeled] - workflow_dispatch: - inputs: - pr: - required: true - description: pr number - co_authored_by: - required: true - description: '`name ` for triggering user' - -# disable all permissions -- we use the PAT's permissions instead -permissions: {} - -jobs: - revert: - runs-on: ubuntu-latest - if: | - github.event_name == 'workflow_dispatch' || github.event.label.name == 'Trigger: Revert' - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - token: ${{ secrets.BUMP_SENTRY_TOKEN }} - - uses: getsentry/action-fast-revert@35b4b6c1f8f91b5911159568b3b15e531b5b8174 # v2.0.1 - with: - pr: ${{ github.event.number || github.event.inputs.pr }} - co_authored_by: ${{ github.event.inputs.co_authored_by || format('{0} <{1}+{0}@users.noreply.github.com>', github.event.sender.login, github.event.sender.id) }} - committer_name: getsentry-bot - committer_email: bot@sentry.io - token: ${{ secrets.BUMP_SENTRY_TOKEN }} - - name: comment on failure - run: | - curl \ - --silent \ - -X POST \ - -H 'Authorization: token ${{ secrets.BUMP_SENTRY_TOKEN }}' \ - -d'{"body": "revert failed (conflict? already reverted?) -- [check the logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"}' \ - https://api.github.com/repositories/${{ github.event.repository.id }}/issues/${{ github.event.number || github.event.inputs.pr }}/comments - if: failure() diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml deleted file mode 100644 index e04529345b68e8..00000000000000 --- a/.github/workflows/frontend.yml +++ /dev/null @@ -1,225 +0,0 @@ -name: frontend - -on: - push: - branches: - - master - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - files-changed: - name: detect what files changed - runs-on: ubuntu-20.04 - timeout-minutes: 3 - # Map a step output to a job output - outputs: - eslint_config: ${{ steps.changes.outputs.eslint_config }} - frontend: ${{ steps.changes.outputs.frontend_all }} - frontend_components_modified_lintable: ${{ steps.changes.outputs.frontend_components_modified_lintable }} - frontend_components_modified_lintable_files: ${{ steps.changes.outputs.frontend_components_modified_lintable_files }} - frontend_modified_lintable_files: ${{ steps.changes.outputs.frontend_modified_lintable_files }} - yarn_lockfile: ${{ steps.changes.outputs.yarn_lockfile }} - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Check for frontend file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - list-files: shell - - typescript-and-lint: - if: needs.files-changed.outputs.frontend == 'true' - needs: files-changed - name: typescript and lint - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Internal github app token - id: token - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - continue-on-error: true - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Install dependencies - id: dependencies - run: yarn install --frozen-lockfile - - # Setup custom tsc matcher, see https://github.com/actions/setup-node/issues/97 - - name: setup matchers - run: | - echo "::remove-matcher owner=masters::" - echo "::add-matcher::.github/tsc.json" - echo "::add-matcher::.github/eslint-stylish.json" - - - name: eslint logic - id: eslint - if: (github.ref == 'refs/heads/master' || needs.files-changed.outputs.eslint_config == 'true' || needs.files-changed.outputs.yarn_lockfile == 'true') - run: echo "all-files=true" >> "$GITHUB_OUTPUT" - - # Lint entire frontend if: - # - this is on main branch - # - eslint configuration in repo has changed - # - yarn lockfile has changed (i.e. we bump our eslint config) - - name: eslint - if: steps.eslint.outputs.all-files == 'true' - env: - # Run relax config on main branch (and stricter config for changed files) - SENTRY_ESLINT_RELAXED: 1 - run: | - yarn lint - yarn lint:css - - # Otherwise... only lint modified files - # Note `eslint --fix` will not fail when it auto fixes files - - name: eslint (changed files only) - if: steps.eslint.outputs.all-files != 'true' - run: | - yarn eslint --fix ${{ needs.files-changed.outputs.frontend_modified_lintable_files }} - - - name: stylelint (changed files only) - if: github.ref != 'refs/heads/master' && needs.files-changed.outputs.frontend_components_modified_lintable == 'true' - run: | - yarn stylelint ${{ needs.files-changed.outputs.frontend_components_modified_lintable_files }} - - # Check (and error) for dirty working tree for forks - # Reason being we need a different token to auto commit changes and - # forks do not have access to said token - - name: Check for dirty git working tree (forks) - if: steps.token.outcome != 'success' && github.ref != 'refs/heads/master' - run: | - git diff --quiet || (echo '::error ::lint produced file changes, run linter locally and try again' && exit 1) - - # If working tree is dirty, commit and update if we have a token - - name: Commit any eslint fixed files - if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' - uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 - with: - github-token: ${{ steps.token.outputs.token }} - message: ':hammer_and_wrench: apply eslint style fixes' - - - name: tsc - id: tsc - if: steps.dependencies.outcome == 'success' - run: yarn tsc -p config/tsconfig.ci.json - - frontend-jest-tests: - if: needs.files-changed.outputs.frontend == 'true' - needs: files-changed - name: Jest - # If you change the runs-on image, you must also change the runner in jest-balance.yml - # so that the balancer runs in the same environment as the tests. - runs-on: ubuntu-20.04 - timeout-minutes: 30 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update CI_NODE_TOTAL. - instance: [0, 1, 2, 3] - - env: - VISUAL_HTML_ENABLE: 1 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: Build CSS - run: NODE_ENV=production yarn build-css - - - name: jest - env: - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} - # XXX: CI_NODE_TOTAL must be hardcoded to the length of strategy.matrix.instance. - # Otherwise, if there are other things in the matrix, using strategy.job-total - # wouldn't be correct. Also, if this increases, make sure to also increase - # `flags.frontend.after_n_builds` in `codecov.yml`. - CI_NODE_TOTAL: 4 - CI_NODE_INDEX: ${{ matrix.instance }} - # Disable testing-library from printing out any of of the DOM to - # stdout. No one actually looks through this in CI, they're just - # going to run it locally. - # - # This quiets up the logs quite a bit. - DEBUG_PRINT_LIMIT: 0 - run: | - JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit - - - name: Save HTML artifacts - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 - with: - retention-days: 14 - name: jest-html - path: .artifacts/visual-snapshots/jest - - - name: Create Images from HTML - uses: getsentry/action-html-to-image@dc153dae538e6e1138f77156d8e62e3b2b897f41 # main - with: - base-path: .artifacts/visual-snapshots/jest - css-path: src/sentry/static/sentry/dist/entrypoints/sentry.css - - # We only upload coverage data for FE changes since it conflicts with - # codecov's carry forward functionality. - # Upload coverage data even if running the tests step fails since - # it reduces large coverage fluctuations. - - name: Handle artifacts - uses: ./.github/actions/artifacts - if: ${{ always() && needs.files-changed.outputs.frontend_all == 'true' }} - with: - files: .artifacts/coverage/* - type: frontend - token: ${{ secrets.CODECOV_TOKEN }} - - # This check runs once all dependant jobs have passed - # It symbolizes that all required Frontend checks have succesfully passed (Or skipped) - # This check is the only required Github check - frontend-required-check: - needs: [files-changed, frontend-jest-tests, typescript-and-lint] - name: Frontend - # This is necessary since a failed/skipped dependent job would cause this job to be skipped - if: always() - runs-on: ubuntu-20.04 - steps: - # If any jobs we depend on fail, we will fail since this is a required check - # NOTE: A timeout is considered a failure - - name: Check for failures - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/getsentry-dispatch.yml b/.github/workflows/getsentry-dispatch.yml deleted file mode 100644 index 3c10409af48957..00000000000000 --- a/.github/workflows/getsentry-dispatch.yml +++ /dev/null @@ -1,81 +0,0 @@ -# Dispatch a request to getsentry to run getsentry test suites -name: getsentry dispatcher - -on: - # XXX: We are using `pull_request_target` instead of `pull_request` because we want - # this to run on forks. It allows forks to access secrets safely by - # only running workflows from the main branch. Prefer to use `pull_request` when possible. - # - # See https://github.com/getsentry/sentry/pull/21600 for more details - pull_request_target: - types: [labeled, opened, reopened, synchronize] - -# disable all other special privileges -permissions: - # needed for `actions/checkout` to clone the code - contents: read - # needed to remove the pull-request label - pull-requests: write - -jobs: - dispatch: - if: "github.event.action != 'labeled' || github.event.label.name == 'Trigger: getsentry tests'" - name: getsentry dispatch - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - persist-credentials: false - - - name: permissions - run: | - python3 -uS .github/workflows/scripts/getsentry-dispatch-setup \ - --repo-id ${{ github.event.repository.id }} \ - --pr ${{ github.event.number }} \ - --event ${{ github.event.action }} \ - --username "$ARG_USERNAME" \ - --label-names "$ARG_LABEL_NAMES" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # these can contain special characters - ARG_USERNAME: ${{ github.event.pull_request.user.login }} - ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} - - - name: Check for file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: getsentry token - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - id: getsentry - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - name: Wait for PR merge commit - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - id: mergecommit - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit`).waitForMergeCommit({ - github, - context, - core, - }); - - - name: Dispatch getsentry tests - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/getsentry-dispatch`).dispatch({ - github, - context, - core, - mergeCommitSha: '${{ steps.mergecommit.outputs.mergeCommitSha }}', - fileChanges: ${{ toJson(steps.changes.outputs) }}, - }); diff --git a/.github/workflows/jest-balance.yml b/.github/workflows/jest-balance.yml deleted file mode 100644 index 2fa4c009d3c123..00000000000000 --- a/.github/workflows/jest-balance.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: jest balancer -on: - workflow_dispatch: - schedule: - - cron: '0 0 1,15 * *' -jobs: - jest-balance: - # Buckle up, this may take a while - timeout-minutes: 60 - # Make sure this matches the runner that runs frontend tests - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Install dependencies - run: yarn install --frozen-lockfile - - - name: Build CSS - run: NODE_ENV=production yarn build-css - - - name: jest balancer - env: - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} - run: JEST_TEST_BALANCER=1 yarn test-ci - - - name: Create Pull Request - uses: peter-evans/create-pull-request@b4d51739f96fca8047ad065eccef63442d8e99f7 # v4.2.0 - with: - token: ${{ secrets.BUMP_SENTRY_TOKEN }} - add-paths: | - tests/js/test-balancer/jest-balance.json - commit-message: 'ci(jest): regenerate jest-balance.json' - branch: 'ci/jest/rebalance-tests' - delete-branch: true - base: master - title: 'ci(jest): regenerate jest-balance.json' - body: | - This PR was auto-generated - it updates the \`jest-balance.json\` file with new test run data from CI. diff --git a/.github/workflows/label-pullrequest.yml b/.github/workflows/label-pullrequest.yml deleted file mode 100644 index 4729a0fcde557e..00000000000000 --- a/.github/workflows/label-pullrequest.yml +++ /dev/null @@ -1,69 +0,0 @@ -# Adds labels to pull requests for the type of change the PR makes -name: meta(labels) - -on: - pull_request_target: - -jobs: - label-pullrequest: - permissions: - contents: read - pull-requests: write - name: labels pull requests (frontend / backend) - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - persist-credentials: false - - - name: Check for file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Add frontend label - uses: getsentry/action-add-labels@54d0cba498c1eaf8bd34985d715504d1b6e2935f - if: steps.changes.outputs.frontend_src == 'true' - with: - labels: 'Scope: Frontend' - - - name: Add backend label - uses: getsentry/action-add-labels@54d0cba498c1eaf8bd34985d715504d1b6e2935f - if: steps.changes.outputs.backend_src == 'true' - with: - labels: 'Scope: Backend' - - - name: Find previous frontend/backend warning comment - uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: '' - - - name: Add frontend/backend warning comment - uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 - if: > - steps.changes.outputs.frontend_src == 'true' && - steps.changes.outputs.backend_src == 'true' && - steps.fc.outputs.comment-id == 0 - with: - issue-number: ${{ github.event.pull_request.number }} - body: > - - - 🚨 **Warning:** This pull request contains Frontend and Backend changes! - - - It's discouraged to make changes to Sentry's Frontend and Backend - in a single pull request. The Frontend and Backend are **not** - atomically deployed. If the changes are interdependent of each - other, they **must** be separated into two pull requests and be made - forward or backwards compatible, such that the Backend or Frontend - can be safely deployed independently. - - - Have questions? Please ask in the [`#discuss-dev-infra` - channel](https://app.slack.com/client/T024ZCV9U/CTJL7358X). diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml deleted file mode 100644 index bd0df9cd60adb0..00000000000000 --- a/.github/workflows/lock.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: 'lock closed issues/PRs' -on: - schedule: - - cron: '* */12 * * *' - workflow_dispatch: -jobs: - lock: - runs-on: ubuntu-latest - steps: - - uses: dessant/lock-threads@be8aa5be94131386884a6da4189effda9b14aa21 # v4.0.1 - with: - github-token: ${{ github.token }} - issue-inactive-days: 15 - issue-lock-reason: '' - pr-inactive-days: 15 - pr-lock-reason: '' diff --git a/.github/workflows/meta-deploys-detect-change-type.yml b/.github/workflows/meta-deploys-detect-change-type.yml deleted file mode 100644 index 66e74bf1881faf..00000000000000 --- a/.github/workflows/meta-deploys-detect-change-type.yml +++ /dev/null @@ -1,33 +0,0 @@ -# Add a GitHub Check to commits in "master" that specifies the "type" of files that were changed -# ex: "only frontend", "only backed", or "fullstack" -name: meta(deploy) - -on: - push: - branches: - - master - -jobs: - files-changed: - name: detect what files changed - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Check for file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - list-files: shell - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Create GitHub job - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - with: - script: | - require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/deploy`).updateChangeType({ - github, - context, - fileChanges: ${{ toJson(steps.changes.outputs) }} - }); diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml deleted file mode 100644 index ae32727703b8e2..00000000000000 --- a/.github/workflows/migrations.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: migrations -on: pull_request - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -defaults: - run: - # the default default is: - # bash --noprofile --norc -eo pipefail {0} - shell: bash --noprofile --norc -eo pipefail -ux {0} - -jobs: - did-migration-change: - name: check if any migration changes - runs-on: ubuntu-20.04 - timeout-minutes: 3 - # Map a step output to a job output - outputs: - added: ${{ steps.changes.outputs.migrations_added }} - modified: ${{ steps.changes.outputs.migrations_modified }} - steps: - - name: Checkout sentry - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Match migration files - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - modified-migration: - name: check if modified migration - runs-on: ubuntu-20.04 - timeout-minutes: 4 - needs: did-migration-change - if: needs.did-migration-change.outputs.modified == 'true' - - steps: - - name: Failure because of modified migration - run: | - echo "If you have a valid reason to modify a migration please get approval" - echo "from @getsentry/owners-migrations." && exit 1 - - sql: - name: Generate SQL - runs-on: ubuntu-20.04 - timeout-minutes: 8 - strategy: - matrix: - pg-version: ['14'] - - needs: did-migration-change - if: needs.did-migration-change.outputs.added == 'true' - - steps: - # Checkout master to run all merged migrations. - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - ref: master - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - with: - pg-version: ${{ matrix.pg-version }} - - - name: Apply migrations - run: | - sentry upgrade --noinput - - # Checkout the current ref - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - clean: false - - - name: Get changed migration files - id: file - run: | - echo 'added<> "$GITHUB_OUTPUT" - git diff --diff-filter=A --name-only origin/master HEAD | grep 'src/sentry/migrations/' >> "$GITHUB_OUTPUT" - echo 'EOF' >> "$GITHUB_OUTPUT" - - - name: Generate SQL for migration - uses: getsentry/action-migrations@f1dc34590460c0fe06ec11c00fec6c16a2159977 # main - env: - SENTRY_LOG_LEVEL: ERROR - with: - githubToken: ${{ secrets.GITHUB_TOKEN }} - migration: ${{ steps.file.outputs.added }} diff --git a/.github/workflows/openapi-diff.yml b/.github/workflows/openapi-diff.yml deleted file mode 100644 index b02f0272cfdf6d..00000000000000 --- a/.github/workflows/openapi-diff.yml +++ /dev/null @@ -1,60 +0,0 @@ -name: openapi-diff -on: - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - check-diff: - name: build api - runs-on: ubuntu-20.04 - timeout-minutes: 90 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - name: Check for python file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - if: steps.changes.outputs.api_docs == 'true' - - - name: Checkout getsentry/sentry-api-schema - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - if: steps.changes.outputs.api_docs == 'true' - with: - ref: 'main' - repository: getsentry/sentry-api-schema - path: sentry-api-schema - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - if: steps.changes.outputs.api_docs == 'true' - - - name: Build OpenAPI Derefed JSON - if: steps.changes.outputs.api_docs == 'true' - # install ts-node for ts build scripts to execute properly without potentially installing - # conflicting deps when running scripts locally - # see: https://github.com/getsentry/sentry/pull/32328/files - run: | - yarn add ts-node && make build-api-docs - - - name: Compare OpenAPI Derefed JSON - if: steps.changes.outputs.api_docs == 'true' - run: | - npx json-diff@0.5.4 --color sentry-api-schema/openapi-derefed.json tests/apidocs/openapi-derefed.json diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml deleted file mode 100644 index f5d0851136bb8a..00000000000000 --- a/.github/workflows/openapi.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: openapi -on: - push: - branches: - - master - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - build_and_deref_json: - runs-on: ubuntu-20.04 - timeout-minutes: 90 - steps: - - name: Getsentry Token - id: getsentry - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - name: Check for python file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - if: steps.changes.outputs.api_docs == 'true' - - - name: Checkout getsentry/sentry-api-schema - if: steps.changes.outputs.api_docs == 'true' - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - ref: 'main' - repository: getsentry/sentry-api-schema - path: sentry-api-schema - token: ${{ steps.getsentry.outputs.token }} - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - if: steps.changes.outputs.api_docs == 'true' - - - name: Build OpenAPI Derefed JSON - if: steps.changes.outputs.api_docs == 'true' - # install ts-node for ts build scripts to execute properly without potentially installing - # conflicting deps when running scripts locally - # see: https://github.com/getsentry/sentry/pull/32328/files - run: | - yarn add ts-node && make build-api-docs - - - name: Copy artifact into getsentry/sentry-api-schema - if: steps.changes.outputs.api_docs == 'true' - run: | - cp tests/apidocs/openapi-derefed.json sentry-api-schema - - - name: Git Commit & Push - uses: stefanzweifel/git-auto-commit-action@0049e3fa4059ca715255fbbcb7dea4516f02ce0a # v4.15.3 - if: steps.changes.outputs.api_docs == 'true' - with: - repository: sentry-api-schema - branch: main - commit_message: Generated - commit_user_email: bot@getsentry.com - commit_user_name: openapi-getsentry-bot diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml deleted file mode 100644 index 51178fe17467fa..00000000000000 --- a/.github/workflows/pre-commit.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: pre-commit - -on: - push: - branches: - - master - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -defaults: - run: - # the default default is: - # bash --noprofile --norc -eo pipefail {0} - shell: bash --noprofile --norc -eo pipefail -ux {0} - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - lint: - name: pre-commit lint - runs-on: ubuntu-20.04 - timeout-minutes: 10 - steps: - - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - id: token - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - name: Get changed files - id: changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - with: - token: ${{ steps.token.outputs.token }} - - # Enable listing of files matching each filter. - # Paths to files will be available in `${FILTER_NAME}_files` output variable. - list-files: json - - # It doesn't make sense to lint deleted files. - # Therefore we specify we are only interested in added or modified files. - filters: | - all: - - added|modified: '**/*' - - - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 - with: - python-version: 3.8.16 - cache-dependency-path: | - requirements-dev.txt - requirements-dev-frozen.txt - install-cmd: pip install -r requirements-dev.txt -c requirements-dev-frozen.txt - - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - with: - path: ~/.cache/pre-commit - key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} - - name: Setup pre-commit - # We don't use make setup-git because we're only interested in installing - # requirements-dev.txt as a fast path. - # We don't need pre-commit install --install-hooks since we're just interested - # in running the hooks. - run: | - pre-commit install-hooks - - - name: Run pre-commit on PR commits - run: | - jq '.[]' --raw-output <<< '${{steps.changes.outputs.all_files}}' | - # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. - xargs pre-commit run --files - - - name: Apply any pre-commit fixed files - if: startsWith(github.ref, 'refs/pull') - uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 - with: - github-token: ${{ steps.token.outputs.token }} - message: ':hammer_and_wrench: apply pre-commit fixes' diff --git a/.github/workflows/react-to-product-owners-yml-changes.yml b/.github/workflows/react-to-product-owners-yml-changes.yml deleted file mode 100644 index 3d49f0656ced53..00000000000000 --- a/.github/workflows/react-to-product-owners-yml-changes.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: React to product-owners.yml changes -on: - # This could be run manually, but the general expectation is that this fires - # from GHA in getsentry/security-as-code on changes there. - - workflow_dispatch: -jobs: - release: - runs-on: ubuntu-latest - name: React to product-owners.yml changes - steps: - - uses: actions/checkout@v2 - - - uses: getsentry/action-setup-venv@v1.0.5 - with: - python-version: 3.11.3 - - - name: Get an auth token - id: token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - name: React to product-owners.yml changes - shell: bash - env: - GITHUB_TOKEN: ${{ steps.token.outputs.token }} - run: ./bin/react-to-product-owners-yml-changes.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index afb2438bd6a239..00000000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,31 +0,0 @@ -name: Release -on: - workflow_dispatch: - inputs: - version: - description: Version to release (optional) - required: false - force: - description: Force a release even when there are release-blockers (optional) - required: false - schedule: - # We want the release to be at 9-10am Pacific Time - # We also want it to be 1 hour before the self-hosted release - - cron: '0 17 15 * *' -jobs: - release: - runs-on: ubuntu-latest - name: 'Release a new version' - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - token: ${{ secrets.GH_RELEASE_PAT }} - fetch-depth: 0 - - name: Prepare release - uses: getsentry/action-prepare-release@d2cc2db3db92bc5b79a90c316f588f2b13626a2b # v1.5.6 - env: - GITHUB_TOKEN: ${{ secrets.GH_RELEASE_PAT }} - with: - version: ${{ github.event.inputs.version }} - force: ${{ github.event.inputs.force }} - calver: true diff --git a/.github/workflows/self-hosted-e2e-tests.yml b/.github/workflows/self-hosted-e2e-tests.yml deleted file mode 100644 index efb5a468314aae..00000000000000 --- a/.github/workflows/self-hosted-e2e-tests.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Self-hosted Sentry end to end tests -on: - push: - branches: - - master - - releases/** - pull_request: - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - self-hosted-end-to-end: - name: self-hosted tests - runs-on: ubuntu-20.04 - # temporary, remove once we are confident the action is working - continue-on-error: true - timeout-minutes: 30 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Check for backend file changes - uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 - id: changes - with: - token: ${{ github.token }} - filters: .github/file-filters.yml - - name: Run Sentry self-hosted e2e CI - if: steps.changes.outputs.backend_all == 'true' - uses: getsentry/action-self-hosted-e2e-tests@3f39ef5bbb432f3f1d163f7e312e8425de2244c7 - with: - project_name: sentry - image_url: us.gcr.io/sentryio/sentry:${{ github.event.pull_request.head.sha || github.sha }} - docker_repo: getsentry/sentry - docker_password: ${{ secrets.DOCKER_HUB_RW_TOKEN }} diff --git a/.github/workflows/sentry-pull-request-bot.yml b/.github/workflows/sentry-pull-request-bot.yml deleted file mode 100644 index 432a66462ae43d..00000000000000 --- a/.github/workflows/sentry-pull-request-bot.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: sentry pull request bot - -# Note this event happens on Issue comments AND PR comments, -# we make sure that we only respond to PR comments. -on: - issue_comment: - types: [created, edited] - pull_request: - types: [opened, edited] - -jobs: - # TODO(billy): Move this into an external action as we add more functionality - test-getsentry: - name: test getsentry - runs-on: ubuntu-20.04 - - # Ensure this bot only responds for pull requests and only for the main repository - if: >- - (github.event.issue.pull_request.url != '' || github.event.pull_request.id != '') && - (contains(github.event.comment.body, '#test-getsentry') || contains(github.event.pull_request.body, '#test-getsentry')) && - github.repository == 'getsentry/sentry' - - steps: - - name: Check getsentry membership - id: org - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - with: - script: | - try { - const result = await github.rest.orgs.checkMembershipForUser({ - org: 'getsentry', - username: context.payload.sender.login, - }) - return result.status == 204; - } catch { - return false; - } - - - name: Fetch getsentry token - if: steps.org.outputs.result == 'true' - id: getsentry - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - - name: Wait for PR merge commit - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - id: mergecommit - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit`).waitForMergeCommit({ - github, - context, - core, - }); - - - name: Dispatch getsentry tests - if: steps.org.outputs.result == 'true' - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - with: - github-token: ${{ steps.getsentry.outputs.token }} - script: | - github.rest.actions.createWorkflowDispatch({ - owner: 'getsentry', - repo: 'getsentry', - workflow_id: 'acceptance.yml', - ref: 'master', - inputs: { - 'sentry-sha': '${{ steps.mergecommit.outputs.mergeCommitSha }}', - 'sentry-pr-sha': '${{ github.event.pull_request.head.sha }}', - } - }) diff --git a/.github/workflows/shuffle-tests.yml b/.github/workflows/shuffle-tests.yml deleted file mode 100644 index 6c4fc0d470abb3..00000000000000 --- a/.github/workflows/shuffle-tests.yml +++ /dev/null @@ -1,55 +0,0 @@ -name: shuffle-tests - -on: - # Allow manually running - workflow_dispatch: - # Run once a week on sunday - schedule: - - cron: '0 1 * * 0' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -env: - SENTRY_SHUFFLE_TESTS: true - -jobs: - backend-test: - name: run backend tests - runs-on: ubuntu-20.04 - timeout-minutes: 40 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2, 3] - pg-version: ['14'] - - env: - # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. - MATRIX_INSTANCE_TOTAL: 4 - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - # Right now, we run so few bigtable related tests that the - # overhead of running bigtable in all backend tests - # is way smaller than the time it would take to run in its own job. - bigtable: true - pg-version: ${{ matrix.pg-version }} - - - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: | - make test-python-ci diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml deleted file mode 100644 index cd721813f7d43f..00000000000000 --- a/.github/workflows/sync-labels.yml +++ /dev/null @@ -1,72 +0,0 @@ -name: meta(labels) -on: - pull_request: - paths: - - .github/labels.yml - push: - branches: - - master - paths: - - .github/labels.yml - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - sync: - name: syncs repository labels - runs-on: ubuntu-latest - permissions: - issues: write - pull-requests: write - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Install github-label-sync - run: yarn global add github-label-sync@2.2.0 - - - name: Run github-label-sync - run: | - github-label-sync \ - --access-token ${{ secrets.GITHUB_TOKEN }} \ - --labels .github/labels.yml \ - ${{ github.event_name == 'pull_request' && '--dry-run' || '' }} \ - ${{ github.repository }} \ - 2>&1 | tee sync-report.txt - - - name: Read sync output into variable - uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 - if: github.event_name == 'pull_request' - id: github-label-sync - with: - script: | - const fs = require('fs'); - return fs.readFileSync('sync-report.txt','utf8').toString(); - result-encoding: string - - - name: Find previous dry-run comment - uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 - if: github.event_name == 'pull_request' - id: fc - with: - issue-number: ${{ github.event.pull_request.number }} - comment-author: 'github-actions[bot]' - body-includes: '' - - - name: Add or update dry-run comment - uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 - if: github.event_name == 'pull_request' - with: - issue-number: ${{ github.event.pull_request.number }} - comment-id: ${{ steps.fc.outputs.comment-id }} - edit-mode: replace - body: > - - - 🏷 The following changes will be made to the repository labels - - ``` - ${{ steps.github-label-sync.outputs.result }} - ``` diff --git a/.github/workflows/visual-snapshots.yml b/.github/workflows/visual-snapshots.yml deleted file mode 100644 index 7c74f8ab5c7452..00000000000000 --- a/.github/workflows/visual-snapshots.yml +++ /dev/null @@ -1,335 +0,0 @@ -# Visual Snapshots compares against artifacts from the same workflow name (on main branch) -name: visual-snapshots -on: - pull_request: - types: [opened, synchronize, reopened, labeled] - -# Cancel in progress workflows on pull_requests. -# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 -env: - SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 - -jobs: - take-master-acceptance-snapshots: - name: Take Default Branch Acceptance Snapshots - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 25 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2] - pg-version: ['14'] - env: - # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. - MATRIX_INSTANCE_TOTAL: 3 - VISUAL_SNAPSHOT_ENABLE: 1 - TEST_GROUP_STRATEGY: roundrobin - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - with: - ref: 'master' - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Step configurations - id: config - run: echo "webpack-path=.webpack_cache" >> "$GITHUB_OUTPUT" - - - name: webpack cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - with: - path: ${{ steps.config.outputs.webpack-path }} - key: ${{ runner.os }}-v2-webpack-cache-${{ hashFiles('webpack.config.ts') }} - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: webpack - env: - WEBPACK_CACHE_PATH: ${{ steps.config.outputs.webpack-path }} - SENTRY_INSTRUMENTATION: 1 - # this is fine to not have for forks, it shouldn't fail - SENTRY_WEBPACK_WEBHOOK_SECRET: ${{ secrets.SENTRY_WEBPACK_WEBHOOK_SECRET }} - run: | - yarn build-acceptance - - name: Build chartcuterie configuration module - run: | - make build-chartcuterie-config - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - chartcuterie: true - pg-version: ${{ matrix.pg-version }} - - - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: | - mkdir -p ${{ steps.setup.outputs.acceptance-dir }} - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-mobile - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-tooltips - make run-acceptance - env: - PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }} - - - name: Save snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - save-only: true - artifact-name: 'acceptance-visual-snapshots-base' - snapshot-path: .artifacts/visual-snapshots - - take-pr-acceptance-snapshots: - name: Take PR Acceptance Snapshots - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 25 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2] - pg-version: ['14'] - env: - # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. - MATRIX_INSTANCE_TOTAL: 3 - VISUAL_SNAPSHOT_ENABLE: 1 - TEST_GROUP_STRATEGY: roundrobin - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Step configurations - id: config - run: echo "webpack-path=.webpack_cache" >> "$GITHUB_OUTPUT" - - - name: webpack cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - with: - path: ${{ steps.config.outputs.webpack-path }} - key: ${{ runner.os }}-v2-webpack-cache-${{ hashFiles('webpack.config.ts') }} - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: webpack - env: - WEBPACK_CACHE_PATH: ${{ steps.config.outputs.webpack-path }} - SENTRY_INSTRUMENTATION: 1 - # this is fine to not have for forks, it shouldn't fail - SENTRY_WEBPACK_WEBHOOK_SECRET: ${{ secrets.SENTRY_WEBPACK_WEBHOOK_SECRET }} - run: | - yarn build-acceptance - - name: Build chartcuterie configuration module - run: | - make build-chartcuterie-config - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - chartcuterie: true - pg-version: ${{ matrix.pg-version }} - - - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: | - mkdir -p ${{ steps.setup.outputs.acceptance-dir }} - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-mobile - mkdir -p ${{ steps.setup.outputs.acceptance-dir }}-tooltips - make run-acceptance - env: - PYTEST_SNAPSHOTS_DIR: ${{ steps.setup.outputs.acceptance-dir }} - - - name: Save snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - save-only: true - artifact-name: 'acceptance-visual-snapshots-pr' - snapshot-path: .artifacts/visual-snapshots - - diff-acceptance-snapshots: - name: Diff Acceptance snapshots - needs: [take-master-acceptance-snapshots, take-pr-acceptance-snapshots] - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 25 - steps: - - name: Diff snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - action-name: 'Visual Snapshot: Acceptance' - api-token: ${{ secrets.VISUAL_SNAPSHOT_SECRET }} - gcs-bucket: 'sentry-visual-snapshots' - gcp-service-account-key: ${{ secrets.SNAPSHOT_GOOGLE_SERVICE_ACCOUNT_KEY }} - base-artifact-name: 'acceptance-visual-snapshots-base' - base-branch: '' - artifact-name: 'acceptance-visual-snapshots-pr' - - take-master-frontend-snapshots: - name: Take Default Branch Frontend Snapshots - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 30 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2, 3] - env: - VISUAL_HTML_ENABLE: 1 - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - with: - ref: 'master' - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: Build CSS - run: NODE_ENV=production yarn build-css - - - name: jest - env: - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} - CI_NODE_TOTAL: 4 - CI_NODE_INDEX: ${{ matrix.instance }} - run: | - SENTRY_PROFILER_LOGGING_MODE=eager JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit - - name: Create Images from HTML - uses: getsentry/action-html-to-image@dc153dae538e6e1138f77156d8e62e3b2b897f41 # main - with: - base-path: .artifacts/visual-snapshots/jest - css-path: src/sentry/static/sentry/dist/entrypoints/sentry.css - - - name: Save snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - save-only: true - artifact-name: 'frontend-visual-snapshots-base' - snapshot-path: .artifacts/visual-snapshots - - take-pr-frontend-snapshots: - name: Take PR Frontend Snapshots - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 30 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - instance: [0, 1, 2, 3] - env: - VISUAL_HTML_ENABLE: 1 - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - name: Checkout sentry - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: node_modules cache - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 - id: nodemodulescache - with: - path: node_modules - key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} - - - name: Install Javascript Dependencies - if: steps.nodemodulescache.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: Build CSS - run: NODE_ENV=production yarn build-css - - - name: jest - env: - GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} - GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} - CI_NODE_TOTAL: 4 - CI_NODE_INDEX: ${{ matrix.instance }} - run: | - SENTRY_PROFILER_LOGGING_MODE=eager JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit - - name: Create Images from HTML - uses: getsentry/action-html-to-image@dc153dae538e6e1138f77156d8e62e3b2b897f41 # main - with: - base-path: .artifacts/visual-snapshots/jest - css-path: src/sentry/static/sentry/dist/entrypoints/sentry.css - - - name: Save snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - save-only: true - artifact-name: 'frontend-visual-snapshots-pr' - snapshot-path: .artifacts/visual-snapshots - - diff-frontend-snapshots: - name: Diff Frontend snapshots - needs: [take-master-frontend-snapshots, take-pr-frontend-snapshots] - runs-on: ubuntu-20.04 - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Visual Snapshot') - timeout-minutes: 25 - steps: - - name: Diff snapshots - uses: getsentry/action-visual-snapshot@f2537eaa143289fc5d246370767fb6cb5eff1e12 - with: - action-name: 'Visual Snapshot: Frontend' - api-token: ${{ secrets.VISUAL_SNAPSHOT_SECRET }} - gcs-bucket: 'sentry-visual-snapshots' - gcp-service-account-key: ${{ secrets.SNAPSHOT_GOOGLE_SERVICE_ACCOUNT_KEY }} - base-artifact-name: 'frontend-visual-snapshots-base' - base-branch: '' - artifact-name: 'frontend-visual-snapshots-pr' diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index a67b847a43a828..2465a119980f9f 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2584,18 +2584,29 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] ), "kafka": lambda settings, options: ( { - "image": "redpandadata/redpanda:v22.3.23", + "image": "confluentinc/cp-kafka:7.5.0", "ports": {"9092/tcp": 9092}, - "command": [ - "redpanda", - "start", - "--kafka-addr", - "internal://0.0.0.0:9093,external://0.0.0.0:9092", - "--advertise-kafka-addr", - "internal://sentry_kafka:9093,external://127.0.0.1:9092", - "--mode", - "dev-container", - ], + # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example + "environment": { + "KAFKA_PROCESS_ROLES": "broker,controller", + "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@{containers[kafka][name]}:29093", + "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", + "KAFKA_NODE_ID": "1", + "CLUSTER_ID": "MkU3OEVBNTcwNTJENDM2Qk", + # "KAFKA_BOOTSTRAP_SERVERS": "127.0.0.1:9092", + "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093", + # can probably remove PLAINTEXT from advert listeners? + "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://{containers[kafka][name]}:29092,EXTERNAL://{containers[kafka]" + "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}", + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT", + "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT", + "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", + "KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS": "1", + "KAFKA_LOG_RETENTION_HOURS": "24", + "KAFKA_MESSAGE_MAX_BYTES": "50000000", + "KAFKA_MAX_REQUEST_SIZE": "50000000", + }, + "volumes": {"kafka": {"bind": "/var/lib/kafka/data"}}, "only_if": "kafka" in settings.SENTRY_EVENTSTREAM or settings.SENTRY_USE_RELAY or settings.SENTRY_DEV_PROCESS_SUBSCRIPTIONS @@ -2643,7 +2654,7 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "CLICKHOUSE_HTTP_PORT": "8123", "DEFAULT_BROKERS": "" if "snuba" in settings.SENTRY_EVENTSTREAM - else "{containers[kafka][name]}:9093", + else "{containers[kafka][name]}:9093", # uh... "REDIS_HOST": "{containers[redis][name]}", "REDIS_PORT": "6379", "REDIS_DB": "1", diff --git a/tools/devservices_healthcheck.py b/tools/devservices_healthcheck.py index f599603301ae33..8796ae3df529b6 100644 --- a/tools/devservices_healthcheck.py +++ b/tools/devservices_healthcheck.py @@ -29,9 +29,10 @@ def main() -> None: "docker", "exec", "sentry_kafka", - "rpk", - "topic", - "list", + "kafka-topics", + "--bootstrap-server", + "127.0.0.1:9092", + "--list", ] healthchecks = [postgres_healthcheck] From 7dd7e0f2315ac5106d84e604105661687407c46b Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 5 Sep 2023 17:28:54 -0700 Subject: [PATCH 04/13] network hostnames no longer necessary --- src/sentry/conf/server.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 8c5b588619abe3..850838ada56210 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2596,15 +2596,12 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example "environment": { "KAFKA_PROCESS_ROLES": "broker,controller", - "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@{containers[kafka][name]}:29093", + "KAFKA_CONTROLLER_QUORUM_VOTERS": "1@127.0.0.1:29093", "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", "KAFKA_NODE_ID": "1", "CLUSTER_ID": "MkU3OEVBNTcwNTJENDM2Qk", - # "KAFKA_BOOTSTRAP_SERVERS": "127.0.0.1:9092", "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093", - # can probably remove PLAINTEXT from advert listeners? - "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://{containers[kafka][name]}:29092,EXTERNAL://{containers[kafka]" - "[ports][9092/tcp][0]}:{containers[kafka][ports][9092/tcp][1]}", + "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://127.0.0.1:29092,EXTERNAL://127.0.0.1:9092", "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT", "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT", "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", @@ -2661,7 +2658,7 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "CLICKHOUSE_HTTP_PORT": "8123", "DEFAULT_BROKERS": "" if "snuba" in settings.SENTRY_EVENTSTREAM - else "{containers[kafka][name]}:9093", # uh... + else "{containers[kafka][name]}:9092", "REDIS_HOST": "{containers[redis][name]}", "REDIS_PORT": "6379", "REDIS_DB": "1", From 5e64d0e83f6022680c0ece9fe77650a3989bf987 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 5 Sep 2023 17:37:53 -0700 Subject: [PATCH 05/13] wups fix --- .github/actions/setup-sentry/action.yml | 1 + .github/workflows/backend.yml | 315 ------------------------ 2 files changed, 1 insertion(+), 315 deletions(-) diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml index ac7603debabf3f..8c568dfa284201 100644 --- a/.github/actions/setup-sentry/action.yml +++ b/.github/actions/setup-sentry/action.yml @@ -171,6 +171,7 @@ runs: if [ "$NEED_KAFKA" = "true" ]; then docker run \ --name sentry_kafka \ + -d --network host \ -e KAFKA_PROCESS_ROLES=broker,controller \ -e KAFKA_CONTROLLER_QUORUM_VOTERS=1@127.0.0.1:29093 \ -e KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER \ diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 3c0009c0941665..cbd0cbe8e6bfe6 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -39,248 +39,6 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml - api-docs: - if: needs.files-changed.outputs.api_docs == 'true' - needs: files-changed - name: api docs test - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 - - - name: Setup sentry python env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - - - name: Run API docs tests - # install ts-node for ts build scripts to execute properly without potentially installing - # conflicting deps when running scripts locally - # see: https://github.com/getsentry/sentry/pull/32328/files - run: | - yarn add ts-node && make test-api-docs - - backend-test: - if: needs.files-changed.outputs.backend == 'true' - needs: files-changed - name: backend test - runs-on: ubuntu-20.04 - timeout-minutes: 40 - strategy: - # This helps not having to run multiple jobs because one fails, thus, reducing resource usage - # and reducing the risk that one of many runs would turn red again (read: intermittent tests) - fail-fast: false - matrix: - # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. - instance: [0, 1, 2, 3, 4, 5, 6] - pg-version: ['14'] - - env: - # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. - # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`. - MATRIX_INSTANCE_TOTAL: 7 - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - name: Update environment for silo databases - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Silo db' ) - run: | - echo "SENTRY_USE_SPLIT_DBS=1" >> "$GITHUB_ENV" - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - # Right now, we run so few bigtable related tests that the - # overhead of running bigtable in all backend tests - # is way smaller than the time it would take to run in its own job. - bigtable: true - pg-version: ${{ matrix.pg-version }} - - - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) - run: | - make test-python-ci - - # Upload coverage data even if running the tests step fails since - # it reduces large coverage fluctuations - - name: Handle artifacts - if: ${{ always() }} - uses: ./.github/actions/artifacts - with: - token: ${{ secrets.CODECOV_TOKEN }} - - backend-migration-tests: - if: needs.files-changed.outputs.backend == 'true' - needs: files-changed - name: backend migration tests - runs-on: ubuntu-20.04 - timeout-minutes: 30 - strategy: - matrix: - pg-version: ['14'] - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - # Avoid codecov error message related to SHA resolution: - # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 - fetch-depth: '2' - - - name: Update environment for silo databases - id: silo_env - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Silo db' ) - run: | - echo "SENTRY_USE_SPLIT_DBS=1" >> "$GITHUB_ENV" - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - pg-version: ${{ matrix.pg-version }} - - - name: run tests - run: | - MIGRATIONS_TEST_MIGRATE=1 PYTEST_ADDOPTS="$PYTEST_ADDOPTS -m migrations" make test-python-ci - - # Upload coverage data even if running the tests step fails since - # it reduces large coverage fluctuations - - name: Handle artifacts - if: ${{ always() }} - uses: ./.github/actions/artifacts - with: - token: ${{ secrets.CODECOV_TOKEN }} - - cli: - if: needs.files-changed.outputs.backend == 'true' - needs: files-changed - name: cli test - runs-on: ubuntu-20.04 - timeout-minutes: 10 - strategy: - matrix: - pg-version: ['14'] - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Update environment for silo databases - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Silo db' ) - run: | - echo "SENTRY_USE_SPLIT_DBS=1" >> "$GITHUB_ENV" - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - pg-version: ${{ matrix.pg-version }} - - - name: Run test - run: | - make test-cli - - # Upload coverage data even if running the tests step fails since - # it reduces large coverage fluctuations - - name: Handle artifacts - if: ${{ always() }} - uses: ./.github/actions/artifacts - with: - token: ${{ secrets.CODECOV_TOKEN }} - - requirements: - if: needs.files-changed.outputs.backend_dependencies == 'true' - needs: files-changed - name: requirements check - runs-on: ubuntu-20.04 - timeout-minutes: 3 - steps: - - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - id: token - continue-on-error: true - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 - with: - python-version: 3.8.16 - cache-depedency: requirements-dev-frozen.txt - install-cmd: pip install -q --constraint requirements-dev-frozen.txt pip-tools - - name: check requirements - run: | - python -S -m tools.freeze_requirements - if ! git diff --exit-code; then - echo $'\n\nrun `make freeze-requirements` locally to update requirements' - exit 1 - fi - - name: apply any requirements changes - if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() - uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 - with: - github-token: ${{ steps.token.outputs.token }} - message: ':snowflake: re-freeze requirements' - - migration: - if: needs.files-changed.outputs.migration_lockfile == 'true' - needs: files-changed - name: check migration - runs-on: ubuntu-20.04 - strategy: - matrix: - pg-version: ['14'] - - steps: - - name: Checkout sentry - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - pg-version: ${{ matrix.pg-version }} - - - name: Migration & lockfile checks - env: - SENTRY_LOG_LEVEL: ERROR - PGPASSWORD: postgres - run: | - ./.github/workflows/scripts/migration-check.sh - - plugins: - if: needs.files-changed.outputs.plugins == 'true' - needs: files-changed - name: plugins test - runs-on: ubuntu-20.04 - timeout-minutes: 10 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Update environment for silo databases - if: | - contains( github.event.pull_request.labels.*.name, 'Trigger: Silo db' ) - run: | - echo "SENTRY_USE_SPLIT_DBS=1" >> "$GITHUB_ENV" - - - name: Setup sentry env - uses: ./.github/actions/setup-sentry - id: setup - with: - snuba: true - - - name: Run test - run: | - make test-plugins - relay: if: needs.files-changed.outputs.backend == 'true' needs: files-changed @@ -417,76 +175,3 @@ jobs: uses: ./.github/actions/artifacts with: token: ${{ secrets.CODECOV_TOKEN }} - - typing: - if: needs.files-changed.outputs.backend == 'true' - needs: files-changed - name: backend typing - runs-on: ubuntu-20.04 - timeout-minutes: 20 - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 - with: - python-version: 3.8.16 - cache-dependency-path: requirements-dev-frozen.txt - install-cmd: pip install -r requirements-dev-frozen.txt - - - name: setup sentry (lite) - run: | - SENTRY_LIGHT_BUILD=1 pip install --no-deps -e . - sentry init - - - run: make backend-typing - id: run - - - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 - id: token - continue-on-error: true - with: - app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} - private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} - - # only if `backend-typing` succeeds should we try and trim the blocklist - - run: | - python3 -m tools.mypy_helpers.make_module_ignores - git diff --exit-code - - - name: apply blocklist changes - if: steps.token.outcome == 'success' && steps.run.outcome == 'success' && github.ref != 'refs/heads/master' && always() - uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 - with: - github-token: ${{ steps.token.outputs.token }} - message: ':knife: regenerate mypy module blocklist' - - # This check runs once all dependent jobs have passed - # It symbolizes that all required Backend checks have succesfully passed (Or skipped) - # This step is the only required backend check - backend-required-check: - needs: - [ - api-docs, - backend-test, - backend-migration-tests, - cli, - files-changed, - requirements, - migration, - plugins, - relay, - snuba, - symbolicator, - typing, - ] - name: Backend - # This is necessary since a failed/skipped dependent job would cause this job to be skipped - if: always() - runs-on: ubuntu-20.04 - steps: - # If any jobs we depend on fail, we will fail since this is a required check - # NOTE: A timeout is considered a failure - - name: Check for failures - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 From 04c71acaaf81a1ec436a9ea38088d41fb5fba075 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 27 Sep 2023 11:54:59 -0700 Subject: [PATCH 06/13] try again after anthony's changes --- .github/actions/setup-sentry/action.yml | 2 +- .github/workflows/backend.yml | 91 +++++++++++++++++++++ tests/tools/test_devservices_healthcheck.py | 16 +--- tools/devservices_healthcheck.py | 10 +-- 4 files changed, 98 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/backend.yml diff --git a/.github/actions/setup-sentry/action.yml b/.github/actions/setup-sentry/action.yml index a81b770dc8354d..911f62da15620e 100644 --- a/.github/actions/setup-sentry/action.yml +++ b/.github/actions/setup-sentry/action.yml @@ -172,7 +172,7 @@ runs: fi if [ "$NEED_KAFKA" = "true" ]; then - services+=(zookeeper kafka) + services+=(kafka) fi sentry devservices up "${services[@]}" diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 00000000000000..b20f9acd74ed95 --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,91 @@ +name: backend + +on: + push: + branches: + - master + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-20.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + api_docs: ${{ steps.changes.outputs.api_docs }} + backend: ${{ steps.changes.outputs.backend_all }} + backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} + backend_any_type: ${{ steps.changes.outputs.backend_any_type }} + migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Check for backend file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + backend-test: + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: backend test + runs-on: ubuntu-20.04 + timeout-minutes: 40 + strategy: + # This helps not having to run multiple jobs because one fails, thus, reducing resource usage + # and reducing the risk that one of many runs would turn red again (read: intermittent tests) + fail-fast: false + matrix: + # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. + instance: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + pg-version: ['14'] + + env: + # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. + # If this increases, make sure to also increase `flags.backend.after_n_builds` in `codecov.yml`. + MATRIX_INSTANCE_TOTAL: 11 + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + kafka: true + snuba: true + symbolicator: true + # Right now, we run so few bigtable related tests that the + # overhead of running bigtable in all backend tests + # is way smaller than the time it would take to run in its own job. + bigtable: true + pg-version: ${{ matrix.pg-version }} + + - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + run: | + make test-python-ci + + # Upload coverage data even if running the tests step fails since + # it reduces large coverage fluctuations + - name: Handle artifacts + if: ${{ always() }} + uses: ./.github/actions/artifacts + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/tests/tools/test_devservices_healthcheck.py b/tests/tools/test_devservices_healthcheck.py index 6e6ff4dd2ba348..364741bc4e8362 100644 --- a/tests/tools/test_devservices_healthcheck.py +++ b/tests/tools/test_devservices_healthcheck.py @@ -73,7 +73,7 @@ def test_postgres_running(mock_subprocess_run: mock.MagicMock) -> None: assert mock_subprocess_run.call_count == 2 -def test_kafka_zookeper_running(mock_subprocess_run: mock.MagicMock) -> None: +def test_kafka_running(mock_subprocess_run: mock.MagicMock) -> None: running = mock.Mock() running.stdout = "running\n" running.code = 0 @@ -89,7 +89,7 @@ def run( "inspect", "-f", "{{.State.Status}}", - "sentry_zookeeper", + "sentry_per", ): return running elif cmd_args == ( @@ -106,16 +106,8 @@ def run( "exec", "sentry_kafka", "kafka-topics", - "--zookeeper", - "sentry_zookeeper:2181", - "--list", - ) or ( - "docker", - "exec", - "sentry_kafka", - "kafka-topics", - "--zookeeper", - "127.0.0.1:2181", + "--bootstrap-server", + "127.0.0.1:9092", "--list", ): return healthcheck diff --git a/tools/devservices_healthcheck.py b/tools/devservices_healthcheck.py index 9898acf70dcf96..1337531e22c566 100644 --- a/tools/devservices_healthcheck.py +++ b/tools/devservices_healthcheck.py @@ -48,8 +48,8 @@ def check_kafka(): "exec", "sentry_kafka", "kafka-topics", - "--zookeeper", - "sentry_zookeeper:2181", + "--bootstrap-server", + "127.0.0.1:9092", "--list", ), check=True, @@ -75,12 +75,6 @@ def check_postgres() -> None: "sentry_kafka", os.getenv("NEED_KAFKA") == "true", check_kafka, - deps=["zookeeper"], - ), - "zookeeper": HealthCheck( - "zookeeper", - "sentry_zookeeper", - os.getenv("NEED_KAFKA") == "true", ), } From f5ab8463db52300a15de26718d252a535f1dcdcd Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Wed, 27 Sep 2023 12:23:39 -0700 Subject: [PATCH 07/13] [skip ci] fix tools test --- tests/tools/test_devservices_healthcheck.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tools/test_devservices_healthcheck.py b/tests/tools/test_devservices_healthcheck.py index 364741bc4e8362..cf17175333fbe4 100644 --- a/tests/tools/test_devservices_healthcheck.py +++ b/tests/tools/test_devservices_healthcheck.py @@ -116,4 +116,4 @@ def run( mock_subprocess_run.side_effect = run check_health(["kafka"]) - assert mock_subprocess_run.call_count == 3 + assert mock_subprocess_run.call_count == 2 From f5e484ee1f050d05605d57a49e22f129e338bef5 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 2 Oct 2023 12:42:23 -0700 Subject: [PATCH 08/13] apply anthonys patch --- src/sentry/conf/server.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 9c14402819cc9f..39892808aa09aa 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2687,9 +2687,9 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "KAFKA_CONTROLLER_LISTENER_NAMES": "CONTROLLER", "KAFKA_NODE_ID": "1", "CLUSTER_ID": "MkU3OEVBNTcwNTJENDM2Qk", - "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093", - "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://127.0.0.1:29092,EXTERNAL://127.0.0.1:9092", - "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT", + "KAFKA_LISTENERS": "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093", + "KAFKA_ADVERTISED_LISTENERS": "PLAINTEXT://127.0.0.1:29092,INTERNAL://sentry_kafka:9093,EXTERNAL://127.0.0.1:9092", + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP": "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT", "KAFKA_INTER_BROKER_LISTENER_NAME": "PLAINTEXT", "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR": "1", "KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS": "1", From 2c9f5027e4e4d688fcf7770dcb5ccc26f78dcc02 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 2 Oct 2023 14:26:14 -0700 Subject: [PATCH 09/13] restore --- .github/workflows/acceptance.yml | 132 +++++++++++ .github/workflows/backend.yml | 217 ++++++++++++++++++ .github/workflows/bootstrap.yml | 17 ++ .../workflows/bump-sentry-in-getsentry.yml | 56 +++++ .github/workflows/bump-version.yml | 74 ++++++ .github/workflows/codecov_ats.yml | 92 ++++++++ .github/workflows/codeql.yml | 80 +++++++ .github/workflows/dependency-review.yml | 19 ++ .github/workflows/development-environment.yml | 121 ++++++++++ .../workflows/enforce-license-compliance.yml | 14 ++ .github/workflows/fast-revert.yml | 40 ++++ .github/workflows/frontend.yml | 208 +++++++++++++++++ .github/workflows/getsentry-dispatch.yml | 81 +++++++ .github/workflows/jest-balance.yml | 39 ++++ .github/workflows/label-pullrequest.yml | 69 ++++++ .github/workflows/lock.yml | 16 ++ .../meta-deploys-detect-change-type.yml | 33 +++ .github/workflows/migrations.yml | 93 ++++++++ .github/workflows/openapi-diff.yml | 60 +++++ .github/workflows/openapi.yml | 73 ++++++ .github/workflows/pre-commit.yml | 87 +++++++ .github/workflows/publish-dockerhub.yml | 49 ++++ .../react-to-product-owners-yml-changes.yml | 24 ++ .github/workflows/release.yml | 31 +++ .github/workflows/sentry-pull-request-bot.yml | 74 ++++++ .github/workflows/shuffle-tests.yml | 55 +++++ .github/workflows/sync-labels.yml | 72 ++++++ 27 files changed, 1926 insertions(+) create mode 100644 .github/workflows/acceptance.yml create mode 100644 .github/workflows/bootstrap.yml create mode 100644 .github/workflows/bump-sentry-in-getsentry.yml create mode 100644 .github/workflows/bump-version.yml create mode 100644 .github/workflows/codecov_ats.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 .github/workflows/dependency-review.yml create mode 100644 .github/workflows/development-environment.yml create mode 100644 .github/workflows/enforce-license-compliance.yml create mode 100644 .github/workflows/fast-revert.yml create mode 100644 .github/workflows/frontend.yml create mode 100644 .github/workflows/getsentry-dispatch.yml create mode 100644 .github/workflows/jest-balance.yml create mode 100644 .github/workflows/label-pullrequest.yml create mode 100644 .github/workflows/lock.yml create mode 100644 .github/workflows/meta-deploys-detect-change-type.yml create mode 100644 .github/workflows/migrations.yml create mode 100644 .github/workflows/openapi-diff.yml create mode 100644 .github/workflows/openapi.yml create mode 100644 .github/workflows/pre-commit.yml create mode 100644 .github/workflows/publish-dockerhub.yml create mode 100644 .github/workflows/react-to-product-owners-yml-changes.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/sentry-pull-request-bot.yml create mode 100644 .github/workflows/shuffle-tests.yml create mode 100644 .github/workflows/sync-labels.yml diff --git a/.github/workflows/acceptance.yml b/.github/workflows/acceptance.yml new file mode 100644 index 00000000000000..5b5624ab05a750 --- /dev/null +++ b/.github/workflows/acceptance.yml @@ -0,0 +1,132 @@ +# Also note that this name *MUST* match the filename because GHA +# only provides the workflow name (https://docs.github.com/en/free-pro-team@latest/actions/reference/environment-variables#default-environment-variables) +# and GH APIs only support querying by workflow *FILENAME* (https://developer.github.com/v3/actions/workflows/#get-a-workflow) +name: acceptance +on: + push: + branches: + - master + - releases/** + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-20.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + acceptance: ${{ steps.changes.outputs.acceptance }} + backend_all: ${{ steps.changes.outputs.backend_all }} + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Check for backend file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + acceptance: + if: needs.files-changed.outputs.acceptance == 'true' + needs: files-changed + name: acceptance + runs-on: ubuntu-20.04 + timeout-minutes: 30 + strategy: + # This helps not having to run multiple jobs because one fails, thus, reducing resource usage + # and reducing the risk that one of many runs would turn red again (read: intermittent tests) + fail-fast: false + matrix: + # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. + instance: [0, 1, 2, 3, 4] + pg-version: ['14'] + env: + # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. + MATRIX_INSTANCE_TOTAL: 5 + TEST_GROUP_STRATEGY: roundrobin + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + name: Checkout sentry + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: Step configurations + id: config + run: echo "webpack-path=.webpack_cache" >> "$GITHUB_OUTPUT" + + - name: webpack cache + uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 + with: + path: ${{ steps.config.outputs.webpack-path }} + key: ${{ runner.os }}-v2-webpack-cache-${{ hashFiles('webpack.config.ts') }} + + - name: node_modules cache + uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 + id: nodemodulescache + with: + path: node_modules + key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} + + - name: Install Javascript Dependencies + if: steps.nodemodulescache.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + + - name: webpack + env: + WEBPACK_CACHE_PATH: ${{ steps.config.outputs.webpack-path }} + SENTRY_INSTRUMENTATION: 1 + # this is fine to not have for forks, it shouldn't fail + SENTRY_WEBPACK_WEBHOOK_SECRET: ${{ secrets.SENTRY_WEBPACK_WEBHOOK_SECRET }} + run: | + yarn build-acceptance + + - name: Build chartcuterie configuration module + run: | + make build-chartcuterie-config + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + snuba: true + chartcuterie: true + pg-version: ${{ matrix.pg-version }} + + - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + run: make run-acceptance + + # This job runs when FE or BE changes happen, however, we only upload coverage data for + # BE changes since it conflicts with codecov's carry forward functionality + # Upload coverage data even if running the tests step fails since + # it reduces large coverage fluctuations + - name: Handle artifacts + uses: ./.github/actions/artifacts + if: ${{ always() && needs.files-changed.outputs.backend_all == 'true' }} + with: + token: ${{ secrets.CODECOV_TOKEN }} + + acceptance-required-checks: + # this is a required check so we need this job to always run and report a status. + if: always() + name: Acceptance + needs: [acceptance, files-changed] + runs-on: ubuntu-20.04 + timeout-minutes: 3 + steps: + - name: Check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index b20f9acd74ed95..2d50464d94013c 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -38,6 +38,29 @@ jobs: token: ${{ github.token }} filters: .github/file-filters.yml + api-docs: + if: needs.files-changed.outputs.api_docs == 'true' + needs: files-changed + name: api docs test + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: Setup sentry python env + uses: ./.github/actions/setup-sentry + id: setup + with: + snuba: true + + - name: Run API docs tests + # install ts-node for ts build scripts to execute properly without potentially installing + # conflicting deps when running scripts locally + # see: https://github.com/getsentry/sentry/pull/32328/files + run: | + yarn add ts-node && make test-api-docs + backend-test: if: needs.files-changed.outputs.backend == 'true' needs: files-changed @@ -89,3 +112,197 @@ jobs: uses: ./.github/actions/artifacts with: token: ${{ secrets.CODECOV_TOKEN }} + + backend-migration-tests: + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: backend migration tests + runs-on: ubuntu-20.04 + timeout-minutes: 30 + strategy: + matrix: + pg-version: ['14'] + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + snuba: true + pg-version: ${{ matrix.pg-version }} + + - name: run tests + run: | + MIGRATIONS_TEST_MIGRATE=1 PYTEST_ADDOPTS="$PYTEST_ADDOPTS -m migrations" make test-python-ci + + # Upload coverage data even if running the tests step fails since + # it reduces large coverage fluctuations + - name: Handle artifacts + if: ${{ always() }} + uses: ./.github/actions/artifacts + with: + token: ${{ secrets.CODECOV_TOKEN }} + + cli: + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: cli test + runs-on: ubuntu-20.04 + timeout-minutes: 10 + strategy: + matrix: + pg-version: ['14'] + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + pg-version: ${{ matrix.pg-version }} + + - name: Run test + run: | + make test-cli + + # Upload coverage data even if running the tests step fails since + # it reduces large coverage fluctuations + - name: Handle artifacts + if: ${{ always() }} + uses: ./.github/actions/artifacts + with: + token: ${{ secrets.CODECOV_TOKEN }} + + requirements: + if: needs.files-changed.outputs.backend_dependencies == 'true' + needs: files-changed + name: requirements check + runs-on: ubuntu-20.04 + timeout-minutes: 3 + steps: + - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + id: token + continue-on-error: true + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 + with: + python-version: 3.8.16 + cache-depedency: requirements-dev-frozen.txt + install-cmd: pip install -q --constraint requirements-dev-frozen.txt pip-tools + - name: check requirements + run: | + python -S -m tools.freeze_requirements + if ! git diff --exit-code; then + echo $'\n\nrun `make freeze-requirements` locally to update requirements' + exit 1 + fi + - name: apply any requirements changes + if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() + uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 + with: + github-token: ${{ steps.token.outputs.token }} + message: ':snowflake: re-freeze requirements' + + migration: + if: needs.files-changed.outputs.migration_lockfile == 'true' + needs: files-changed + name: check migration + runs-on: ubuntu-20.04 + strategy: + matrix: + pg-version: ['14'] + + steps: + - name: Checkout sentry + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + pg-version: ${{ matrix.pg-version }} + + - name: Migration & lockfile checks + env: + SENTRY_LOG_LEVEL: ERROR + PGPASSWORD: postgres + run: | + ./.github/workflows/scripts/migration-check.sh + + typing: + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + name: backend typing + runs-on: ubuntu-20.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 + with: + python-version: 3.8.16 + cache-dependency-path: requirements-dev-frozen.txt + install-cmd: pip install -r requirements-dev-frozen.txt + + - name: setup sentry (lite) + run: | + SENTRY_LIGHT_BUILD=1 pip install --no-deps -e . + sentry init + + - run: mypy + id: run + + - uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + id: token + continue-on-error: true + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + # only if `mypy` succeeds should we try and trim the blocklist + - run: | + python3 -m tools.mypy_helpers.make_module_ignores + git diff --exit-code + + - name: apply blocklist changes + if: steps.token.outcome == 'success' && steps.run.outcome == 'success' && github.ref != 'refs/heads/master' && always() + uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 + with: + github-token: ${{ steps.token.outputs.token }} + message: ':knife: regenerate mypy module blocklist' + + # This check runs once all dependent jobs have passed + # It symbolizes that all required Backend checks have succesfully passed (Or skipped) + # This step is the only required backend check + backend-required-check: + needs: + [ + api-docs, + backend-test, + backend-migration-tests, + cli, + files-changed, + requirements, + migration, + typing, + ] + name: Backend + # This is necessary since a failed/skipped dependent job would cause this job to be skipped + if: always() + runs-on: ubuntu-20.04 + steps: + # If any jobs we depend on fail, we will fail since this is a required check + # NOTE: A timeout is considered a failure + - name: Check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/bootstrap.yml b/.github/workflows/bootstrap.yml new file mode 100644 index 00000000000000..aaa1096d95747a --- /dev/null +++ b/.github/workflows/bootstrap.yml @@ -0,0 +1,17 @@ +name: bootstrap +on: + schedule: + # Run on Mondays in hopes to fix things before new engineers run the script + # It's far more likely that somethings regresses because there's new software released than + # by our code changes regressing it + - cron: '0 3 * * 1' + +jobs: + bootstrap-script: + name: bootstrap + runs-on: macos-13 + timeout-minutes: 30 + steps: + - name: Run bootstrap scripts + run: | + bash <(curl -s https://raw.githubusercontent.com/getsentry/bootstrap-sentry/master/bootstrap.sh) diff --git a/.github/workflows/bump-sentry-in-getsentry.yml b/.github/workflows/bump-sentry-in-getsentry.yml new file mode 100644 index 00000000000000..51643299296e44 --- /dev/null +++ b/.github/workflows/bump-sentry-in-getsentry.yml @@ -0,0 +1,56 @@ +name: bump sentry in getsentry + +# Serializes the execution of this workflow. +concurrency: + group: ${{ github.workflow }} + +on: + push: + branches: + - master + +defaults: + run: + # the default default is: + # bash --noprofile --norc -eo pipefail {0} + shell: bash --noprofile --norc -eo pipefail -ux {0} + +jobs: + bump-sentry: + runs-on: ubuntu-20.04 + steps: + - name: checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # For getsentry/bin/bump-sentry, sentry needs to be at ../sentry relative to getsentry. + path: sentry + + - name: checkout getsentry + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + repository: 'getsentry/getsentry' + path: getsentry + # This PAT (Personal Access Token) belongs to getsentry-bot, + # who can write to getsentry and is SAML+SSO ready. + token: ${{ secrets.BUMP_SENTRY_TOKEN }} + + - name: bump-sentry ${{ github.sha }} + run: | + cd getsentry + + python -S -m bin.bump_sentry ${{ github.sha }} + + # If getsentry is pushed to by any other means while we were here, + # we won't be able to push. + for i in 1 2 3 4 5; do + git push origin master && exit 0 + # There's a little bit of network delay here that suffices + # as a small sleep. + git \ + -c user.name=getsentry-bot \ + -c user.email=bot@sentry.io \ + pull --rebase origin master + done + + # 5th and final attempt. + git push origin master diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml new file mode 100644 index 00000000000000..5c44b8ab4c1178 --- /dev/null +++ b/.github/workflows/bump-version.yml @@ -0,0 +1,74 @@ +on: + workflow_dispatch: + inputs: + package: + required: true + type: string + description: package name such as `sentry-arroyo` + version: + required: true + type: string + description: desired version such as `1.2.3`, or `latest` to pull the latest version from PyPI + pr_options: + type: string + default: '' + description: additional options for gh pr create, such as for asking for specific reviewers + + # for use in other (cron/scheduled) workflows to bump specific + # company-internal dependencies on a more aggressive schedule + workflow_call: + inputs: + package: + required: true + type: string + version: + required: true + type: string + pr_options: + type: string + default: '' + +# disable all permissions -- we use the PAT's permissions instead +permissions: {} + +jobs: + bump-version: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + token: ${{ secrets.BUMP_SENTRY_TOKEN }} + - run: | + set -euxo pipefail + + if [ "$VERSION" = latest ]; then + VERSION="$(curl -sL https://pypi.org/pypi/$PACKAGE/json | jq -r .info.version)" + fi + + git checkout -b "bot/bump-version/$PACKAGE/$VERSION" + + re="$(sed 's/[_-]/[_-]/g' <<< "$PACKAGE")" + sed -i "s/^$re==.*/$PACKAGE==$VERSION/g" -- requirements*.txt + sed -i "s/^$re>=.*/$PACKAGE>=$VERSION/g" -- requirements*.txt + + if git diff --exit-code; then + exit 0 + fi + + git \ + -c user.name=getsentry-bot \ + -c user.email='10587625+getsentry-bot@users.noreply.github.com' \ + commit \ + --all \ + --message "ref: bump $PACKAGE to $VERSION" \ + --message "Co-Authored-By: $SENDER <$SENDER_ID+$SENDER@users.noreply.github.com>" + + git push origin HEAD --quiet + + gh pr create --fill ${{ inputs.pr_options }} + env: + GH_TOKEN: ${{ secrets.BUMP_SENTRY_TOKEN }} + PACKAGE: ${{ inputs.package }} + VERSION: ${{ inputs.version }} + SENDER: ${{ github.event.sender.login }} + SENDER_ID: ${{ github.event.sender.id }} diff --git a/.github/workflows/codecov_ats.yml b/.github/workflows/codecov_ats.yml new file mode 100644 index 00000000000000..ab145cb19b6b4c --- /dev/null +++ b/.github/workflows/codecov_ats.yml @@ -0,0 +1,92 @@ + +name: IGNORE ME codecov-ats + +on: + pull_request: + +env: + CLI_VERSION: v0.1.5 + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-20.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + api_docs: ${{ steps.changes.outputs.api_docs }} + backend: ${{ steps.changes.outputs.backend_all }} + backend_dependencies: ${{ steps.changes.outputs.backend_dependencies }} + backend_any_type: ${{ steps.changes.outputs.backend_any_type }} + migration_lockfile: ${{ steps.changes.outputs.migration_lockfile }} + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Check for backend file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + coverage-ats: + # Temporary test + if: needs.files-changed.outputs.backend == 'true' + needs: files-changed + timeout-minutes: 40 + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Python 3.10.10 + uses: actions/setup-python@v4 + with: + python-version: "3.10.10" + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + snuba: true + # Right now, we run so few bigtable related tests that the + # overhead of running bigtable in all backend tests + # is way smaller than the time it would take to run in its own job. + bigtable: true + pg-version: 14 + - name: Download Codecov CLI + run: | + pip install --extra-index-url https://pypi.org/simple --no-cache-dir pytest codecov-cli + # Creates the commit and report objects in codecov + - name: Codecov startup + run: | + codecovcli create-commit + codecovcli create-report + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + # Sends static analysis information to codecov + - name: Static Analysis + run: | + codecovcli static-analysis --token=${CODECOV_STATIC_TOKEN} \ + --folders-to-exclude .artifacts \ + --folders-to-exclude .github \ + --folders-to-exclude .venv \ + --folders-to-exclude static \ + --folders-to-exclude bin + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} + # Run label analysis IN DRY MODE (no tests will actually run) + - name: Label Analysis + continue-on-error: true + run: | + BASE_COMMIT=$(git merge-base ${{ github.sha }}^ origin/master) + echo $BASE_COMMIT + codecovcli --codecov-yml-path=codecov.yml label-analysis --dry-run --token=${CODECOV_STATIC_TOKEN} --base-sha=${BASE_COMMIT} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + CODECOV_STATIC_TOKEN: ${{ secrets.CODECOV_STATIC_TOKEN }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 00000000000000..8145557b79a644 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,80 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: 'CodeQL' + +on: + push: + branches: ['master'] + pull_request: + # The branches below must be a subset of the branches above + branches: ['master'] + paths: + - '**.js' + - '**.ejs' + - '**.tsx' + - '**.ts' + - '**.py' + - '!tests/**' + schedule: + - cron: '44 12 * * 1' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: ['javascript', 'python'] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Learn more about CodeQL language support at https://git.io/codeql-language-support + + steps: + - name: Checkout repository + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 + with: + config-file: ./.github/codeql/codeql-config.yml + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + # queries: security-extended + + # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language + + #- run: | + # make bootstrap + # make release + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@9a866ed4524fc3422c3af1e446dab8efa3503411 # v2.12.7 + with: + category: '/language:${{matrix.language}}' diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 00000000000000..2f1c34516a4c74 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,19 @@ +name: 'Dependency Review' +on: + pull_request: + branches: ['master'] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + - name: Dependency Review + uses: actions/dependency-review-action@c090f4e553673e6e505ea70d6a95362ee12adb94 # v3.0.3 + with: + # Possible values: "critical", "high", "moderate", "low" + fail-on-severity: high diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml new file mode 100644 index 00000000000000..5655b0b1d5d130 --- /dev/null +++ b/.github/workflows/development-environment.yml @@ -0,0 +1,121 @@ +name: dev env +on: + pull_request: + paths: + - '.pre-commit-config.yaml' + - 'Makefile' + - '.github/workflows/development-environment.yml' + - 'requirements-*.txt' + - '.python-version' + - '.envrc' + - 'Brewfile' + - 'scripts/**' + - 'tools/**' + - 'src/sentry/runner/commands/devserver.py' + - 'src/sentry/runner/commands/devservices.py' + - 'bin/load-mocks' + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + # This workflow is optimized to test the dev env with dev services as fast as possible + # The bootstrap workflow (see last workflow) tests the experience of first time engineers + docker-setup: + name: Docker set up + runs-on: macos-13 + timeout-minutes: 40 + env: + # Make the environment more similar to what Mac defaults to + SHELL: /bin/zsh + + steps: + - name: Checkout sentry + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Set up + id: info + run: | + echo "yarn-cache-dir=$(yarn cache dir)" >> "$GITHUB_OUTPUT" + + # Trick for unattended Docker installations + # https://github.com/docker/for-mac/issues/2359#issuecomment-943131345 + # NOTE: This can sometimes take up to 10 minutes + - name: Install Docker + run: | + # d4m 4.11+ never starts: https://github.com/docker/for-mac/issues/6450 + curl -o /tmp/docker.rb https://raw.githubusercontent.com/Homebrew/homebrew-cask/fe866ec0765de141599745f03e215452db7f511b/Casks/docker.rb + HOMEBREW_NO_AUTO_UPDATE=1 brew install -v --HEAD -s /tmp/docker.rb + sudo /Applications/Docker.app/Contents/MacOS/Docker --unattended --install-privileged-components + open -a /Applications/Docker.app --args --unattended --accept-license + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + # This handles Python's cache + - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 + with: + python-version: 3.8.16 + cache-dependency-path: requirements-dev-frozen.txt + + # This tests starting up the dev services, loading mocks and pre-commit installation + # This can take over 15 minutes + - name: make bootstrap + # GHA pythons are miscompiled and report macos 10.16 + env: + SYSTEM_VERSION_COMPAT: 0 + run: make bootstrap + + # The pyenv set up takes long, thus, separating it into its own + pyenv-setup: + name: pyenv set up + runs-on: macos-13 + timeout-minutes: 25 + env: + # This is to support this code https://github.com/getsentry/sentry/blob/47b837a69c38e190a7555de81e6d7d216498b854/scripts/pyenv_setup.sh#L17-L40 + SHELL: /bin/zsh + + steps: + - name: Checkout sentry + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Install pyenv + run: | + HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv + + # Only after we source ~/.zprofile that the right Python will be selected + - name: Set up pyenv + run: | + make setup-pyenv + [[ $(which python) != "${HOME}/.pyenv/shims/python" ]] + source ~/.zprofile + [[ $(which python) == "${HOME}/.pyenv/shims/python" ]] + [[ $(python -V) == "Python $(cat .python-version)" ]] + python -m venv .venv + source .venv/bin/activate + [[ $(python -V) == "Python $(cat .python-version)" ]] + + tools-tests: + runs-on: ubuntu-20.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 + with: + python-version: 3.8.16 + cache-dependency-path: | + requirements-dev.txt + requirements-dev-frozen.txt + install-cmd: pip install -r requirements-dev.txt -c requirements-dev-frozen.txt + - name: run tests + run: make test-tools + - name: Handle artifacts + uses: ./.github/actions/artifacts + with: + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/enforce-license-compliance.yml b/.github/workflows/enforce-license-compliance.yml new file mode 100644 index 00000000000000..3c7ec30fa0aa98 --- /dev/null +++ b/.github/workflows/enforce-license-compliance.yml @@ -0,0 +1,14 @@ +name: Enforce License Compliance + +on: + pull_request: + branches: [master, main] + +jobs: + enforce-license-compliance: + runs-on: ubuntu-latest + steps: + - name: 'Enforce License Compliance' + uses: getsentry/action-enforce-license-compliance@520fb640b532c27b4da9644116d102b579ef84f5 # main + with: + fossa_api_key: ${{ secrets.FOSSA_API_KEY }} diff --git a/.github/workflows/fast-revert.yml b/.github/workflows/fast-revert.yml new file mode 100644 index 00000000000000..c87ecee323a63b --- /dev/null +++ b/.github/workflows/fast-revert.yml @@ -0,0 +1,40 @@ +on: + pull_request_target: + types: [labeled] + workflow_dispatch: + inputs: + pr: + required: true + description: pr number + co_authored_by: + required: true + description: '`name ` for triggering user' + +# disable all permissions -- we use the PAT's permissions instead +permissions: {} + +jobs: + revert: + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || github.event.label.name == 'Trigger: Revert' + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + token: ${{ secrets.BUMP_SENTRY_TOKEN }} + - uses: getsentry/action-fast-revert@35b4b6c1f8f91b5911159568b3b15e531b5b8174 # v2.0.1 + with: + pr: ${{ github.event.number || github.event.inputs.pr }} + co_authored_by: ${{ github.event.inputs.co_authored_by || format('{0} <{1}+{0}@users.noreply.github.com>', github.event.sender.login, github.event.sender.id) }} + committer_name: getsentry-bot + committer_email: bot@sentry.io + token: ${{ secrets.BUMP_SENTRY_TOKEN }} + - name: comment on failure + run: | + curl \ + --silent \ + -X POST \ + -H 'Authorization: token ${{ secrets.BUMP_SENTRY_TOKEN }}' \ + -d'{"body": "revert failed (conflict? already reverted?) -- [check the logs](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})"}' \ + https://api.github.com/repositories/${{ github.event.repository.id }}/issues/${{ github.event.number || github.event.inputs.pr }}/comments + if: failure() diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml new file mode 100644 index 00000000000000..2b6cc0fb58c7e0 --- /dev/null +++ b/.github/workflows/frontend.yml @@ -0,0 +1,208 @@ +name: frontend + +on: + push: + branches: + - master + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + NODE_OPTIONS: '--max-old-space-size=4096' + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-20.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + eslint_config: ${{ steps.changes.outputs.eslint_config }} + frontend: ${{ steps.changes.outputs.frontend_all }} + frontend_components_modified_lintable: ${{ steps.changes.outputs.frontend_components_modified_lintable }} + frontend_components_modified_lintable_files: ${{ steps.changes.outputs.frontend_components_modified_lintable_files }} + frontend_modified_lintable_files: ${{ steps.changes.outputs.frontend_modified_lintable_files }} + yarn_lockfile: ${{ steps.changes.outputs.yarn_lockfile }} + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Check for frontend file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + list-files: shell + + typescript-and-lint: + if: needs.files-changed.outputs.frontend == 'true' + needs: files-changed + name: typescript and lint + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Internal github app token + id: token + uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + continue-on-error: true + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: Install dependencies + id: dependencies + run: yarn install --frozen-lockfile + + # Setup custom tsc matcher, see https://github.com/actions/setup-node/issues/97 + - name: setup matchers + run: | + echo "::remove-matcher owner=masters::" + echo "::add-matcher::.github/tsc.json" + echo "::add-matcher::.github/eslint-stylish.json" + + - name: eslint logic + id: eslint + if: (github.ref == 'refs/heads/master' || needs.files-changed.outputs.eslint_config == 'true' || needs.files-changed.outputs.yarn_lockfile == 'true') + run: echo "all-files=true" >> "$GITHUB_OUTPUT" + + # Lint entire frontend if: + # - this is on main branch + # - eslint configuration in repo has changed + # - yarn lockfile has changed (i.e. we bump our eslint config) + - name: eslint + if: steps.eslint.outputs.all-files == 'true' + env: + # Run relax config on main branch (and stricter config for changed files) + SENTRY_ESLINT_RELAXED: 1 + run: | + yarn lint + yarn lint:css + + # Otherwise... only lint modified files + # Note `eslint --fix` will not fail when it auto fixes files + - name: eslint (changed files only) + if: steps.eslint.outputs.all-files != 'true' + run: | + yarn eslint --fix ${{ needs.files-changed.outputs.frontend_modified_lintable_files }} + + - name: stylelint (changed files only) + if: github.ref != 'refs/heads/master' && needs.files-changed.outputs.frontend_components_modified_lintable == 'true' + run: | + yarn stylelint ${{ needs.files-changed.outputs.frontend_components_modified_lintable_files }} + + # Check (and error) for dirty working tree for forks + # Reason being we need a different token to auto commit changes and + # forks do not have access to said token + - name: Check for dirty git working tree (forks) + if: steps.token.outcome != 'success' && github.ref != 'refs/heads/master' + run: | + git diff --quiet || (echo '::error ::lint produced file changes, run linter locally and try again' && exit 1) + + # If working tree is dirty, commit and update if we have a token + - name: Commit any eslint fixed files + if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' + uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 + with: + github-token: ${{ steps.token.outputs.token }} + message: ':hammer_and_wrench: apply eslint style fixes' + + - name: tsc + id: tsc + if: steps.dependencies.outcome == 'success' + run: yarn tsc -p config/tsconfig.ci.json + + frontend-jest-tests: + if: needs.files-changed.outputs.frontend == 'true' + needs: files-changed + name: Jest + # If you change the runs-on image, you must also change the runner in jest-balance.yml + # so that the balancer runs in the same environment as the tests. + runs-on: ubuntu-20.04 + timeout-minutes: 30 + strategy: + # This helps not having to run multiple jobs because one fails, thus, reducing resource usage + # and reducing the risk that one of many runs would turn red again (read: intermittent tests) + fail-fast: false + matrix: + # XXX: When updating this, make sure you also update CI_NODE_TOTAL. + instance: [0, 1, 2, 3] + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + name: Checkout sentry + + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: node_modules cache + uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 + id: nodemodulescache + with: + path: node_modules + key: ${{ runner.os }}-node-modules-${{ hashFiles('yarn.lock', 'api-docs/yarn.lock') }} + + - name: Install Javascript Dependencies + if: steps.nodemodulescache.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + + - name: jest + env: + GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} + # XXX: CI_NODE_TOTAL must be hardcoded to the length of strategy.matrix.instance. + # Otherwise, if there are other things in the matrix, using strategy.job-total + # wouldn't be correct. Also, if this increases, make sure to also increase + # `flags.frontend.after_n_builds` in `codecov.yml`. + CI_NODE_TOTAL: 4 + CI_NODE_INDEX: ${{ matrix.instance }} + # Disable testing-library from printing out any of of the DOM to + # stdout. No one actually looks through this in CI, they're just + # going to run it locally. + # + # This quiets up the logs quite a bit. + DEBUG_PRINT_LIMIT: 0 + run: | + JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit + + # We only upload coverage data for FE changes since it conflicts with + # codecov's carry forward functionality. + # Upload coverage data even if running the tests step fails since + # it reduces large coverage fluctuations. + - name: Handle artifacts + uses: ./.github/actions/artifacts + if: ${{ always() && needs.files-changed.outputs.frontend_all == 'true' }} + with: + files: .artifacts/coverage/* + type: frontend + token: ${{ secrets.CODECOV_TOKEN }} + + # This check runs once all dependant jobs have passed + # It symbolizes that all required Frontend checks have succesfully passed (Or skipped) + # This check is the only required Github check + frontend-required-check: + needs: [files-changed, frontend-jest-tests, typescript-and-lint] + name: Frontend + # This is necessary since a failed/skipped dependent job would cause this job to be skipped + if: always() + runs-on: ubuntu-20.04 + steps: + # If any jobs we depend on fail, we will fail since this is a required check + # NOTE: A timeout is considered a failure + - name: Check for failures + if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') + run: | + echo "One of the dependent jobs have failed. You may need to re-run it." && exit 1 diff --git a/.github/workflows/getsentry-dispatch.yml b/.github/workflows/getsentry-dispatch.yml new file mode 100644 index 00000000000000..3c10409af48957 --- /dev/null +++ b/.github/workflows/getsentry-dispatch.yml @@ -0,0 +1,81 @@ +# Dispatch a request to getsentry to run getsentry test suites +name: getsentry dispatcher + +on: + # XXX: We are using `pull_request_target` instead of `pull_request` because we want + # this to run on forks. It allows forks to access secrets safely by + # only running workflows from the main branch. Prefer to use `pull_request` when possible. + # + # See https://github.com/getsentry/sentry/pull/21600 for more details + pull_request_target: + types: [labeled, opened, reopened, synchronize] + +# disable all other special privileges +permissions: + # needed for `actions/checkout` to clone the code + contents: read + # needed to remove the pull-request label + pull-requests: write + +jobs: + dispatch: + if: "github.event.action != 'labeled' || github.event.label.name == 'Trigger: getsentry tests'" + name: getsentry dispatch + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: permissions + run: | + python3 -uS .github/workflows/scripts/getsentry-dispatch-setup \ + --repo-id ${{ github.event.repository.id }} \ + --pr ${{ github.event.number }} \ + --event ${{ github.event.action }} \ + --username "$ARG_USERNAME" \ + --label-names "$ARG_LABEL_NAMES" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # these can contain special characters + ARG_USERNAME: ${{ github.event.pull_request.user.login }} + ARG_LABEL_NAMES: ${{ toJSON(github.event.pull_request.labels.*.name) }} + + - name: Check for file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: getsentry token + uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + id: getsentry + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - name: Wait for PR merge commit + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + id: mergecommit + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit`).waitForMergeCommit({ + github, + context, + core, + }); + + - name: Dispatch getsentry tests + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/getsentry-dispatch`).dispatch({ + github, + context, + core, + mergeCommitSha: '${{ steps.mergecommit.outputs.mergeCommitSha }}', + fileChanges: ${{ toJson(steps.changes.outputs) }}, + }); diff --git a/.github/workflows/jest-balance.yml b/.github/workflows/jest-balance.yml new file mode 100644 index 00000000000000..ec1e6e0abb043e --- /dev/null +++ b/.github/workflows/jest-balance.yml @@ -0,0 +1,39 @@ +name: jest balancer +on: + workflow_dispatch: + schedule: + - cron: '0 0 1,15 * *' +jobs: + jest-balance: + # Buckle up, this may take a while + timeout-minutes: 60 + # Make sure this matches the runner that runs frontend tests + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + name: Checkout sentry + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: Install dependencies + run: yarn install --frozen-lockfile + + - name: jest balancer + env: + GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }} + run: JEST_TEST_BALANCER=1 yarn test-ci + + - name: Create Pull Request + uses: peter-evans/create-pull-request@b4d51739f96fca8047ad065eccef63442d8e99f7 # v4.2.0 + with: + token: ${{ secrets.BUMP_SENTRY_TOKEN }} + add-paths: | + tests/js/test-balancer/jest-balance.json + commit-message: 'ci(jest): regenerate jest-balance.json' + branch: 'ci/jest/rebalance-tests' + delete-branch: true + base: master + title: 'ci(jest): regenerate jest-balance.json' + body: | + This PR was auto-generated - it updates the \`jest-balance.json\` file with new test run data from CI. diff --git a/.github/workflows/label-pullrequest.yml b/.github/workflows/label-pullrequest.yml new file mode 100644 index 00000000000000..4729a0fcde557e --- /dev/null +++ b/.github/workflows/label-pullrequest.yml @@ -0,0 +1,69 @@ +# Adds labels to pull requests for the type of change the PR makes +name: meta(labels) + +on: + pull_request_target: + +jobs: + label-pullrequest: + permissions: + contents: read + pull-requests: write + name: labels pull requests (frontend / backend) + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + persist-credentials: false + + - name: Check for file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Add frontend label + uses: getsentry/action-add-labels@54d0cba498c1eaf8bd34985d715504d1b6e2935f + if: steps.changes.outputs.frontend_src == 'true' + with: + labels: 'Scope: Frontend' + + - name: Add backend label + uses: getsentry/action-add-labels@54d0cba498c1eaf8bd34985d715504d1b6e2935f + if: steps.changes.outputs.backend_src == 'true' + with: + labels: 'Scope: Backend' + + - name: Find previous frontend/backend warning comment + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Add frontend/backend warning comment + uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 + if: > + steps.changes.outputs.frontend_src == 'true' && + steps.changes.outputs.backend_src == 'true' && + steps.fc.outputs.comment-id == 0 + with: + issue-number: ${{ github.event.pull_request.number }} + body: > + + + 🚨 **Warning:** This pull request contains Frontend and Backend changes! + + + It's discouraged to make changes to Sentry's Frontend and Backend + in a single pull request. The Frontend and Backend are **not** + atomically deployed. If the changes are interdependent of each + other, they **must** be separated into two pull requests and be made + forward or backwards compatible, such that the Backend or Frontend + can be safely deployed independently. + + + Have questions? Please ask in the [`#discuss-dev-infra` + channel](https://app.slack.com/client/T024ZCV9U/CTJL7358X). diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 00000000000000..bd0df9cd60adb0 --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,16 @@ +name: 'lock closed issues/PRs' +on: + schedule: + - cron: '* */12 * * *' + workflow_dispatch: +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@be8aa5be94131386884a6da4189effda9b14aa21 # v4.0.1 + with: + github-token: ${{ github.token }} + issue-inactive-days: 15 + issue-lock-reason: '' + pr-inactive-days: 15 + pr-lock-reason: '' diff --git a/.github/workflows/meta-deploys-detect-change-type.yml b/.github/workflows/meta-deploys-detect-change-type.yml new file mode 100644 index 00000000000000..66e74bf1881faf --- /dev/null +++ b/.github/workflows/meta-deploys-detect-change-type.yml @@ -0,0 +1,33 @@ +# Add a GitHub Check to commits in "master" that specifies the "type" of files that were changed +# ex: "only frontend", "only backed", or "fullstack" +name: meta(deploy) + +on: + push: + branches: + - master + +jobs: + files-changed: + name: detect what files changed + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Check for file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + list-files: shell + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Create GitHub job + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + with: + script: | + require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/deploy`).updateChangeType({ + github, + context, + fileChanges: ${{ toJson(steps.changes.outputs) }} + }); diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml new file mode 100644 index 00000000000000..ae32727703b8e2 --- /dev/null +++ b/.github/workflows/migrations.yml @@ -0,0 +1,93 @@ +name: migrations +on: pull_request + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + # the default default is: + # bash --noprofile --norc -eo pipefail {0} + shell: bash --noprofile --norc -eo pipefail -ux {0} + +jobs: + did-migration-change: + name: check if any migration changes + runs-on: ubuntu-20.04 + timeout-minutes: 3 + # Map a step output to a job output + outputs: + added: ${{ steps.changes.outputs.migrations_added }} + modified: ${{ steps.changes.outputs.migrations_modified }} + steps: + - name: Checkout sentry + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Match migration files + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + modified-migration: + name: check if modified migration + runs-on: ubuntu-20.04 + timeout-minutes: 4 + needs: did-migration-change + if: needs.did-migration-change.outputs.modified == 'true' + + steps: + - name: Failure because of modified migration + run: | + echo "If you have a valid reason to modify a migration please get approval" + echo "from @getsentry/owners-migrations." && exit 1 + + sql: + name: Generate SQL + runs-on: ubuntu-20.04 + timeout-minutes: 8 + strategy: + matrix: + pg-version: ['14'] + + needs: did-migration-change + if: needs.did-migration-change.outputs.added == 'true' + + steps: + # Checkout master to run all merged migrations. + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + ref: master + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + with: + pg-version: ${{ matrix.pg-version }} + + - name: Apply migrations + run: | + sentry upgrade --noinput + + # Checkout the current ref + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + clean: false + + - name: Get changed migration files + id: file + run: | + echo 'added<> "$GITHUB_OUTPUT" + git diff --diff-filter=A --name-only origin/master HEAD | grep 'src/sentry/migrations/' >> "$GITHUB_OUTPUT" + echo 'EOF' >> "$GITHUB_OUTPUT" + + - name: Generate SQL for migration + uses: getsentry/action-migrations@f1dc34590460c0fe06ec11c00fec6c16a2159977 # main + env: + SENTRY_LOG_LEVEL: ERROR + with: + githubToken: ${{ secrets.GITHUB_TOKEN }} + migration: ${{ steps.file.outputs.added }} diff --git a/.github/workflows/openapi-diff.yml b/.github/workflows/openapi-diff.yml new file mode 100644 index 00000000000000..b02f0272cfdf6d --- /dev/null +++ b/.github/workflows/openapi-diff.yml @@ -0,0 +1,60 @@ +name: openapi-diff +on: + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + check-diff: + name: build api + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - name: Check for python file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + if: steps.changes.outputs.api_docs == 'true' + + - name: Checkout getsentry/sentry-api-schema + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + if: steps.changes.outputs.api_docs == 'true' + with: + ref: 'main' + repository: getsentry/sentry-api-schema + path: sentry-api-schema + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + if: steps.changes.outputs.api_docs == 'true' + + - name: Build OpenAPI Derefed JSON + if: steps.changes.outputs.api_docs == 'true' + # install ts-node for ts build scripts to execute properly without potentially installing + # conflicting deps when running scripts locally + # see: https://github.com/getsentry/sentry/pull/32328/files + run: | + yarn add ts-node && make build-api-docs + + - name: Compare OpenAPI Derefed JSON + if: steps.changes.outputs.api_docs == 'true' + run: | + npx json-diff@0.5.4 --color sentry-api-schema/openapi-derefed.json tests/apidocs/openapi-derefed.json diff --git a/.github/workflows/openapi.yml b/.github/workflows/openapi.yml new file mode 100644 index 00000000000000..f5d0851136bb8a --- /dev/null +++ b/.github/workflows/openapi.yml @@ -0,0 +1,73 @@ +name: openapi +on: + push: + branches: + - master + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + build_and_deref_json: + runs-on: ubuntu-20.04 + timeout-minutes: 90 + steps: + - name: Getsentry Token + id: getsentry + uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - name: Check for python file changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + id: changes + with: + token: ${{ github.token }} + filters: .github/file-filters.yml + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + if: steps.changes.outputs.api_docs == 'true' + + - name: Checkout getsentry/sentry-api-schema + if: steps.changes.outputs.api_docs == 'true' + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + ref: 'main' + repository: getsentry/sentry-api-schema + path: sentry-api-schema + token: ${{ steps.getsentry.outputs.token }} + + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + if: steps.changes.outputs.api_docs == 'true' + + - name: Build OpenAPI Derefed JSON + if: steps.changes.outputs.api_docs == 'true' + # install ts-node for ts build scripts to execute properly without potentially installing + # conflicting deps when running scripts locally + # see: https://github.com/getsentry/sentry/pull/32328/files + run: | + yarn add ts-node && make build-api-docs + + - name: Copy artifact into getsentry/sentry-api-schema + if: steps.changes.outputs.api_docs == 'true' + run: | + cp tests/apidocs/openapi-derefed.json sentry-api-schema + + - name: Git Commit & Push + uses: stefanzweifel/git-auto-commit-action@0049e3fa4059ca715255fbbcb7dea4516f02ce0a # v4.15.3 + if: steps.changes.outputs.api_docs == 'true' + with: + repository: sentry-api-schema + branch: main + commit_message: Generated + commit_user_email: bot@getsentry.com + commit_user_name: openapi-getsentry-bot diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 00000000000000..fe5b9dc63ed5b1 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,87 @@ +name: pre-commit + +on: + push: + branches: + - master + pull_request: + +# Cancel in progress workflows on pull_requests. +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +defaults: + run: + # the default default is: + # bash --noprofile --norc -eo pipefail {0} + shell: bash --noprofile --norc -eo pipefail -ux {0} + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + # workaround: secrets cannot be directly referenced in `if` + # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#example-using-secrets + SECRET_ACCESS: ${{toJSON(secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY != null)}} + +jobs: + lint: + name: pre-commit lint + runs-on: ubuntu-20.04 + timeout-minutes: 10 + steps: + - # get a non-default github token so that any changes are verified by CI + if: env.SECRET_ACCESS == 'true' + uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + id: token + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - name: Get changed files + id: changes + uses: getsentry/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 + with: + # Enable listing of files matching each filter. + # Paths to files will be available in `${FILTER_NAME}_files` output variable. + list-files: json + + # It doesn't make sense to lint deleted files. + # Therefore we specify we are only interested in added or modified files. + filters: | + all: + - added|modified: '**/*' + + - uses: getsentry/action-setup-venv@9e3bbae3836b1b6f129955bf55a19e1d99a61c67 # v1.0.5 + with: + python-version: 3.8.16 + cache-dependency-path: | + requirements-dev.txt + requirements-dev-frozen.txt + install-cmd: pip install -r requirements-dev.txt -c requirements-dev-frozen.txt + - uses: actions/cache@9b0c1fce7a93df8e3bb8926b0d6e9d89e92f20a7 # v3.0.11 + with: + path: ~/.cache/pre-commit + key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: Setup pre-commit + # We don't use make setup-git because we're only interested in installing + # requirements-dev.txt as a fast path. + # We don't need pre-commit install --install-hooks since we're just interested + # in running the hooks. + run: | + pre-commit install-hooks + + - name: Run pre-commit on PR commits + run: | + jq '.[]' --raw-output <<< '${{steps.changes.outputs.all_files}}' | + # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. + xargs pre-commit run --files + + - name: Apply any pre-commit fixed files + # note: this runs "always" or else it's skipped when pre-commit fails + if: env.SECRET_ACCESS == 'true' && startsWith(github.ref, 'refs/pull') && always() + uses: getsentry/action-github-commit@748c31dd78cffe76f51bef49a0be856b6effeda7 # v1.1.0 + with: + github-token: ${{ steps.token.outputs.token }} + message: ':hammer_and_wrench: apply pre-commit fixes' diff --git a/.github/workflows/publish-dockerhub.yml b/.github/workflows/publish-dockerhub.yml new file mode 100644 index 00000000000000..946427b25a9d81 --- /dev/null +++ b/.github/workflows/publish-dockerhub.yml @@ -0,0 +1,49 @@ +name: Publish Sentry image to DockerHub +on: + push: + branches: + - master + - releases/** +jobs: + publish-sentry: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - name: Pull the test image + id: image_pull + env: + IMAGE_URL: us.gcr.io/sentryio/sentry:${{ github.sha }} + shell: bash + run: | + echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out." + echo "Polling for $IMAGE_URL" + timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done' + - name: Get short SHA for docker tag + id: short_sha + shell: bash + run: | + SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") + if [[ -z "$SHORT_SHA" ]]; then + echo "Short SHA empty? Re-running rev-parse." + git rev-parse --short "$GITHUB_SHA" + else + echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT + fi + - name: Push built docker image + shell: bash + env: + SHORT_SHA: ${{ steps.short_sha.outputs.sha }} + IMAGE_URL: us.gcr.io/sentryio/sentry:${{ github.sha }} + run: | + # only login if the password is set + if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi + # We push 3 tags to Dockerhub: + # first, the full sha of the commit + docker tag ${IMAGE_URL} getsentry/sentry:${GITHUB_SHA} + docker push getsentry/sentry:${GITHUB_SHA} + # second, the short sha of the commit + docker tag ${IMAGE_URL} getsentry/sentry:${SHORT_SHA} + docker push getsentry/sentry:${SHORT_SHA} + # finally, nightly + docker tag ${IMAGE_URL} getsentry/sentry:nightly + docker push getsentry/sentry:nightly diff --git a/.github/workflows/react-to-product-owners-yml-changes.yml b/.github/workflows/react-to-product-owners-yml-changes.yml new file mode 100644 index 00000000000000..85aa22ec5de64f --- /dev/null +++ b/.github/workflows/react-to-product-owners-yml-changes.yml @@ -0,0 +1,24 @@ +name: React to product-owners.yml changes +on: + # This could be run manually, but the general expectation is that this fires + # from GHA in getsentry/security-as-code on changes there. + + workflow_dispatch: +jobs: + release: + runs-on: ubuntu-latest + name: React to product-owners.yml changes + steps: + - uses: actions/checkout@v2 + + - uses: getsentry/action-setup-venv@v1.0.5 + with: + python-version: 3.11.3 + + - name: React to product-owners.yml changes + shell: bash + env: + GITHUB_TOKEN: ${{ secrets.BUMP_SENTRY_TOKEN }} + COMMITTER_NAME: getsentry-bot + COMMITTER_EMAIL: bot@sentry.io + run: ./bin/react-to-product-owners-yml-changes.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000000000..afb2438bd6a239 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,31 @@ +name: Release +on: + workflow_dispatch: + inputs: + version: + description: Version to release (optional) + required: false + force: + description: Force a release even when there are release-blockers (optional) + required: false + schedule: + # We want the release to be at 9-10am Pacific Time + # We also want it to be 1 hour before the self-hosted release + - cron: '0 17 15 * *' +jobs: + release: + runs-on: ubuntu-latest + name: 'Release a new version' + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + token: ${{ secrets.GH_RELEASE_PAT }} + fetch-depth: 0 + - name: Prepare release + uses: getsentry/action-prepare-release@d2cc2db3db92bc5b79a90c316f588f2b13626a2b # v1.5.6 + env: + GITHUB_TOKEN: ${{ secrets.GH_RELEASE_PAT }} + with: + version: ${{ github.event.inputs.version }} + force: ${{ github.event.inputs.force }} + calver: true diff --git a/.github/workflows/sentry-pull-request-bot.yml b/.github/workflows/sentry-pull-request-bot.yml new file mode 100644 index 00000000000000..432a66462ae43d --- /dev/null +++ b/.github/workflows/sentry-pull-request-bot.yml @@ -0,0 +1,74 @@ +name: sentry pull request bot + +# Note this event happens on Issue comments AND PR comments, +# we make sure that we only respond to PR comments. +on: + issue_comment: + types: [created, edited] + pull_request: + types: [opened, edited] + +jobs: + # TODO(billy): Move this into an external action as we add more functionality + test-getsentry: + name: test getsentry + runs-on: ubuntu-20.04 + + # Ensure this bot only responds for pull requests and only for the main repository + if: >- + (github.event.issue.pull_request.url != '' || github.event.pull_request.id != '') && + (contains(github.event.comment.body, '#test-getsentry') || contains(github.event.pull_request.body, '#test-getsentry')) && + github.repository == 'getsentry/sentry' + + steps: + - name: Check getsentry membership + id: org + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + with: + script: | + try { + const result = await github.rest.orgs.checkMembershipForUser({ + org: 'getsentry', + username: context.payload.sender.login, + }) + return result.status == 204; + } catch { + return false; + } + + - name: Fetch getsentry token + if: steps.org.outputs.result == 'true' + id: getsentry + uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 + with: + app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} + private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} + + - name: Wait for PR merge commit + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + id: mergecommit + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + require(`${process.env.GITHUB_WORKSPACE}/.github/workflows/scripts/wait-for-merge-commit`).waitForMergeCommit({ + github, + context, + core, + }); + + - name: Dispatch getsentry tests + if: steps.org.outputs.result == 'true' + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + with: + github-token: ${{ steps.getsentry.outputs.token }} + script: | + github.rest.actions.createWorkflowDispatch({ + owner: 'getsentry', + repo: 'getsentry', + workflow_id: 'acceptance.yml', + ref: 'master', + inputs: { + 'sentry-sha': '${{ steps.mergecommit.outputs.mergeCommitSha }}', + 'sentry-pr-sha': '${{ github.event.pull_request.head.sha }}', + } + }) diff --git a/.github/workflows/shuffle-tests.yml b/.github/workflows/shuffle-tests.yml new file mode 100644 index 00000000000000..b6ae44264cae9b --- /dev/null +++ b/.github/workflows/shuffle-tests.yml @@ -0,0 +1,55 @@ +name: shuffle-tests + +on: + # Allow manually running + workflow_dispatch: + # Run once a week on sunday + schedule: + - cron: '0 1 * * 0' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +env: + SENTRY_SHUFFLE_TESTS: true + +jobs: + backend-test: + name: run backend tests + runs-on: ubuntu-20.04 + timeout-minutes: 90 + strategy: + # This helps not having to run multiple jobs because one fails, thus, reducing resource usage + # and reducing the risk that one of many runs would turn red again (read: intermittent tests) + fail-fast: false + matrix: + # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL. + instance: [0, 1, 2, 3, 4, 5, 6] + pg-version: ['14'] + + env: + # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance. + MATRIX_INSTANCE_TOTAL: 7 + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + # Avoid codecov error message related to SHA resolution: + # https://github.com/codecov/codecov-bash/blob/7100762afbc822b91806a6574658129fe0d23a7d/codecov#L891 + fetch-depth: '2' + + - name: Setup sentry env + uses: ./.github/actions/setup-sentry + id: setup + with: + snuba: true + # Right now, we run so few bigtable related tests that the + # overhead of running bigtable in all backend tests + # is way smaller than the time it would take to run in its own job. + bigtable: true + pg-version: ${{ matrix.pg-version }} + + - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }}) + run: | + make test-python-ci diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml new file mode 100644 index 00000000000000..cd721813f7d43f --- /dev/null +++ b/.github/workflows/sync-labels.yml @@ -0,0 +1,72 @@ +name: meta(labels) +on: + pull_request: + paths: + - .github/labels.yml + push: + branches: + - master + paths: + - .github/labels.yml + +# hack for https://github.com/actions/cache/issues/810#issuecomment-1222550359 +env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 3 + +jobs: + sync: + name: syncs repository labels + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + - uses: getsentry/action-setup-volta@54775a59c41065f54ecc76d1dd5f2cdc7a1550cb # v1.1.0 + + - name: Install github-label-sync + run: yarn global add github-label-sync@2.2.0 + + - name: Run github-label-sync + run: | + github-label-sync \ + --access-token ${{ secrets.GITHUB_TOKEN }} \ + --labels .github/labels.yml \ + ${{ github.event_name == 'pull_request' && '--dry-run' || '' }} \ + ${{ github.repository }} \ + 2>&1 | tee sync-report.txt + + - name: Read sync output into variable + uses: actions/github-script@d556feaca394842dc55e4734bf3bb9f685482fa0 # v6.3.3 + if: github.event_name == 'pull_request' + id: github-label-sync + with: + script: | + const fs = require('fs'); + return fs.readFileSync('sync-report.txt','utf8').toString(); + result-encoding: string + + - name: Find previous dry-run comment + uses: peter-evans/find-comment@a54c31d7fa095754bfef525c0c8e5e5674c4b4b1 # v2.4.0 + if: github.event_name == 'pull_request' + id: fc + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: '' + + - name: Add or update dry-run comment + uses: peter-evans/create-or-update-comment@b95e16d2859ad843a14218d1028da5b2c4cbc4b4 + if: github.event_name == 'pull_request' + with: + issue-number: ${{ github.event.pull_request.number }} + comment-id: ${{ steps.fc.outputs.comment-id }} + edit-mode: replace + body: > + + + 🏷 The following changes will be made to the repository labels + + ``` + ${{ steps.github-label-sync.outputs.result }} + ``` From e140bb4316e79ab33ef6605a3a7423d6cd65fcaa Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 2 Oct 2023 14:27:02 -0700 Subject: [PATCH 10/13] fix --- tests/tools/test_devservices_healthcheck.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/tests/tools/test_devservices_healthcheck.py b/tests/tools/test_devservices_healthcheck.py index cf17175333fbe4..1eb0039defe078 100644 --- a/tests/tools/test_devservices_healthcheck.py +++ b/tests/tools/test_devservices_healthcheck.py @@ -84,15 +84,6 @@ def run( cmd_args: List[str], capture_output: bool = False, text: bool = False, check: bool = False ) -> mock.Mock: if cmd_args == ( - "docker", - "container", - "inspect", - "-f", - "{{.State.Status}}", - "sentry_per", - ): - return running - elif cmd_args == ( "docker", "container", "inspect", From 8ade1e4763e096f4638dfdd861882624b0d0f900 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 2 Oct 2023 20:58:56 -0700 Subject: [PATCH 11/13] back to internal access --- src/sentry/conf/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index 46cc94e6ae3457..aa9ac6b449c454 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2743,7 +2743,7 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] "CLICKHOUSE_HTTP_PORT": "8123", "DEFAULT_BROKERS": "" if "snuba" in settings.SENTRY_EVENTSTREAM - else "{containers[kafka][name]}:9092", + else "{containers[kafka][name]}:9093", "REDIS_HOST": "{containers[redis][name]}", "REDIS_PORT": "6379", "REDIS_DB": "1", From 15d166ff1faa0185dc472d79437dc919087bb6fb Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Mon, 2 Oct 2023 20:59:26 -0700 Subject: [PATCH 12/13] update --- src/sentry/conf/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index aa9ac6b449c454..4a791e7abbbad5 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -2676,7 +2676,7 @@ def build_cdc_postgres_init_db_volume(settings: Any) -> dict[str, dict[str, str] ), "kafka": lambda settings, options: ( { - "image": "confluentinc/cp-kafka:7.5.0", + "image": "ghcr.io/getsentry/image-mirror-confluentinc-cp-kafka:7.5.0", "ports": {"9092/tcp": 9092}, # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example "environment": { From b99dd2c803cded66ce1fa71fd21c0ad381523c8f Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 3 Oct 2023 10:33:47 -0700 Subject: [PATCH 13/13] temporary compat code --- src/sentry/runner/commands/devservices.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentry/runner/commands/devservices.py b/src/sentry/runner/commands/devservices.py index 6a263474ceec53..e884685ee3c242 100644 --- a/src/sentry/runner/commands/devservices.py +++ b/src/sentry/runner/commands/devservices.py @@ -235,6 +235,8 @@ def up( if services: for service in services: if service not in containers: + if service == "zookeeper": + continue click.secho( f"Service `{service}` is not known or not enabled.\n", err=True,