Ease modification of base image (#1479)

Author: spawnia

.github/workflows/test.yml

@@ -58,15 +58,7 @@ jobs:
           sudo chmod +x "${{ matrix.compose_path }}/docker-compose"
 
       - name: Integration Test
-        run: |
-          echo "Testing initial install"
-          ./install.sh
-          ./_integration-test/run.sh
-          echo "Testing in-place upgrade"
-          # Also test plugin installation here
-          echo "sentry-auth-oidc" >> sentry/requirements.txt
-          ./install.sh --minimize-downtime
-          ./_integration-test/run.sh
+        run: ./integration-test.sh
 
       - name: Inspect failure
         if: failure()

.gitignore

@@ -80,6 +80,7 @@ data/
 sentry/sentry.conf.py
 sentry/config.yml
 sentry/*.bak
+sentry/enhance-image.sh
 sentry/requirements.txt
 relay/credentials.json
 relay/config.yml

CONTRIBUTING.md

@@ -0,0 +1,7 @@
+## Testing
+
+Validate changes to the setup by running the integration test:
+
+```shell
+./integration-test.sh
+```

README.md

@@ -12,6 +12,16 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke
 
 ## Setup
 
+### Installation
+
+To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2.
+
+During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`.
+
+Thinking of not managing this yourself? Check out the [SaaS migration docs](https://docs.sentry.io/product/sentry-basics/migration/) or [contact us](https://sentry.io/from/self-hosted) for help.
+
+Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else.
+
 ### Customize DotEnv (.env) file
 
 Environment specific configurations can be done in the `.env.custom` file. It will be located in the root directory of the Sentry installation.
@@ -20,15 +30,13 @@ By default, there exists no `.env.custom` file. In this case, you can manually a
 
 Please keep in mind to check the `.env` file for changes, when you perform an upgrade of Sentry, so that you can adjust your `.env.custom` accordingly, if required.
 
-### Installation
+### Enhance Sentry image
 
-To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2.
+To install plugins and their dependencies or make other modifications to the Sentry base image,
+copy `sentry/enhance-image.example.sh` to `sentry/enhance-image.sh` and add necessary steps there.
+For example, you can use `apt-get` to install dependencies and use `pip` to install plugins.
 
-During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --skip-user-prompt`.
-
-Thinking of not managing this yourself? Check out the [SaaS migration docs](https://docs.sentry.io/product/sentry-basics/migration/) or [contact us](https://sentry.io/from/self-hosted) for help.
-
-Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else.
+After making modifications to `sentry/enhance-image.sh`, run `./install.sh` again to apply them.
 
 ## Tips & Tricks
 
@@ -40,7 +48,7 @@ Sentry comes with a cleanup cron job that prunes events older than `90 days` by
 
 If you want to install a specific release of Sentry, use the tags/releases on this repo.
 
-We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`:
+We continuously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`:
 
 ```shell
 SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh
@@ -57,7 +65,3 @@ sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh
 ```
 
 Where you replace `83b1380` with the sha you want to use.
-
-[build-status-image]: https://github.com/getsentry/self-hosted/workflows/test/badge.svg
-[build-status-url]: https://github.com/getsentry/self-hosted/actions?query=workflow%3Atest+branch%3Amaster+event%3Apush
-

_integration-test/ensure-customizations-not-present.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -ex
+
+source "$(dirname $0)/../install/_lib.sh"
+
+source ../install/dc-detect-version.sh
+
+# Negated version of ensure-customizations-work.sh, make changes in sync
+echo "${_group}Ensure customizations not present"
+! $dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi"
+! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi"
+! $dcr --no-deps web python -c "import ldap"
+! $dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap"
+echo "${_endgroup}"

_integration-test/ensure-customizations-work.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -ex
+
+source "$(dirname $0)/../install/_lib.sh"
+
+source ../install/dc-detect-version.sh
+
+# Negated version of ensure-customizations-not-present.sh, make changes in sync
+echo "${_group}Ensure customizations work"
+$dcr --no-deps web bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi"
+$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup bash -c "if [ ! -e /created-by-enhance-image ]; then exit 1; fi"
+$dcr --no-deps web python -c "import ldap"
+$dcr --no-deps --entrypoint=/etc/sentry/entrypoint.sh sentry-cleanup python -c "import ldap"
+echo "${_endgroup}"

docker-compose.yml

@@ -16,7 +16,11 @@ x-healthcheck-defaults: &healthcheck_defaults
   start_period: 10s
 x-sentry-defaults: &sentry_defaults
   <<: *restart_policy
-  image: "$SENTRY_IMAGE"
+  image: sentry-self-hosted-local
+  build:
+    context: ./sentry
+    args:
+      - SENTRY_IMAGE
   depends_on:
     redis:
       <<: *depends_on-healthy
@@ -321,7 +325,7 @@ services:
     build:
       context: ./cron
       args:
-        BASE_IMAGE: "$SENTRY_IMAGE"
+        BASE_IMAGE: sentry-self-hosted-local
     entrypoint: "/entrypoint.sh"
     command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"'
   nginx:

install/build-docker-images.sh

@@ -1,6 +1,9 @@
 echo "${_group}Building and tagging Docker images ..."
 
 echo ""
+# Build any service that provides the image sentry-self-hosted-local first,
+# as it is used as the base image for sentry-cleanup-self-hosted-local.
+$dc build --force-rm web
 $dc build --force-rm
 echo ""
 echo "Docker images built."

install/ensure-files-from-examples.sh

@@ -1,8 +1,7 @@
 echo "${_group}Ensuring files from examples ..."
 
-ensure_file_from_example $SENTRY_CONFIG_PY
-ensure_file_from_example $SENTRY_CONFIG_YML
+ensure_file_from_example "$SENTRY_CONFIG_PY"
+ensure_file_from_example "$SENTRY_CONFIG_YML"
 ensure_file_from_example '../symbolicator/config.yml'
-ensure_file_from_example '../sentry/requirements.txt'
 
 echo "${_endgroup}"

integration-test.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -ex
+
+echo "Reset customizations"
+rm -f sentry/enhance-image.sh
+rm -f sentry/requirements.txt
+
+echo "Testing initial install"
+./install.sh
+./_integration-test/run.sh
+./_integration-test/ensure-customizations-not-present.sh
+
+echo "Make customizations"
+cat <<EOT > sentry/enhance-image.sh
+#!/bin/bash
+touch /created-by-enhance-image
+apt-get update
+apt-get install -y gcc libsasl2-dev python-dev libldap2-dev libssl-dev
+EOT
+chmod +x sentry/enhance-image.sh
+printf "python-ldap" > sentry/requirements.txt
+
+echo "Testing in-place upgrade and customizations"
+./install.sh --minimize-downtime
+./_integration-test/run.sh
+./_integration-test/ensure-customizations-work.sh