From 7b5074da96fd51f770a2a8cfbaad02e794fd09f2 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 18:33:14 -0700 Subject: [PATCH 01/15] Set up pg_hba --- docker-compose.yml | 17 ++++++++++++++++ postgres/init_hba.sh | 7 +++++++ postgres/postgres-entrypoint.sh | 36 +++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100755 postgres/init_hba.sh create mode 100755 postgres/postgres-entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index c981be9b4f9..388ae90c9dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,10 +71,27 @@ services: postgres: <<: *restart_policy image: "postgres:9.6" + command: + - "postgres" + - "-c" + - "wal_level=logical" + - "-c" + - "max_replication_slots=1" + - "-c" + - "max_wal_senders=1" environment: POSTGRES_HOST_AUTH_METHOD: "trust" + entrypoint: /cdc/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" + - type: bind + read_only: true + source: ./postgres/init_hba.sh + target: /docker-entrypoint-initdb.d/init_hba.sh + - type: bind + read_only: true + source: ./postgres/postgres-entrypoint.sh + target: /cdc/postgres-entrypoint.sh zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" diff --git a/postgres/init_hba.sh b/postgres/init_hba.sh new file mode 100755 index 00000000000..f4b332abfab --- /dev/null +++ b/postgres/init_hba.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Initializes the pg_hba file with access permissions to the replication +# slots. + +set -e + +{ echo "host replication all all trust"; } >> "$PGDATA/pg_hba.conf" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh new file mode 100755 index 00000000000..5c96b18027f --- /dev/null +++ b/postgres/postgres-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# This script replaces the default docker entrypoint for postgres in the +# development environment. +# Its job is to ensure postgres is properly configured to support the +# Change Data Capture pipeline (by setting access permissions and installing +# the replication plugin we use for CDC). Unfortunately the default +# Postgres image does not allow this level of configurability so we need +# to do it this way in order not to have to publish and maintain our own +# Postgres image. +# +# This then, at the end, transfers control to the default entrypoint. + +set -e + +cdc_setup_hba_conf() { + # Ensure pg-hba is properly configured to allow connections + # to the replication slots. + + PG_HBA="$PGDATA/pg_hba.conf" + if [ ! -f "$PG_HBA" ]; then + echo "DB not initialized. Postgres will take care of pg_hba" + elif [ "$(grep -c -E "^host\s+replication" "$PGDATA"/pg_hba.conf)" != 0 ]; then + echo "Replication config already present in pg_hba. Not changing anything." + else + # Execute the same script we run on DB initialization + /docker-entrypoint-initdb.d/init_hba.sh + fi +} + + +echo "Setting up Change Data Capture" + +if [ "$1" = 'postgres' ]; then + cdc_setup_hba_conf +fi +exec /docker-entrypoint.sh "$@" From 66844f877e02ae42dab191311e8157eda4fde64e Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 19:47:42 -0700 Subject: [PATCH 02/15] Download wal2json --- .gitignore | 3 +++ docker-compose.yml | 4 ++++ install.sh | 1 + install/install-wal2json.sh | 29 +++++++++++++++++++++++++++++ postgres/postgres-entrypoint.sh | 5 +++++ 5 files changed, 42 insertions(+) create mode 100644 install/install-wal2json.sh diff --git a/.gitignore b/.gitignore index 707622f4251..8a169049fa5 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock + +# wal2json download +postgres/wal2json diff --git a/docker-compose.yml b/docker-compose.yml index 388ae90c9dc..3c6711f4997 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,10 @@ services: read_only: true source: ./postgres/postgres-entrypoint.sh target: /cdc/postgres-entrypoint.sh + - type: bind + read_only: true + source: ./postgres/wal2json/wal2json.so + target: /wal2json/wal2json.so zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" diff --git a/install.sh b/install.sh index 171851b59da..3886463061f 100755 --- a/install.sh +++ b/install.sh @@ -18,6 +18,7 @@ source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh +source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh new file mode 100644 index 00000000000..a61ce4d59d4 --- /dev/null +++ b/install/install-wal2json.sh @@ -0,0 +1,29 @@ +echo "${_group}Downloading and installing wal2json ..." + +LATEST_VERSION_FILE="../postgres/wal2json/wal2json.so" +ARCH=$(uname -m) +FILE_NAME="wal2json-Linux-$ARCH.so" +LATEST_VERSION=$( + wget "https://api.github.com/repos/getsentry/wal2json/releases/latest" -O - | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' +) + +mkdir -p ../postgres/wal2json +if [[ $LATEST_VERSION ]]; then + if [ ! -f "../postgres/wal2json/$LATEST_VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$LATEST_VERSION" + if wget \ + "https://github.com/getsentry/wal2json/releases/download/$LATEST_VERSION/$FILE_NAME" \ + -P "../postgres/wal2json/$LATEST_VERSION/"; then + ln -s "`pwd`/../postgres/wal2json/$LATEST_VERSION/$FILE_NAME" "$LATEST_VERSION_FILE" + fi + fi +else + echo "wal2json is not installed and cannot download latest version" + exit 1 +fi + +set -e + +echo "${_endgroup}" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh index 5c96b18027f..98bedf39eaf 100755 --- a/postgres/postgres-entrypoint.sh +++ b/postgres/postgres-entrypoint.sh @@ -27,10 +27,15 @@ cdc_setup_hba_conf() { fi } +bind_wal2json() { + # Create the symlink to wal2json.so + ln -s /wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so +} echo "Setting up Change Data Capture" if [ "$1" = 'postgres' ]; then cdc_setup_hba_conf + bind_wal2json fi exec /docker-entrypoint.sh "$@" From 57b2d1a08a317ccc49331d78c0fdcb7f2c494987 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 19:55:12 -0700 Subject: [PATCH 03/15] Fix symlink --- postgres/postgres-entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh index 98bedf39eaf..3312e53b5f4 100755 --- a/postgres/postgres-entrypoint.sh +++ b/postgres/postgres-entrypoint.sh @@ -29,7 +29,7 @@ cdc_setup_hba_conf() { bind_wal2json() { # Create the symlink to wal2json.so - ln -s /wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so + ln -sf /wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so } echo "Setting up Change Data Capture" From 47e95d607b70a4ab5c0fffe9f288a8ad9b97731d Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 22:24:41 -0700 Subject: [PATCH 04/15] Pin version when releasing --- .env | 1 + install/install-wal2json.sh | 38 +++++++++++++++++++------------------ scripts/bump-version.sh | 3 +++ 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/.env b/.env index 3848c8c0eba..d49dae9c4f8 100644 --- a/.env +++ b/.env @@ -7,3 +7,4 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +WAL2JSON_VERSION=latest diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index a61ce4d59d4..68a560b7801 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -1,29 +1,31 @@ echo "${_group}Downloading and installing wal2json ..." -LATEST_VERSION_FILE="../postgres/wal2json/wal2json.so" +VERSION_FILE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) FILE_NAME="wal2json-Linux-$ARCH.so" -LATEST_VERSION=$( - wget "https://api.github.com/repos/getsentry/wal2json/releases/latest" -O - | - grep '"tag_name":' | - sed -E 's/.*"([^"]+)".*/\1/' -) -mkdir -p ../postgres/wal2json -if [[ $LATEST_VERSION ]]; then - if [ ! -f "../postgres/wal2json/$LATEST_VERSION/$FILE_NAME" ]; then - mkdir -p "../postgres/wal2json/$LATEST_VERSION" - if wget \ - "https://github.com/getsentry/wal2json/releases/download/$LATEST_VERSION/$FILE_NAME" \ - -P "../postgres/wal2json/$LATEST_VERSION/"; then - ln -s "`pwd`/../postgres/wal2json/$LATEST_VERSION/$FILE_NAME" "$LATEST_VERSION_FILE" - fi +if [[ $WAL2JSON_VERSION == "latest" ]]; then + VERSION=$( + wget "https://api.github.com/repos/getsentry/wal2json/releases/latest" -O - | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' + ) + + if [[ ! $VERSION ]]; then + echo "Cannot find wal2json latest version" + exit 1 fi else - echo "wal2json is not installed and cannot download latest version" - exit 1 + VERSION=$WAL2JSON_VERSION fi -set -e +mkdir -p ../postgres/wal2json +if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$VERSION" + wget \ + "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ + -P "../postgres/wal2json/$VERSION/"; then + ln -s "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$VERSION_FILE" +fi echo "${_endgroup}" diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4c8bb5a5d30..b5d4586c422 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -8,11 +8,14 @@ OLD_VERSION="$1" NEW_VERSION="$2" SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} +WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" +echo "New wal2json version: $WAL2JSON_VERSION" From dbc94c61c447771052528410aa52fd6762c71a3a Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 22:51:09 -0700 Subject: [PATCH 05/15] Change variable name --- install/install-wal2json.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 68a560b7801..b5121de2de0 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -1,6 +1,6 @@ echo "${_group}Downloading and installing wal2json ..." -VERSION_FILE="../postgres/wal2json/wal2json.so" +FILE_TO_USE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) FILE_NAME="wal2json-Linux-$ARCH.so" @@ -25,7 +25,7 @@ if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then wget \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -P "../postgres/wal2json/$VERSION/"; then - ln -s "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$VERSION_FILE" + ln -s "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi echo "${_endgroup}" From d82b2161897b69e074e610237fe2d0ccf6df9f67 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Wed, 28 Apr 2021 22:53:21 -0700 Subject: [PATCH 06/15] Fix script --- install/install-wal2json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index b5121de2de0..cc1e55600da 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -24,7 +24,7 @@ if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" wget \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ - -P "../postgres/wal2json/$VERSION/"; then + -P "../postgres/wal2json/$VERSION/" ln -s "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi From 8844e8d522b76e4b23e4ab03f5e70939e67600c1 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Fri, 30 Apr 2021 15:45:38 -0700 Subject: [PATCH 07/15] Address review comments --- docker-compose.yml | 23 ++++------------------- install/install-wal2json.sh | 2 +- postgres/postgres-entrypoint.sh | 11 ++++++++--- 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3c6711f4997..60f922e6066 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,31 +71,16 @@ services: postgres: <<: *restart_policy image: "postgres:9.6" - command: - - "postgres" - - "-c" - - "wal_level=logical" - - "-c" - - "max_replication_slots=1" - - "-c" - - "max_wal_senders=1" + command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] environment: POSTGRES_HOST_AUTH_METHOD: "trust" - entrypoint: /cdc/postgres-entrypoint.sh + entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" - type: bind read_only: true - source: ./postgres/init_hba.sh - target: /docker-entrypoint-initdb.d/init_hba.sh - - type: bind - read_only: true - source: ./postgres/postgres-entrypoint.sh - target: /cdc/postgres-entrypoint.sh - - type: bind - read_only: true - source: ./postgres/wal2json/wal2json.so - target: /wal2json/wal2json.so + source: ./postgres/ + target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index cc1e55600da..9056bd2e645 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -25,7 +25,7 @@ if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then wget \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -P "../postgres/wal2json/$VERSION/" - ln -s "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi echo "${_endgroup}" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh index 3312e53b5f4..0b0d98a9640 100755 --- a/postgres/postgres-entrypoint.sh +++ b/postgres/postgres-entrypoint.sh @@ -12,6 +12,10 @@ set -e +prep_init_db() { + cp /opt/sentry/init_hba.sh /docker-entrypoint-initdb.d/init_hba.sh +} + cdc_setup_hba_conf() { # Ensure pg-hba is properly configured to allow connections # to the replication slots. @@ -23,17 +27,18 @@ cdc_setup_hba_conf() { echo "Replication config already present in pg_hba. Not changing anything." else # Execute the same script we run on DB initialization - /docker-entrypoint-initdb.d/init_hba.sh + /opt/sentry/init_hba.sh fi } bind_wal2json() { - # Create the symlink to wal2json.so - ln -sf /wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so + # Copy the file in the right place + cp /opt/sentry/wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so } echo "Setting up Change Data Capture" +prep_init_db if [ "$1" = 'postgres' ]; then cdc_setup_hba_conf bind_wal2json From 9d2f8a70638ca0c69f03153c01e266a64ea5761c Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 18:11:02 -0700 Subject: [PATCH 08/15] Do not rely on having wget --- install/install-wal2json.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 9056bd2e645..5bb3b04e34e 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -4,9 +4,12 @@ FILE_TO_USE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) FILE_NAME="wal2json-Linux-$ARCH.so" +DOCKER="docker run --rm" +CURL="curlimages/curl" + if [[ $WAL2JSON_VERSION == "latest" ]]; then VERSION=$( - wget "https://api.github.com/repos/getsentry/wal2json/releases/latest" -O - | + $DOCKER $CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' ) @@ -22,10 +25,10 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" - wget \ + $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION:/tmp" $CURL \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ - -P "../postgres/wal2json/$VERSION/" - cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + -o /tmp/$FILE_NAME + cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi echo "${_endgroup}" From a31b8e11b1909c518763342a6e0599f53c87ce85 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 18:23:19 -0700 Subject: [PATCH 09/15] Fix path --- install/install-wal2json.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 5bb3b04e34e..016393e7d02 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -25,9 +25,11 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" - $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION:/tmp" $CURL \ + mkdir -p "/tmp/wal2json" + $DOCKER -v "/tmp/wal2json:/tmp" $CURL \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -o /tmp/$FILE_NAME + mv "/tmp/wal2json/$FILE_NAME" "../postgres/wal2json/$VERSION/$FILE_NAME" cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi From 66d0b0cf1ea943f425f559e6193cdea94dd249c5 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 18:33:15 -0700 Subject: [PATCH 10/15] Ensure the user can write --- install/install-wal2json.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 016393e7d02..25d95f41e93 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -25,12 +25,11 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" - mkdir -p "/tmp/wal2json" - $DOCKER -v "/tmp/wal2json:/tmp" $CURL \ + $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME:/tmp" \ + --user=`whoami` $CURL \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -o /tmp/$FILE_NAME - mv "/tmp/wal2json/$FILE_NAME" "../postgres/wal2json/$VERSION/$FILE_NAME" - cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi echo "${_endgroup}" From 47906d960697688c893d1a7ecf8a02c8489c3f56 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 18:39:48 -0700 Subject: [PATCH 11/15] Ensure the user can write --- install/install-wal2json.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 25d95f41e93..1e4ea0f5b2f 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -25,8 +25,9 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" + ls -al "../postgres/wal2json/" $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME:/tmp" \ - --user=`whoami` $CURL \ + $CURL \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -o /tmp/$FILE_NAME cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" From 2f6259541f6d5388267f5d0d771280f17b06d51e Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 18:50:02 -0700 Subject: [PATCH 12/15] Ensure the user can write --- install/install-wal2json.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 1e4ea0f5b2f..6e04e13dd77 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -26,8 +26,9 @@ mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" ls -al "../postgres/wal2json/" + id $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME:/tmp" \ - $CURL \ + $CURL -u $(id -u ${USER}):$(id -g ${USER})\ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ -o /tmp/$FILE_NAME cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" From b2a48a9a47d3e659e07cedbb8c434d90d2f62494 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 19:04:32 -0700 Subject: [PATCH 13/15] Try by redirecting output --- install/install-wal2json.sh | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 6e04e13dd77..3e2e3246e58 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -4,12 +4,11 @@ FILE_TO_USE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) FILE_NAME="wal2json-Linux-$ARCH.so" -DOCKER="docker run --rm" -CURL="curlimages/curl" +DOCKER_CURL="docker run --rm curlimages/curl" if [[ $WAL2JSON_VERSION == "latest" ]]; then VERSION=$( - $DOCKER $CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | + $DOCKER_CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' ) @@ -25,12 +24,10 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" - ls -al "../postgres/wal2json/" - id - $DOCKER -v "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME:/tmp" \ - $CURL -u $(id -u ${USER}):$(id -g ${USER})\ + $DOCKER_CURL -L \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ - -o /tmp/$FILE_NAME + > "../postgres/wal2json/$VERSION/$FILE_NAME" + cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi From b10d57ca7670c24d85f58f56d319ec86c6178795 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 4 May 2021 19:13:25 -0700 Subject: [PATCH 14/15] nit --- install/install-wal2json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 3e2e3246e58..73e3cbe3189 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -28,7 +28,7 @@ if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ > "../postgres/wal2json/$VERSION/$FILE_NAME" - cp "`pwd`/../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" fi echo "${_endgroup}" From 9e3a60a29a638cb4b82ce6447f67944835c9e434 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Thu, 13 May 2021 17:09:55 -0700 Subject: [PATCH 15/15] Change lib name --- install/install-wal2json.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 73e3cbe3189..5ed58005e01 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -2,7 +2,7 @@ echo "${_group}Downloading and installing wal2json ..." FILE_TO_USE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) -FILE_NAME="wal2json-Linux-$ARCH.so" +FILE_NAME="wal2json-Linux-$ARCH-glibc.so" DOCKER_CURL="docker run --rm curlimages/curl"