From bb9ce957802a20224a0793f7a0a895b13d5a96fa Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 13:15:37 +0300 Subject: [PATCH 001/188] First branch manipulation implementation --- .../actions/ensure-release-branch/action.yml | 12 +++ .../ensure-release-branch.sh | 84 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 16 ++++ 3 files changed, 112 insertions(+) create mode 100644 .github/actions/ensure-release-branch/action.yml create mode 100755 .github/actions/ensure-release-branch/ensure-release-branch.sh create mode 100644 .github/workflows/release_build_and_test.yml diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml new file mode 100644 index 000000000..52ab05d4a --- /dev/null +++ b/.github/actions/ensure-release-branch/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh new file mode 100755 index 000000000..465e20f25 --- /dev/null +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -0,0 +1,84 @@ +#!/bin/bash +set -e + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Configure Git to use GITHUB_TOKEN for authentication +if [ -n "$GITHUB_TOKEN" ]; then + echo "Configuring Git with GITHUB_TOKEN..." + git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" + + # Set Git user for commits (required for GitHub Actions) + git config --global user.name "github-actions[bot]" + git config --global user.email "github-actions[bot]@users.noreply.github.com" +else + echo "Warning: GITHUB_TOKEN not found. Git operations may fail if authentication is required." +fi + +# Define RELEASE_VERSION_BRANCH which is the same as TAG +RELEASE_VERSION_BRANCH="$TAG" + +echo "TAG: $TAG" +echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" + +# Check if RELEASE_VERSION_BRANCH exists in origin +if git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" | grep -q "$RELEASE_VERSION_BRANCH"; then + echo "Branch $RELEASE_VERSION_BRANCH exists in origin, checking out..." + git fetch origin "$RELEASE_VERSION_BRANCH" + git checkout "$RELEASE_VERSION_BRANCH" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + exit 0 +fi + +echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" + +# Detect RELEASE_BRANCH name (release/X.Y format) +RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" +echo "RELEASE_BRANCH: $RELEASE_BRANCH" + +# Check if RELEASE_BRANCH exists in origin +if git ls-remote --heads origin "$RELEASE_BRANCH" | grep -q "$RELEASE_BRANCH"; then + echo "Branch $RELEASE_BRANCH exists in origin" + git fetch origin "$RELEASE_BRANCH" + git checkout "$RELEASE_BRANCH" +else + echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" + + # Detect base branch (previous existing branch for the version) + MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') + MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) + MINOR=$(echo "$MAJOR_MINOR" | cut -d. -f2) + + # Find the previous existing release branch + BASE_BRANCH=$(git ls-remote --heads origin "release/$MAJOR.[0-9]" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) + echo git ls-remote --heads origin "release/$MAJOR.[0-9]" + + if [ -z "$BASE_BRANCH" ]; then + echo "Error: Could not find a base branch for $RELEASE_BRANCH" + exit 1 + fi + + echo "Using base branch: $BASE_BRANCH" + + # Create new branch based on base branch and push to origin + git fetch origin "$BASE_BRANCH" + git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + git push origin "$RELEASE_BRANCH" + echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" +fi + +# At this point, we should be on RELEASE_BRANCH +echo "Current branch: $(git branch --show-current)" + +# Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin +git checkout -b "$RELEASE_VERSION_BRANCH" +git push origin "$RELEASE_VERSION_BRANCH" +echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" + +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml new file mode 100644 index 000000000..6a7edd108 --- /dev/null +++ b/.github/workflows/release_build_and_test.yml @@ -0,0 +1,16 @@ +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to build' + required: true + +jobs: + build-and-test: + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Ensure Release Branch + uses: ./.github/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From bced1dabf834b49ac7a89d418cf0e396a1de3171 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 13:27:40 +0300 Subject: [PATCH 002/188] Fix workflow --- .github/workflows/release_build_and_test.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6a7edd108..b0a1853e8 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,12 +1,13 @@ on: workflow_dispatch: - inputs: - release_tag: - description: 'Release tag to build' - required: true + inputs: + release_tag: + description: 'Release tag to build' + required: true jobs: build-and-test: + runs-on: ["self-hosted"] steps: - name: Checkout code uses: actions/checkout@v4 From 36d63fc32680ad63990fab4ef9e5728f60095644 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 14:22:57 +0300 Subject: [PATCH 003/188] Fix yaml --- .github/workflows/release_build_and_test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b0a1853e8..6f7009e03 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -7,11 +7,11 @@ on: jobs: build-and-test: - runs-on: ["self-hosted"] - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Ensure Release Branch - uses: ./.github/ensure-release-branch - with: - release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file + runs-on: ["self-hosted"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Ensure Release Branch + uses: ./.github/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From 6d12d22e06488a7ca2ec725bd384267c8aa5561c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 14:25:30 +0300 Subject: [PATCH 004/188] Fix path to actions --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6f7009e03..6856b8460 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -12,6 +12,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure Release Branch - uses: ./.github/ensure-release-branch + uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file From d067295e4a9fa195ac3f3625ebaeb0fa1cfd3569 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 17:11:09 +0300 Subject: [PATCH 005/188] Fix push empty branches --- .../actions/ensure-release-branch/ensure-release-branch.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 465e20f25..973108741 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +#set -x # Input TAG is expected in $1 TAG="$1" @@ -69,7 +70,7 @@ else # Create new branch based on base branch and push to origin git fetch origin "$BASE_BRANCH" git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - git push origin "$RELEASE_BRANCH" + git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi @@ -78,7 +79,7 @@ echo "Current branch: $(git branch --show-current)" # Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin git checkout -b "$RELEASE_VERSION_BRANCH" -git push origin "$RELEASE_VERSION_BRANCH" +git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From b82ea9d86eb182865b814131aeb0e9d6d63bc474 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 17:18:36 +0300 Subject: [PATCH 006/188] Use different runner --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6856b8460..9febc219f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -7,7 +7,7 @@ on: jobs: build-and-test: - runs-on: ["self-hosted"] + runs-on: ["ubuntu-latest"] steps: - name: Checkout code uses: actions/checkout@v4 From 32823d7cfcba567423c52b5866848da1215ae4a7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 20:30:55 +0300 Subject: [PATCH 007/188] Pretty output --- .../ensure-release-branch.sh | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 973108741..cd14e32c7 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -2,6 +2,19 @@ set -e #set -x +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/helpers.sh" + # Input TAG is expected in $1 TAG="$1" @@ -9,19 +22,6 @@ if [ -z "$TAG" ]; then echo "Error: TAG is required as first argument" exit 1 fi - -# Configure Git to use GITHUB_TOKEN for authentication -if [ -n "$GITHUB_TOKEN" ]; then - echo "Configuring Git with GITHUB_TOKEN..." - git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.com/".insteadOf "https://github.com/" - - # Set Git user for commits (required for GitHub Actions) - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" -else - echo "Warning: GITHUB_TOKEN not found. Git operations may fail if authentication is required." -fi - # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" @@ -29,10 +29,10 @@ echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" # Check if RELEASE_VERSION_BRANCH exists in origin -if git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" | grep -q "$RELEASE_VERSION_BRANCH"; then - echo "Branch $RELEASE_VERSION_BRANCH exists in origin, checking out..." - git fetch origin "$RELEASE_VERSION_BRANCH" - git checkout "$RELEASE_VERSION_BRANCH" +execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then + execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" exit 0 fi @@ -44,21 +44,21 @@ RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" # Check if RELEASE_BRANCH exists in origin -if git ls-remote --heads origin "$RELEASE_BRANCH" | grep -q "$RELEASE_BRANCH"; then +execute_command git ls-remote --heads origin "$RELEASE_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - git fetch origin "$RELEASE_BRANCH" - git checkout "$RELEASE_BRANCH" + execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git checkout "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" # Detect base branch (previous existing branch for the version) MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) - MINOR=$(echo "$MAJOR_MINOR" | cut -d. -f2) # Find the previous existing release branch - BASE_BRANCH=$(git ls-remote --heads origin "release/$MAJOR.[0-9]" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) - echo git ls-remote --heads origin "release/$MAJOR.[0-9]" + execute_command git ls-remote --heads origin "release/$MAJOR.[0-9]" + BASE_BRANCH=$(echo "$last_cmd_stdout" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) if [ -z "$BASE_BRANCH" ]; then echo "Error: Could not find a base branch for $RELEASE_BRANCH" @@ -68,9 +68,9 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - git fetch origin "$BASE_BRANCH" - git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - git push origin HEAD:"$RELEASE_BRANCH" + execute command git fetch origin "$BASE_BRANCH" + execute command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + execute command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi @@ -78,8 +78,8 @@ fi echo "Current branch: $(git branch --show-current)" # Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin -git checkout -b "$RELEASE_VERSION_BRANCH" -git push origin HEAD:"$RELEASE_VERSION_BRANCH" +execute_command git checkout -b "$RELEASE_VERSION_BRANCH" +execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From efe9b17fe14c77572087b096af11964de57150e0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 31 Jul 2025 20:32:19 +0300 Subject: [PATCH 008/188] Fix execut command --- .../actions/ensure-release-branch/ensure-release-branch.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index cd14e32c7..9e7bd5ab1 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,9 +68,9 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute command git fetch origin "$BASE_BRANCH" - execute command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - execute command git push origin HEAD:"$RELEASE_BRANCH" + execute_command git fetch origin "$BASE_BRANCH" + execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" + execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi From 68817c7f9a152e001064664b0946d5ef85e5b27b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 12:20:16 +0300 Subject: [PATCH 009/188] Use common base --- .github/actions/common/helpers.sh | 83 +++++++++++++++++++ .../ensure-release-branch.sh | 8 +- 2 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 .github/actions/common/helpers.sh diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh new file mode 100644 index 000000000..3a74f3062 --- /dev/null +++ b/.github/actions/common/helpers.sh @@ -0,0 +1,83 @@ +# Function to execute command from array and capture output +execute_command() { +# turn off errexit (set -e) if it is active and restore it later + echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" + + local cmd + + # Check if no arguments provided + if [ $# -eq 0 ]; then + # Check if cmd_array variable exists and is an array + if declare -p cmd_array 2>/dev/null | grep -q "declare -a"; then + # Use the existing cmd_array variable + cmd=("${cmd_array[@]}") + else + echo "Error: No arguments provided and cmd_array variable not found or not an array" >&2 + return 1 + fi + else + cmd=("$@") + fi + + # Create temporary files for stdout and stderr + local stdout_file stderr_file + stdout_file=$(mktemp) + stderr_file=$(mktemp) + + # Execute command and capture output + console_output 1 gray "Executing command: ${cmd[*]}" + "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" + last_cmd_result=$? + + # Read captured output + last_cmd_stdout=$(cat "$stdout_file") + last_cmd_stderr=$(cat "$stderr_file") + + if [ "$last_cmd_result" -ne 0 ]; then + console_output 0 red "Command failed with exit code $last_cmd_result" + console_output 0 red "Standard Output:" + console_output 0 red "$last_cmd_stdout" + console_output 0 red "Standard Error:" + console_output 0 red "$last_cmd_stderr" + fi + + # Clean up temporary files + rm -f "$stdout_file" "$stderr_file" + + [ "$restore_errexit" ] && set -e + return $last_cmd_result +} + +# Helper function to output multiline variables with color +console_output() { + local verbosity_level="$1" + local color="$2" + local content="$3" + local current_verbosity="${VERBOSITY:-0}" + + # Check if we should output based on verbosity level + if [ "$current_verbosity" -ge "$verbosity_level" ]; then + local color_code="" + local reset_code="\033[0m" + + case "$color" in + "gray"|"grey") + color_code="\033[90m" + ;; + "white") + color_code="\033[97m" + ;; + "red") + color_code="\033[91m" + ;; + *) + color_code="\033[0m" # Default to no color + ;; + esac + + # Output each line with 4-space indent and color + while IFS= read -r line || [ -n "$line" ]; do + printf "${color_code} %s${reset_code}\n" "$line" + done <<< "$content" + fi +} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 9e7bd5ab1..e3baf7673 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,4 +1,10 @@ #!/bin/bash + +# This script ensures that a release branch and release version branch exist for a given release tag. +# It creates and pushes both branches if they do not exist. +# It also checks out the release version branch at the end. +# https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation + set -e #set -x @@ -13,7 +19,7 @@ VERBOSITY=1 SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$SCRIPT_DIR/helpers.sh" +. "$SCRIPT_DIR/../common/helpers.sh" # Input TAG is expected in $1 TAG="$1" From 3041ccdb126d42e65587bcfbcb502eb7ba1f33b9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 12:57:15 +0300 Subject: [PATCH 010/188] Add validate archive action --- .../validate-redis-release-archive/action.yml | 12 +++++ .../validate-redis-release-archive.sh | 47 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 5 ++ 3 files changed, 64 insertions(+) create mode 100644 .github/actions/validate-redis-release-archive/action.yml create mode 100644 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/validate-redis-release-archive/action.yml b/.github/actions/validate-redis-release-archive/action.yml new file mode 100644 index 000000000..fcd775ce7 --- /dev/null +++ b/.github/actions/validate-redis-release-archive/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/validate-redis-release-archive.sh ${{ inputs.release_tag }} \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh new file mode 100644 index 000000000..8d038cade --- /dev/null +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -e + +# This script validates a Redis release archive by downloading it and calculating its SHA256 sum. +# It constructs the URL, downloads the file, calculates the hash, and exports environment variables. + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Construct Redis archive URL +REDIS_ARCHIVE_URL="https://github.com/redis/redis/archive/refs/tags/${TAG}.tar.gz" +echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" + +# Download the Redis archive +TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" +echo "Downloading Redis archive to $TEMP_ARCHIVE..." +if ! curl -L -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then + echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" + exit 1 +fi + +# Calculate SHA256 sum +echo "Calculating SHA256 sum..." +REDIS_ARCHIVE_SHA=$(sha256sum "$TEMP_ARCHIVE" | cut -d' ' -f1) +echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" + +# Write variables to GITHUB_ENV +if [ -n "$GITHUB_ENV" ]; then + echo "REDIS_ARCHIVE_URL=$REDIS_ARCHIVE_URL" >> "$GITHUB_ENV" + echo "REDIS_ARCHIVE_SHA=$REDIS_ARCHIVE_SHA" >> "$GITHUB_ENV" + echo "Environment variables written to $GITHUB_ENV" +else + echo "Error: GITHUB_ENV not set" + # Clean up temporary file + rm -f "$TEMP_ARCHIVE" + exit 1 +fi + +# Clean up temporary file +rm -f "$TEMP_ARCHIVE" + +echo "Redis archive validation completed successfully" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 9febc219f..7451198c4 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -11,6 +11,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Validate Redis Release Archive + uses: ./.github/actions/validate-redis-release-archive + with: + release_tag: ${{ github.event.inputs.release_tag }} + - name: Ensure Release Branch uses: ./.github/actions/ensure-release-branch with: From 5637a313a6e415d064fec19dc137d7da9c87d16d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:29:50 +0300 Subject: [PATCH 011/188] Make executable --- .../validate-redis-release-archive.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh old mode 100644 new mode 100755 From c17467a24300e09bd84af4c8d20d54c2cbda5b85 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:32:35 +0300 Subject: [PATCH 012/188] Make curl return error --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index 8d038cade..be450a4f6 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -L -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -fiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From 1d007596a833f80cdb0a11215c138ceaa700d8a4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 14:33:33 +0300 Subject: [PATCH 013/188] Make curl silent --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index be450a4f6..d258369aa 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -fiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -sfiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From a86989c8bc6a4762c2247511cbe6743fd488a40c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:25:39 +0300 Subject: [PATCH 014/188] Try to use pre-merge.yml --- .../ensure-release-branch.sh | 42 ++++++++++++++++--- .github/workflows/release_build_and_test.yml | 8 +++- 2 files changed, 43 insertions(+), 7 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index e3baf7673..037248e92 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,12 +1,12 @@ #!/bin/bash +set -e +#set -x # This script ensures that a release branch and release version branch exist for a given release tag. # It creates and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation -set -e -#set -x # shellcheck disable=SC2034 last_cmd_stdout="" @@ -17,17 +17,43 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 + SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" -# Input TAG is expected in $1 -TAG="$1" +# Parse arguments +CREATE="" +TAG="" + +while [[ $# -gt 0 ]]; do + case $1 in + --create) + CREATE=1 + shift + ;; + -*) + echo "Error: Unknown option $1" + exit 1 + ;; + *) + if [ -z "$TAG" ]; then + TAG="$1" + else + echo "Error: Multiple TAG arguments provided" + exit 1 + fi + shift + ;; + esac +done if [ -z "$TAG" ]; then - echo "Error: TAG is required as first argument" + echo "Error: TAG is required as argument" + echo "Usage: $0 [--no-write] " exit 1 fi + # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" @@ -44,6 +70,10 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" +if [ -z "$CREATE" ]; then + echo "Refuse to modify repository without --create option" + exit 1 +fi # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" @@ -88,4 +118,4 @@ execute_command git checkout -b "$RELEASE_VERSION_BRANCH" execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 7451198c4..72f28679c 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -11,6 +11,7 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Validate Redis Release Archive uses: ./.github/actions/validate-redis-release-archive with: @@ -19,4 +20,9 @@ jobs: - name: Ensure Release Branch uses: ./.github/actions/ensure-release-branch with: - release_tag: ${{ github.event.inputs.release_tag }} \ No newline at end of file + release_tag: ${{ github.event.inputs.release_tag }} + + - name: Apply Docker Version + uses: ./.github/actions/apply-docker-version + with: + release_tag: ${{ github.event.inputs.release_tag }} From 5f6fccc7234ab72f1e3e958f418c2515a4295049 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:27:38 +0300 Subject: [PATCH 015/188] allow-modify flag --- .github/actions/ensure-release-branch/action.yml | 5 ++++- .../ensure-release-branch/ensure-release-branch.sh | 14 +++++++------- .github/workflows/pre-merge.yml | 13 ++++++++++--- .github/workflows/release_build_and_test.yml | 11 ++++++++++- 4 files changed, 31 insertions(+), 12 deletions(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 52ab05d4a..8426db2a8 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -2,6 +2,9 @@ inputs: release_tag: description: 'Release tag to build' required: true + allow_modify: + description: 'Allow modifying the repository' + default: false runs: using: "composite" @@ -9,4 +12,4 @@ runs: - name: Ensure Release Branch shell: bash run: | - ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.release_tag }} + ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 037248e92..9a41004cf 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -3,7 +3,7 @@ set -e #set -x # This script ensures that a release branch and release version branch exist for a given release tag. -# It creates and pushes both branches if they do not exist. +# It allow-modifys and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation @@ -23,13 +23,13 @@ SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" . "$SCRIPT_DIR/../common/helpers.sh" # Parse arguments -CREATE="" +ALLOW_MODIFY="" TAG="" while [[ $# -gt 0 ]]; do case $1 in - --create) - CREATE=1 + --allow-modify) + ALLOW_MODIFY=1 shift ;; -*) @@ -50,7 +50,7 @@ done if [ -z "$TAG" ]; then echo "Error: TAG is required as argument" - echo "Usage: $0 [--no-write] " + echo "Usage: $0 [--allow-modify] " exit 1 fi @@ -70,8 +70,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$CREATE" ]; then - echo "Refuse to modify repository without --create option" +if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" exit 1 fi diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 2b0a81525..7cb53b149 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -4,9 +4,11 @@ on: branches: - master - release/* - push: - branches: - - release/8.2 + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to build' + required: true jobs: build-and-test: @@ -40,6 +42,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Ensure release branch + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: ./.github/actions/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 72f28679c..24cae9095 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -6,7 +6,7 @@ on: required: true jobs: - build-and-test: + prepare-release: runs-on: ["ubuntu-latest"] steps: - name: Checkout code @@ -21,8 +21,17 @@ jobs: uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: true - name: Apply Docker Version uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} + + build-and-test: + needs: prepare-release + runs-on: ["ubuntu-latest"] + use: pre-merge.yml + with: + release_tag: ${{ github.event.inputs.release_tag }} + From 8e1e9ad126b0eee82782c85e11db2288cf0e7359 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:28:56 +0300 Subject: [PATCH 016/188] Fix workflow --- .github/workflows/release_build_and_test.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 24cae9095..5fbf9f46d 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -28,10 +28,10 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} - build-and-test: - needs: prepare-release - runs-on: ["ubuntu-latest"] - use: pre-merge.yml - with: - release_tag: ${{ github.event.inputs.release_tag }} + build-and-test: + needs: prepare-release + runs-on: ["ubuntu-latest"] + use: pre-merge.yml + with: + release_tag: ${{ github.event.inputs.release_tag }} From b981b62dd78cdb4f8504cc70a939b032789811e0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:31:34 +0300 Subject: [PATCH 017/188] Fix worflow reuse --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 7cb53b149..fe2734330 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -4,7 +4,7 @@ on: branches: - master - release/* - workflow_dispatch: + workflow_call: inputs: release_tag: description: 'Release tag to build' @@ -43,7 +43,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure release branch - if: ${{ github.event_name == 'workflow_dispatch' }} + if: ${{ github.event_name == 'workflow_call' }} uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5fbf9f46d..ee620a802 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -29,9 +29,10 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: - needs: prepare-release runs-on: ["ubuntu-latest"] - use: pre-merge.yml + needs: prepare-release + uses: ./.github/workflows/pre-merge.yml + secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 07869702053c4eaef3d1ab65605c9a4879df6993 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:34:07 +0300 Subject: [PATCH 018/188] Fixed required property type --- .github/workflows/pre-merge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index fe2734330..51a50a3a2 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -9,6 +9,7 @@ on: release_tag: description: 'Release tag to build' required: true + type: string jobs: build-and-test: From d7507d3b88eff8e5d65f8cfbc91065449edff95c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:38:55 +0300 Subject: [PATCH 019/188] Remove runs-on --- .github/workflows/release_build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ee620a802..10ea16c37 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -29,7 +29,6 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: - runs-on: ["ubuntu-latest"] needs: prepare-release uses: ./.github/workflows/pre-merge.yml secrets: inherit From d3d5043e6b9b3224c48f04d17f9d39c7c7ad2a6e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 16:39:37 +0300 Subject: [PATCH 020/188] Add apply-docker-version --- .../actions/apply-docker-version/action.yml | 12 +++ .../apply-docker-version.sh | 88 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 .github/actions/apply-docker-version/action.yml create mode 100755 .github/actions/apply-docker-version/apply-docker-version.sh diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml new file mode 100644 index 000000000..fd02d56ba --- /dev/null +++ b/.github/actions/apply-docker-version/action.yml @@ -0,0 +1,12 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + +runs: + using: "composite" + steps: + - name: Ensure Release Branch + shell: bash + run: | + ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh new file mode 100755 index 000000000..6fe9e3eef --- /dev/null +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -0,0 +1,88 @@ +#!/bin/bash +set -e + +# This script updates Redis version in Dockerfiles using environment variables +# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Check if required environment variables are set +if [ -z "$REDIS_ARCHIVE_URL" ]; then + echo "Error: REDIS_ARCHIVE_URL environment variable is not set" + exit 1 +fi + +if [ -z "$REDIS_ARCHIVE_SHA" ]; then + echo "Error: REDIS_ARCHIVE_SHA environment variable is not set" + exit 1 +fi + +echo "TAG: $TAG" +echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" +echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" + +# Function to update Dockerfile +update_dockerfile() { + local dockerfile="$1" + local updated=false + + if [ ! -f "$dockerfile" ]; then + echo "Warning: $dockerfile not found, skipping" + return 1 + fi + + echo "Updating $dockerfile..." + + # Update REDIS_DOWNLOAD_URL + if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile" + updated=true + echo " Updated REDIS_DOWNLOAD_URL" + fi + + # Update REDIS_DOWNLOAD_SHA + if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile" + updated=true + echo " Updated REDIS_DOWNLOAD_SHA" + fi + + if [ "$updated" = true ]; then + echo " $dockerfile updated successfully" + return 0 + else + echo " No changes needed in $dockerfile" + return 1 + fi +} + +# Track if any files were modified +files_modified=false + +# Update debian/Dockerfile +if update_dockerfile "debian/Dockerfile"; then + files_modified=true +fi + +# Update alpine/Dockerfile +if update_dockerfile "alpine/Dockerfile"; then + files_modified=true +fi + +# Commit changes if any files were modified +if [ "$files_modified" = true ]; then + echo "Files were modified, committing changes..." + git add debian/Dockerfile alpine/Dockerfile + git commit -m "$TAG" + echo "Changes committed with message: $TAG" +else + echo "No files were modified, nothing to commit" +fi + +echo "Docker version update completed" From 21faa0e128d89dee74bab8dc0cebf33d6e0e1705 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 17:18:36 +0300 Subject: [PATCH 021/188] Use explicit branch name --- .github/workflows/release_build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 10ea16c37..5ea610233 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,24 +13,24 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive + uses: ./.github/actions/validate-redis-release-archive@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch - uses: ./.github/actions/ensure-release-branch + uses: ./.github/actions/ensure-release-branch@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true - name: Apply Docker Version - uses: ./.github/actions/apply-docker-version + uses: ./.github/actions/apply-docker-version@Peter-Sh_release_automation_RED-165688 with: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml + uses: ./.github/workflows/pre-merge.yml@Peter-Sh_release_automation_RED-165688 secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 880577793873b94c385192f1ff97364974c91eaa Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 17:52:05 +0300 Subject: [PATCH 022/188] Branches doesn't work with local versions --- .github/workflows/release_build_and_test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5ea610233..10ea16c37 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,24 +13,24 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/validate-redis-release-archive with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch - uses: ./.github/actions/ensure-release-branch@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true - name: Apply Docker Version - uses: ./.github/actions/apply-docker-version@Peter-Sh_release_automation_RED-165688 + uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml@Peter-Sh_release_automation_RED-165688 + uses: ./.github/workflows/pre-merge.yml secrets: inherit with: release_tag: ${{ github.event.inputs.release_tag }} From 3a85e67886a63733a39dcce7bc2b281d96f3f448 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 18:00:41 +0300 Subject: [PATCH 023/188] Add bot name --- .github/actions/apply-docker-version/apply-docker-version.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6fe9e3eef..6f973f077 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,6 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." + git config --global user.email "relesase-bot@redis.com" + git config --global user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git commit -m "$TAG" echo "Changes committed with message: $TAG" From 2262399b779a1a3e95ed058ca4c10a304e026a3c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 18:02:32 +0300 Subject: [PATCH 024/188] Try locl config --- .github/actions/apply-docker-version/apply-docker-version.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6f973f077..ae15cb93a 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,8 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config --global user.email "relesase-bot@redis.com" - git config --global user.name "Release Bot" + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git commit -m "$TAG" echo "Changes committed with message: $TAG" From cc1dd2adba92fab8d2c0de73dbd6ad74c2fa6e61 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:26:18 +0300 Subject: [PATCH 025/188] Use --author --- .github/actions/apply-docker-version/apply-docker-version.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index ae15cb93a..470fb97c4 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,10 +78,8 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git commit -m "$TAG" + git commit -m "$TAG" --author "Redis Release Bot " echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" From 0a7159a9b7c08743fa5c19522bad224a2fccb33d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:33:44 +0300 Subject: [PATCH 026/188] debug output --- .github/actions/apply-docker-version/apply-docker-version.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 470fb97c4..e8bae42c0 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -78,8 +78,11 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git commit -m "$TAG" --author "Redis Release Bot " + git config --list --show-origin + git commit -m "$TAG" echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" From d158d12a7913821d0516fe1d8543d06d9a8fb7bd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:54:30 +0300 Subject: [PATCH 027/188] Auto merge from release branch --- .../apply-docker-version.sh | 2 +- .../ensure-release-branch.sh | 81 ++++++++++++++----- 2 files changed, 63 insertions(+), 20 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index e8bae42c0..7bbee17c3 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -81,7 +81,7 @@ if [ "$files_modified" = true ]; then git config user.email "relesase-bot@redis.com" git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile - git config --list --show-origin + git diff --cached git commit -m "$TAG" echo "Changes committed with message: $TAG" else diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 9a41004cf..baa0668a5 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -1,12 +1,12 @@ #!/bin/bash -set -e -#set -x # This script ensures that a release branch and release version branch exist for a given release tag. # It allow-modifys and pushes both branches if they do not exist. # It also checks out the release version branch at the end. # https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation +set -e +#set -x # shellcheck disable=SC2034 last_cmd_stdout="" @@ -60,21 +60,6 @@ RELEASE_VERSION_BRANCH="$TAG" echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" -# Check if RELEASE_VERSION_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "$RELEASE_VERSION_BRANCH" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" - exit 0 -fi - -echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 -fi - # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" @@ -84,9 +69,12 @@ execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" execute_command git fetch origin "$RELEASE_BRANCH" - execute_command git checkout "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" + if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" + exit 1 + fi # Detect base branch (previous existing branch for the version) MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') @@ -110,6 +98,61 @@ else echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" fi +# Check if RELEASE_VERSION_BRANCH exists in origin +execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" +if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then + execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "$RELEASE_VERSION_BRANCH" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + + # Check if there are changes in release branch that are not in release version branch + echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." + execute_command git fetch origin "$RELEASE_BRANCH" + + # Compare the two branches to see if there are commits in release branch not in release version branch + execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') + + if [ "$COMMITS_BEHIND" -gt 0 ]; then + echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + + if [ -z "$ALLOW_MODIFY" ]; then + echo "Changes detected but refusing to merge without --allow-modify option" + exit 1 + fi + + echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." + + # Try to merge the release branch into the current release version branch + execute_command git merge "origin/$RELEASE_BRANCH" --no-edit + if [ $last_cmd_result -eq 0 ]; then + echo "Successfully merged changes from $RELEASE_BRANCH" + + # Push the merged changes to origin + execute_command git push origin "$RELEASE_VERSION_BRANCH" + if [ $last_cmd_result -eq 0 ]; then + echo "Successfully pushed merged changes to origin/$RELEASE_VERSION_BRANCH" + else + echo "Error: Failed to push merged changes to origin" + exit 1 + fi + else + echo "Error: Merge failed. There may be conflicts that need manual resolution." + exit 1 + fi + fi + + exit 0 + +fi + +echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" +if [ -z "$ALLOW_MODIFY" ]; then + echo "Refuse to modify repository without --allow-modify option" + exit 1 +fi + +execute_command git checkout "$RELEASE_BRANCH" # At this point, we should be on RELEASE_BRANCH echo "Current branch: $(git branch --show-current)" @@ -118,4 +161,4 @@ execute_command git checkout -b "$RELEASE_VERSION_BRANCH" execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" +echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file From a75454ead3bc9b1a54fab03be2ef60c72d855865 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 19:59:41 +0300 Subject: [PATCH 028/188] Push after merge, use helpers --- .../apply-docker-version.sh | 18 +++++++++++++++++- .../ensure-release-branch.sh | 18 ++---------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 7bbee17c3..6c05391d9 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -4,6 +4,21 @@ set -e # This script updates Redis version in Dockerfiles using environment variables # REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/helpers.sh" + + # Input TAG is expected in $1 TAG="$1" @@ -82,7 +97,8 @@ if [ "$files_modified" = true ]; then git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git diff --cached - git commit -m "$TAG" + execute_command git commit -m "$TAG" + execute_command git push origin "$TAG" echo "Changes committed with message: $TAG" else echo "No files were modified, nothing to commit" diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index baa0668a5..47339864a 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -122,24 +122,10 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then fi echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." - # Try to merge the release branch into the current release version branch execute_command git merge "origin/$RELEASE_BRANCH" --no-edit - if [ $last_cmd_result -eq 0 ]; then - echo "Successfully merged changes from $RELEASE_BRANCH" - - # Push the merged changes to origin - execute_command git push origin "$RELEASE_VERSION_BRANCH" - if [ $last_cmd_result -eq 0 ]; then - echo "Successfully pushed merged changes to origin/$RELEASE_VERSION_BRANCH" - else - echo "Error: Failed to push merged changes to origin" - exit 1 - fi - else - echo "Error: Merge failed. There may be conflicts that need manual resolution." - exit 1 - fi + # Push the merged changes to origin + execute_command git push origin "$RELEASE_VERSION_BRANCH" fi exit 0 From f7699664a5827043affb47e944ae0fa7e415dc73 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:09:06 +0300 Subject: [PATCH 029/188] debug output --- .../ensure-release-branch.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 47339864a..daeabd64c 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -123,6 +123,21 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." # Try to merge the release branch into the current release version branch + echo origin/$RELEASE_BRANCH + git log --graph --decorate -5 origin/$RELEASE_BRANCH + + echo origin/$RELEASE_VERSION_BRANCH + git log --graph --decorate -5 origin/$RELEASE_VERSION_BRANCH + + echo $RELEASE_BRANCH + git log --graph --decorate -5 $RELEASE_BRANCH + + echo $RELEASE_VERSION_BRANCH + git log --graph --decorate -5 $RELEASE_VERSION_BRANCH + + echo "HEAD" + git log --graph --decorate -5 + execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" From 9be223f751467a4cecc16c687bef3e7065161e5a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:22:16 +0300 Subject: [PATCH 030/188] Try to use --unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index daeabd64c..4bf556fb3 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,7 +68,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +92,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch origin "$BASE_BRANCH" + execute_command git fetch --unshallow origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +101,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From 49e31123419a438a5f4df8fb9a872a22eb091c6e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:29:28 +0300 Subject: [PATCH 031/188] Try to combine unshallow and not unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 4bf556fb3..bc3badce4 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -68,7 +68,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +92,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch --unshallow origin "$BASE_BRANCH" + execute_command git fetch --unshallow origin "$BASE_BRANCH" || execute_command git fetch origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +101,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" || execute_command git fetch origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" + execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From c7b84a2e7a83c3f2e349a4ada3ce1e62a70c025d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:33:12 +0300 Subject: [PATCH 032/188] Use function to fetch unshallow --- .../ensure-release-branch/ensure-release-branch.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index bc3badce4..848669566 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -48,6 +48,10 @@ while [[ $# -gt 0 ]]; do esac done +git_fetch_unshallow() { + git fetch --unshallow "$@" 2>/dev/null || git fetch "$@" +} + if [ -z "$TAG" ]; then echo "Error: TAG is required as argument" echo "Usage: $0 [--allow-modify] " @@ -68,7 +72,7 @@ echo "RELEASE_BRANCH: $RELEASE_BRANCH" execute_command git ls-remote --heads origin "$RELEASE_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" else echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" if [ -z "$ALLOW_MODIFY" ]; then @@ -92,7 +96,7 @@ else echo "Using base branch: $BASE_BRANCH" # Create new branch based on base branch and push to origin - execute_command git fetch --unshallow origin "$BASE_BRANCH" || execute_command git fetch origin "$BASE_BRANCH" + execute_command git_fetch_unshallow origin "$BASE_BRANCH" execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" execute_command git push origin HEAD:"$RELEASE_BRANCH" echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" @@ -101,13 +105,13 @@ fi # Check if RELEASE_VERSION_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git fetch --unshallow origin "$RELEASE_VERSION_BRANCH" || execute_command git fetch origin "$RELEASE_VERSION_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" execute_command git checkout "$RELEASE_VERSION_BRANCH" echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git fetch --unshallow origin "$RELEASE_BRANCH" || execute_command git fetch origin "$RELEASE_BRANCH" + execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From dec5f36f673df35038a33c2e25879019f11af8ef Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:34:09 +0300 Subject: [PATCH 033/188] Set author for merge --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 848669566..693d363ef 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -142,6 +142,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "HEAD" git log --graph --decorate -5 + git config user.email "relesase-bot@redis.com" + git config user.name "Release Bot" execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" From 4e05f49113ba18c028861a8697d58004765e4421 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:38:45 +0300 Subject: [PATCH 034/188] Fix sha calculation --- .../validate-redis-release-archive.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh index d258369aa..187769731 100755 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh @@ -19,7 +19,7 @@ echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" # Download the Redis archive TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -sfiL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then +if ! curl -sfL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" exit 1 fi From e86576dad368ef70b1b51e0acb5ee0552f3002de Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:58:18 +0300 Subject: [PATCH 035/188] Fix ensure branch call when building --- .github/workflows/pre-merge.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 51a50a3a2..82c9414f8 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -44,10 +44,10 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - name: Ensure release branch - if: ${{ github.event_name == 'workflow_call' }} + if: ${{ inputs.release_tag }} uses: ./.github/actions/ensure-release-branch with: - release_tag: ${{ github.event.inputs.release_tag }} + release_tag: ${{ inputs.release_tag }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} From 017408251c8c767cc30778c1074b749b983842cf Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 1 Aug 2025 20:58:18 +0300 Subject: [PATCH 036/188] Fix ensure branch call when building --- .github/actions/apply-docker-version/action.yml | 4 +++- .../apply-docker-version.sh | 2 -- .../actions/configure_git_commiter/action.yml | 8 ++++++++ .../actions/ensure-release-branch/action.yml | 2 ++ .../ensure-release-branch.sh | 17 ----------------- .github/workflows/release_build_and_test.yml | 8 +++++++- 6 files changed, 20 insertions(+), 21 deletions(-) create mode 100644 .github/actions/configure_git_commiter/action.yml diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index fd02d56ba..b1fe46582 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -6,7 +6,9 @@ inputs: runs: using: "composite" steps: - - name: Ensure Release Branch + - name: Configure Git Commiter + uses: ./.github/actions/configure_git_commiter + - name: Apply docker version shell: bash run: | ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 6c05391d9..f5a86d986 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -93,8 +93,6 @@ fi # Commit changes if any files were modified if [ "$files_modified" = true ]; then echo "Files were modified, committing changes..." - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" git add debian/Dockerfile alpine/Dockerfile git diff --cached execute_command git commit -m "$TAG" diff --git a/.github/actions/configure_git_commiter/action.yml b/.github/actions/configure_git_commiter/action.yml new file mode 100644 index 000000000..3ab5c0fe0 --- /dev/null +++ b/.github/actions/configure_git_commiter/action.yml @@ -0,0 +1,8 @@ +runs: + using: "composite" + steps: + - name: Configure bot commiter + shell: bash + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 8426db2a8..160960ec3 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -9,6 +9,8 @@ inputs: runs: using: "composite" steps: + - name: Conigure Git Commiter + uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch shell: bash run: | diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 693d363ef..b6c7f4efa 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -127,23 +127,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." # Try to merge the release branch into the current release version branch - echo origin/$RELEASE_BRANCH - git log --graph --decorate -5 origin/$RELEASE_BRANCH - - echo origin/$RELEASE_VERSION_BRANCH - git log --graph --decorate -5 origin/$RELEASE_VERSION_BRANCH - - echo $RELEASE_BRANCH - git log --graph --decorate -5 $RELEASE_BRANCH - - echo $RELEASE_VERSION_BRANCH - git log --graph --decorate -5 $RELEASE_VERSION_BRANCH - - echo "HEAD" - git log --graph --decorate -5 - - git config user.email "relesase-bot@redis.com" - git config user.name "Release Bot" execute_command git merge "origin/$RELEASE_BRANCH" --no-edit # Push the merged changes to origin execute_command git push origin "$RELEASE_VERSION_BRANCH" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 10ea16c37..012342dab 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,3 +1,9 @@ +# This worjflow is a part of release automation process. +# It is intended to be run with workflow_dispatch event by the automation. + +# Warning: Workflow does switch branches and this may lead to confusion when changing workflow actions. +# The usual safety rule is to make changes to workflow or actions in base branch (e.g, release/8.X) +# Version branches (e.g, 8.0.10-rc5-int8) will merge changes from base branch automatically. on: workflow_dispatch: inputs: @@ -33,5 +39,5 @@ jobs: uses: ./.github/workflows/pre-merge.yml secrets: inherit with: - release_tag: ${{ github.event.inputs.release_tag }} + release_tag: ${{ inputs.release_tag }} From 8ea64de61dff8725b6a5f4be1f57d86b0cf43080 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 2 Aug 2025 12:56:36 +0300 Subject: [PATCH 037/188] Try to create verified commit --- .../actions/apply-docker-version/action.yml | 15 +++++++-- .../apply-docker-version.sh | 33 +++++++++++-------- .github/workflows/release_build_and_test.yml | 1 + 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index b1fe46582..3d93f8326 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -3,12 +3,23 @@ inputs: description: 'Release tag to build' required: true +outputs: + changed_files: + description: 'List of files that were modified' + value: ${{ steps.apply-version.outputs.changed_files }} + runs: using: "composite" steps: - - name: Configure Git Commiter - uses: ./.github/actions/configure_git_commiter - name: Apply docker version + id: apply-version shell: bash run: | ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} + + - name: Create verified commit + if: steps.apply-version.outputs.changed_files != '' + uses: iarekylew00t/verified-bot-commit@v1 + with: + message: ${{ inputs.release_tag }} + files: ${{ steps.apply-version.outputs.changed_files }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index f5a86d986..130c29e2d 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -77,29 +77,36 @@ update_dockerfile() { fi } -# Track if any files were modified -files_modified=false +# Track which files were modified +changed_files=() # Update debian/Dockerfile if update_dockerfile "debian/Dockerfile"; then - files_modified=true + changed_files+=("debian/Dockerfile") fi # Update alpine/Dockerfile if update_dockerfile "alpine/Dockerfile"; then - files_modified=true + changed_files+=("alpine/Dockerfile") fi -# Commit changes if any files were modified -if [ "$files_modified" = true ]; then - echo "Files were modified, committing changes..." - git add debian/Dockerfile alpine/Dockerfile - git diff --cached - execute_command git commit -m "$TAG" - execute_command git push origin "$TAG" - echo "Changes committed with message: $TAG" +# Output the list of changed files for GitHub Actions +if [ ${#changed_files[@]} -gt 0 ]; then + echo "Files were modified:" + printf '%s\n' "${changed_files[@]}" + + # Set GitHub Actions output + changed_files_output=$(printf '%s\n' "${changed_files[@]}") + { + echo "changed_files<> "$GITHUB_OUTPUT" + + echo "Changed files output set for next step" else - echo "No files were modified, nothing to commit" + echo "No files were modified" + echo "changed_files=" >> "$GITHUB_OUTPUT" fi echo "Docker version update completed" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 012342dab..3b1f96fb2 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -36,6 +36,7 @@ jobs: build-and-test: needs: prepare-release + if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From 4bb6e39033489187f67005370d0f7a6342689c6b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 2 Aug 2025 13:04:23 +0300 Subject: [PATCH 038/188] Push to release_version_branch instead of release_branch --- .github/actions/apply-docker-version/action.yml | 4 ++++ .github/actions/ensure-release-branch/action.yml | 6 ++++++ .../actions/ensure-release-branch/ensure-release-branch.sh | 3 +++ .github/workflows/release_build_and_test.yml | 2 ++ 4 files changed, 15 insertions(+) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 3d93f8326..315c56513 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -2,6 +2,9 @@ inputs: release_tag: description: 'Release tag to build' required: true + release_version_branch: + description: 'Release version branch to commit to' + required: true outputs: changed_files: @@ -23,3 +26,4 @@ runs: with: message: ${{ inputs.release_tag }} files: ${{ steps.apply-version.outputs.changed_files }} + ref: ${{ inputs.release_version_branch }} diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 160960ec3..e7a51bca0 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -6,12 +6,18 @@ inputs: description: 'Allow modifying the repository' default: false +outputs: + release_version_branch: + description: 'The release version branch name' + value: ${{ steps.ensure-branch.outputs.release_version_branch }} + runs: using: "composite" steps: - name: Conigure Git Commiter uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch + id: ensure-branch shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index b6c7f4efa..be9ccd48e 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -61,6 +61,9 @@ fi # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" +# Output the release version branch for GitHub Actions +echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" + echo "TAG: $TAG" echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3b1f96fb2..816f0b7e6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,7 @@ jobs: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch + id: ensure-branch uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} @@ -33,6 +34,7 @@ jobs: uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} + release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: needs: prepare-release From 0fda96a2e9c206ce0e56b432b750c9d5d6b24eac Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:15:36 +0300 Subject: [PATCH 039/188] Optional workflow_uuid --- .github/workflows/release_build_and_test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 816f0b7e6..be90a71dd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -10,9 +10,13 @@ on: release_tag: description: 'Release tag to build' required: true + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false jobs: prepare-release: + name: ${{ github.event.inputs.workflow_uuid && format('prepare-release {0}', github.event.inputs.workflow_uuid) || 'prepare-release' }} runs-on: ["ubuntu-latest"] steps: - name: Checkout code From ab10d32e7c136e6b13d6d2d96cb3c571b522c40f Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:35:08 +0300 Subject: [PATCH 040/188] Use run-name --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index be90a71dd..972fc9abd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,9 +14,10 @@ on: description: 'Optional UUID to identify this workflow run' required: false +run-name: "Release Build and Test ${{ github.event.inputs.workflow_uuid && format(' {0}', github.event.inputs.workflow_uuid) || '' }}" + jobs: prepare-release: - name: ${{ github.event.inputs.workflow_uuid && format('prepare-release {0}', github.event.inputs.workflow_uuid) || 'prepare-release' }} runs-on: ["ubuntu-latest"] steps: - name: Checkout code From 8f8e26b818f0df0dba5cf787f0945d05c4e1fd1c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 15:38:19 +0300 Subject: [PATCH 041/188] Fix double space --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 972fc9abd..80bc78340 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,7 +14,7 @@ on: description: 'Optional UUID to identify this workflow run' required: false -run-name: "Release Build and Test ${{ github.event.inputs.workflow_uuid && format(' {0}', github.event.inputs.workflow_uuid) || '' }}" +run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" jobs: prepare-release: From 87e8b8237b4551b3428df7e3d31bc19521e99b8c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 17:33:10 +0300 Subject: [PATCH 042/188] Use ghcr.io to push images --- .github/workflows/pre-merge.yml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 82c9414f8..6dfdb2fb0 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -22,15 +22,15 @@ jobs: - alpine platform: - linux/amd64 - - linux/i386 - - linux/arm/v5 - - linux/arm/v6 - - linux/arm/v7 - - linux/mips64le - - linux/ppc64le - - linux/s390x - - linux/arm64 - - linux/riscv64 + # - linux/i386 + # - linux/arm/v5 + # - linux/arm/v6 + # - linux/arm/v7 + # - linux/mips64le + # - linux/ppc64le + # - linux/s390x + # - linux/arm64 + # - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -52,7 +52,7 @@ jobs: with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} - registry_username: ${{ vars.REGISTRY_USERNAME }} - registry_password: ${{ secrets.REGISTRY_PASSWORD }} + registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} + registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY }} \ No newline at end of file From c3dcc3e4aeeb9b1d23cc77d88c6acc0e9a580126 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 17:35:36 +0300 Subject: [PATCH 043/188] Enable build again for testing --- .github/workflows/release_build_and_test.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 80bc78340..ea9dbb2f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,6 @@ jobs: build-and-test: needs: prepare-release - if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From d5c2133eebad8a20f441a65f806f824042d35ed3 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 19:00:45 +0300 Subject: [PATCH 044/188] Try to fix ghcr.io tag --- .github/actions/build-and-tag-locally/action.yml | 15 +++++++++------ .github/workflows/pre-merge.yml | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 35f234b2f..78175ce86 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -19,6 +19,9 @@ inputs: registry_repository: description: 'Repository to push the image to' required: false + release_tag: + description: 'Release tag to build' + required: false runs: using: "composite" @@ -102,7 +105,7 @@ runs: shell: bash run: | docker save -o /tmp/image-${{ steps.platform.outputs.display_name }}.tar ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Upload image uses: actions/upload-artifact@v4 with: @@ -115,7 +118,7 @@ runs: if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} run: | docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Container Logs if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -128,7 +131,7 @@ runs: run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server - + - name: Verify installed modules if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -148,7 +151,7 @@ runs: echo "The following modules are missing: ${missing_modules[*]}" exit 1 fi - + - name: Test RedisBloom if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -158,7 +161,7 @@ runs: [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:hash")" = "1" ] || { echo "RedisBloom test failed: 'redis:hash' not found"; exit 1; } [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:list")" = "0" ] || { echo "RedisBloom test failed: 'redis:list' found unexpectedly"; exit 1; } echo "RedisBloom test passed successfully" - + - name: Test RediSearch if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -230,6 +233,6 @@ runs: with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '' }}${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha cache-to: type=gha,mode=max diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6dfdb2fb0..f3f27f5d7 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,4 +55,4 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY }} \ No newline at end of file + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} From 81b938aafe667029ed060d5bbd03a6eb559accdb Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 20:49:37 +0300 Subject: [PATCH 045/188] Fix release_tag propagation --- .github/workflows/pre-merge.yml | 2 ++ .github/workflows/release_build_and_test.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f3f27f5d7..1f82c5538 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -56,3 +56,5 @@ jobs: registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + release_tag: ${{ inputs.release_tag }} + diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ea9dbb2f6..e6d2405d6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -46,5 +46,5 @@ jobs: uses: ./.github/workflows/pre-merge.yml secrets: inherit with: - release_tag: ${{ inputs.release_tag }} + release_tag: ${{ github.event.inputs.release_tag }} From e21a84d3621957093d90b1384402d40ecf79659b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 20:51:28 +0300 Subject: [PATCH 046/188] Fix unexpected end of expression --- .github/actions/build-and-tag-locally/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 78175ce86..79db0e772 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -233,6 +233,6 @@ runs: with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '' }}${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}${{ github.sha }}-${{ inputs.distribution }} cache-from: type=gha cache-to: type=gha,mode=max From f26e8eac9f613b3865da3847a9f168772909ba78 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 21:30:46 +0300 Subject: [PATCH 047/188] registry tags must be lowercase --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 1f82c5538..55e01b788 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,6 +55,6 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', toLower(github.repository)) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} From 7db9d6693996ee0c5708976fdb4a685d26c4238d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:28:51 +0300 Subject: [PATCH 048/188] Step to correctly format registry tag --- .github/actions/build-and-tag-locally/action.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 79db0e772..472a17a07 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -227,12 +227,22 @@ runs: path: test/report-entrypoint.xml reporter: java-junit + - name: Format registry tag + id: format-registry-tag + run: | + printf "tag=%s:%s%s-%s" \ + "${{ inputs.registry_repository }}" \ + "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \ + "${{ github.sha }}" \ + "${{ inputs.distribution }}" \ + | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" + - name: Push image uses: docker/build-push-action@v6 if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ steps.format-registry-tag.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max From 507d27ae916293905832b8c02b575d10b374e967 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:30:37 +0300 Subject: [PATCH 049/188] Remove unsupported toLower() --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 55e01b788..1f82c5538 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -55,6 +55,6 @@ jobs: registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', toLower(github.repository)) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} From 5276cb94c0e3022fff2796d0fe9aa806ae2429ab Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 22:32:07 +0300 Subject: [PATCH 050/188] Fix step shell --- .github/actions/build-and-tag-locally/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 472a17a07..060384d5b 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -229,6 +229,7 @@ runs: - name: Format registry tag id: format-registry-tag + shell: bash run: | printf "tag=%s:%s%s-%s" \ "${{ inputs.registry_repository }}" \ From be7d64252403cdd534d5bf3e623739996150a38d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:14:33 +0300 Subject: [PATCH 051/188] Try verified merge --- .../ensure-release-branch.sh | 37 ++++++++++++++++--- .github/workflows/release_build_and_test.yml | 1 + 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index be9ccd48e..547ae9fe4 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -127,12 +127,37 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Changes detected but refusing to merge without --allow-modify option" exit 1 fi - - echo "Attempting to merge changes from $RELEASE_BRANCH into $RELEASE_VERSION_BRANCH..." - # Try to merge the release branch into the current release version branch - execute_command git merge "origin/$RELEASE_BRANCH" --no-edit - # Push the merged changes to origin - execute_command git push origin "$RELEASE_VERSION_BRANCH" + # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) + API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" + + read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:16:18 +0300 Subject: [PATCH 052/188] Fix heredoc --- .../ensure-release-branch/ensure-release-branch.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 547ae9fe4..4572b4d14 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -132,11 +132,11 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:19:53 +0300 Subject: [PATCH 053/188] Show commits --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 4572b4d14..fb77db9f5 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -122,6 +122,9 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then if [ "$COMMITS_BEHIND" -gt 0 ]; then echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + console_output 1 gray "Commits mising:" + console_output 1 gray "$last_cmd_stdout" if [ -z "$ALLOW_MODIFY" ]; then echo "Changes detected but refusing to merge without --allow-modify option" From baf14b24b12b4abd0cc2ab6cf2e09d5edc6cea5b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:25:50 +0300 Subject: [PATCH 054/188] Debug --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index fb77db9f5..76e99722d 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,6 +121,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then + set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" From af2fa706a743df2ee0b8b2d1ef818d8c6658b96c Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:27:37 +0300 Subject: [PATCH 055/188] debg --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 76e99722d..a000fed05 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,6 +121,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then + # debug set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" From bcb994bb8d737c7a33c2f29800c27cc9d91e7144 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:28:18 +0300 Subject: [PATCH 056/188] Format oneline --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index a000fed05..870b53f8b 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -124,7 +124,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # debug set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" - execute_command git log "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" + execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" console_output 1 gray "$last_cmd_stdout" From 52f21d98052065ecb3c136c179ffb88b36e9c645 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:35:58 +0300 Subject: [PATCH 057/188] Remove heredoc --- .../ensure-release-branch/ensure-release-branch.sh | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 870b53f8b..d9510bc3f 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,13 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" - read -r -d '' PAYLOAD < Date: Thu, 28 Aug 2025 23:42:06 +0300 Subject: [PATCH 058/188] debug3 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index d9510bc3f..a8e8e710a 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,6 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" + # t PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" # Make the request and capture status code + body From 0dd9300cd7225a97a662c5dfb856c5cf86dd7b40 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:44:57 +0300 Subject: [PATCH 059/188] debug 4 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index a8e8e710a..97b96695c 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,8 +135,8 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" - # t PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" + echo "$PAYLOAD" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ From debc6a172b099f5ee1d56025bb09eb93b7e26098 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:47:35 +0300 Subject: [PATCH 060/188] debyg 5 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 97b96695c..755d8ed59 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,6 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" + echo gh_token=$(gh auth token) PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" echo "$PAYLOAD" From 3d5e032728907b173e30f425c551eacd5a7bb409 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:52:08 +0300 Subject: [PATCH 061/188] Add GITHUB_TOKEN --- .github/actions/ensure-release-branch/action.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index e7a51bca0..38f0edf03 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -18,6 +18,8 @@ runs: uses: ./.github/actions/configure_git_commiter - name: Ensure Release Branch id: ensure-branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} From a9e5e2598934fa501a0fb6080e7ad627d4ae87df Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Thu, 28 Aug 2025 23:56:19 +0300 Subject: [PATCH 062/188] Pass gh_token --- .github/actions/ensure-release-branch/action.yml | 5 ++++- .github/workflows/pre-merge.yml | 1 + .github/workflows/release_build_and_test.yml | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 38f0edf03..8daebfb97 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -5,6 +5,9 @@ inputs: allow_modify: description: 'Allow modifying the repository' default: false + gh_token: + description: 'GitHub token to use' + required: true outputs: release_version_branch: @@ -19,7 +22,7 @@ runs: - name: Ensure Release Branch id: ensure-branch env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 1f82c5538..8494c316b 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -48,6 +48,7 @@ jobs: uses: ./.github/actions/ensure-release-branch with: release_tag: ${{ inputs.release_tag }} + gh_token: ${{ secrets.GITHUB_TOKEN }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 1992c2a63..9c39623a1 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -34,6 +34,7 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true + gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Apply Docker Version uses: ./.github/actions/apply-docker-version From e708852a6a60c1c5fca72c8430673124ef9b0a7f Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:02:22 +0300 Subject: [PATCH 063/188] debug 6 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 755d8ed59..d9510bc3f 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -135,9 +135,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" - echo gh_token=$(gh auth token) PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" - echo "$PAYLOAD" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ From c266e09acac58c72b66ec750963c518bd52cce5b Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:08:16 +0300 Subject: [PATCH 064/188] fix tkn --- .github/actions/ensure-release-branch/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 8daebfb97..814ba6059 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -22,7 +22,7 @@ runs: - name: Ensure Release Branch id: ensure-branch env: - GITHUB_TOKEN: ${{ inputs.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ inputs.gh_token }} shell: bash run: | ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} From 4ddc546d00397fd93bcf3f57132bd7b5168f6f5d Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:09:07 +0300 Subject: [PATCH 065/188] debug 7 --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index d9510bc3f..7943f7d20 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -132,6 +132,7 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then echo "Changes detected but refusing to merge without --allow-modify option" exit 1 fi + # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" From 1b671ac9a90e6c6cf5d9b203448d39a3897c0c72 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 00:13:40 +0300 Subject: [PATCH 066/188] remove set -x --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 7943f7d20..e4b940838 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -121,8 +121,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') if [ "$COMMITS_BEHIND" -gt 0 ]; then - # debug - set -x echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" console_output 1 gray "Commits mising:" From 8658fa6a22a5399b6dc8c9bb10170bcca3981152 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:10:03 +0300 Subject: [PATCH 067/188] Moved verfied merge into function --- .github/actions/common/github_helpers.sh | 46 +++++++++++++++++++ .../ensure-release-branch.sh | 39 ++++------------ 2 files changed, 54 insertions(+), 31 deletions(-) create mode 100644 .github/actions/common/github_helpers.sh diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh new file mode 100644 index 000000000..5f67fe089 --- /dev/null +++ b/.github/actions/common/github_helpers.sh @@ -0,0 +1,46 @@ +github_create_verified_merge() { + BASE_BRANCH= + HEAD_BRANCH= + while [[ $# -gt 0 ]]; do + case $1 in + --from) + HEAD_BRANCH="$2" + shift + ;; + --to) + BASE_BRANCH="$2" + shift + ;; + -*) + echo "Error: Unknown option $1" + exit 1 + ;; + esac + done + + if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then + echo "Error: Missing required arguments --from and --to" + exit 1 + fi + + # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) + API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" + + PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" + + # Make the request and capture status code + body + HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "$API_URL" \ + -d "$PAYLOAD") + + case "$HTTP_CODE" in + 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; + 204) echo "āœ”ļø Already up to date (no merge necessary)";; + 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; + *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; + esac +} \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index e4b940838..0e67e0b69 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -21,6 +21,8 @@ VERBOSITY=1 SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/github_helpers.sh" # Parse arguments ALLOW_MODIFY="" @@ -109,8 +111,6 @@ fi execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "$RELEASE_VERSION_BRANCH" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." @@ -119,11 +119,9 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') - if [ "$COMMITS_BEHIND" -gt 0 ]; then - echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH" + echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH:" execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - console_output 1 gray "Commits mising:" console_output 1 gray "$last_cmd_stdout" if [ -z "$ALLOW_MODIFY" ]; then @@ -131,35 +129,14 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then exit 1 fi - # Create a verified merge commit on GitHub (RELEASE_BRANCH -> RELEASE_VERSION_BRANCH) - API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" - - PAYLOAD="{\"base\":\"${RELEASE_VERSION_BRANCH}\",\"head\":\"${RELEASE_BRANCH}\",\"commit_message\":\"Merge ${RELEASE_BRANCH} into ${RELEASE_VERSION_BRANCH} (bot)\"}" - - # Make the request and capture status code + body - HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "$API_URL" \ - -d "$PAYLOAD") - - case "$HTTP_CODE" in - 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; - 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; - esac - - # Update the runner's working copy to the server-side result (optional but handy) - git fetch origin - git checkout "${RELEASE_VERSION_BRANCH}" - git reset --hard "origin/${RELEASE_VERSION_BRANCH}" + github_create_verified_merge --from "$RELEASE_BRANCH" --to "$RELEASE_VERSION_BRANCH" fi - exit 0 + execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" + execute_command git checkout "${RELEASE_VERSION_BRANCH}" + echo "Successfully checked out to $RELEASE_VERSION_BRANCH" + exit 0 fi echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" From 89d5e41fbbcb159a0ae5b6233eff574edb54cc77 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:52:44 +0300 Subject: [PATCH 068/188] Just a change --- .github/actions/ensure-release-branch/ensure-release-branch.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 0e67e0b69..6cfd998b1 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -115,7 +115,6 @@ if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then # Check if there are changes in release branch that are not in release version branch echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" - # Compare the two branches to see if there are commits in release branch not in release version branch execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') From 3a29c1b9daa36ba0120777b84b519517a9391162 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 10:58:03 +0300 Subject: [PATCH 069/188] Fix arguments parsing --- .github/actions/common/github_helpers.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index 5f67fe089..dc4450d5e 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -11,8 +11,8 @@ github_create_verified_merge() { BASE_BRANCH="$2" shift ;; - -*) - echo "Error: Unknown option $1" + *) + echo "Error: Unknown argument $1" exit 1 ;; esac From f9bedc14095ca64048a5f2f015ee0ad8ad283391 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:02:02 +0300 Subject: [PATCH 070/188] Add second shift --- .github/actions/common/github_helpers.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index dc4450d5e..ef193603d 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -6,10 +6,12 @@ github_create_verified_merge() { --from) HEAD_BRANCH="$2" shift + shift ;; --to) BASE_BRANCH="$2" shift + shift ;; *) echo "Error: Unknown argument $1" From 71a636219c04d0382aca36974465cf24e8db3360 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:39:13 +0300 Subject: [PATCH 071/188] Fix docker files updating --- .../apply-docker-version.sh | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 130c29e2d..5ca21d469 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -57,38 +57,30 @@ update_dockerfile() { # Update REDIS_DOWNLOAD_URL if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile" - updated=true - echo " Updated REDIS_DOWNLOAD_URL" + else + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_URL not found" + return 1 fi + # Update REDIS_DOWNLOAD_SHA if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile" - updated=true - echo " Updated REDIS_DOWNLOAD_SHA" - fi - - if [ "$updated" = true ]; then - echo " $dockerfile updated successfully" - return 0 else - echo " No changes needed in $dockerfile" + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_SHA not found" return 1 fi } +docker_files=("debian/Dockerfile" "alpine/Dockerfile") # Track which files were modified changed_files=() -# Update debian/Dockerfile -if update_dockerfile "debian/Dockerfile"; then - changed_files+=("debian/Dockerfile") -fi +for dockerfile in "${docker_files[@]}"; do + update_dockerfile "$dockerfile" +done -# Update alpine/Dockerfile -if update_dockerfile "alpine/Dockerfile"; then - changed_files+=("alpine/Dockerfile") -fi +changed_files=($(git diff --name-only "${docker_files[@]}")) # Output the list of changed files for GitHub Actions if [ ${#changed_files[@]} -gt 0 ]; then @@ -107,6 +99,4 @@ if [ ${#changed_files[@]} -gt 0 ]; then else echo "No files were modified" echo "changed_files=" >> "$GITHUB_OUTPUT" -fi - -echo "Docker version update completed" +fi \ No newline at end of file From 4e317e62e5fc48d7b97632b48e97908e360ee5c2 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 11:44:36 +0300 Subject: [PATCH 072/188] Test modification --- debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/Dockerfile b/debian/Dockerfile index 0e7d71718..ad615c384 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz -ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e +ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz +ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 5aa2d5dcc9b2a3374059d8037eeabf60ac0d4c51 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 12:13:33 +0300 Subject: [PATCH 073/188] Merge back --- .../actions/ensure-release-branch/action.yml | 5 +- .../ensure-release-branch.sh | 2 +- .../merge-branches-verified/action.yml | 15 ++++++ .../merge-branches-verified.sh | 49 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 26 +++++++++- 5 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 .github/actions/merge-branches-verified/action.yml create mode 100644 .github/actions/merge-branches-verified/merge-branches-verified.sh diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml index 814ba6059..50e7db536 100644 --- a/.github/actions/ensure-release-branch/action.yml +++ b/.github/actions/ensure-release-branch/action.yml @@ -11,8 +11,11 @@ inputs: outputs: release_version_branch: - description: 'The release version branch name' + description: 'The release version branch name, e.g.: 8.X.X-int1' value: ${{ steps.ensure-branch.outputs.release_version_branch }} + release_branch: + description: 'The release branch name, e.g.: release/8.X' + value: ${{ steps.ensure-branch.outputs.release_branch }} runs: using: "composite" diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh index 6cfd998b1..ac51293a0 100755 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ b/.github/actions/ensure-release-branch/ensure-release-branch.sh @@ -63,7 +63,6 @@ fi # Define RELEASE_VERSION_BRANCH which is the same as TAG RELEASE_VERSION_BRANCH="$TAG" -# Output the release version branch for GitHub Actions echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" echo "TAG: $TAG" @@ -72,6 +71,7 @@ echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" # Detect RELEASE_BRANCH name (release/X.Y format) RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" echo "RELEASE_BRANCH: $RELEASE_BRANCH" +echo "release_branch=$RELEASE_BRANCH" >> "$GITHUB_OUTPUT" # Check if RELEASE_BRANCH exists in origin execute_command git ls-remote --heads origin "$RELEASE_BRANCH" diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml new file mode 100644 index 000000000..56465948f --- /dev/null +++ b/.github/actions/merge-branches-verified/action.yml @@ -0,0 +1,15 @@ +inputs: + from_branch: + description: 'Branch to merge from' + required: true + to_branch: + description: 'Branch to merge into' + required: true + +runs: + using: "composite" + steps: + - name: Do verified merge + shell: bash + run: | + ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh new file mode 100644 index 000000000..025fd142c --- /dev/null +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e + +# This script updates Redis version in Dockerfiles using environment variables +# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. + +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/helpers.sh" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/github_helpers.sh" + +FROM_BRANCH= +TO_BRANCH= +while [[ $# -gt 0 ]]; do + case $1 in + --from) + FROM_BRANCH="$2" + shift + shift + ;; + --to) + TO_BRANCH="$2" + shift + shift + ;; + *) + echo "Error: Unknown argument $1" + exit 1 + ;; + esac +done + +if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then + echo "Error: Missing required arguments --from and --to" + exit 1 +fi + +execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 9c39623a1..6e709bd9e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -1,4 +1,4 @@ -# This worjflow is a part of release automation process. +# This workflow is a part of release automation process. # It is intended to be run with workflow_dispatch event by the automation. # Warning: Workflow does switch branches and this may lead to confusion when changing workflow actions. @@ -19,6 +19,8 @@ run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format jobs: prepare-release: runs-on: ["ubuntu-latest"] + outputs: + changed_files: ${{ steps.apply-version.outputs.changed_files }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -37,6 +39,7 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Apply Docker Version + id: apply-version uses: ./.github/actions/apply-docker-version with: release_tag: ${{ github.event.inputs.release_tag }} @@ -50,3 +53,24 @@ jobs: with: release_tag: ${{ github.event.inputs.release_tag }} + merge-back-to-release-branch: + needs: [prepare-release, build-and-test] + if: success() && needs.prepare-release.outputs.changed_files != '' + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure Release Branch + id: ensure-branch + uses: ./.github/actions/ensure-release-branch + with: + release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: false + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge back to release branch + uses: ./.github/actions/merge-branches-verified + with: + from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} + to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From c4121f6035d0b1cbcd71ed99b257296d97d48c2a Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 12:16:43 +0300 Subject: [PATCH 074/188] Enable build --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6e709bd9e..105018a82 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -47,7 +47,7 @@ jobs: build-and-test: needs: prepare-release - if: false + #if: false uses: ./.github/workflows/pre-merge.yml secrets: inherit with: From d54f8acaf0fb858073dfc115ab0ed9075c91f1b0 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:04:39 +0300 Subject: [PATCH 075/188] Set execute bit --- .../actions/merge-branches-verified/merge-branches-verified.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/actions/merge-branches-verified/merge-branches-verified.sh diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh old mode 100644 new mode 100755 From ee242c2c778449fc68e7921a28f27399ae2936ce Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:23:51 +0300 Subject: [PATCH 076/188] Add debug --- .github/actions/common/github_helpers.sh | 1 + .../merge-branches-verified/merge-branches-verified.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index ef193603d..ae225524a 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -1,4 +1,5 @@ github_create_verified_merge() { + set -x BASE_BRANCH= HEAD_BRANCH= while [[ $# -gt 0 ]]; do diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 025fd142c..7965f1d9c 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e - -# This script updates Redis version in Dockerfiles using environment variables -# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. +set -x # shellcheck disable=SC2034 last_cmd_stdout="" From 3e47704f39b22c57eda9413d2c0feffd532e59e7 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:47:27 +0300 Subject: [PATCH 077/188] Skip build for debug --- .../merge-branches-verified.sh | 2 +- .github/workflows/release_build_and_test.yml | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 7965f1d9c..05029c304 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -44,4 +44,4 @@ if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then exit 1 fi -execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file +github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 105018a82..ff52de8cd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -47,11 +47,17 @@ jobs: build-and-test: needs: prepare-release + runs-on: ["ubuntu-latest"] #if: false - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} + #uses: ./.github/workflows/pre-merge.yml + #secrets: inherit + #with: + #release_tag: ${{ github.event.inputs.release_tag }} + steps: + - name: Test + run: | + echo Poop + merge-back-to-release-branch: needs: [prepare-release, build-and-test] From bda1955500f8c3911acd1e42f4ab9a7eea717766 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:48:08 +0300 Subject: [PATCH 078/188] Disable fail on error --- .../actions/merge-branches-verified/merge-branches-verified.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 05029c304..75eb9e77b 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -e +#set -e set -x # shellcheck disable=SC2034 From b6d53ec2547dbc19a8daf87ada54a54504241ce4 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 13:55:04 +0300 Subject: [PATCH 079/188] Further debug --- .github/actions/common/github_helpers.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index ae225524a..2e315e765 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -31,6 +31,7 @@ github_create_verified_merge() { PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" + echo "Going to debug this" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ -X POST \ @@ -40,6 +41,8 @@ github_create_verified_merge() { "$API_URL" \ -d "$PAYLOAD") + echo "HTTP_CODE: $HTTP_CODE" + case "$HTTP_CODE" in 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; 204) echo "āœ”ļø Already up to date (no merge necessary)";; From 28f12813e93e91ffe0482b1a497e203be901b000 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:06:47 +0300 Subject: [PATCH 080/188] Fix merge function and add gh token --- .github/actions/common/github_helpers.sh | 12 ++++-------- .github/actions/common/helpers.sh | 4 +++- .github/actions/merge-branches-verified/action.yml | 5 +++++ .../merge-branches-verified.sh | 6 ++---- .github/workflows/release_build_and_test.yml | 1 + 5 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh index 2e315e765..fe6d09adf 100644 --- a/.github/actions/common/github_helpers.sh +++ b/.github/actions/common/github_helpers.sh @@ -1,5 +1,4 @@ github_create_verified_merge() { - set -x BASE_BRANCH= HEAD_BRANCH= while [[ $# -gt 0 ]]; do @@ -16,14 +15,14 @@ github_create_verified_merge() { ;; *) echo "Error: Unknown argument $1" - exit 1 + return 1 ;; esac done if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then echo "Error: Missing required arguments --from and --to" - exit 1 + return 1 fi # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) @@ -31,7 +30,6 @@ github_create_verified_merge() { PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" - echo "Going to debug this" # Make the request and capture status code + body HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ -X POST \ @@ -41,12 +39,10 @@ github_create_verified_merge() { "$API_URL" \ -d "$PAYLOAD") - echo "HTTP_CODE: $HTTP_CODE" - case "$HTTP_CODE" in 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; exit 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; exit 1;; + 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; return 1;; + *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; return 1;; esac } \ No newline at end of file diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh index 3a74f3062..cf0039fcd 100644 --- a/.github/actions/common/helpers.sh +++ b/.github/actions/common/helpers.sh @@ -1,6 +1,6 @@ # Function to execute command from array and capture output execute_command() { -# turn off errexit (set -e) if it is active and restore it later + # turn off errexit (set -e) if it is active and restore it later echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" local cmd @@ -26,6 +26,7 @@ execute_command() { # Execute command and capture output console_output 1 gray "Executing command: ${cmd[*]}" + "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" last_cmd_result=$? @@ -33,6 +34,7 @@ execute_command() { last_cmd_stdout=$(cat "$stdout_file") last_cmd_stderr=$(cat "$stderr_file") + if [ "$last_cmd_result" -ne 0 ]; then console_output 0 red "Command failed with exit code $last_cmd_result" console_output 0 red "Standard Output:" diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml index 56465948f..a5c1c95f4 100644 --- a/.github/actions/merge-branches-verified/action.yml +++ b/.github/actions/merge-branches-verified/action.yml @@ -1,4 +1,7 @@ inputs: + gh_token: + description: 'GitHub token to use' + required: true from_branch: description: 'Branch to merge from' required: true @@ -8,6 +11,8 @@ inputs: runs: using: "composite" + env: + GITHUB_TOKEN: ${{ inputs.gh_token }} steps: - name: Do verified merge shell: bash diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh index 75eb9e77b..597a9ccb0 100755 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ b/.github/actions/merge-branches-verified/merge-branches-verified.sh @@ -1,6 +1,5 @@ #!/bin/bash -#set -e -set -x +set -e # shellcheck disable=SC2034 last_cmd_stdout="" @@ -11,7 +10,6 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 - SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../common/helpers.sh" @@ -44,4 +42,4 @@ if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then exit 1 fi -github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file +execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ff52de8cd..caee9e83e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -80,3 +80,4 @@ jobs: with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} + gh_token: ${{ secrets.GITHUB_TOKEN }} From f6bbb64bb973978d10dc884006aa57fa3e14a3a2 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:09:37 +0300 Subject: [PATCH 081/188] Fix env --- .github/actions/merge-branches-verified/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml index a5c1c95f4..669312a76 100644 --- a/.github/actions/merge-branches-verified/action.yml +++ b/.github/actions/merge-branches-verified/action.yml @@ -11,10 +11,10 @@ inputs: runs: using: "composite" - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} steps: - name: Do verified merge + env: + GITHUB_TOKEN: ${{ inputs.gh_token }} shell: bash run: | ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file From 970c2aa9435fdd1b4d47ba0dab49bb4a25c6edfb Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:42:45 +0300 Subject: [PATCH 082/188] Move reusable actions to redis-oss-release-automation --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 8494c316b..e35821d27 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -45,11 +45,11 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: ./.github/actions/build-and-tag-locally + - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index caee9e83e..2321628c3 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -26,13 +26,13 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: ./.github/actions/validate-redis-release-archive + uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -69,14 +69,14 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: ./.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch - uses: ./.github/actions/merge-branches-verified + uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From 707d66badf9f4cf3a86f968683dc793f6a2b8873 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 16:49:40 +0300 Subject: [PATCH 083/188] Fix actions refs --- .github/workflows/pre-merge.yml | 4 ++-- .github/workflows/release_build_and_test.yml | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index e35821d27..164357a44 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -45,11 +45,11 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally + - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally@main with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 2321628c3..a87e3bd1e 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -26,13 +26,13 @@ jobs: uses: actions/checkout@v4 - name: Validate Redis Release Archive - uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive + uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -69,14 +69,14 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch + uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch - uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified + uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} From 5d1810f2d0d502fe1d921c8e92d13940c23671d6 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:26:08 +0300 Subject: [PATCH 084/188] Remove actions that were transerred to release automation --- .github/actions/common/github_helpers.sh | 48 ------ .github/actions/common/helpers.sh | 85 ---------- .../actions/configure_git_commiter/action.yml | 8 - .../actions/ensure-release-branch/action.yml | 31 ---- .../ensure-release-branch.sh | 156 ------------------ .../merge-branches-verified/action.yml | 20 --- .../merge-branches-verified.sh | 45 ----- .../validate-redis-release-archive/action.yml | 12 -- .../validate-redis-release-archive.sh | 47 ------ 9 files changed, 452 deletions(-) delete mode 100644 .github/actions/common/github_helpers.sh delete mode 100644 .github/actions/common/helpers.sh delete mode 100644 .github/actions/configure_git_commiter/action.yml delete mode 100644 .github/actions/ensure-release-branch/action.yml delete mode 100755 .github/actions/ensure-release-branch/ensure-release-branch.sh delete mode 100644 .github/actions/merge-branches-verified/action.yml delete mode 100755 .github/actions/merge-branches-verified/merge-branches-verified.sh delete mode 100644 .github/actions/validate-redis-release-archive/action.yml delete mode 100755 .github/actions/validate-redis-release-archive/validate-redis-release-archive.sh diff --git a/.github/actions/common/github_helpers.sh b/.github/actions/common/github_helpers.sh deleted file mode 100644 index fe6d09adf..000000000 --- a/.github/actions/common/github_helpers.sh +++ /dev/null @@ -1,48 +0,0 @@ -github_create_verified_merge() { - BASE_BRANCH= - HEAD_BRANCH= - while [[ $# -gt 0 ]]; do - case $1 in - --from) - HEAD_BRANCH="$2" - shift - shift - ;; - --to) - BASE_BRANCH="$2" - shift - shift - ;; - *) - echo "Error: Unknown argument $1" - return 1 - ;; - esac - done - - if [ -z "$BASE_BRANCH" ] || [ -z "$HEAD_BRANCH" ]; then - echo "Error: Missing required arguments --from and --to" - return 1 - fi - - # Create a verified merge commit on GitHub (HEAD_BRANCH -> BASE_BRANCH) - API_URL="https://api.github.com/repos/${GITHUB_REPOSITORY}/merges" - - PAYLOAD="{\"base\":\"${BASE_BRANCH}\",\"head\":\"${HEAD_BRANCH}\",\"commit_message\":\"Merge ${HEAD_BRANCH} into ${BASE_BRANCH} (bot)\"}" - - # Make the request and capture status code + body - HTTP_CODE=$(curl -sS -w "%{http_code}" -o /tmp/merge.json \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${GITHUB_TOKEN}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "$API_URL" \ - -d "$PAYLOAD") - - case "$HTTP_CODE" in - 201) echo "āœ… Verified merge created: $(jq -r '.sha' /tmp/merge.json)";; - 204) echo "āœ”ļø Already up to date (no merge necessary)";; - 409) echo "āŒ Merge conflict; open a PR to resolve"; cat /tmp/merge.json; return 1;; - *) echo "āŒ Unexpected status $HTTP_CODE"; cat /tmp/merge.json; return 1;; - esac -} \ No newline at end of file diff --git a/.github/actions/common/helpers.sh b/.github/actions/common/helpers.sh deleted file mode 100644 index cf0039fcd..000000000 --- a/.github/actions/common/helpers.sh +++ /dev/null @@ -1,85 +0,0 @@ -# Function to execute command from array and capture output -execute_command() { - # turn off errexit (set -e) if it is active and restore it later - echo $SHELLOPTS | grep -q errexit && restore_errexit="1" && set +e || restore_errexit="" - - local cmd - - # Check if no arguments provided - if [ $# -eq 0 ]; then - # Check if cmd_array variable exists and is an array - if declare -p cmd_array 2>/dev/null | grep -q "declare -a"; then - # Use the existing cmd_array variable - cmd=("${cmd_array[@]}") - else - echo "Error: No arguments provided and cmd_array variable not found or not an array" >&2 - return 1 - fi - else - cmd=("$@") - fi - - # Create temporary files for stdout and stderr - local stdout_file stderr_file - stdout_file=$(mktemp) - stderr_file=$(mktemp) - - # Execute command and capture output - console_output 1 gray "Executing command: ${cmd[*]}" - - "${cmd[@]}" >"$stdout_file" 2>"$stderr_file" - last_cmd_result=$? - - # Read captured output - last_cmd_stdout=$(cat "$stdout_file") - last_cmd_stderr=$(cat "$stderr_file") - - - if [ "$last_cmd_result" -ne 0 ]; then - console_output 0 red "Command failed with exit code $last_cmd_result" - console_output 0 red "Standard Output:" - console_output 0 red "$last_cmd_stdout" - console_output 0 red "Standard Error:" - console_output 0 red "$last_cmd_stderr" - fi - - # Clean up temporary files - rm -f "$stdout_file" "$stderr_file" - - [ "$restore_errexit" ] && set -e - return $last_cmd_result -} - -# Helper function to output multiline variables with color -console_output() { - local verbosity_level="$1" - local color="$2" - local content="$3" - local current_verbosity="${VERBOSITY:-0}" - - # Check if we should output based on verbosity level - if [ "$current_verbosity" -ge "$verbosity_level" ]; then - local color_code="" - local reset_code="\033[0m" - - case "$color" in - "gray"|"grey") - color_code="\033[90m" - ;; - "white") - color_code="\033[97m" - ;; - "red") - color_code="\033[91m" - ;; - *) - color_code="\033[0m" # Default to no color - ;; - esac - - # Output each line with 4-space indent and color - while IFS= read -r line || [ -n "$line" ]; do - printf "${color_code} %s${reset_code}\n" "$line" - done <<< "$content" - fi -} diff --git a/.github/actions/configure_git_commiter/action.yml b/.github/actions/configure_git_commiter/action.yml deleted file mode 100644 index 3ab5c0fe0..000000000 --- a/.github/actions/configure_git_commiter/action.yml +++ /dev/null @@ -1,8 +0,0 @@ -runs: - using: "composite" - steps: - - name: Configure bot commiter - shell: bash - run: | - git config user.name "github-actions[bot]" - git config user.email "41898282+github-actions[bot]@users.noreply.github.com" \ No newline at end of file diff --git a/.github/actions/ensure-release-branch/action.yml b/.github/actions/ensure-release-branch/action.yml deleted file mode 100644 index 50e7db536..000000000 --- a/.github/actions/ensure-release-branch/action.yml +++ /dev/null @@ -1,31 +0,0 @@ -inputs: - release_tag: - description: 'Release tag to build' - required: true - allow_modify: - description: 'Allow modifying the repository' - default: false - gh_token: - description: 'GitHub token to use' - required: true - -outputs: - release_version_branch: - description: 'The release version branch name, e.g.: 8.X.X-int1' - value: ${{ steps.ensure-branch.outputs.release_version_branch }} - release_branch: - description: 'The release branch name, e.g.: release/8.X' - value: ${{ steps.ensure-branch.outputs.release_branch }} - -runs: - using: "composite" - steps: - - name: Conigure Git Commiter - uses: ./.github/actions/configure_git_commiter - - name: Ensure Release Branch - id: ensure-branch - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} - shell: bash - run: | - ${{ github.action_path }}/ensure-release-branch.sh ${{ inputs.allow_modify == 'true' && '--allow-modify' || '' }} ${{ inputs.release_tag }} diff --git a/.github/actions/ensure-release-branch/ensure-release-branch.sh b/.github/actions/ensure-release-branch/ensure-release-branch.sh deleted file mode 100755 index ac51293a0..000000000 --- a/.github/actions/ensure-release-branch/ensure-release-branch.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -# This script ensures that a release branch and release version branch exist for a given release tag. -# It allow-modifys and pushes both branches if they do not exist. -# It also checks out the release version branch at the end. -# https://redislabs.atlassian.net/wiki/spaces/RED/pages/5293342875/Redis+OSS+release+automation - -set -e -#set -x - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -VERBOSITY=1 - - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/github_helpers.sh" - -# Parse arguments -ALLOW_MODIFY="" -TAG="" - -while [[ $# -gt 0 ]]; do - case $1 in - --allow-modify) - ALLOW_MODIFY=1 - shift - ;; - -*) - echo "Error: Unknown option $1" - exit 1 - ;; - *) - if [ -z "$TAG" ]; then - TAG="$1" - else - echo "Error: Multiple TAG arguments provided" - exit 1 - fi - shift - ;; - esac -done - -git_fetch_unshallow() { - git fetch --unshallow "$@" 2>/dev/null || git fetch "$@" -} - -if [ -z "$TAG" ]; then - echo "Error: TAG is required as argument" - echo "Usage: $0 [--allow-modify] " - exit 1 -fi - -# Define RELEASE_VERSION_BRANCH which is the same as TAG -RELEASE_VERSION_BRANCH="$TAG" - -echo "release_version_branch=$RELEASE_VERSION_BRANCH" >> "$GITHUB_OUTPUT" - -echo "TAG: $TAG" -echo "RELEASE_VERSION_BRANCH: $RELEASE_VERSION_BRANCH" - -# Detect RELEASE_BRANCH name (release/X.Y format) -RELEASE_BRANCH="release/$(echo "$TAG" | grep -Po '^\d+\.\d+')" -echo "RELEASE_BRANCH: $RELEASE_BRANCH" -echo "release_branch=$RELEASE_BRANCH" >> "$GITHUB_OUTPUT" - -# Check if RELEASE_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_BRANCH"; then - echo "Branch $RELEASE_BRANCH exists in origin" - execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" -else - echo "Branch $RELEASE_BRANCH does not exist in origin, need to create it" - if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 - fi - - # Detect base branch (previous existing branch for the version) - MAJOR_MINOR=$(echo "$TAG" | grep -Po '^\d+\.\d+') - MAJOR=$(echo "$MAJOR_MINOR" | cut -d. -f1) - - # Find the previous existing release branch - execute_command git ls-remote --heads origin "release/$MAJOR.[0-9]" - BASE_BRANCH=$(echo "$last_cmd_stdout" | grep -oP 'release/\d+\.\d+' | sort -V | tail -n 1) - - if [ -z "$BASE_BRANCH" ]; then - echo "Error: Could not find a base branch for $RELEASE_BRANCH" - exit 1 - fi - - echo "Using base branch: $BASE_BRANCH" - - # Create new branch based on base branch and push to origin - execute_command git_fetch_unshallow origin "$BASE_BRANCH" - execute_command git checkout -b "$RELEASE_BRANCH" "origin/$BASE_BRANCH" - execute_command git push origin HEAD:"$RELEASE_BRANCH" - echo "Created and pushed $RELEASE_BRANCH based on $BASE_BRANCH" -fi - -# Check if RELEASE_VERSION_BRANCH exists in origin -execute_command git ls-remote --heads origin "$RELEASE_VERSION_BRANCH" -if echo "$last_cmd_stdout" | grep -q "$RELEASE_VERSION_BRANCH"; then - execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - - # Check if there are changes in release branch that are not in release version branch - echo "Checking for differences between $RELEASE_BRANCH and $RELEASE_VERSION_BRANCH..." - execute_command git_fetch_unshallow origin "$RELEASE_BRANCH" - # Compare the two branches to see if there are commits in release branch not in release version branch - execute_command git rev-list --count "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - COMMITS_BEHIND=$(echo "$last_cmd_stdout" | tr -d '[:space:]') - if [ "$COMMITS_BEHIND" -gt 0 ]; then - echo "Found $COMMITS_BEHIND commit(s) in $RELEASE_BRANCH that are not in $RELEASE_VERSION_BRANCH:" - execute_command git log --oneline "origin/$RELEASE_VERSION_BRANCH..origin/$RELEASE_BRANCH" - console_output 1 gray "$last_cmd_stdout" - - if [ -z "$ALLOW_MODIFY" ]; then - echo "Changes detected but refusing to merge without --allow-modify option" - exit 1 - fi - - github_create_verified_merge --from "$RELEASE_BRANCH" --to "$RELEASE_VERSION_BRANCH" - fi - - execute_command git_fetch_unshallow origin "$RELEASE_VERSION_BRANCH" - execute_command git checkout "${RELEASE_VERSION_BRANCH}" - echo "Successfully checked out to $RELEASE_VERSION_BRANCH" - - exit 0 -fi - -echo "Branch $RELEASE_VERSION_BRANCH does not exist in origin" -if [ -z "$ALLOW_MODIFY" ]; then - echo "Refuse to modify repository without --allow-modify option" - exit 1 -fi - -execute_command git checkout "$RELEASE_BRANCH" -# At this point, we should be on RELEASE_BRANCH -echo "Current branch: $(git branch --show-current)" - -# Create RELEASE_VERSION_BRANCH based on RELEASE_BRANCH and push to origin -execute_command git checkout -b "$RELEASE_VERSION_BRANCH" -execute_command git push origin HEAD:"$RELEASE_VERSION_BRANCH" -echo "Created and pushed $RELEASE_VERSION_BRANCH based on $RELEASE_BRANCH" - -echo "Successfully set up $RELEASE_VERSION_BRANCH - working directory now points to this branch" \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/action.yml b/.github/actions/merge-branches-verified/action.yml deleted file mode 100644 index 669312a76..000000000 --- a/.github/actions/merge-branches-verified/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -inputs: - gh_token: - description: 'GitHub token to use' - required: true - from_branch: - description: 'Branch to merge from' - required: true - to_branch: - description: 'Branch to merge into' - required: true - -runs: - using: "composite" - steps: - - name: Do verified merge - env: - GITHUB_TOKEN: ${{ inputs.gh_token }} - shell: bash - run: | - ${{ github.action_path }}/merge-branches-verified.sh --from ${{ inputs.from_branch }} --to ${{ inputs.to_branch }} \ No newline at end of file diff --git a/.github/actions/merge-branches-verified/merge-branches-verified.sh b/.github/actions/merge-branches-verified/merge-branches-verified.sh deleted file mode 100755 index 597a9ccb0..000000000 --- a/.github/actions/merge-branches-verified/merge-branches-verified.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/bash -set -e - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -VERBOSITY=1 - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/github_helpers.sh" - -FROM_BRANCH= -TO_BRANCH= -while [[ $# -gt 0 ]]; do - case $1 in - --from) - FROM_BRANCH="$2" - shift - shift - ;; - --to) - TO_BRANCH="$2" - shift - shift - ;; - *) - echo "Error: Unknown argument $1" - exit 1 - ;; - esac -done - -if [ -z "$FROM_BRANCH" ] || [ -z "$TO_BRANCH" ]; then - echo "Error: Missing required arguments --from and --to" - exit 1 -fi - -execute_command github_create_verified_merge --from "$FROM_BRANCH" --to "$TO_BRANCH" \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/action.yml b/.github/actions/validate-redis-release-archive/action.yml deleted file mode 100644 index fcd775ce7..000000000 --- a/.github/actions/validate-redis-release-archive/action.yml +++ /dev/null @@ -1,12 +0,0 @@ -inputs: - release_tag: - description: 'Release tag to build' - required: true - -runs: - using: "composite" - steps: - - name: Ensure Release Branch - shell: bash - run: | - ${{ github.action_path }}/validate-redis-release-archive.sh ${{ inputs.release_tag }} \ No newline at end of file diff --git a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh b/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh deleted file mode 100755 index 187769731..000000000 --- a/.github/actions/validate-redis-release-archive/validate-redis-release-archive.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -set -e - -# This script validates a Redis release archive by downloading it and calculating its SHA256 sum. -# It constructs the URL, downloads the file, calculates the hash, and exports environment variables. - -# Input TAG is expected in $1 -TAG="$1" - -if [ -z "$TAG" ]; then - echo "Error: TAG is required as first argument" - exit 1 -fi - -# Construct Redis archive URL -REDIS_ARCHIVE_URL="https://github.com/redis/redis/archive/refs/tags/${TAG}.tar.gz" -echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" - -# Download the Redis archive -TEMP_ARCHIVE="/tmp/redis-${TAG}.tar.gz" -echo "Downloading Redis archive to $TEMP_ARCHIVE..." -if ! curl -sfL -o "$TEMP_ARCHIVE" "$REDIS_ARCHIVE_URL"; then - echo "Error: Failed to download Redis archive from $REDIS_ARCHIVE_URL" - exit 1 -fi - -# Calculate SHA256 sum -echo "Calculating SHA256 sum..." -REDIS_ARCHIVE_SHA=$(sha256sum "$TEMP_ARCHIVE" | cut -d' ' -f1) -echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" - -# Write variables to GITHUB_ENV -if [ -n "$GITHUB_ENV" ]; then - echo "REDIS_ARCHIVE_URL=$REDIS_ARCHIVE_URL" >> "$GITHUB_ENV" - echo "REDIS_ARCHIVE_SHA=$REDIS_ARCHIVE_SHA" >> "$GITHUB_ENV" - echo "Environment variables written to $GITHUB_ENV" -else - echo "Error: GITHUB_ENV not set" - # Clean up temporary file - rm -f "$TEMP_ARCHIVE" - exit 1 -fi - -# Clean up temporary file -rm -f "$TEMP_ARCHIVE" - -echo "Redis archive validation completed successfully" \ No newline at end of file From 68bfae02286141a329978b24235f7addba7ef142 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:29:45 +0300 Subject: [PATCH 085/188] Return commented out actions --- .github/workflows/pre-merge.yml | 18 +++++++++--------- .github/workflows/release_build_and_test.yml | 16 +++++----------- 2 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 164357a44..bf8e71bbb 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -22,15 +22,15 @@ jobs: - alpine platform: - linux/amd64 - # - linux/i386 - # - linux/arm/v5 - # - linux/arm/v6 - # - linux/arm/v7 - # - linux/mips64le - # - linux/ppc64le - # - linux/s390x - # - linux/arm64 - # - linux/riscv64 + - linux/i386 + - linux/arm/v5 + - linux/arm/v6 + - linux/arm/v7 + - linux/mips64le + - linux/ppc64le + - linux/s390x + - linux/arm64 + - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a87e3bd1e..26e7a46bf 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -14,6 +14,7 @@ on: description: 'Optional UUID to identify this workflow run' required: false +# UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" jobs: @@ -47,17 +48,10 @@ jobs: build-and-test: needs: prepare-release - runs-on: ["ubuntu-latest"] - #if: false - #uses: ./.github/workflows/pre-merge.yml - #secrets: inherit - #with: - #release_tag: ${{ github.event.inputs.release_tag }} - steps: - - name: Test - run: | - echo Poop - + uses: ./.github/workflows/pre-merge.yml + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] From 51910ee1b0fe3d8073ce85b443ede39aef9d7b3a Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:45:23 +0300 Subject: [PATCH 086/188] Use common from another repo --- .github/actions/apply-docker-version/action.yml | 7 +++++++ .../actions/apply-docker-version/apply-docker-version.sh | 5 +---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 315c56513..07cf7b3ec 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -14,6 +14,13 @@ outputs: runs: using: "composite" steps: + - name: Checkout common functions + uses: actions/checkout@v4 + with: + repository: redis/redis-oss-release-automation + ref: main + path: redis-oss-release-automation + - name: Apply docker version id: apply-version shell: bash diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 5ca21d469..243e645eb 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -13,11 +13,8 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/helpers.sh" - +. "$GITHUB_WORKSPACE/redis-oss-release-automation/.github/actions/common/helpers.sh" # Input TAG is expected in $1 TAG="$1" From 9b9253d229baf5e17f0f9683b8034153f71c37d9 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 17:50:31 +0300 Subject: [PATCH 087/188] Fix remote action usage instead of local --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index bf8e71bbb..0924c7eb3 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -49,7 +49,7 @@ jobs: with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - uses: redis/redis-oss-release-automation/.github/actions/build-and-tag-locally@main + - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} From 59788b31d37eba43ddc1077e41fd37d00cd79fbe Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 18:11:22 +0300 Subject: [PATCH 088/188] Test merge back without changed files --- .github/workflows/release_build_and_test.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 26e7a46bf..b54ade886 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -20,8 +20,6 @@ run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format jobs: prepare-release: runs-on: ["ubuntu-latest"] - outputs: - changed_files: ${{ steps.apply-version.outputs.changed_files }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -48,14 +46,18 @@ jobs: build-and-test: needs: prepare-release - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} + # uses: ./.github/workflows/pre-merge.yml + # secrets: inherit + # with: + # release_tag: ${{ github.event.inputs.release_tag }} + runs-on: [ubuntu-latest] + steps: + - run: | + echo "Hello World" merge-back-to-release-branch: needs: [prepare-release, build-and-test] - if: success() && needs.prepare-release.outputs.changed_files != '' + if: success() runs-on: ["ubuntu-latest"] steps: - name: Checkout code @@ -74,4 +76,4 @@ jobs: with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} - gh_token: ${{ secrets.GITHUB_TOKEN }} + gh_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file From d3f4cb661bd6699424f0e13115090ad6bd88d2d9 Mon Sep 17 00:00:00 2001 From: Peter Sh Date: Fri, 29 Aug 2025 18:24:59 +0300 Subject: [PATCH 089/188] Return full build --- .github/workflows/release_build_and_test.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b54ade886..3cf591b98 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -46,14 +46,11 @@ jobs: build-and-test: needs: prepare-release - # uses: ./.github/workflows/pre-merge.yml - # secrets: inherit - # with: - # release_tag: ${{ github.event.inputs.release_tag }} - runs-on: [ubuntu-latest] - steps: - - run: | - echo "Hello World" + uses: ./.github/workflows/pre-merge.yml + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} + merge-back-to-release-branch: needs: [prepare-release, build-and-test] From a531b37d0d9c0a68d57b30b65190f41320a604d1 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Tue, 9 Sep 2025 21:02:08 +0300 Subject: [PATCH 090/188] Prepare stackbrew librirary intermediate commit --- .../apply-docker-version.sh | 7 +- .github/actions/common/func.sh | 269 ++++++++++++++++++ .../generate-stackbrew-library.sh | 54 ++++ test/run-shell-func-tests.sh | 246 ++++++++++++++++ 4 files changed, 575 insertions(+), 1 deletion(-) create mode 100644 .github/actions/common/func.sh create mode 100755 .github/actions/create-library-pr/generate-stackbrew-library.sh create mode 100755 test/run-shell-func-tests.sh diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh index 243e645eb..0078b05cb 100755 --- a/.github/actions/apply-docker-version/apply-docker-version.sh +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -13,8 +13,13 @@ last_cmd_result=0 # shellcheck disable=SC2034 VERBOSITY=1 + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 -. "$GITHUB_WORKSPACE/redis-oss-release-automation/.github/actions/common/helpers.sh" +. "$SCRIPT_DIR/../common/func.sh" + +source_helper_file helpers.sh # Input TAG is expected in $1 TAG="$1" diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh new file mode 100644 index 000000000..d0afa9d0e --- /dev/null +++ b/.github/actions/common/func.sh @@ -0,0 +1,269 @@ +#!/bin/bash + +source_helper_file() { + local helper_file="$1" + local helper_errors="" + for dir in "GITHUB_WORKSPACE:$GITHUB_WORKSPACE/redis-oss-release-automation" "RELEASE_AUTOMATION_DIR:$RELEASE_AUTOMATION_DIR" ":../redis-oss-release-automation"; do + local var_name="${dir%%:*}" + local dir="${dir#*:}" + if [ -n "$var_name" ]; then + var_name="\$$var_name" + fi + local helper_path="$dir/.github/actions/common/$helper_file" + if [ -f "$helper_path" ]; then + helper_errors="" + # shellcheck disable=SC1090 + . "$helper_path" + break + else + helper_errors=$(printf "%s\n %s: %s" "$helper_errors" "$var_name" "$helper_path") + fi + done + if [ -n "$helper_errors" ]; then + echo "Error: $helper_file not found in any of the following locations: $helper_errors" >&2 + exit 1 + fi +} + +git_ls_remote_major_release_version_branches() { + local remote="$1" + local major_version="$2" + execute_command --no-std -- git ls-remote --heads "$remote" "release/$major_version.*" + if [ -z "$last_cmd_stdout" ]; then + console_output 0 red "Error: No release branches found for major_version=$major_version" + return 1 + fi + echo "$last_cmd_stdout" +} + +git_ls_remote_tags() { + local remote="$1" + local major_version="$2" + execute_command git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" +} + +filter_major_release_version_branches() { + local major_version="$1" + while read -r line; do + local ref="$(echo "$line" | awk '{print $2}')" + local commit="$(echo "$line" | awk '{print $1}')" + if echo "$ref" | grep -q "release/$major_version\.[0-9][0-9]*$"; then + echo "$ref $commit" + fi + done | sort -Vr +} + +filter_actual_major_redis_versions() { + local major_version="$1" + local last_minor="" last_is_milestone="" + local ref commit version_tag + + while read -r commit ref; do + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + echo "$version_tag $ref $commit" + done | sort -Vr | while read -r version_tag ref commit; do + local major minor patch suffix is_milestone + IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") + + if [ -n "$suffix" ]; then + is_milestone=1 + else + is_milestone="" + fi + + if [ "$last_minor" = "$minor" ] && [ "$last_is_milestone" = "$is_milestone" ]; then + console_output 2 gray "Skipping $version_tag, already have minor=$last_minor is_milestone=$last_is_milestone" + continue + fi + last_minor="$minor" + last_is_milestone="$is_milestone" + + console_output 2 gray "$version_tag $commit" + echo "$version_tag $commit" + done +} + +get_major_release_version_branches () { + local remote="$1" + local major_version="$2" + execute_command git_ls_remote_major_release_version_branches "$remote" "$major_version" | execute_command filter_major_release_version_branches "$major_version" +} + +get_actual_major_redis_versions() { + local remote="$1" + local major_version="$2" + execute_command git_ls_remote_tags "$remote" "$major_version" | execute_command filter_actual_major_redis_versions "$major_version" +} + +git_fetch_unshallow_refs() { + local remote="$1" + local refs_to_fetch + while read -r line; do + local ref="$(echo "$line" | awk '{print $1}')" + refs_to_fetch="$refs_to_fetch $ref" + done + # shellcheck disable=SC2086 + execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch +} + +extract_distro_name_from_dockerfile() { + local base_img + base_img="$(grep -m1 -i '^from' | awk '{print $2}')" + + increase_indent_level + console_output 2 gray "Extracting distro from dockerfile" + + if echo "$base_img" | grep -q 'alpine:'; then + distro="$(echo "$base_img" | tr -d ':')" + elif echo "$base_img" | grep -q 'debian:'; then + distro="$(echo "${base_img//-slim/}" | awk -F: '{print $2}')" + else + console_output 0 red "Error: Unknown base image $base_img" + decrease_indent_level + return 1 + fi + console_output 2 gray "distro=$distro" + decrease_indent_level + echo "$distro" +} + +extract_redis_version_from_dockerfile() { + increase_indent_level + console_output 2 gray "Extracting redis version from dockerfile" + local redis_version + redis_version=$(grep -m1 -i '^ENV REDIS_DOWNLOAD_URL.*https*:.*tar' \ + | sed 's/ENV.*REDIS_DOWNLOAD_URL.*[-/]\([1-9][0-9]*\..*\)\.tar\.gz/\1/g' \ + | grep -E '^[1-9][0-9]*\.' + ) + console_output 2 gray "redis_version=$redis_version" + if [ -z "$redis_version" ]; then + console_output 0 red "Error: Failed to extract redis version from dockerfile" + decrease_indent_level + return 1 + fi + echo "$redis_version" + decrease_indent_level +} + +redis_version_split() { + local version + local numerics + # shellcheck disable=SC2001 + version=$(echo "$1" | sed 's/^v//') + + numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)') + if [ -z "$numerics" ]; then + console_output 2 red "Cannot split version '$version', incorrect version format" + return 1 + fi + local major minor patch suffix + IFS=. read -r major minor patch < <(echo "$numerics") + suffix=${version:${#numerics}} + printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" +} + + +git_show_file_from_ref() { + local ref=$1 + local file=$2 + execute_command git show "$ref:$file" +} + +generate_tags_list() { + local redis_version=$1 + local distro_names=$2 + local is_latest=$3 + local is_default=$4 + + local tags versions + + local major minor patch suffix + IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") + + local mainline_version + mainline_version="$major.$minor" + + versions=("$redis_version" "$mainline_version") + if [ "$is_latest" = 1 ]; then + versions+=("$major") + fi + + if [ "$is_default" = 1 ]; then + tags=("${versions[@]}") + fi + + for distro_name in $distro_names; do + for v in "${versions[@]}"; do + tags+=("$v-$distro_name") + done + done + + if [ "$is_latest" = 1 ]; then + if [ "$is_default" = 1 ]; then + tags+=("latest") + fi + # shellcheck disable=SC2206 + tags+=($distro_names) + fi + # shellcheck disable=SC2001 + echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" +} + +generate_stackbrew_library() { + local commit redis_version distro distro_version + local is_latest="unset" is_default + + local stackbrew_content + + while read -r commit redis_version distro distro_version; do + local major minor patch suffix + IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") + + # assigning latest to the first non milestone (empty suffix) version from top + if [ "$is_latest" = "unset" ]; then + if [ -z "$suffix" ]; then + is_latest=1 + fi + else + is_latest="" + fi + + if echo "$distro" | grep -q 'alpine'; then + is_default="" + distro_names="$distro $distro_version" + else + is_default=1 + distro_names="$distro_version" + fi + + local tags + tags=$(generate_tags_list "$redis_version" "$distro_names" "$is_latest" "$is_default") + printf -v stackbrew_content "%s%s\n" "$stackbrew_content" "$tags" + done + printf %s "$stackbrew_content" + console_output 2 gray "$stackbrew_content" +} + +prepare_releases_list() { + local redis_version commit + local debug_output version_line + while read -r redis_version commit; do + for distro in debian alpine; do + local dockerfile distro_version redis_version + dockerfile=$(git_show_file_from_ref "$commit" "$distro/Dockerfile") + console_output 3 gray "$dockerfile" + + distro_version=$(echo "$dockerfile" | extract_distro_name_from_dockerfile) + # validate version + redis_version_split "$redis_version" >/dev/null + + printf -v version_line "%s %s %s %s\n" "$commit" "$redis_version" "$distro" "$distro_version" + printf "%s" "$version_line" + printf -v debug_output "%s%s" "$debug_output" "$version_line" + done + done + console_output 2 gray "Final Releases list:" + increase_indent_level + console_output 2 gray "$debug_output" + decrease_indent_level +} \ No newline at end of file diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh new file mode 100755 index 000000000..011ec7914 --- /dev/null +++ b/.github/actions/create-library-pr/generate-stackbrew-library.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -e + +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +if [ -z "$VERBOSITY" ]; then + VERBOSITY=1 +fi + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/func.sh" + +source_helper_file helpers.sh +source_helper_file github_helpers.sh + +init_console_output + +MAJOR_VERSION="" +REMOTE="origin" +while [[ $# -gt 0 ]]; do + case $1 in + --major-version) + MAJOR_VERSION=$2 + shift + shift + ;; + --remote) + REMOTE=$2 + shift + shift + ;; + *) + echo "Error: Unknown option $1" + exit 1 + ;; + esac +done + +if [ -z "$MAJOR_VERSION" ]; then + echo "Error: --major-version M is required as argument" + exit 1 +fi + +execute_command git fetch +redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") +echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" +echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library + diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh new file mode 100755 index 000000000..ef7e0746f --- /dev/null +++ b/test/run-shell-func-tests.sh @@ -0,0 +1,246 @@ +#!/bin/bash +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../.github/actions/common/func.sh" + +source_helper_file "helpers.sh" + +test_get_distro_name_from_dockerfile() { + distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) + assertEquals "alpine3.22" "$distro_name" + + distro_name=$(echo 'FROM debian:bookworm-slim' | extract_distro_name_from_dockerfile) + assertEquals "bookworm" "$distro_name" + + distro_name=$(echo 'FROM debian:bookworm' | extract_distro_name_from_dockerfile) + assertEquals "bookworm" "$distro_name" +} + +test_filter_major_release_version_branches() { + local input + input=$(cat </dev/null) + local ret=$? + assertNotEquals "extract_redis_version_from_dockerfile returned non zero" "0" "$ret" +} + +test_generate_tags_list() { + local tags + tags=$(generate_tags_list 8.2.1 bookworm 1 1) + assertEquals "8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm" "$tags" + + tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 0) + assertEquals "8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22" "$tags" +} + +test_redis_version_split() { + local major minor patch suffix + local version + + version="8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="v8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="v8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" + + version="v8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" +} + +test_filter_actual_major_release_version() { + version=$(cat < Date: Wed, 10 Sep 2025 14:57:20 +0300 Subject: [PATCH 091/188] Support for EOLing versions, fixes and update tests --- .github/actions/common/func.sh | 97 ++++++++++---- test/run-shell-func-tests.sh | 226 +++++++++++++++++++++------------ 2 files changed, 212 insertions(+), 111 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index d0afa9d0e..7d548247e 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -39,7 +39,12 @@ git_ls_remote_major_release_version_branches() { git_ls_remote_tags() { local remote="$1" local major_version="$2" - execute_command git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" + execute_command --no-std -- git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" + if [ -z "$last_cmd_stdout" ]; then + console_output 0 red "Error: No tags found for major_version=$major_version" + return 1 + fi + echo "$last_cmd_stdout" } filter_major_release_version_branches() { @@ -53,6 +58,51 @@ filter_major_release_version_branches() { done | sort -Vr } +sort_version_tags() { + local major_version="$1" + local version_tag commit ref + while read -r commit ref; do + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + printf "%s %s\n" "$version_tag" "$commit" + done | sort -Vr +} + +filter_out_eol_versions() { + local major_version="$1" + local version_tag commit + local last_minor skip_minor minors + local major minor patch suffix + local versions + + mapfile -t versions + for line in "${versions[@]}"; do + read -r version_tag commit < <(echo "$line") + IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") + + if [ "$minor" != "$last_minor" ] && [ -n "$last_minor" ]; then + if [ -z "$skip_minor" ]; then + printf "%s" "$minors" + else + console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" + fi + minors="" + skip_minor="" + fi + last_minor="$minor" + + printf -v minors "%s%s\n" "$minors" "$version_tag $commit" + + if echo "$suffix" | grep -qi "-eol$"; then + skip_minor="$minor" + fi + done + if [ -z "$skip_minor" ]; then + printf "%s" "$minors" + else + console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" + fi +} + filter_actual_major_redis_versions() { local major_version="$1" local last_minor="" last_is_milestone="" @@ -127,24 +177,6 @@ extract_distro_name_from_dockerfile() { echo "$distro" } -extract_redis_version_from_dockerfile() { - increase_indent_level - console_output 2 gray "Extracting redis version from dockerfile" - local redis_version - redis_version=$(grep -m1 -i '^ENV REDIS_DOWNLOAD_URL.*https*:.*tar' \ - | sed 's/ENV.*REDIS_DOWNLOAD_URL.*[-/]\([1-9][0-9]*\..*\)\.tar\.gz/\1/g' \ - | grep -E '^[1-9][0-9]*\.' - ) - console_output 2 gray "redis_version=$redis_version" - if [ -z "$redis_version" ]; then - console_output 0 red "Error: Failed to extract redis version from dockerfile" - decrease_indent_level - return 1 - fi - echo "$redis_version" - decrease_indent_level -} - redis_version_split() { local version local numerics @@ -175,6 +207,8 @@ generate_tags_list() { local is_latest=$3 local is_default=$4 + console_output 2 gray "Generate tags redis_version=$redis_version distro_names=$distro_names is_latest=$is_latest id_default=$is_default" + local tags versions local major minor patch suffix @@ -183,8 +217,13 @@ generate_tags_list() { local mainline_version mainline_version="$major.$minor" - versions=("$redis_version" "$mainline_version") - if [ "$is_latest" = 1 ]; then + versions=("$redis_version") + # generate mainline version tag only for GA releases, e.g 8.2 and 8.2-distro + # tags will be generated only for 8.2.1 but not for 8.2.1-m01 + if [ -z "$suffix" ]; then + versions+=("$mainline_version") + fi + if [ "$is_latest" != "" ]; then versions+=("$major") fi @@ -198,7 +237,7 @@ generate_tags_list() { done done - if [ "$is_latest" = 1 ]; then + if [ "$is_latest" != "" ]; then if [ "$is_default" = 1 ]; then tags+=("latest") fi @@ -211,20 +250,24 @@ generate_tags_list() { generate_stackbrew_library() { local commit redis_version distro distro_version - local is_latest="unset" is_default + local is_latest="" is_latest_unset=1 is_default local stackbrew_content - while read -r commit redis_version distro distro_version; do + mapfile -t releases + for line in "${releases[@]}"; do + read -r commit redis_version distro distro_version < <(echo "$line") + local major minor patch suffix IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") # assigning latest to the first non milestone (empty suffix) version from top - if [ "$is_latest" = "unset" ]; then + if [ "$is_latest_unset" = 1 ]; then if [ -z "$suffix" ]; then - is_latest=1 + is_latest="$minor" + is_latest_unset="" fi - else + elif [ "$is_latest" != "$minor" ]; then is_latest="" fi diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index ef7e0746f..008f25d5a 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -5,6 +5,8 @@ SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" source_helper_file "helpers.sh" +init_console_output + test_get_distro_name_from_dockerfile() { distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) assertEquals "alpine3.22" "$distro_name" @@ -44,77 +46,19 @@ EXPECTED assertEquals "$expected" "$output" } -test_extract_redis_version_from_dockerfile_github() { - local dockerfile_content - dockerfile_content=$(cat </dev/null) - local ret=$? - assertNotEquals "extract_redis_version_from_dockerfile returned non zero" "0" "$ret" -} - test_generate_tags_list() { local tags tags=$(generate_tags_list 8.2.1 bookworm 1 1) assertEquals "8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm" "$tags" - tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 0) + tags=$(generate_tags_list 8.2.1 "alpine alpine3.22" 1 "") assertEquals "8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22" "$tags" + + tags=$(generate_tags_list 8.0-m03 "bookworm" "" 1) + assertEquals "8.0-m03, 8.0-m03-bookworm" "$tags" + + tags=$(generate_tags_list 8.0-m03 "alpine alpine3.21" "" 1) + assertEquals "8.0-m03, 8.0-m03-alpine, 8.0-m03-alpine3.21" "$tags" } test_redis_version_split() { @@ -170,14 +114,24 @@ test_redis_version_split() { assertEquals "suffix of $version" "-m03-int" "$suffix" } +test_redis_version_split_fail() { + IFS=: read -r major minor patch suffix < <(redis_version_split 8.x.x) + assertNotEquals "return code" "0" "$?" +} + + test_filter_actual_major_release_version() { version=$(cat < Date: Thu, 11 Sep 2025 17:03:42 +0300 Subject: [PATCH 092/188] Comments and fixes to the shell version --- .github/actions/common/func.sh | 63 ++++++++++++------- .../generate-stackbrew-library.sh | 3 +- test/run-shell-func-tests.sh | 3 + 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 7d548247e..efb4ccee3 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Sources a helper file from multiple possible locations (GITHUB_WORKSPACE, RELEASE_AUTOMATION_DIR, or relative path) source_helper_file() { local helper_file="$1" local helper_errors="" @@ -25,6 +26,7 @@ source_helper_file() { fi } +# Lists remote release branches for a specific major version (e.g., release/8.*) git_ls_remote_major_release_version_branches() { local remote="$1" local major_version="$2" @@ -36,6 +38,7 @@ git_ls_remote_major_release_version_branches() { echo "$last_cmd_stdout" } +# Lists remote tags for a specific major version (e.g., v8.*) git_ls_remote_tags() { local remote="$1" local major_version="$2" @@ -47,6 +50,7 @@ git_ls_remote_tags() { echo "$last_cmd_stdout" } +# Filters and sorts release branches by major version in reverse version order filter_major_release_version_branches() { local major_version="$1" while read -r line; do @@ -58,19 +62,29 @@ filter_major_release_version_branches() { done | sort -Vr } +# Sorts version tags in reverse version order for a specific major version +# stdin: commit ref (git ls-remote) +# stdout: version commit (vX.X.X sha1) - sorted by version sort_version_tags() { local major_version="$1" local version_tag commit ref while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" + version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*" || :)" + if [ -z "$version_tag" ]; then + console_output 2 red "Incorrect reference format: $ref" + return 1 + fi printf "%s %s\n" "$version_tag" "$commit" done | sort -Vr } +# Filters out end-of-life (EOL) versions by skipping entire minor version series marked with -eol suffix +# stdin: version commit (vX.X.X sha1) - must be sorted by version +# stdout: version commit (vX.X.X sha1) filter_out_eol_versions() { local major_version="$1" local version_tag commit - local last_minor skip_minor minors + local last_minor="" skip_minor="" minors="" local major minor patch suffix local versions @@ -103,15 +117,15 @@ filter_out_eol_versions() { fi } +# Filters Redis versions to keep only the latest patch version (and optionally the latest milestone) for each minor version +# stdin: version commit (vX.X.X sha1) - must be sorted by version +# stdout: version commit (vX.X.X sha1) filter_actual_major_redis_versions() { local major_version="$1" local last_minor="" last_is_milestone="" local ref commit version_tag - - while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*")" - echo "$version_tag $ref $commit" - done | sort -Vr | while read -r version_tag ref commit; do + console_output 2 gray "filter_actual_major_redis_versions" + while read -r version_tag commit; do local major minor patch suffix is_milestone IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") @@ -133,21 +147,20 @@ filter_actual_major_redis_versions() { done } -get_major_release_version_branches () { - local remote="$1" - local major_version="$2" - execute_command git_ls_remote_major_release_version_branches "$remote" "$major_version" | execute_command filter_major_release_version_branches "$major_version" -} - +# Gets and filters actual Redis versions (tags) from a remote repository for a major version get_actual_major_redis_versions() { local remote="$1" local major_version="$2" - execute_command git_ls_remote_tags "$remote" "$major_version" | execute_command filter_actual_major_redis_versions "$major_version" + execute_command git_ls_remote_tags "$remote" "$major_version" \ + | execute_command sort_version_tags "$major_version" \ + | execute_command filter_out_eol_versions "$major_version" \ + | execute_command filter_actual_major_redis_versions "$major_version" } +# Fetches unshallow refs from a remote repository for the provided list of references git_fetch_unshallow_refs() { local remote="$1" - local refs_to_fetch + local refs_to_fetch="" while read -r line; do local ref="$(echo "$line" | awk '{print $1}')" refs_to_fetch="$refs_to_fetch $ref" @@ -156,6 +169,7 @@ git_fetch_unshallow_refs() { execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch } +# Extracts the distribution name from a Dockerfile's FROM statement (supports Alpine and Debian) extract_distro_name_from_dockerfile() { local base_img base_img="$(grep -m1 -i '^from' | awk '{print $2}')" @@ -177,13 +191,14 @@ extract_distro_name_from_dockerfile() { echo "$distro" } +# Splits a Redis version string into major:minor:patch:suffix components redis_version_split() { local version local numerics # shellcheck disable=SC2001 version=$(echo "$1" | sed 's/^v//') - numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)') + numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)' || :) if [ -z "$numerics" ]; then console_output 2 red "Cannot split version '$version', incorrect version format" return 1 @@ -194,21 +209,23 @@ redis_version_split() { printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" } - +# Shows a file from a specific git reference (commit/branch/tag) git_show_file_from_ref() { local ref=$1 local file=$2 execute_command git show "$ref:$file" } +# Generates a comma-separated list of Docker tags for a Redis version and distribution +# args: redis_version distro_names is_latest is_default +# is_latest empty for non-latest, otherwise latest +# is_default 1 for default distro, otherwise not default generate_tags_list() { local redis_version=$1 local distro_names=$2 local is_latest=$3 local is_default=$4 - console_output 2 gray "Generate tags redis_version=$redis_version distro_names=$distro_names is_latest=$is_latest id_default=$is_default" - local tags versions local major minor patch suffix @@ -248,11 +265,13 @@ generate_tags_list() { echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" } +# Generates stackbrew library content (for specific major version) +# stdin: commit redis_version distro distro_version (sha1 vX.X.X alpine alpine3.21) generate_stackbrew_library() { local commit redis_version distro distro_version local is_latest="" is_latest_unset=1 is_default - local stackbrew_content + local stackbrew_content="" mapfile -t releases for line in "${releases[@]}"; do @@ -287,9 +306,11 @@ generate_stackbrew_library() { console_output 2 gray "$stackbrew_content" } +# Prepares a list of releases with commit, Redis version, distro, and distro version information +# stdin: redis_version commit prepare_releases_list() { local redis_version commit - local debug_output version_line + local debug_output="" version_line while read -r redis_version commit; do for distro in debian alpine; do local dockerfile distro_version redis_version diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh index 011ec7914..ed1fffdcc 100755 --- a/.github/actions/create-library-pr/generate-stackbrew-library.sh +++ b/.github/actions/create-library-pr/generate-stackbrew-library.sh @@ -1,5 +1,6 @@ #!/bin/bash set -e +set -o pipefail # shellcheck disable=SC2034 last_cmd_stdout="" @@ -47,7 +48,7 @@ if [ -z "$MAJOR_VERSION" ]; then exit 1 fi -execute_command git fetch +set -u redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index 008f25d5a..1f4c98c94 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -1,10 +1,13 @@ #!/bin/bash +set -e -o pipefail SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" # shellcheck disable=SC1091 . "$SCRIPT_DIR/../.github/actions/common/func.sh" source_helper_file "helpers.sh" +set -u + init_console_output test_get_distro_name_from_dockerfile() { From bbd49d2d9904b62c733f7a4464ad121e27f94e35 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:33:19 +0300 Subject: [PATCH 093/188] Python release-automation implementation --- .../workflows/build_release_automation.yml | 80 ++++++++ release-automation/README.md | 17 ++ release-automation/pyproject.toml | 86 ++++++++ .../src/stackbrew_generator/__init__.py | 3 + .../src/stackbrew_generator/cli.py | 139 +++++++++++++ .../src/stackbrew_generator/distribution.py | 117 +++++++++++ .../src/stackbrew_generator/exceptions.py | 138 +++++++++++++ .../src/stackbrew_generator/git_operations.py | 156 ++++++++++++++ .../src/stackbrew_generator/logging_config.py | 95 +++++++++ .../src/stackbrew_generator/models.py | 177 ++++++++++++++++ .../src/stackbrew_generator/stackbrew.py | 131 ++++++++++++ .../src/stackbrew_generator/version_filter.py | 153 ++++++++++++++ release-automation/tests/__init__.py | 1 + release-automation/tests/test_integration.py | 100 +++++++++ release-automation/tests/test_models.py | 179 ++++++++++++++++ release-automation/tests/test_stackbrew.py | 193 ++++++++++++++++++ 16 files changed, 1765 insertions(+) create mode 100644 .github/workflows/build_release_automation.yml create mode 100644 release-automation/README.md create mode 100644 release-automation/pyproject.toml create mode 100644 release-automation/src/stackbrew_generator/__init__.py create mode 100644 release-automation/src/stackbrew_generator/cli.py create mode 100644 release-automation/src/stackbrew_generator/distribution.py create mode 100644 release-automation/src/stackbrew_generator/exceptions.py create mode 100644 release-automation/src/stackbrew_generator/git_operations.py create mode 100644 release-automation/src/stackbrew_generator/logging_config.py create mode 100644 release-automation/src/stackbrew_generator/models.py create mode 100644 release-automation/src/stackbrew_generator/stackbrew.py create mode 100644 release-automation/src/stackbrew_generator/version_filter.py create mode 100644 release-automation/tests/__init__.py create mode 100644 release-automation/tests/test_integration.py create mode 100644 release-automation/tests/test_models.py create mode 100644 release-automation/tests/test_stackbrew.py diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml new file mode 100644 index 000000000..f20adf8eb --- /dev/null +++ b/.github/workflows/build_release_automation.yml @@ -0,0 +1,80 @@ +name: Build Release Automation Docker Image + +on: + workflow_dispatch: + inputs: + image_tag: + description: 'Docker image tag (default: latest)' + required: false + default: 'latest' + push_to_ghcr: + description: 'Push image to GHCR' + required: false + default: true + type: boolean + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/release-automation + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Log in to Container Registry + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ github.event.inputs.image_tag }} + type=raw,value=latest,enable={{is_default_branch}} + type=sha,prefix={{branch}}- + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: ./release-automation + file: ./release-automation/docker/Dockerfile + push: ${{ github.event.inputs.push_to_ghcr == 'true' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Output image details + run: | + echo "## Docker Image Built Successfully! 🐳" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Tags:**" >> $GITHUB_STEP_SUMMARY + echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/' | sed 's/$/`/' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [[ "${{ github.event.inputs.push_to_ghcr }}" == "true" ]]; then + echo "āœ… **Image pushed to GHCR**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "To pull the image:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + else + echo "ā„¹ļø **Image built locally only (not pushed)**" >> $GITHUB_STEP_SUMMARY + fi diff --git a/release-automation/README.md b/release-automation/README.md new file mode 100644 index 000000000..70aec8fce --- /dev/null +++ b/release-automation/README.md @@ -0,0 +1,17 @@ +# Release automation helper + +## Installation + +### From Source + +```bash +cd release-automation +pip install -e . +``` + +### Development Installation + +```bash +cd release-automation +pip install -e ".[dev]" +``` diff --git a/release-automation/pyproject.toml b/release-automation/pyproject.toml new file mode 100644 index 000000000..50714fc96 --- /dev/null +++ b/release-automation/pyproject.toml @@ -0,0 +1,86 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stackbrew-library-generator" +version = "0.1.0" +description = "Stackbrew Library Generator for Redis Docker Images" +authors = [ + {name = "Redis Team", email = "team@redis.io"}, +] +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "typer[all]>=0.9.0", + "rich>=13.0.0", + "pydantic>=2.0.0", + "packaging>=21.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "isort>=5.12.0", + "mypy>=1.0.0", + "pre-commit>=3.0.0", +] + +[project.scripts] +release-automation = "stackbrew_generator.cli:app" + +[project.urls] +Homepage = "https://github.com/redis/docker-library-redis" +Repository = "https://github.com/redis/docker-library-redis" +Issues = "https://github.com/redis/docker-library-redis/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/stackbrew_generator"] + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/README.md", + "/pyproject.toml", +] + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--cov=stackbrew_generator", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] diff --git a/release-automation/src/stackbrew_generator/__init__.py b/release-automation/src/stackbrew_generator/__init__.py new file mode 100644 index 000000000..38a1b1c6e --- /dev/null +++ b/release-automation/src/stackbrew_generator/__init__.py @@ -0,0 +1,3 @@ +"""Stackbrew Library Generator for Redis Docker Images.""" + +__version__ = "0.1.0" diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py new file mode 100644 index 000000000..cdc1721b4 --- /dev/null +++ b/release-automation/src/stackbrew_generator/cli.py @@ -0,0 +1,139 @@ +"""CLI interface for stackbrew library generator.""" + +import typer +from rich.console import Console +from rich.traceback import install + +from .distribution import DistributionDetector +from .exceptions import StackbrewGeneratorError +from .git_operations import GitClient +from .logging_config import setup_logging +from .stackbrew import StackbrewGenerator +from .version_filter import VersionFilter + +# Install rich traceback handler +install(show_locals=True) + +app = typer.Typer( + name="release-automation", + help="Generate stackbrew library content for Redis Docker images", + add_completion=False, +) + +# Console for logging and user messages (stderr) +console = Console(stderr=True) + + +@app.command() +def generate( + major_version: int = typer.Argument( + ..., + help="Redis major version to process (e.g., 8 for Redis 8.x)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Show generated content without outputting to stdout" + ), +) -> None: + """Generate stackbrew library content for Redis Docker images. + + This command: + 1. Fetches Redis version tags from the specified remote + 2. Filters versions to remove EOL and select latest patches + 3. Extracts distribution information from Dockerfiles + 4. Generates appropriate Docker tags for each version/distribution + 5. Outputs stackbrew library content + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if verbose: + console.print(f"[bold blue]Stackbrew Library Generator[/bold blue]") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") + if dry_run: + console.print("[yellow]DRY RUN MODE - Generated content will be shown but not output to stdout[/yellow]") + + try: + # Initialize components + git_client = GitClient(remote=remote) + version_filter = VersionFilter(git_client) + distribution_detector = DistributionDetector(git_client) + stackbrew_generator = StackbrewGenerator() + + # Get actual Redis versions to process + versions = version_filter.get_actual_major_redis_versions(major_version) + + if not versions: + console.print(f"[red]No versions found for Redis {major_version}.x[/red]") + raise typer.Exit(1) + + # Fetch required refs + refs_to_fetch = [commit for _, commit in versions] + git_client.fetch_refs(refs_to_fetch) + + # Prepare releases list with distribution information + releases = distribution_detector.prepare_releases_list(versions) + + if not releases: + console.print("[red]No releases prepared[/red]") + raise typer.Exit(1) + + # Generate stackbrew library content + entries = stackbrew_generator.generate_stackbrew_library(releases) + output = stackbrew_generator.format_stackbrew_output(entries) + + if dry_run: + console.print(f"[yellow]DRY RUN: Would generate stackbrew library with {len(entries)} entries[/yellow]") + if verbose: + console.print("[yellow]Generated content:[/yellow]") + console.print(output) + else: + if output: + # Output the stackbrew library content + print(output) + + if verbose: + console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + else: + console.print("[yellow]No stackbrew content generated[/yellow]") + + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + + +@app.command() +def version() -> None: + """Show version information.""" + from . import __version__ + console.print(f"stackbrew-library-generator {__version__}") + + +if __name__ == "__main__": + app() diff --git a/release-automation/src/stackbrew_generator/distribution.py b/release-automation/src/stackbrew_generator/distribution.py new file mode 100644 index 000000000..187e8f521 --- /dev/null +++ b/release-automation/src/stackbrew_generator/distribution.py @@ -0,0 +1,117 @@ +"""Distribution detection from Dockerfiles.""" + +from typing import List, Tuple + +from rich.console import Console + +from .exceptions import DistributionError +from .git_operations import GitClient +from .models import Distribution, RedisVersion, Release + +console = Console(stderr=True) + + +class DistributionDetector: + """Detects distribution information from Dockerfiles.""" + + def __init__(self, git_client: GitClient): + """Initialize distribution detector. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def extract_distribution_from_dockerfile(self, dockerfile_content: str) -> Distribution: + """Extract distribution information from Dockerfile content. + + Args: + dockerfile_content: Content of the Dockerfile + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + # Find the FROM line + from_line = None + for line in dockerfile_content.split('\n'): + line = line.strip() + if line.upper().startswith('FROM '): + from_line = line + break + + if not from_line: + raise DistributionError("No FROM line found in Dockerfile") + + try: + return Distribution.from_dockerfile_line(from_line) + except ValueError as e: + raise DistributionError(f"Failed to parse distribution from FROM line: {e}") from e + + def get_distribution_for_commit(self, commit: str, distro_type: str) -> Distribution: + """Get distribution information for a specific commit and distro type. + + Args: + commit: Git commit hash + distro_type: Distribution type ("debian" or "alpine") + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + dockerfile_path = f"{distro_type}/Dockerfile" + + try: + dockerfile_content = self.git_client.show_file(commit, dockerfile_path) + console.print(f"[dim]Retrieved {dockerfile_path} from {commit[:8]}[/dim]") + + distribution = self.extract_distribution_from_dockerfile(dockerfile_content) + console.print(f"[dim]Detected distribution: {distribution.type.value} {distribution.name}[/dim]") + + return distribution + + except Exception as e: + raise DistributionError( + f"Failed to get distribution for {distro_type} from {commit}: {e}" + ) from e + + def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> List[Release]: + """Prepare list of releases with distribution information. + + Args: + versions: List of (RedisVersion, commit) tuples + + Returns: + List of Release objects with distribution information + """ + console.print("[blue]Preparing releases list with distribution information[/blue]") + + releases = [] + distro_types = ["debian", "alpine"] + + for version, commit in versions: + console.print(f"[dim]Processing [bold yellow]{version}[/bold yellow] - {commit[:8]}[/dim]") + + for distro_type in distro_types: + try: + distribution = self.get_distribution_for_commit(commit, distro_type) + + release = Release( + commit=commit, + version=version, + distribution=distribution + ) + + releases.append(release) + console.print(f"[dim] Added: {release.console_repr()}[/dim]", highlight=False) + + except DistributionError as e: + console.print(f"[yellow]Warning: Failed to process {distro_type} for {version}: {e}[/yellow]") + continue + + console.print(f"[green]Prepared {len(releases)} releases[/green]") + return releases diff --git a/release-automation/src/stackbrew_generator/exceptions.py b/release-automation/src/stackbrew_generator/exceptions.py new file mode 100644 index 000000000..f5b11b2a6 --- /dev/null +++ b/release-automation/src/stackbrew_generator/exceptions.py @@ -0,0 +1,138 @@ +"""Custom exceptions for stackbrew library generation.""" + +from typing import Optional, Any, Dict + + +class StackbrewGeneratorError(Exception): + """Base exception for stackbrew generator errors. + + Provides structured error information with context and suggestions. + """ + + def __init__( + self, + message: str, + context: Optional[Dict[str, Any]] = None, + suggestion: Optional[str] = None, + original_error: Optional[Exception] = None + ): + """Initialize error with context. + + Args: + message: Error message + context: Additional context information + suggestion: Suggested fix or next steps + original_error: Original exception that caused this error + """ + super().__init__(message) + self.context = context or {} + self.suggestion = suggestion + self.original_error = original_error + + def get_detailed_message(self) -> str: + """Get detailed error message with context and suggestions.""" + parts = [str(self)] + + if self.context: + parts.append("Context:") + for key, value in self.context.items(): + parts.append(f" {key}: {value}") + + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + + if self.original_error: + parts.append(f"Original error: {self.original_error}") + + return "\n".join(parts) + + +class GitOperationError(StackbrewGeneratorError): + """Exception raised for Git operation failures.""" + + def __init__( + self, + message: str, + command: Optional[str] = None, + exit_code: Optional[int] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if command: + context['command'] = command + if exit_code is not None: + context['exit_code'] = exit_code + + suggestion = kwargs.get('suggestion') + if not suggestion and command: + if 'ls-remote' in command: + suggestion = "Check that the remote repository exists and is accessible" + elif 'fetch' in command: + suggestion = "Ensure you have network access and proper Git credentials" + elif 'show' in command: + suggestion = "Verify that the commit exists and contains the requested file" + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class VersionParsingError(StackbrewGeneratorError): + """Exception raised for version parsing failures.""" + + def __init__(self, message: str, version_string: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if version_string: + context['version_string'] = version_string + + suggestion = kwargs.get('suggestion', + "Version should be in format 'X.Y.Z' or 'vX.Y.Z' with optional suffix") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class DistributionError(StackbrewGeneratorError): + """Exception raised for distribution detection failures.""" + + def __init__( + self, + message: str, + dockerfile_path: Optional[str] = None, + from_line: Optional[str] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if dockerfile_path: + context['dockerfile_path'] = dockerfile_path + if from_line: + context['from_line'] = from_line + + suggestion = kwargs.get('suggestion', + "Dockerfile should have a FROM line with supported base image (alpine:* or debian:*)") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class ValidationError(StackbrewGeneratorError): + """Exception raised for validation failures.""" + + def __init__(self, message: str, field: Optional[str] = None, value: Optional[Any] = None, **kwargs): + context = kwargs.get('context', {}) + if field: + context['field'] = field + if value is not None: + context['value'] = value + + super().__init__(message, context=context, **kwargs) + + +class ConfigurationError(StackbrewGeneratorError): + """Exception raised for configuration errors.""" + + def __init__(self, message: str, config_key: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if config_key: + context['config_key'] = config_key + + suggestion = kwargs.get('suggestion', + "Check your configuration and environment variables") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) diff --git a/release-automation/src/stackbrew_generator/git_operations.py b/release-automation/src/stackbrew_generator/git_operations.py new file mode 100644 index 000000000..14dab035c --- /dev/null +++ b/release-automation/src/stackbrew_generator/git_operations.py @@ -0,0 +1,156 @@ +"""Git operations for stackbrew library generation.""" + +import re +import subprocess +from typing import Dict, List, Tuple + +from rich.console import Console + +from .exceptions import GitOperationError +from .models import RedisVersion + +console = Console(stderr=True) + + +class GitClient: + """Client for Git operations.""" + + def __init__(self, remote: str = "origin"): + """Initialize Git client. + + Args: + remote: Git remote name to use + """ + self.remote = remote + + def _run_command(self, cmd: List[str], capture_output: bool = True) -> subprocess.CompletedProcess: + """Run a git command with error handling. + + Args: + cmd: Command and arguments to run + capture_output: Whether to capture stdout/stderr + + Returns: + CompletedProcess result + + Raises: + GitOperationError: If command fails + """ + try: + result = subprocess.run( + cmd, + capture_output=capture_output, + text=True, + check=True, + ) + return result + except subprocess.CalledProcessError as e: + error_msg = f"Git command failed: {' '.join(cmd)}" + if e.stderr: + error_msg += f"\nError: {e.stderr.strip()}" + raise GitOperationError(error_msg) from e + except FileNotFoundError as e: + raise GitOperationError("Git command not found. Is git installed?") from e + + def list_remote_tags(self, major_version: int) -> List[Tuple[str, str]]: + """List remote tags for a specific major version. + + Args: + major_version: Major version to filter tags for + + Returns: + List of (commit, tag_ref) tuples + + Raises: + GitOperationError: If no tags found or git operation fails + """ + console.print(f"[dim]Listing remote tags for v{major_version}.*[/dim]") + + cmd = [ + "git", "ls-remote", "--refs", "--tags", + self.remote, f"refs/tags/v{major_version}.*" + ] + + result = self._run_command(cmd) + + if not result.stdout.strip(): + raise GitOperationError(f"No tags found for major version {major_version}") + + tags = [] + for line in result.stdout.strip().split('\n'): + if line: + commit, ref = line.split('\t', 1) + tags.append((commit, ref)) + + console.print(f"[dim]Found {len(tags)} tags[/dim]") + return tags + + def fetch_refs(self, refs: List[str]) -> None: + """Fetch specific refs from remote. + + Args: + refs: List of refs to fetch + + Raises: + GitOperationError: If fetch operation fails + """ + if not refs: + return + + console.print(f"[dim]Fetching {len(refs)} refs[/dim]") + + # Use git fetch with unshallow to ensure we have full history + cmd = ["git", "fetch", "--unshallow", self.remote] + refs + + try: + self._run_command(cmd, capture_output=False) + except GitOperationError: + # If --unshallow fails (repo already unshallow), try without it + cmd = ["git", "fetch", self.remote] + refs + self._run_command(cmd, capture_output=False) + + def show_file(self, commit: str, file_path: str) -> str: + """Show file content from a specific commit. + + Args: + commit: Git commit hash + file_path: Path to file in repository + + Returns: + File content as string + + Raises: + GitOperationError: If file cannot be retrieved + """ + cmd = ["git", "show", f"{commit}:{file_path}"] + + try: + result = self._run_command(cmd) + return result.stdout + except GitOperationError as e: + raise GitOperationError(f"Failed to get {file_path} from {commit}: {e}") from e + + def extract_version_from_tag(self, tag_ref: str, major_version: int) -> RedisVersion: + """Extract Redis version from tag reference. + + Args: + tag_ref: Git tag reference (e.g., refs/tags/v8.2.1) + major_version: Expected major version for validation + + Returns: + Parsed RedisVersion + + Raises: + GitOperationError: If tag format is invalid + """ + # Extract version from tag reference + match = re.search(rf"v{major_version}\.\d+(?:\.\d+)?.*", tag_ref) + if not match: + raise GitOperationError(f"Invalid tag format: {tag_ref}") + + version_str = match.group(0) + + try: + return RedisVersion.parse(version_str) + except ValueError as e: + raise GitOperationError(f"Failed to parse version from {tag_ref}: {e}") from e diff --git a/release-automation/src/stackbrew_generator/logging_config.py b/release-automation/src/stackbrew_generator/logging_config.py new file mode 100644 index 000000000..b261e7ae7 --- /dev/null +++ b/release-automation/src/stackbrew_generator/logging_config.py @@ -0,0 +1,95 @@ +"""Logging configuration for stackbrew generator.""" + +import logging +from typing import Optional + +from rich.console import Console +from rich.logging import RichHandler + + +def setup_logging( + level: str = "INFO", + verbose: bool = False, + console: Optional[Console] = None +) -> logging.Logger: + """Set up logging configuration. + + Args: + level: Logging level (DEBUG, INFO, WARNING, ERROR) + verbose: Enable verbose logging + console: Rich console instance to use (should use stderr) + + Returns: + Configured logger instance + """ + if console is None: + # Create console that outputs to stderr + console = Console(stderr=True) + + # Determine log level + if verbose: + log_level = logging.DEBUG + else: + log_level = getattr(logging, level.upper(), logging.INFO) + + # Configure root logger + logging.basicConfig( + level=log_level, + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler( + console=console, + show_path=verbose, + show_time=verbose, + rich_tracebacks=True, + tracebacks_show_locals=verbose, + ) + ], + ) + + # Get logger for our package + logger = logging.getLogger("stackbrew_generator") + logger.setLevel(log_level) + + return logger + + +def get_logger(name: str) -> logging.Logger: + """Get a logger instance for a specific module. + + Args: + name: Logger name (usually __name__) + + Returns: + Logger instance + """ + return logging.getLogger(f"stackbrew_generator.{name}") + + +class LoggingMixin: + """Mixin class to add logging capabilities to other classes.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = get_logger(self.__class__.__name__) + + def log_debug(self, message: str, *args, **kwargs) -> None: + """Log debug message.""" + self.logger.debug(message, *args, **kwargs) + + def log_info(self, message: str, *args, **kwargs) -> None: + """Log info message.""" + self.logger.info(message, *args, **kwargs) + + def log_warning(self, message: str, *args, **kwargs) -> None: + """Log warning message.""" + self.logger.warning(message, *args, **kwargs) + + def log_error(self, message: str, *args, **kwargs) -> None: + """Log error message.""" + self.logger.error(message, *args, **kwargs) + + def log_exception(self, message: str, *args, **kwargs) -> None: + """Log exception with traceback.""" + self.logger.exception(message, *args, **kwargs) diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py new file mode 100644 index 000000000..599bbe9c9 --- /dev/null +++ b/release-automation/src/stackbrew_generator/models.py @@ -0,0 +1,177 @@ +"""Data models for stackbrew library generation.""" + +import re +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, Field, validator + + +class DistroType(str, Enum): + """Distribution type enumeration.""" + + ALPINE = "alpine" + DEBIAN = "debian" + + +class RedisVersion(BaseModel): + """Represents a parsed Redis version.""" + + major: int = Field(..., ge=1, description="Major version number") + minor: int = Field(..., ge=0, description="Minor version number") + patch: Optional[int] = Field(None, ge=0, description="Patch version number") + suffix: str = Field("", description="Version suffix (e.g., -m01, -rc1, -eol)") + + @classmethod + def parse(cls, version_str: str) -> "RedisVersion": + """Parse a version string into components. + + Args: + version_str: Version string (e.g., "v8.2.1-m01", "8.2", "7.4.0-eol") + + Returns: + RedisVersion instance + + Raises: + ValueError: If version string format is invalid + """ + # Remove 'v' prefix if present + version = version_str.lstrip("v") + + # Extract numeric part and suffix + match = re.match(r"^([1-9]\d*\.\d+(?:\.\d+)?)(.*)", version) + if not match: + raise ValueError(f"Invalid version format: {version_str}") + + numeric_part, suffix = match.groups() + + # Parse numeric components + parts = numeric_part.split(".") + major = int(parts[0]) + minor = int(parts[1]) + patch = int(parts[2]) if len(parts) > 2 else None + + return cls(major=major, minor=minor, patch=patch, suffix=suffix) + + @property + def is_milestone(self) -> bool: + """Check if this is a milestone version (has suffix).""" + return bool(self.suffix) + + @property + def is_eol(self) -> bool: + """Check if this version is end-of-life.""" + return self.suffix.lower().endswith("-eol") + + @property + def mainline_version(self) -> str: + """Get the mainline version string (major.minor).""" + return f"{self.major}.{self.minor}" + + def __str__(self) -> str: + """String representation of the version.""" + version = f"{self.major}.{self.minor}" + if self.patch is not None: + version += f".{self.patch}" + return version + self.suffix + + def __lt__(self, other: "RedisVersion") -> bool: + """Compare versions for sorting.""" + if not isinstance(other, RedisVersion): + return NotImplemented + + # Compare major.minor.patch first + self_tuple = (self.major, self.minor, self.patch or 0) + other_tuple = (other.major, other.minor, other.patch or 0) + + if self_tuple != other_tuple: + return self_tuple < other_tuple + + # If numeric parts are equal, compare suffixes + # Empty suffix (GA) comes after suffixes (milestones) + if not self.suffix and other.suffix: + return False + if self.suffix and not other.suffix: + return True + + return self.suffix < other.suffix + + +class Distribution(BaseModel): + """Represents a Linux distribution.""" + + type: DistroType = Field(..., description="Distribution type") + name: str = Field(..., description="Distribution name/version") + + @classmethod + def from_dockerfile_line(cls, from_line: str) -> "Distribution": + """Parse distribution from Dockerfile FROM line. + + Args: + from_line: FROM line from Dockerfile (e.g., "FROM alpine:3.22") + + Returns: + Distribution instance + + Raises: + ValueError: If FROM line format is not supported + """ + # Extract base image from FROM line + parts = from_line.strip().split() + if len(parts) < 2 or parts[0].upper() != "FROM": + raise ValueError(f"Invalid FROM line: {from_line}") + + base_img = parts[1] + + if "alpine:" in base_img: + # Extract alpine version (e.g., alpine:3.22 -> alpine3.22) + version = base_img.split(":", 1)[1] + return cls(type=DistroType.ALPINE, name=f"alpine{version}") + elif "debian:" in base_img: + # Extract debian version, remove -slim suffix + version = base_img.split(":", 1)[1].replace("-slim", "") + return cls(type=DistroType.DEBIAN, name=version) + else: + raise ValueError(f"Unsupported base image: {base_img}") + + @property + def is_default(self) -> bool: + """Check if this is the default distribution (Debian).""" + return self.type == DistroType.DEBIAN + + @property + def tag_names(self) -> List[str]: + """Get tag name components for this distribution.""" + if self.type == DistroType.ALPINE: + return [self.type.value, self.name] + else: + return [self.name] + + +class Release(BaseModel): + """Represents a Redis release with distribution information.""" + + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + + def __str__(self) -> str: + """String representation of the release.""" + return f"{self.commit[:8]} {self.version} {self.distribution.type.value} {self.distribution.name}" + + def console_repr(self) -> str: + """Rich console representation with markup.""" + return f"{self.commit[:8]} [bold yellow]{self.version}[/bold yellow] {self.distribution.type.value} [bold yellow]{self.distribution.name}[/bold yellow]" + + +class StackbrewEntry(BaseModel): + """Represents a stackbrew library entry with tags.""" + + tags: List[str] = Field(..., description="Docker tags for this entry") + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + + def __str__(self) -> str: + """String representation as comma-separated tags.""" + return ", ".join(self.tags) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py new file mode 100644 index 000000000..e40bd4a6a --- /dev/null +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -0,0 +1,131 @@ +"""Stackbrew library generation.""" + +from typing import List + +from rich.console import Console + +from .models import Release, StackbrewEntry + +console = Console(stderr=True) + + +class StackbrewGenerator: + """Generates stackbrew library content.""" + + def generate_tags_for_release( + self, + release: Release, + is_latest: bool = False + ) -> List[str]: + """Generate Docker tags for a release. + + Args: + release: Release to generate tags for + is_latest: Whether this is the latest version + + Returns: + List of Docker tags + """ + tags = [] + version = release.version + distribution = release.distribution + + # Base version tags + version_tags = [str(version)] + + # Add mainline version tag only for GA releases (no suffix) + if not version.is_milestone: + version_tags.append(version.mainline_version) + + # Add major version tag for latest versions + if is_latest: + version_tags.append(str(version.major)) + + # For default distribution (Debian), add version tags without distro suffix + if distribution.is_default: + tags.extend(version_tags) + + # Add distro-specific tags + for distro_name in distribution.tag_names: + for version_tag in version_tags: + tags.append(f"{version_tag}-{distro_name}") + + # Add special latest tags + if is_latest: + if distribution.is_default: + tags.append("latest") + # Add bare distro names as tags + tags.extend(distribution.tag_names) + + return tags + + def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewEntry]: + """Generate stackbrew library entries from releases. + + Args: + releases: List of releases to process + + Returns: + List of StackbrewEntry objects + """ + console.print("[blue]Generating stackbrew library content[/blue]") + + if not releases: + console.print("[yellow]No releases to process[/yellow]") + return [] + + entries = [] + latest_minor = None + latest_minor_unset = True + + for release in releases: + # Determine latest version following bash logic: + # - Set latest_minor to the minor version of the first non-milestone version + # - Clear latest_minor if subsequent versions have different minor versions + if latest_minor_unset: + if not release.version.is_milestone: + latest_minor = release.version.minor + latest_minor_unset = False + console.print(f"[dim]Latest minor version set to: {latest_minor}[/dim]") + elif latest_minor != release.version.minor: + latest_minor = None + + # Check if this release should get latest tags + is_latest = latest_minor is not None + + # Generate tags for this release + tags = self.generate_tags_for_release(release, is_latest) + + if tags: + entry = StackbrewEntry( + tags=tags, + commit=release.commit, + version=release.version, + distribution=release.distribution + ) + entries.append(entry) + + console.print(f"[dim]{release.console_repr()} -> {len(tags)} tags[/dim]") + else: + console.print(f"[yellow]No tags generated for {release}[/yellow]") + + console.print(f"[green]Generated {len(entries)} stackbrew entries[/green]") + return entries + + def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: + """Format stackbrew entries as output string. + + Args: + entries: List of stackbrew entries + + Returns: + Formatted stackbrew library content + """ + if not entries: + return "" + + lines = [] + for entry in entries: + lines.append(str(entry)) + + return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py new file mode 100644 index 000000000..d928e0e83 --- /dev/null +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -0,0 +1,153 @@ +"""Version filtering and processing for Redis releases.""" + +from typing import Dict, List, Tuple + +from packaging.version import Version +from rich.console import Console + +from .git_operations import GitClient +from .models import RedisVersion + +console = Console(stderr=True) + + +class VersionFilter: + """Filters and processes Redis versions.""" + + def __init__(self, git_client: GitClient): + """Initialize version filter. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + """Get Redis versions from git tags. + + Args: + major_version: Major version to filter for + + Returns: + List of (RedisVersion, commit) tuples sorted by version (newest first) + """ + console.print(f"[blue]Getting Redis versions for major version {major_version}[/blue]") + + # Get remote tags + tags = self.git_client.list_remote_tags(major_version) + + # Parse versions from tags + versions = [] + for commit, tag_ref in tags: + try: + version = self.git_client.extract_version_from_tag(tag_ref, major_version) + versions.append((version, commit)) + except Exception as e: + console.print(f"[yellow]Warning: Skipping invalid tag {tag_ref}: {e}[/yellow]") + continue + + # Sort by version (newest first) + versions.sort(key=lambda x: x[0], reverse=True) + + console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") + return versions + + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + """Filter out end-of-life versions. + + Args: + versions: List of (RedisVersion, commit) tuples + + Returns: + Filtered list with EOL minor versions removed + """ + console.print("[blue]Filtering out EOL versions[/blue]") + + # Group versions by minor version + minor_versions: Dict[str, List[Tuple[RedisVersion, str]]] = {} + for version, commit in versions: + minor_key = version.mainline_version + if minor_key not in minor_versions: + minor_versions[minor_key] = [] + minor_versions[minor_key].append((version, commit)) + + # Check each minor version for EOL marker + filtered_versions = [] + for minor_key, minor_group in minor_versions.items(): + # Check if any version in this minor series is marked as EOL + has_eol = any(version.is_eol for version, _ in minor_group) + + if has_eol: + console.print(f"[yellow]Skipping minor version {minor_key}.* due to EOL[/yellow]") + else: + filtered_versions.extend(minor_group) + + # Sort again after filtering + filtered_versions.sort(key=lambda x: x[0], reverse=True) + + console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") + return filtered_versions + + def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + """Filter to keep only the latest patch version for each minor version and milestone status. + + Args: + versions: List of (RedisVersion, commit) tuples (should be sorted newest first) + + Returns: + Filtered list with only the latest versions for each minor/milestone combination + """ + console.print("[blue]Filtering to actual versions (latest patch per minor/milestone)[/blue]") + + seen_combinations = set() + filtered_versions = [] + + for version, commit in versions: + # Create a key for minor version + milestone status + combination_key = (version.mainline_version, version.is_milestone) + + if combination_key not in seen_combinations: + seen_combinations.add(combination_key) + filtered_versions.append((version, commit)) + + milestone_str = "milestone" if version.is_milestone else "GA" + console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") + else: + milestone_str = "milestone" if version.is_milestone else "GA" + console.print(f"[dim]Skipping {version} ({milestone_str}) - already have this minor/milestone combination[/dim]") + + console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") + return filtered_versions + + def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + """Get the actual Redis versions to process for a major version. + + This is the main entry point that combines all filtering steps: + 1. Get versions from git tags + 2. Filter out EOL versions + 3. Filter to actual versions (latest patch per minor/milestone) + + Args: + major_version: Major version to process + + Returns: + List of (RedisVersion, commit) tuples for processing + """ + console.print(f"[bold blue]Processing Redis {major_version}.x versions[/bold blue]") + + # Get all versions from tags + versions = self.get_redis_versions_from_tags(major_version) + + if not versions: + console.print(f"[red]No versions found for major version {major_version}[/red]") + return [] + + # Apply filters + versions = self.filter_eol_versions(versions) + versions = self.filter_actual_versions(versions) + + console.print(f"[green]Final selection: {len(versions)} versions to process[/green]") + for version, commit in versions: + console.print(f"[green] [bold yellow]{version}[/bold yellow] - {commit[:8]}[/green]") + + return versions diff --git a/release-automation/tests/__init__.py b/release-automation/tests/__init__.py new file mode 100644 index 000000000..a60351fa0 --- /dev/null +++ b/release-automation/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for stackbrew library generator.""" diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py new file mode 100644 index 000000000..066392eaf --- /dev/null +++ b/release-automation/tests/test_integration.py @@ -0,0 +1,100 @@ +"""Integration tests for the stackbrew generator.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType +from typer.testing import CliRunner + + +class TestIntegration: + """Integration tests for the complete workflow.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + @patch('stackbrew_generator.distribution.DistributionDetector') + @patch('stackbrew_generator.git_operations.GitClient') + def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distribution_detector_class): + """Test complete workflow in dry run mode.""" + # Mock git client + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + # Mock distribution detector + mock_distribution_detector = Mock() + mock_distribution_detector_class.return_value = mock_distribution_detector + + # Mock git operations + mock_git_client.list_remote_tags.return_value = [ + ("abc123", "refs/tags/v8.2.1"), + ("def456", "refs/tags/v8.2.0"), + ] + + mock_git_client.extract_version_from_tag.side_effect = [ + RedisVersion.parse("8.2.1"), + RedisVersion.parse("8.2.0"), + ] + + # Mock releases + from stackbrew_generator.models import Release, Distribution, DistroType + mock_releases = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + # Run command in dry run mode + result = self.runner.invoke(app, ["generate", "8", "--dry-run", "--verbose"]) + + # Check that it completed successfully + assert result.exit_code == 0 + assert "DRY RUN: Would generate stackbrew library" in result.stderr + assert "Generated content:" in result.stderr + + def test_version_command(self): + """Test version command.""" + result = self.runner.invoke(app, ["version"]) + assert result.exit_code == 0 + assert "stackbrew-library-generator" in result.stderr + + def test_invalid_major_version(self): + """Test handling of invalid major version.""" + result = self.runner.invoke(app, ["generate", "0"]) + assert result.exit_code != 0 + + @patch('stackbrew_generator.git_operations.GitClient') + def test_no_tags_found(self, mock_git_client_class): + """Test handling when no tags are found.""" + # Mock git client to return no tags + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + mock_git_client.list_remote_tags.return_value = [] + + result = self.runner.invoke(app, ["generate", "99"]) + assert result.exit_code == 1 + assert "No tags found" in result.stderr + + @patch('stackbrew_generator.version_filter.VersionFilter.get_actual_major_redis_versions') + def test_no_versions_found(self, mock_get_versions): + """Test handling when no versions are found.""" + # Mock git client to return no tags + mock_get_versions.return_value = [] + + result = self.runner.invoke(app, ["generate", "8"]) + #assert result.exit_code == 1 + assert "No versions found" in result.stderr + + def test_help_output(self): + """Test help output.""" + result = self.runner.invoke(app, ["generate", "--help"]) + assert result.exit_code == 0 + assert "Generate stackbrew library content" in result.stdout + assert "--remote" in result.stdout + assert "--verbose" in result.stdout + assert "--dry-run" in result.stdout diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py new file mode 100644 index 000000000..bc55fa9ab --- /dev/null +++ b/release-automation/tests/test_models.py @@ -0,0 +1,179 @@ +"""Tests for data models.""" + +import pytest + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release + + +class TestRedisVersion: + """Tests for RedisVersion model.""" + + def test_parse_basic_version(self): + """Test parsing basic version strings.""" + version = RedisVersion.parse("8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_v_prefix(self): + """Test parsing version with 'v' prefix.""" + version = RedisVersion.parse("v8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_suffix(self): + """Test parsing version with suffix.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "-m01" + + def test_parse_version_without_patch(self): + """Test parsing version without patch number.""" + version = RedisVersion.parse("8.2") + assert version.major == 8 + assert version.minor == 2 + assert version.patch is None + assert version.suffix == "" + + def test_parse_eol_version(self): + """Test parsing EOL version.""" + version = RedisVersion.parse("7.4.0-eol") + assert version.major == 7 + assert version.minor == 4 + assert version.patch == 0 + assert version.suffix == "-eol" + assert version.is_eol is True + + def test_parse_invalid_version(self): + """Test parsing invalid version strings.""" + with pytest.raises(ValueError): + RedisVersion.parse("invalid") + + with pytest.raises(ValueError): + RedisVersion.parse("0.1.0") # Major version must be >= 1 + + def test_is_milestone(self): + """Test milestone detection.""" + ga_version = RedisVersion.parse("8.2.1") + milestone_version = RedisVersion.parse("8.2.1-m01") + + assert ga_version.is_milestone is False + assert milestone_version.is_milestone is True + + def test_mainline_version(self): + """Test mainline version property.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.mainline_version == "8.2" + + def test_string_representation(self): + """Test string representation.""" + version1 = RedisVersion.parse("8.2.1") + version2 = RedisVersion.parse("8.2.1-m01") + version3 = RedisVersion.parse("8.2") + + assert str(version1) == "8.2.1" + assert str(version2) == "8.2.1-m01" + assert str(version3) == "8.2" + + def test_version_comparison(self): + """Test version comparison for sorting.""" + v1 = RedisVersion.parse("8.2.1") + v2 = RedisVersion.parse("8.2.2") + v3 = RedisVersion.parse("8.2.1-m01") + v4 = RedisVersion.parse("8.3.0") + + # Test numeric comparison + assert v1 < v2 + assert v2 < v4 + + # Test milestone vs GA (GA comes after milestone) + assert v3 < v1 + + # Test sorting + versions = [v4, v1, v3, v2] + sorted_versions = sorted(versions) + assert sorted_versions == [v3, v1, v2, v4] + + +class TestDistribution: + """Tests for Distribution model.""" + + def test_from_dockerfile_alpine(self): + """Test parsing Alpine distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM alpine:3.22") + assert distro.type == DistroType.ALPINE + assert distro.name == "alpine3.22" + + def test_from_dockerfile_debian(self): + """Test parsing Debian distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_debian_slim(self): + """Test parsing Debian slim distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm-slim") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_invalid(self): + """Test parsing invalid Dockerfile lines.""" + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("INVALID LINE") + + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("FROM unsupported:latest") + + def test_is_default(self): + """Test default distribution detection.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.is_default is False + assert debian.is_default is True + + def test_tag_names(self): + """Test tag name generation.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.tag_names == ["alpine", "alpine3.22"] + assert debian.tag_names == ["bookworm"] + + +class TestRelease: + """Tests for Release model.""" + + def test_release_creation(self): + """Test creating a Release instance.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution + ) + + assert release.commit == "abc123def456" + assert release.version == version + assert release.distribution == distribution + + def test_release_string_representation(self): + """Test Release string representation.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution + ) + + expected = "abc123de 8.2.1 debian bookworm" + assert str(release) == expected diff --git a/release-automation/tests/test_stackbrew.py b/release-automation/tests/test_stackbrew.py new file mode 100644 index 000000000..2d065513e --- /dev/null +++ b/release-automation/tests/test_stackbrew.py @@ -0,0 +1,193 @@ +"""Tests for stackbrew library generation.""" + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release +from stackbrew_generator.stackbrew import StackbrewGenerator + + +class TestStackbrewGenerator: + """Tests for StackbrewGenerator.""" + + def setup_method(self): + """Set up test fixtures.""" + self.generator = StackbrewGenerator() + + def test_generate_tags_debian_ga_latest(self): + """Test tag generation for Debian GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1", # Full version + "8.2", # Mainline version (GA only) + "8", # Major version (latest only) + "8.2.1-bookworm", # Version with distro + "8.2-bookworm", # Mainline with distro + "8-bookworm", # Major with distro + "latest", # Latest tag (default distro only) + "bookworm" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_debian_ga_not_latest(self): + """Test tag generation for Debian GA version (not latest).""" + version = RedisVersion.parse("7.4.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + expected_tags = [ + "7.4.1", # Full version + "7.4", # Mainline version (GA only) + "7.4.1-bookworm", # Version with distro + "7.4-bookworm" # Mainline with distro + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_alpine_ga_latest(self): + """Test tag generation for Alpine GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1-alpine", # Version with distro type + "8.2.1-alpine3.22", # Version with full distro name + "8.2-alpine", # Mainline with distro type + "8.2-alpine3.22", # Mainline with full distro name + "8-alpine", # Major with distro type + "8-alpine3.22", # Major with full distro name + "alpine", # Bare distro type (latest only) + "alpine3.22" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_milestone_version(self): + """Test tag generation for milestone version.""" + version = RedisVersion.parse("8.2.1-m01") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution) + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + # Milestone versions should not get mainline version tags or major version tags + expected_tags = [ + "8.2.1-m01", # Full version only + "8.2.1-m01-bookworm", # Version with distro + ] + + assert set(tags) == set(expected_tags) + + + + def test_generate_stackbrew_library(self): + """Test complete stackbrew library generation.""" + releases = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ), + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22") + ), + Release( + commit="def456", + version=RedisVersion.parse("8.1.5"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + assert len(entries) == 3 + + # Check that the 8.2.1 versions are marked as latest + debian_8_2_1 = next(e for e in entries if e.version.patch == 1 and e.distribution.type == DistroType.DEBIAN) + assert "latest" in debian_8_2_1.tags + assert "8" in debian_8_2_1.tags + + # Check that 8.1.5 is not marked as latest + debian_8_1_5 = next(e for e in entries if e.version.minor == 1) + assert "latest" not in debian_8_1_5.tags + assert "8" not in debian_8_1_5.tags + + def test_format_stackbrew_output(self): + """Test stackbrew output formatting.""" + entries = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + ) + ] + + stackbrew_entries = self.generator.generate_stackbrew_library(entries) + output = self.generator.format_stackbrew_output(stackbrew_entries) + + assert isinstance(output, str) + assert len(output) > 0 + # Should contain comma-separated tags + assert "," in output + + def test_generate_stackbrew_library_with_head_milestone(self): + """Test stackbrew generation with milestone at head (matches bash test).""" + # This matches the bash test case: test_generate_stackbrew_library_with_head_milestone + releases = [ + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.21") + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + # Expected tags based on bash test + expected_tags = [ + ["8.2.2-m01-int1", "8.2.2-m01-int1-bookworm"], # milestone - no major/mainline tags + ["8.2.2-m01-int1-alpine", "8.2.2-m01-int1-alpine3.22"], # milestone - no major/mainline tags + ["8.2.1", "8.2", "8", "8.2.1-bookworm", "8.2-bookworm", "8-bookworm", "latest", "bookworm"], # GA - gets all tags + ["8.2.1-alpine", "8.2-alpine", "8-alpine", "8.2.1-alpine3.22", "8.2-alpine3.22", "8-alpine3.22", "alpine", "alpine3.22"], # GA - gets all tags + ["8.0.3", "8.0", "8.0.3-bookworm", "8.0-bookworm"], # different minor - no major tags + ["8.0.3-alpine", "8.0-alpine", "8.0.3-alpine3.21", "8.0-alpine3.21"] # different minor - no major tags + ] + + assert len(entries) == 6 + for i, entry in enumerate(entries): + assert set(entry.tags) == set(expected_tags[i]), f"Tags mismatch for entry {i}: {entry.tags} != {expected_tags[i]}" From c06c3452f123abf6c243c8cb5472b4dce583efba Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:44:08 +0300 Subject: [PATCH 094/188] Triggering workflow --- .github/workflows/release_build_and_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3cf591b98..6715153a9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,6 +13,9 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false + push: + branches: + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 672c2aeb0b3a88557a6b6bd8f81f7cff921dc98c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:46:33 +0300 Subject: [PATCH 095/188] Fix trigger --- .github/workflows/build_release_automation.yml | 3 +++ .github/workflows/release_build_and_test.yml | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index f20adf8eb..d07da428f 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,6 +12,9 @@ on: required: false default: true type: boolean + push: + branches: + - release/** env: REGISTRY: ghcr.io diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6715153a9..3cf591b98 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -13,9 +13,6 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From f5d5d0c23fdee29e5064b7e9264cedfa10e8f349 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:49:55 +0300 Subject: [PATCH 096/188] Docker for releasr-automation --- .github/workflows/build_release_automation.yml | 3 --- release-automation/docker/Dockerfile | 11 +++++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) create mode 100644 release-automation/docker/Dockerfile diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index d07da428f..f20adf8eb 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -12,9 +12,6 @@ on: required: false default: true type: boolean - push: - branches: - - release/** env: REGISTRY: ghcr.io diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile new file mode 100644 index 000000000..653200a2a --- /dev/null +++ b/release-automation/docker/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.11-slim-trixie + +RUN apt update && apt -y install git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +COPY . /release-automation +RUN pip install -e /release-automation + +ENTRYPOINT ["release-automation"] +CMD ["--help"] From 1501da1408156670f74b570cefeda343ee52f446 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:56:39 +0300 Subject: [PATCH 097/188] Test release-automation docker --- .github/workflows/release_build_and_test.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3cf591b98..a84a86843 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,26 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Test Release Automation Docker Image + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ + -e GITHUB_ACTOR="${{ github.actor }}" \ + -e GITHUB_REPOSITORY="${{ github.repository }}" \ + ghcr.io/${{ github.repository }}/release-automation:latest \ + generate 8 + - name: Validate Redis Release Archive uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: From 13648c511477bcb79e1b5a91f0992d40e4d03fb7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 17:59:38 +0300 Subject: [PATCH 098/188] Fix image url --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a84a86843..3de2cd658 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -41,7 +41,7 @@ jobs: -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ -e GITHUB_ACTOR="${{ github.actor }}" \ -e GITHUB_REPOSITORY="${{ github.repository }}" \ - ghcr.io/${{ github.repository }}/release-automation:latest \ + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 - name: Validate Redis Release Archive From 3c5896d366d28238190d9b012fa0ebc7b1b59828 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:07:07 +0300 Subject: [PATCH 099/188] Test remote names --- .github/workflows/release_build_and_test.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3de2cd658..b0c98b0ad 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,6 +35,16 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ + -e GITHUB_ACTOR="${{ github.actor }}" \ + -e GITHUB_REPOSITORY="${{ github.repository }}" \ + --entrypoint /bin/bash + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ + -c 'git remote -v' + docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ From 5b2eb4c66c27c1675bf3179482159d8e85246222 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:08:37 +0300 Subject: [PATCH 100/188] fix --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b0c98b0ad..ef1963a63 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -41,7 +41,7 @@ jobs: -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ -e GITHUB_ACTOR="${{ github.actor }}" \ -e GITHUB_REPOSITORY="${{ github.repository }}" \ - --entrypoint /bin/bash + --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ -c 'git remote -v' From ee0636fb40b346132b222bba728bd773d15b9525 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:10:24 +0300 Subject: [PATCH 101/188] fix add to safe directory --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index ef1963a63..48a324222 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'git remote -v' + -c 'git config --global --add safe.directory /workspace && git remote -v' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From 555d12472d34cdd011826746f0270922d1cacb2e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:14:13 +0300 Subject: [PATCH 102/188] Debug output --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 48a324222..d164c7338 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'git config --global --add safe.directory /workspace && git remote -v' + -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From de932b0d672a9b375003b8b2a6c9b5024f6c5e69 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:16:09 +0300 Subject: [PATCH 103/188] Continue debugging --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index d164c7338..2c299ed6f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -43,7 +43,7 @@ jobs: -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v' + -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From a7a9434398bbcc45b3a9eb8c6f13d64c8ba34f78 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:22:44 +0300 Subject: [PATCH 104/188] Remove env to test --- .github/workflows/release_build_and_test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 2c299ed6f..a2258a0fa 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -38,9 +38,6 @@ jobs: docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ - -e GITHUB_ACTOR="${{ github.actor }}" \ - -e GITHUB_REPOSITORY="${{ github.repository }}" \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' From 7b91991a5ab7611d2aa38528ec9eecdd7ec7cd3e Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:24:09 +0300 Subject: [PATCH 105/188] Removed safe dir --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index a2258a0fa..0c8af74c9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -40,7 +40,7 @@ jobs: -w /workspace \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git config --global --add safe.directory /workspace; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' + -c 'cat .git/config; set -x; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From ee9823822bafdd94146cde0c50762f0ddcb12f94 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:34:39 +0300 Subject: [PATCH 106/188] debug outpu --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 0c8af74c9..88ba0365f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -40,7 +40,7 @@ jobs: -w /workspace \ --entrypoint /bin/bash\ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; git remote -v; git ls-remote --refs --tags origin refs/tags/v8.*' + -c 'cat .git/config; set -x; whoami; ls -al;' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From 11fced5c5910a9d0ff292882430a3aaaf7d3c0cb Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:39:05 +0300 Subject: [PATCH 107/188] Debug output --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 88ba0365f..3412336b3 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -39,8 +39,9 @@ jobs: -v ${{ github.workspace }}:/workspace \ -w /workspace \ --entrypoint /bin/bash\ + --user $(id -u):0 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; whoami; ls -al;' + -c 'cat .git/config; set -x; id; whoami; ls -al;' docker run --rm \ -v ${{ github.workspace }}:/workspace \ From abada2975fdb04d00ef9167d1004772a563b74f1 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:51:38 +0300 Subject: [PATCH 108/188] Add global safe dir and run tests --- .../workflows/build_release_automation.yml | 60 ++++++++++++++----- release-automation/docker/Dockerfile | 3 +- 2 files changed, 46 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index f20adf8eb..1aa495683 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -18,7 +18,7 @@ env: IMAGE_NAME: ${{ github.repository }}/release-automation jobs: - build-and-push: + build-test-and-push: runs-on: ubuntu-latest permissions: contents: read @@ -28,13 +28,8 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Log in to Container Registry - if: ${{ github.event.inputs.push_to_ghcr == 'true' }} - uses: docker/login-action@v3 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - name: Extract metadata id: meta @@ -46,24 +41,57 @@ jobs: type=raw,value=latest,enable={{is_default_branch}} type=sha,prefix={{branch}}- - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Build and push Docker image + - name: Build Docker image (without pushing) uses: docker/build-push-action@v5 with: context: ./release-automation - file: ./release-automation/docker/Dockerfile - push: ${{ github.event.inputs.push_to_ghcr == 'true' }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + file: ./release-automation/docker/Dockerfile.prod + push: false + tags: test-image:latest + load: true cache-from: type=gha cache-to: type=gha,mode=max + - name: Test the built image + run: | + # Start container and install dev dependencies for testing + docker run --rm \ + -v ${{ github.workspace }}/release-automation:/workspace \ + -w /workspace \ + --entrypoint /bin/bash \ + test-image:latest \ + -c " + set -e + echo '=== Installing test dependencies ===' + pip install pytest pytest-cov + echo '=== Running tests ===' + pytest -v tests/ + " + + - name: Log in to Container Registry + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag and push image + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + run: | + # Tag the tested image with the proper tags + echo '${{ steps.meta.outputs.tags }}' | while read -r tag; do + docker tag test-image:latest "$tag" + docker push "$tag" + done + - name: Output image details run: | echo "## Docker Image Built Successfully! 🐳" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY + echo "āœ… **Tests passed**" >> $GITHUB_STEP_SUMMARY + echo "šŸ—ļø **Production image built** (without dev dependencies)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY echo "**Tags:**" >> $GITHUB_STEP_SUMMARY echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/' | sed 's/$/`/' >> $GITHUB_STEP_SUMMARY diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile index 653200a2a..2d1c06650 100644 --- a/release-automation/docker/Dockerfile +++ b/release-automation/docker/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.11-slim-trixie RUN apt update && apt -y install git && \ apt-get clean && \ - rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + && git config --global --add safe.directory '*' COPY . /release-automation RUN pip install -e /release-automation From e5cc7b32c0bc3c966d5d636392c9377f3aac53ee Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:54:13 +0300 Subject: [PATCH 109/188] Fix docker image --- .github/workflows/build_release_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 1aa495683..cb845ed8a 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -45,7 +45,7 @@ jobs: uses: docker/build-push-action@v5 with: context: ./release-automation - file: ./release-automation/docker/Dockerfile.prod + file: ./release-automation/docker/Dockerfile push: false tags: test-image:latest load: true From 50884e52e4ee5eaffa1f10559ff56b249ff5a861 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:57:37 +0300 Subject: [PATCH 110/188] Try to fix workspace --- .github/workflows/build_release_automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index cb845ed8a..6288b8b74 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -56,7 +56,7 @@ jobs: run: | # Start container and install dev dependencies for testing docker run --rm \ - -v ${{ github.workspace }}/release-automation:/workspace \ + -v ${{ github.workspace }}:/workspace \ -w /workspace \ --entrypoint /bin/bash \ test-image:latest \ From 35c8bc89aa71fa7023b908c144a71c9e8a802c9c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 18:59:01 +0300 Subject: [PATCH 111/188] cd to release-automation for testing --- .github/workflows/build_release_automation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 6288b8b74..3df169dd8 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -61,6 +61,7 @@ jobs: --entrypoint /bin/bash \ test-image:latest \ -c " + cd release-automation set -e echo '=== Installing test dependencies ===' pip install pytest pytest-cov From 7a430bd99d8a252a53768d19b13ab06fddf78317 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 20:34:38 +0300 Subject: [PATCH 112/188] Comment --- .github/workflows/build_release_automation.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml index 3df169dd8..a9c0ff005 100644 --- a/.github/workflows/build_release_automation.yml +++ b/.github/workflows/build_release_automation.yml @@ -52,6 +52,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max + # Integration tests do need access to git repository - name: Test the built image run: | # Start container and install dev dependencies for testing From f4f91bc6e108c264893cadd59edd837a81ebcc75 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 11 Sep 2025 21:12:19 +0300 Subject: [PATCH 113/188] Full stackbrew format support --- .../src/stackbrew_generator/cli.py | 2 +- .../src/stackbrew_generator/distribution.py | 9 +++-- .../src/stackbrew_generator/models.py | 18 ++++++++- .../src/stackbrew_generator/stackbrew.py | 7 +++- .../src/stackbrew_generator/version_filter.py | 32 ++++++++-------- release-automation/tests/test_integration.py | 3 +- release-automation/tests/test_models.py | 32 ++++++++-------- release-automation/tests/test_stackbrew.py | 38 ++++++++++++------- 8 files changed, 86 insertions(+), 55 deletions(-) diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py index cdc1721b4..953a29048 100644 --- a/release-automation/src/stackbrew_generator/cli.py +++ b/release-automation/src/stackbrew_generator/cli.py @@ -81,7 +81,7 @@ def generate( raise typer.Exit(1) # Fetch required refs - refs_to_fetch = [commit for _, commit in versions] + refs_to_fetch = [commit for _, commit, _ in versions] git_client.fetch_refs(refs_to_fetch) # Prepare releases list with distribution information diff --git a/release-automation/src/stackbrew_generator/distribution.py b/release-automation/src/stackbrew_generator/distribution.py index 187e8f521..5e2080e8e 100644 --- a/release-automation/src/stackbrew_generator/distribution.py +++ b/release-automation/src/stackbrew_generator/distribution.py @@ -79,11 +79,11 @@ def get_distribution_for_commit(self, commit: str, distro_type: str) -> Distribu f"Failed to get distribution for {distro_type} from {commit}: {e}" ) from e - def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> List[Release]: + def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Release]: """Prepare list of releases with distribution information. Args: - versions: List of (RedisVersion, commit) tuples + versions: List of (RedisVersion, commit, tag_ref) tuples Returns: List of Release objects with distribution information @@ -93,7 +93,7 @@ def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> Lis releases = [] distro_types = ["debian", "alpine"] - for version, commit in versions: + for version, commit, tag_ref in versions: console.print(f"[dim]Processing [bold yellow]{version}[/bold yellow] - {commit[:8]}[/dim]") for distro_type in distro_types: @@ -103,7 +103,8 @@ def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str]]) -> Lis release = Release( commit=commit, version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref=tag_ref ) releases.append(release) diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 599bbe9c9..37d2a1c89 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -154,6 +154,7 @@ class Release(BaseModel): commit: str = Field(..., description="Git commit hash") version: RedisVersion = Field(..., description="Redis version") distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") def __str__(self) -> str: """String representation of the release.""" @@ -171,7 +172,20 @@ class StackbrewEntry(BaseModel): commit: str = Field(..., description="Git commit hash") version: RedisVersion = Field(..., description="Redis version") distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") + + # Hard-coded architectures as specified + architectures: List[str] = Field( + default_factory=lambda: ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"], + description="Supported architectures" + ) def __str__(self) -> str: - """String representation as comma-separated tags.""" - return ", ".join(self.tags) + """String representation in stackbrew format.""" + lines = [] + lines.append(f"Tags: {', '.join(self.tags)}") + lines.append(f"Architectures: {', '.join(self.architectures)}") + lines.append(f"GitCommit: {self.commit}") + lines.append(f"GitFetch: {self.git_fetch_ref}") + lines.append(f"Directory: {self.distribution.type.value}") + return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index e40bd4a6a..5eb95213b 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -101,7 +101,8 @@ def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewE tags=tags, commit=release.commit, version=release.version, - distribution=release.distribution + distribution=release.distribution, + git_fetch_ref=release.git_fetch_ref ) entries.append(entry) @@ -125,7 +126,9 @@ def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: return "" lines = [] - for entry in entries: + for i, entry in enumerate(entries): + if i > 0: + lines.append("") # Add blank line between entries lines.append(str(entry)) return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index d928e0e83..23cb0ed3b 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -22,14 +22,14 @@ def __init__(self, git_client: GitClient): """ self.git_client = git_client - def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: """Get Redis versions from git tags. Args: major_version: Major version to filter for Returns: - List of (RedisVersion, commit) tuples sorted by version (newest first) + List of (RedisVersion, commit, tag_ref) tuples sorted by version (newest first) """ console.print(f"[blue]Getting Redis versions for major version {major_version}[/blue]") @@ -41,7 +41,7 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe for commit, tag_ref in tags: try: version = self.git_client.extract_version_from_tag(tag_ref, major_version) - versions.append((version, commit)) + versions.append((version, commit, tag_ref)) except Exception as e: console.print(f"[yellow]Warning: Skipping invalid tag {tag_ref}: {e}[/yellow]") continue @@ -52,11 +52,11 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") return versions - def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter out end-of-life versions. Args: - versions: List of (RedisVersion, commit) tuples + versions: List of (RedisVersion, commit, tag_ref) tuples Returns: Filtered list with EOL minor versions removed @@ -64,18 +64,18 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[ console.print("[blue]Filtering out EOL versions[/blue]") # Group versions by minor version - minor_versions: Dict[str, List[Tuple[RedisVersion, str]]] = {} - for version, commit in versions: + minor_versions: Dict[str, List[Tuple[RedisVersion, str, str]]] = {} + for version, commit, tag_ref in versions: minor_key = version.mainline_version if minor_key not in minor_versions: minor_versions[minor_key] = [] - minor_versions[minor_key].append((version, commit)) + minor_versions[minor_key].append((version, commit, tag_ref)) # Check each minor version for EOL marker filtered_versions = [] for minor_key, minor_group in minor_versions.items(): # Check if any version in this minor series is marked as EOL - has_eol = any(version.is_eol for version, _ in minor_group) + has_eol = any(version.is_eol for version, _, _ in minor_group) if has_eol: console.print(f"[yellow]Skipping minor version {minor_key}.* due to EOL[/yellow]") @@ -88,11 +88,11 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[ console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") return filtered_versions - def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> List[Tuple[RedisVersion, str]]: + def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter to keep only the latest patch version for each minor version and milestone status. Args: - versions: List of (RedisVersion, commit) tuples (should be sorted newest first) + versions: List of (RedisVersion, commit, tag_ref) tuples (should be sorted newest first) Returns: Filtered list with only the latest versions for each minor/milestone combination @@ -102,13 +102,13 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> Li seen_combinations = set() filtered_versions = [] - for version, commit in versions: + for version, commit, tag_ref in versions: # Create a key for minor version + milestone status combination_key = (version.mainline_version, version.is_milestone) if combination_key not in seen_combinations: seen_combinations.add(combination_key) - filtered_versions.append((version, commit)) + filtered_versions.append((version, commit, tag_ref)) milestone_str = "milestone" if version.is_milestone else "GA" console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") @@ -119,7 +119,7 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str]]) -> Li console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") return filtered_versions - def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str]]: + def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: """Get the actual Redis versions to process for a major version. This is the main entry point that combines all filtering steps: @@ -131,7 +131,7 @@ def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[Redi major_version: Major version to process Returns: - List of (RedisVersion, commit) tuples for processing + List of (RedisVersion, commit, tag_ref) tuples for processing """ console.print(f"[bold blue]Processing Redis {major_version}.x versions[/bold blue]") @@ -147,7 +147,7 @@ def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[Redi versions = self.filter_actual_versions(versions) console.print(f"[green]Final selection: {len(versions)} versions to process[/green]") - for version, commit in versions: + for version, commit, tag_ref in versions: console.print(f"[green] [bold yellow]{version}[/bold yellow] - {commit[:8]}[/green]") return versions diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py index 066392eaf..4d5ebd2a4 100644 --- a/release-automation/tests/test_integration.py +++ b/release-automation/tests/test_integration.py @@ -44,7 +44,8 @@ def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distributio Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ) ] mock_distribution_detector.prepare_releases_list.return_value = mock_releases diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py index bc55fa9ab..1ca03a1ab 100644 --- a/release-automation/tests/test_models.py +++ b/release-automation/tests/test_models.py @@ -53,7 +53,7 @@ def test_parse_invalid_version(self): """Test parsing invalid version strings.""" with pytest.raises(ValueError): RedisVersion.parse("invalid") - + with pytest.raises(ValueError): RedisVersion.parse("0.1.0") # Major version must be >= 1 @@ -61,7 +61,7 @@ def test_is_milestone(self): """Test milestone detection.""" ga_version = RedisVersion.parse("8.2.1") milestone_version = RedisVersion.parse("8.2.1-m01") - + assert ga_version.is_milestone is False assert milestone_version.is_milestone is True @@ -75,7 +75,7 @@ def test_string_representation(self): version1 = RedisVersion.parse("8.2.1") version2 = RedisVersion.parse("8.2.1-m01") version3 = RedisVersion.parse("8.2") - + assert str(version1) == "8.2.1" assert str(version2) == "8.2.1-m01" assert str(version3) == "8.2" @@ -86,14 +86,14 @@ def test_version_comparison(self): v2 = RedisVersion.parse("8.2.2") v3 = RedisVersion.parse("8.2.1-m01") v4 = RedisVersion.parse("8.3.0") - + # Test numeric comparison assert v1 < v2 assert v2 < v4 - + # Test milestone vs GA (GA comes after milestone) assert v3 < v1 - + # Test sorting versions = [v4, v1, v3, v2] sorted_versions = sorted(versions) @@ -125,7 +125,7 @@ def test_from_dockerfile_invalid(self): """Test parsing invalid Dockerfile lines.""" with pytest.raises(ValueError): Distribution.from_dockerfile_line("INVALID LINE") - + with pytest.raises(ValueError): Distribution.from_dockerfile_line("FROM unsupported:latest") @@ -133,7 +133,7 @@ def test_is_default(self): """Test default distribution detection.""" alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") debian = Distribution(type=DistroType.DEBIAN, name="bookworm") - + assert alpine.is_default is False assert debian.is_default is True @@ -141,7 +141,7 @@ def test_tag_names(self): """Test tag name generation.""" alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") debian = Distribution(type=DistroType.DEBIAN, name="bookworm") - + assert alpine.tag_names == ["alpine", "alpine3.22"] assert debian.tag_names == ["bookworm"] @@ -153,13 +153,14 @@ def test_release_creation(self): """Test creating a Release instance.""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - + release = Release( commit="abc123def456", version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" ) - + assert release.commit == "abc123def456" assert release.version == version assert release.distribution == distribution @@ -168,12 +169,13 @@ def test_release_string_representation(self): """Test Release string representation.""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - + release = Release( commit="abc123def456", version=version, - distribution=distribution + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" ) - + expected = "abc123de 8.2.1 debian bookworm" assert str(release) == expected diff --git a/release-automation/tests/test_stackbrew.py b/release-automation/tests/test_stackbrew.py index 2d065513e..e0864a41f 100644 --- a/release-automation/tests/test_stackbrew.py +++ b/release-automation/tests/test_stackbrew.py @@ -15,7 +15,7 @@ def test_generate_tags_debian_ga_latest(self): """Test tag generation for Debian GA version (latest).""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") tags = self.generator.generate_tags_for_release(release, is_latest=True) @@ -36,7 +36,7 @@ def test_generate_tags_debian_ga_not_latest(self): """Test tag generation for Debian GA version (not latest).""" version = RedisVersion.parse("7.4.1") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v7.4.1") tags = self.generator.generate_tags_for_release(release, is_latest=False) @@ -53,7 +53,7 @@ def test_generate_tags_alpine_ga_latest(self): """Test tag generation for Alpine GA version (latest).""" version = RedisVersion.parse("8.2.1") distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") tags = self.generator.generate_tags_for_release(release, is_latest=True) @@ -74,7 +74,7 @@ def test_generate_tags_milestone_version(self): """Test tag generation for milestone version.""" version = RedisVersion.parse("8.2.1-m01") distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") - release = Release(commit="abc123", version=version, distribution=distribution) + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1-m01") tags = self.generator.generate_tags_for_release(release, is_latest=False) @@ -94,17 +94,20 @@ def test_generate_stackbrew_library(self): Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22") + distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="def456", version=RedisVersion.parse("8.1.5"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.1.5" ) ] @@ -128,7 +131,8 @@ def test_format_stackbrew_output(self): Release( commit="abc123", version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm") + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ) ] @@ -147,32 +151,38 @@ def test_generate_stackbrew_library_with_head_milestone(self): Release( commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", version=RedisVersion.parse("8.2.2-m01-int1"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" ), Release( commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", version=RedisVersion.parse("8.2.2-m01-int1"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" ), Release( commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", version=RedisVersion.parse("8.2.1"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", version=RedisVersion.parse("8.2.1"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.22") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.1" ), Release( commit="101262a8cf05b98137d88bc17e77db90c24cc783", version=RedisVersion.parse("8.0.3"), - distribution=Distribution.from_dockerfile_line("FROM debian:bookworm") + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.0.3" ), Release( commit="101262a8cf05b98137d88bc17e77db90c24cc783", version=RedisVersion.parse("8.0.3"), - distribution=Distribution.from_dockerfile_line("FROM alpine:3.21") + distribution=Distribution.from_dockerfile_line("FROM alpine:3.21"), + git_fetch_ref="refs/tags/v8.0.3" ) ] From c140a5bee7283a5b41079b0c8bafe9856802ed0a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 09:57:19 +0300 Subject: [PATCH 114/188] Try interactive mode --- .github/workflows/release_build_and_test.yml | 13 +------------ release-automation/docker/Dockerfile | 1 + 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 3412336b3..c982d9449 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,20 +35,9 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm \ + docker run --rm -it \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ - --entrypoint /bin/bash\ - --user $(id -u):0 \ - $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - -c 'cat .git/config; set -x; id; whoami; ls -al;' - - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - -w /workspace \ - -e GITHUB_TOKEN="${GITHUB_TOKEN}" \ - -e GITHUB_ACTOR="${{ github.actor }}" \ - -e GITHUB_REPOSITORY="${{ github.repository }}" \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile index 2d1c06650..2a6f061c2 100644 --- a/release-automation/docker/Dockerfile +++ b/release-automation/docker/Dockerfile @@ -3,6 +3,7 @@ FROM python:3.11-slim-trixie RUN apt update && apt -y install git && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + # avoid dubious permissions problem in github CI && git config --global --add safe.directory '*' COPY . /release-automation From 45aa7b15d32a568e984c0aa220a833f12672486d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 10:02:00 +0300 Subject: [PATCH 115/188] Use only -i --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c982d9449..444b76788 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,7 +35,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm -it \ + docker run --rm -i \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ From 688c2c33a135ef80d34ddc54a2ccaef7ead14415 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 10:05:45 +0300 Subject: [PATCH 116/188] Try force color --- .github/workflows/release_build_and_test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 444b76788..e8711056c 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -35,9 +35,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - docker run --rm -i \ + docker run --rm \ -v ${{ github.workspace }}:/workspace \ -w /workspace \ + -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ generate 8 From 1b9bf327e42c0de082db04adc78080f052f0caa4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 14:10:11 +0300 Subject: [PATCH 117/188] Readme --- release-automation/README.md | 59 +++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/release-automation/README.md b/release-automation/README.md index 70aec8fce..e37e7189a 100644 --- a/release-automation/README.md +++ b/release-automation/README.md @@ -1,4 +1,55 @@ -# Release automation helper +# Docker release process and release automation description + +This readme covers relase process for versions 8 and above. + +In version 8 the docker-library structure has changed. Static Dockerfiles are used instead of templating. Versions live in a different mainline branches and are marked with tags. + +The docker release process goal is to create a PR in official-docker library for library/redis file. + +library/redis stackbrew file should reflect the tags in redis/docker-library-redis repository. + +## Branches and tags + +Mainline branches are named `release/Major.Minor` (e.g. `release/8.2`) + +Each version release is tagged with `vMajor.Minor.Patch` (e.g. `v8.2.1`) + +Milestone releases are tagged with `vMajor.Minor.Patch-Milestone` (e.g. `v8.2.1-m01`). Any suffix after patch version is considered a milestone. + +Tags without suffix are considered GA (General Availability) releases (e.g. `v8.2.1`). + +Internal releases are milestone releases containing `-int` in their name (e.g. `v8.2.1-m01-int1` or `8.4.0-int3`). They are not released to the public. + +Milestone releases never get latest or any other default tags, like `8`, `8.2`, `8.2.1`, `latest`, `bookworm`, etc. + +For each mainline only one GA release and optionally one milestone release should be published in official-library. The most latest versions. + +End of life versions are marked with `-eol` suffix (e.g. `v8.0.3-eol`). When there is a at least one minor version tagged with eol all versions in this minor series are considered EOL and are not included in the release file. + +## Creating a release manually + +This process is automated using github workflows. However, it's useful to understand the manual process. + +Determine a mainline branch, e.g `release/8.2` for version `8.2.2`. + +Optionally create a release branch from the mainline branch, e.g. `8.2.2`. + +Modify dockerfiles. + +Test dockerfiles. + +If release branch was created, merge it back to mainline branch. + +Tag commit with `vMajor.Minor.Patch` (e.g. `v8.2.1`) in the mainline branch. + +Push your changes to redis/docker-library-redis repository. + +Create a PR to official-library refering the tag and commit you created. + + +# Release automation tool + +Release automation tool is used to generate library/redis file for official-library. It uses origin repository as a source of truth and follows the process described above. ## Installation @@ -15,3 +66,9 @@ pip install -e . cd release-automation pip install -e ".[dev]" ``` + +## Usage + +```bash +release-automation --help +``` From 77a5c189520dc5a4e7b680bf412dadeace0a35f4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 15:37:19 +0300 Subject: [PATCH 118/188] Refactoring the code --- .../src/stackbrew_generator/cli.py | 207 +++++++++++++----- .../src/stackbrew_generator/models.py | 15 +- .../src/stackbrew_generator/stackbrew.py | 160 ++++++++++++++ release-automation/tests/test_integration.py | 52 +---- release-automation/tests/test_models.py | 60 ++++- 5 files changed, 391 insertions(+), 103 deletions(-) diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py index 953a29048..f572bee7a 100644 --- a/release-automation/src/stackbrew_generator/cli.py +++ b/release-automation/src/stackbrew_generator/cli.py @@ -1,6 +1,7 @@ """CLI interface for stackbrew library generator.""" import typer +from pathlib import Path from rich.console import Console from rich.traceback import install @@ -8,7 +9,7 @@ from .exceptions import StackbrewGeneratorError from .git_operations import GitClient from .logging_config import setup_logging -from .stackbrew import StackbrewGenerator +from .stackbrew import StackbrewGenerator, StackbrewUpdater from .version_filter import VersionFilter # Install rich traceback handler @@ -24,8 +25,63 @@ console = Console(stderr=True) -@app.command() -def generate( +def _generate_stackbrew_content(major_version: int, remote: str, verbose: bool) -> str: + """Generate stackbrew content for a major version. + + This helper function contains the common logic for generating stackbrew content + that is used by both generate-stackbrew-content and update-stackbrew-file commands. + + Args: + major_version: Redis major version to process + remote: Git remote to use + verbose: Whether to enable verbose output + + Returns: + Generated stackbrew content as string + + Raises: + typer.Exit: If no versions found or other errors occur + """ + # Initialize components + git_client = GitClient(remote=remote) + version_filter = VersionFilter(git_client) + distribution_detector = DistributionDetector(git_client) + stackbrew_generator = StackbrewGenerator() + + # Get actual Redis versions to process + versions = version_filter.get_actual_major_redis_versions(major_version) + + if not versions: + console.print(f"[red]No versions found for Redis {major_version}.x[/red]") + raise typer.Exit(1) + + # Fetch required refs + refs_to_fetch = [commit for _, commit, _ in versions] + git_client.fetch_refs(refs_to_fetch) + + # Prepare releases list with distribution information + releases = distribution_detector.prepare_releases_list(versions) + + if not releases: + console.print("[red]No releases prepared[/red]") + raise typer.Exit(1) + + # Generate stackbrew library content + entries = stackbrew_generator.generate_stackbrew_library(releases) + output = stackbrew_generator.format_stackbrew_output(entries) + + if not output: + console.print("[yellow]No stackbrew content generated[/yellow]") + raise typer.Exit(1) + + if verbose: + console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + + return output + + +@app.command(name="generate-stackbrew-content") +def generate_stackbrew_content( major_version: int = typer.Argument( ..., help="Redis major version to process (e.g., 8 for Redis 8.x)" @@ -41,11 +97,6 @@ def generate( "-v", help="Enable verbose output" ), - dry_run: bool = typer.Option( - False, - "--dry-run", - help="Show generated content without outputting to stdout" - ), ) -> None: """Generate stackbrew library content for Redis Docker images. @@ -63,52 +114,117 @@ def generate( console.print(f"[bold blue]Stackbrew Library Generator[/bold blue]") console.print(f"Major version: {major_version}") console.print(f"Remote: {remote}") - if dry_run: - console.print("[yellow]DRY RUN MODE - Generated content will be shown but not output to stdout[/yellow]") try: - # Initialize components - git_client = GitClient(remote=remote) - version_filter = VersionFilter(git_client) - distribution_detector = DistributionDetector(git_client) - stackbrew_generator = StackbrewGenerator() + # Generate stackbrew content using the helper function + output = _generate_stackbrew_content(major_version, remote, verbose) - # Get actual Redis versions to process - versions = version_filter.get_actual_major_redis_versions(major_version) + # Output the stackbrew library content + print(output) - if not versions: - console.print(f"[red]No versions found for Redis {major_version}.x[/red]") - raise typer.Exit(1) + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) - # Fetch required refs - refs_to_fetch = [commit for _, commit, _ in versions] - git_client.fetch_refs(refs_to_fetch) - # Prepare releases list with distribution information - releases = distribution_detector.prepare_releases_list(versions) +@app.command() +def version() -> None: + """Show version information.""" + from . import __version__ + console.print(f"stackbrew-library-generator {__version__}") - if not releases: - console.print("[red]No releases prepared[/red]") - raise typer.Exit(1) - # Generate stackbrew library content - entries = stackbrew_generator.generate_stackbrew_library(releases) - output = stackbrew_generator.format_stackbrew_output(entries) +@app.command() +def update_stackbrew_file( + major_version: int = typer.Argument( + ..., + help="Redis major version to update (e.g., 8 for Redis 8.x)" + ), + input_file: Path = typer.Option( + ..., + "--input", + "-i", + help="Path to the stackbrew library file to update" + ), + output_file: Path = typer.Option( + None, + "--output", + "-o", + help="Output file path (defaults to stdout)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), +) -> None: + """Update stackbrew library file by replacing entries for a specific major version. - if dry_run: - console.print(f"[yellow]DRY RUN: Would generate stackbrew library with {len(entries)} entries[/yellow]") - if verbose: - console.print("[yellow]Generated content:[/yellow]") - console.print(output) + This command: + 1. Reads the existing stackbrew library file + 2. Generates new stackbrew content for the specified major version + 3. Replaces all entries related to that major version in their original position + 4. Preserves the header and entries for other major versions + 5. Outputs to stdout by default, or to specified output file + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if not input_file.exists(): + console.print(f"[red]Input file does not exist: {input_file}[/red]") + raise typer.Exit(1) + + if verbose: + console.print(f"[bold blue]Stackbrew Library File Updater[/bold blue]") + console.print(f"Input file: {input_file}") + if output_file: + console.print(f"Output file: {output_file}") else: - if output: - # Output the stackbrew library content - print(output) + console.print("Output: stdout") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") - if verbose: - console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + try: + # Generate new stackbrew content for the major version using helper function + new_content = _generate_stackbrew_content(major_version, remote, verbose) + + # Update the stackbrew file content + updater = StackbrewUpdater() + updated_content = updater.update_stackbrew_content( + input_file, major_version, new_content, verbose + ) + + # Write the updated content + if output_file: + output_file.write_text(updated_content, encoding='utf-8') + if verbose: + console.print(f"[green]Successfully updated {output_file} for Redis {major_version}.x[/green]") else: - console.print("[yellow]No stackbrew content generated[/yellow]") + console.print(f"[green]Updated {output_file}[/green]") + else: + # Output to stdout + print(updated_content) + if verbose: + console.print(f"[green]Generated updated stackbrew content for Redis {major_version}.x[/green]") except StackbrewGeneratorError as e: if verbose and hasattr(e, 'get_detailed_message'): @@ -128,12 +244,5 @@ def generate( raise typer.Exit(1) -@app.command() -def version() -> None: - """Show version information.""" - from . import __version__ - console.print(f"stackbrew-library-generator {__version__}") - - if __name__ == "__main__": app() diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 37d2a1c89..6d7c88991 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -174,11 +174,16 @@ class StackbrewEntry(BaseModel): distribution: Distribution = Field(..., description="Linux distribution") git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") - # Hard-coded architectures as specified - architectures: List[str] = Field( - default_factory=lambda: ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"], - description="Supported architectures" - ) + @property + def architectures(self) -> List[str]: + """Get supported architectures based on distribution type.""" + if self.distribution.type == DistroType.DEBIAN: + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + elif self.distribution.type == DistroType.ALPINE: + return ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + else: + # Fallback to debian architectures for unknown distributions + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] def __str__(self) -> str: """String representation in stackbrew format.""" diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index 5eb95213b..d872b65e5 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -1,5 +1,7 @@ """Stackbrew library generation.""" +import re +from pathlib import Path from typing import List from rich.console import Console @@ -132,3 +134,161 @@ def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: lines.append(str(entry)) return "\n".join(lines) + + +class StackbrewUpdater: + """Updates stackbrew library files by replacing entries for specific major versions.""" + + def __init__(self): + """Initialize the updater.""" + pass + + def update_stackbrew_content(self, input_file: Path, major_version: int, new_content: str, verbose: bool = False) -> str: + """Update stackbrew file content by replacing entries for a specific major version. + + Args: + input_file: Path to the input stackbrew file + major_version: Major version to replace entries for + new_content: New stackbrew content to insert + verbose: Whether to print verbose output + + Returns: + Updated stackbrew file content + """ + content = input_file.read_text(encoding='utf-8') + lines = content.split('\n') + + # Find header (everything before the first Tags: line) + header_lines = [] + content_start_idx = 0 + + for i, line in enumerate(lines): + if line.startswith('Tags:'): + content_start_idx = i + break + header_lines.append(line) + + if content_start_idx == 0 and not any(line.startswith('Tags:') for line in lines): + # No existing entries, just append new content + if verbose: + console.print("[dim]No existing entries found, appending new content[/dim]") + return content.rstrip() + '\n\n' + new_content + + # Parse entries and find where target major version entries start and end + entries = self._parse_stackbrew_entries(lines[content_start_idx:]) + target_entries = [] + other_entries_before = [] + other_entries_after = [] + target_start_found = False + target_end_found = False + removed_count = 0 + + for entry in entries: + if self._entry_belongs_to_major_version(entry, major_version): + target_entries.append(entry) + removed_count += 1 + if not target_start_found: + target_start_found = True + elif not target_start_found: + # Entries before target major version + other_entries_before.append(entry) + else: + # Entries after target major version + other_entries_after.append(entry) + if not target_end_found: + target_end_found = True + + if verbose: + if removed_count > 0: + console.print(f"[dim]Removed {removed_count} existing entries for Redis {major_version}.x[/dim]") + else: + console.print(f"[dim]No existing entries found for Redis {major_version}.x, placing at end[/dim]") + + # Reconstruct the file + result_lines = header_lines[:] + + # Add entries before target major version + for entry in other_entries_before: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + # Add new content for the target major version + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(new_content.split('\n')) + + # Add entries after target major version + for entry in other_entries_after: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + return '\n'.join(result_lines) + + def _parse_stackbrew_entries(self, lines: List[str]) -> List[List[str]]: + """Parse stackbrew entries from lines, returning list of entry line groups. + + Args: + lines: Lines to parse + + Returns: + List of entry line groups + """ + entries = [] + current_entry = [] + + for line in lines: + line = line.rstrip() + + if line.startswith('Tags:') and current_entry: + # Start of new entry, save the previous one + entries.append(current_entry) + current_entry = [line] + elif line.startswith('Tags:'): + # First entry + current_entry = [line] + elif current_entry and (line.startswith(('Architectures:', 'GitCommit:', 'GitFetch:', 'Directory:')) or line.strip() == ''): + # Part of current entry + current_entry.append(line) + elif not line.strip() and not current_entry: + # Empty line before any entry starts, skip + continue + elif not line.strip() and current_entry: + # Empty line after entry content - end of entry + if current_entry: + entries.append(current_entry) + current_entry = [] + + # Don't forget the last entry + if current_entry: + entries.append(current_entry) + + return entries + + def _entry_belongs_to_major_version(self, entry_lines: List[str], major_version: int) -> bool: + """Check if a stackbrew entry belongs to the specified major version. + + Args: + entry_lines: Lines of the stackbrew entry + major_version: Major version to check for + + Returns: + True if the entry belongs to the major version + """ + for line in entry_lines: + if line.startswith('Tags:'): + tags_line = line[5:].strip() # Remove 'Tags:' prefix + tags = [tag.strip() for tag in tags_line.split(',')] + + # Check if any tag indicates this major version + for tag in tags: + # Look for patterns like "8", "8.2", "8.2.1", "8-alpine", etc. + if re.match(rf'^{major_version}(?:\.|$|-)', tag): + return True + # Also check for "latest" tag which typically belongs to the highest major version + # But we'll be conservative and not assume latest belongs to our major version + # unless we have other evidence + break + + return False diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py index 4d5ebd2a4..2a11548a1 100644 --- a/release-automation/tests/test_integration.py +++ b/release-automation/tests/test_integration.py @@ -15,49 +15,6 @@ def setup_method(self): """Set up test fixtures.""" self.runner = CliRunner() - @patch('stackbrew_generator.distribution.DistributionDetector') - @patch('stackbrew_generator.git_operations.GitClient') - def test_complete_workflow_dry_run(self, mock_git_client_class, mock_distribution_detector_class): - """Test complete workflow in dry run mode.""" - # Mock git client - mock_git_client = Mock() - mock_git_client_class.return_value = mock_git_client - - # Mock distribution detector - mock_distribution_detector = Mock() - mock_distribution_detector_class.return_value = mock_distribution_detector - - # Mock git operations - mock_git_client.list_remote_tags.return_value = [ - ("abc123", "refs/tags/v8.2.1"), - ("def456", "refs/tags/v8.2.0"), - ] - - mock_git_client.extract_version_from_tag.side_effect = [ - RedisVersion.parse("8.2.1"), - RedisVersion.parse("8.2.0"), - ] - - # Mock releases - from stackbrew_generator.models import Release, Distribution, DistroType - mock_releases = [ - Release( - commit="abc123", - version=RedisVersion.parse("8.2.1"), - distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), - git_fetch_ref="refs/tags/v8.2.1" - ) - ] - mock_distribution_detector.prepare_releases_list.return_value = mock_releases - - # Run command in dry run mode - result = self.runner.invoke(app, ["generate", "8", "--dry-run", "--verbose"]) - - # Check that it completed successfully - assert result.exit_code == 0 - assert "DRY RUN: Would generate stackbrew library" in result.stderr - assert "Generated content:" in result.stderr - def test_version_command(self): """Test version command.""" result = self.runner.invoke(app, ["version"]) @@ -66,7 +23,7 @@ def test_version_command(self): def test_invalid_major_version(self): """Test handling of invalid major version.""" - result = self.runner.invoke(app, ["generate", "0"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "0"]) assert result.exit_code != 0 @patch('stackbrew_generator.git_operations.GitClient') @@ -77,7 +34,7 @@ def test_no_tags_found(self, mock_git_client_class): mock_git_client_class.return_value = mock_git_client mock_git_client.list_remote_tags.return_value = [] - result = self.runner.invoke(app, ["generate", "99"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "99"]) assert result.exit_code == 1 assert "No tags found" in result.stderr @@ -87,15 +44,14 @@ def test_no_versions_found(self, mock_get_versions): # Mock git client to return no tags mock_get_versions.return_value = [] - result = self.runner.invoke(app, ["generate", "8"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "8"]) #assert result.exit_code == 1 assert "No versions found" in result.stderr def test_help_output(self): """Test help output.""" - result = self.runner.invoke(app, ["generate", "--help"]) + result = self.runner.invoke(app, ["generate-stackbrew-content", "--help"]) assert result.exit_code == 0 assert "Generate stackbrew library content" in result.stdout assert "--remote" in result.stdout assert "--verbose" in result.stdout - assert "--dry-run" in result.stdout diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py index 1ca03a1ab..0feef1f94 100644 --- a/release-automation/tests/test_models.py +++ b/release-automation/tests/test_models.py @@ -2,7 +2,7 @@ import pytest -from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release, StackbrewEntry class TestRedisVersion: @@ -179,3 +179,61 @@ def test_release_string_representation(self): expected = "abc123de 8.2.1 debian bookworm" assert str(release) == expected + + +class TestStackbrewEntry: + """Tests for StackbrewEntry model.""" + + def test_debian_architectures(self): + """Test that Debian distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + entry = StackbrewEntry( + tags=["8.2.1", "latest"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + assert entry.architectures == expected_architectures + + def test_alpine_architectures(self): + """Test that Alpine distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + assert entry.architectures == expected_architectures + + def test_stackbrew_entry_string_format(self): + """Test that StackbrewEntry formats correctly with architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + output = str(entry) + + # Check that it contains the expected Alpine architectures + assert "amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x" in output + assert "Tags: 8.2.1-alpine, alpine" in output + assert "GitCommit: abc123def456" in output + assert "GitFetch: refs/tags/v8.2.1" in output + assert "Directory: alpine" in output From 316f0c4783b6ca89a0e225d11bcd1566424a06bf Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 17:01:20 +0300 Subject: [PATCH 119/188] Temporary disable the build, test tag creation --- .github/workflows/release_build_and_test.yml | 46 ++++++++++++-------- 1 file changed, 29 insertions(+), 17 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index e8711056c..0013bc9b0 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -31,17 +31,6 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Test Release Automation Docker Image - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - docker run --rm \ - -v ${{ github.workspace }}:/workspace \ - -w /workspace \ - -e FORCE_COLOR=1 \ - $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - generate 8 - - name: Validate Redis Release Archive uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: @@ -63,12 +52,16 @@ jobs: release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: + runs-on: ["ubuntu-latest"] + steps: + - shell: bash + run: | + echo Hi needs: prepare-release - uses: ./.github/workflows/pre-merge.yml - secrets: inherit - with: - release_tag: ${{ github.event.inputs.release_tag }} - + #uses: ./.github/workflows/pre-merge.yml + #secrets: inherit + #with: + # release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] @@ -87,8 +80,27 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Merge back to release branch + id: merge-back uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} - gh_token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create version tag + uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + with: + tag: v${{ github.event.inputs.release_tag }} + ref: ${{ steps.merge-back.outputs.merge_commit_sha }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + + # - name: Test Release Automation Docker Image + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # run: | + # docker run --rm \ + # -v ${{ github.workspace }}:/workspace \ + # -w /workspace \ + # -e FORCE_COLOR=1 \ + # $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ + # generate 8 \ No newline at end of file From 4c0e47005d3a64360e660c53f7471b6374c8f8dd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 17:42:59 +0300 Subject: [PATCH 120/188] Create tag only if merge was done --- .github/workflows/release_build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 0013bc9b0..c029ae4f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -88,6 +88,7 @@ jobs: gh_token: ${{ secrets.GITHUB_TOKEN }} - name: Create version tag + if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main with: tag: v${{ github.event.inputs.release_tag }} From 9faae5f7b6cafedaa99f51f899f36f10d6b4cf58 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 22:23:47 +0300 Subject: [PATCH 121/188] Debug output --- .github/workflows/release_build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c029ae4f6..caf9e94b9 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,6 +87,14 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} + - name: Debug + shell: bash + run: | + echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" + echo "Merge commit SHA length: ${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c + echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : + echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} + - name: Create version tag if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main From 0631debc12aec9f62e7a9c3edfecd2e20b39c147 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:02:38 +0300 Subject: [PATCH 122/188] lilbit dbg --- .github/workflows/release_build_and_test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index caf9e94b9..408471261 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -91,8 +91,8 @@ jobs: shell: bash run: | echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: ${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c - echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : + echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}"" | wc -c) + echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} - name: Create version tag From c81711689d3d2ba04ea467ad62aeb01303e7f4ad Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:04:46 +0300 Subject: [PATCH 123/188] fix dblquoute --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 408471261..5c33be479 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -91,7 +91,7 @@ jobs: shell: bash run: | echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}"" | wc -c) + echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c) echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} From 9d6435dca10190e7ec70ca9705e777eca3e472be Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 12 Sep 2025 23:09:51 +0300 Subject: [PATCH 124/188] Remove debug --- .github/workflows/release_build_and_test.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 5c33be479..c029ae4f6 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,14 +87,6 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - name: Debug - shell: bash - run: | - echo "Merge commit SHA: ${{ steps.merge-back.outputs.merge_commit_sha }}" - echo "Merge commit SHA length: $(echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | wc -c) - echo -n "${{ steps.merge-back.outputs.merge_commit_sha }}" | hexdump || : - echo if:${{ steps.merge-back.outputs.merge_commit_sha != '' }} - - name: Create version tag if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main From c626c11407daf99c9bb64ce97de6424e163d4f58 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 12:33:07 +0300 Subject: [PATCH 125/188] Release handle --- .github/workflows/release_build_and_test.yml | 43 +++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index c029ae4f6..db0500579 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -87,13 +87,44 @@ jobs: to_branch: ${{ steps.ensure-branch.outputs.release_branch }} gh_token: ${{ secrets.GITHUB_TOKEN }} - - name: Create version tag - if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} - uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + - name: Create release handle JSON + shell: bash + run: | + if [ -n "${{ steps.merge-back.outputs.merge_commit_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.merge_commit_sha }}" + elif [ -n "${{ steps.merge-back.outputs.target_before_merge_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.target_before_merge_sha }}" + else + echo "Error: No commit SHA found, both merge_commit_sha and target_before_merge_sha are empty" >&2 + exit 1 + fi + + cat > release_handle.json << EOF + { + "release_commit_sha": "$RELEASE_COMMIT_SHA", + "release_version": "${{ github.event.inputs.release_tag }}", + "release_version_branch": "${{ steps.ensure-branch.outputs.release_version_branch }}", + "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}" + } + EOF + + echo "Created release_handle.json:" + cat release_handle.json + + - name: Upload release handle artifact + uses: actions/upload-artifact@v4 with: - tag: v${{ github.event.inputs.release_tag }} - ref: ${{ steps.merge-back.outputs.merge_commit_sha }} - gh_token: ${{ secrets.GITHUB_TOKEN }} + name: release-handle + path: release_handle.json + retention-days: 30 + + # - name: Create version tag + # if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} + # uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + # with: + # tag: v${{ github.event.inputs.release_tag }} + # ref: ${{ steps.merge-back.outputs.merge_commit_sha }} + # gh_token: ${{ secrets.GITHUB_TOKEN }} # - name: Test Release Automation Docker Image # env: From 256448ccae3d1e698cc27aa001ddc3111e0e47f6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 14:01:23 +0300 Subject: [PATCH 126/188] Rename release_handle --- .github/workflows/release_build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index db0500579..75d8e4ccd 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -114,7 +114,7 @@ jobs: - name: Upload release handle artifact uses: actions/upload-artifact@v4 with: - name: release-handle + name: release_handle path: release_handle.json retention-days: 30 From 8ccb38d912c93ac4451131a03f98a113c7ed1f12 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:16:17 +0300 Subject: [PATCH 127/188] Release publish first run --- .github/workflows/release_publish.yml | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 .github/workflows/release_publish.yml diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml new file mode 100644 index 000000000..4b902309e --- /dev/null +++ b/.github/workflows/release_publish.yml @@ -0,0 +1,69 @@ +# This workflow publishes a release by creating a version tag. +# It is intended to be run with workflow_dispatch event by the automation. + +on: + workflow_dispatch: + inputs: + release_handle: + description: 'Release handle JSON string containing release information' + required: true + type: string + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false + push: + branches: + - release/8* + +# UUID is used to help automation to identify workflow run in the list of workflow runs. +run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" + +jobs: + publish-release: + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Parse release handle and validate + id: parse-release + shell: bash + run: | + # Parse the JSON input + RELEASE_HANDLE='${{ github.event.inputs.release_handle }}' + echo "Parsing release handle JSON:" + echo "$RELEASE_HANDLE" | jq . + + # Extract release_commit_sha + RELEASE_COMMIT_SHA=$(echo "$RELEASE_HANDLE" | jq -r '.release_commit_sha // empty') + + # Validate that release_commit_sha exists and is not empty + if [ -z "$RELEASE_COMMIT_SHA" ] || [ "$RELEASE_COMMIT_SHA" = "null" ]; then + echo "Error: release_commit_sha is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + # Extract release_version for tag creation + RELEASE_VERSION=$(echo "$RELEASE_HANDLE" | jq -r '.release_version // empty') + + if [ -z "$RELEASE_VERSION" ] || [ "$RELEASE_VERSION" = "null" ]; then + echo "Error: release_version is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + echo "Successfully parsed release handle:" + echo " release_commit_sha: $RELEASE_COMMIT_SHA" + echo " release_version: $RELEASE_VERSION" + + # Set outputs for next steps + echo "release_commit_sha=$RELEASE_COMMIT_SHA" >> $GITHUB_OUTPUT + echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT + + - name: Create version tag + uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + with: + tag: v${{ steps.parse-release.outputs.release_version }} + ref: ${{ steps.parse-release.outputs.release_commit_sha }} + gh_token: ${{ secrets.GITHUB_TOKEN }} From 9ab8c8eee58b59c05b7cf40ed0ec75b4b6738ecb Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:18:07 +0300 Subject: [PATCH 128/188] Fix push condition --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 4b902309e..70dd798a1 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,7 +13,7 @@ on: required: false push: branches: - - release/8* + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 2613793b942b2b8c0c681b40bacbfb68e1aedbc9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:21:38 +0300 Subject: [PATCH 129/188] Fix yaml for release_publish workflow --- .github/workflows/release_publish.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 70dd798a1..dc4d91047 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -11,9 +11,9 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** + push: + branches: + - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 3e6e240fd56fd958656794a87458bccd3c8b67e6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 15:22:40 +0300 Subject: [PATCH 130/188] Remove push run --- .github/workflows/release_publish.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index dc4d91047..d0f0f111c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -11,9 +11,6 @@ on: workflow_uuid: description: 'Optional UUID to identify this workflow run' required: false - push: - branches: - - release/** # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From eb29459d4a31b7656d9c89b640c9a17a2cd46522 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 20:54:23 +0300 Subject: [PATCH 131/188] Creating a PR: first attempt --- .github/workflows/release_publish.yml | 44 +++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index d0f0f111c..cc7f03c98 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -64,3 +64,47 @@ jobs: tag: v${{ steps.parse-release.outputs.release_version }} ref: ${{ steps.parse-release.outputs.release_commit_sha }} gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout official-images repo + uses: actions/checkout@v4 + with: + path: official-images + repository: Sh-Peter/official-library + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Generate stackbrew library content + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Extract major version from release version (e.g., "8.2.1" -> "8") + MAJOR_VERSION=$(echo "${{ steps.parse-release.outputs.release_version }}" | cut -d. -f1) + echo "Major version: $MAJOR_VERSION" + + # Generate updated stackbrew content using the release automation Docker image + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e FORCE_COLOR=1 \ + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ + update-stackbrew-file --input official-images/library/redis --output official-images/library/redis + + - name: Create pull request to official-images + id: create-pr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GITHUB_TOKEN }} + push-to-fork: Peter-Sh/official-images + path: official-images + branch: redis-${{ steps.parse-release.outputs.release_version }} + commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + body: | + Automated update for Redis ${{ steps.parse-release.outputs.release_version }} + + Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} + Release tag: v${{ steps.parse-release.outputs.release_version }} + + - name: PR creation results + run: | + echo "Pull Request Number: ${{ steps.create-pr.outputs.pull-request-number }}" + echo "Pull Request URL: ${{ steps.create-pr.outputs.pull-request-url }}" From 2291feeb2dc3d4d1c48a0c1ae677a75b97b9b6b6 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:11:10 +0300 Subject: [PATCH 132/188] Fix official images url --- .github/workflows/release_publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index cc7f03c98..82e20de30 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -69,8 +69,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Sh-Peter/official-library - token: ${{ secrets.GITHUB_TOKEN }} + repository: Sh-Peter/official-images - name: Generate stackbrew library content env: From fd6c12f317f34d9be4388251a94382c7aa650259 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:18:56 +0300 Subject: [PATCH 133/188] Fix update-stackbrew-file args --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 82e20de30..3cabb1f11 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -85,7 +85,7 @@ jobs: -w /workspace \ -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ - update-stackbrew-file --input official-images/library/redis --output official-images/library/redis + update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis - name: Create pull request to official-images id: create-pr From 75794726da9a23c51977aa7abb5110527f5c8704 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:36:10 +0300 Subject: [PATCH 134/188] Swap remotes --- .github/workflows/release_publish.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 3cabb1f11..b6a4dcdb4 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Sh-Peter/official-images + repository: Peter-Sh/official-images - name: Generate stackbrew library content env: @@ -86,13 +86,15 @@ jobs: -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis + cd official-images && git diff + cd - - name: Create pull request to official-images id: create-pr uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.GITHUB_TOKEN }} - push-to-fork: Peter-Sh/official-images + push-to-fork: Sh-Peter/official-images path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" From 480669b1fb60b421cd0981b7f515107f7ed863bc Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:40:31 +0300 Subject: [PATCH 135/188] Use token for fork --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index b6a4dcdb4..aa4afb53b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -93,7 +93,7 @@ jobs: id: create-pr uses: peter-evans/create-pull-request@v7 with: - token: ${{ secrets.GITHUB_TOKEN }} + token: ${{ secrets.gh_token_for_fork }} push-to-fork: Sh-Peter/official-images path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} From 8654ea2aaa7c1c876975b92667c709e1ed2c1d47 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:40:57 +0300 Subject: [PATCH 136/188] Show color diff --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index aa4afb53b..38b31ce21 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -86,7 +86,7 @@ jobs: -e FORCE_COLOR=1 \ $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis - cd official-images && git diff + cd official-images && git diff --color cd - - name: Create pull request to official-images From f33f94a667e89adfc9b87a3cfcb5690c9431dcfd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:47:31 +0300 Subject: [PATCH 137/188] Add generated output entries --- release-automation/src/stackbrew_generator/stackbrew.py | 1 + 1 file changed, 1 insertion(+) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py index d872b65e5..faac43b94 100644 --- a/release-automation/src/stackbrew_generator/stackbrew.py +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -113,6 +113,7 @@ def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewE console.print(f"[yellow]No tags generated for {release}[/yellow]") console.print(f"[green]Generated {len(entries)} stackbrew entries[/green]") + console.print(f"[dim]{self.format_stackbrew_output(entries)}[/dim]") return entries def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: From 4faa3a487769d4db05dc6ff421eb658ab67a52be Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Sat, 13 Sep 2025 21:57:29 +0300 Subject: [PATCH 138/188] Upload release_info --- .github/workflows/release_publish.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 38b31ce21..41913426a 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -109,3 +109,21 @@ jobs: run: | echo "Pull Request Number: ${{ steps.create-pr.outputs.pull-request-number }}" echo "Pull Request URL: ${{ steps.create-pr.outputs.pull-request-url }}" + + # Create release_info.json artifact + cat > release_info.json << EOF + { + "pull_request_number": "${{ steps.create-pr.outputs.pull-request-number }}", + "pull_request_url": "${{ steps.create-pr.outputs.pull-request-url }}" + } + EOF + + echo "Created release_info.json:" + cat release_info.json + + - name: Upload release info artifact + uses: actions/upload-artifact@v4 + with: + name: release_info + path: release_info.json + retention-days: 30 From 7f38f4ec2803d25bfbd0cc34b339133e1cce9b8f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 09:59:07 +0300 Subject: [PATCH 139/188] Increase artifact lifetime, remove unused yaml --- .github/workflows/release_build_and_test.yml | 21 +------------------- .github/workflows/release_publish.yml | 2 +- 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 75d8e4ccd..8c232ea8f 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -116,23 +116,4 @@ jobs: with: name: release_handle path: release_handle.json - retention-days: 30 - - # - name: Create version tag - # if: ${{ steps.merge-back.outputs.merge_commit_sha != '' }} - # uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main - # with: - # tag: v${{ github.event.inputs.release_tag }} - # ref: ${{ steps.merge-back.outputs.merge_commit_sha }} - # gh_token: ${{ secrets.GITHUB_TOKEN }} - - # - name: Test Release Automation Docker Image - # env: - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # run: | - # docker run --rm \ - # -v ${{ github.workspace }}:/workspace \ - # -w /workspace \ - # -e FORCE_COLOR=1 \ - # $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]')\ - # generate 8 \ No newline at end of file + retention-days: 400 \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 41913426a..0f6837d43 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -126,4 +126,4 @@ jobs: with: name: release_info path: release_info.json - retention-days: 30 + retention-days: 400 From e7d4b01ec1a74e758ff2ad21fc67f57fb9f1cee5 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 10:07:02 +0300 Subject: [PATCH 140/188] Use redis-developer fork, but target personal repo for testing --- .github/workflows/release_publish.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 0f6837d43..219fafabe 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -12,6 +12,11 @@ on: description: 'Optional UUID to identify this workflow run' required: false +env: + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + FORKED_OFFICIAL_IMAGES_REPO: redis-developer/official-images + # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" @@ -69,7 +74,7 @@ jobs: uses: actions/checkout@v4 with: path: official-images - repository: Peter-Sh/official-images + repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} - name: Generate stackbrew library content env: @@ -94,7 +99,8 @@ jobs: uses: peter-evans/create-pull-request@v7 with: token: ${{ secrets.gh_token_for_fork }} - push-to-fork: Sh-Peter/official-images + draft: true + push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" From d2af09d58364ddbda42b62449034e53c8d5b4742 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:03:27 +0300 Subject: [PATCH 141/188] Use personal token for PR --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 219fafabe..3ca0194d0 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -98,7 +98,7 @@ jobs: id: create-pr uses: peter-evans/create-pull-request@v7 with: - token: ${{ secrets.gh_token_for_fork }} + token: ${{ secrets.GH_TOKEN_FOR_PR }} draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images From 800abcba816cd0c91cf36180f35d077b6c38d4d9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:10:52 +0300 Subject: [PATCH 142/188] Fix fork url --- .github/workflows/release_publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 3ca0194d0..f1fe15395 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -15,7 +15,7 @@ on: env: #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images - FORKED_OFFICIAL_IMAGES_REPO: redis-developer/official-images + FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 2de5de1fe61cf45a87d600ee0cba74c054e6ea6f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 13:28:26 +0300 Subject: [PATCH 143/188] Add a DRAFT note --- .github/workflows/release_publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f1fe15395..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -106,6 +106,9 @@ jobs: commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | + THIS IS A DRAFT PR FOR AUTOMATION TESTING + PLEASE IGNORE AND SORRY FOR BOTHERING + Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} From 333a41b3adfdbe17bfed4a61480102b4748d5b54 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:02:09 +0300 Subject: [PATCH 144/188] Try syncing fork and pushing to master --- .github/workflows/release_publish.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..a3bd66969 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -76,6 +76,16 @@ jobs: path: official-images repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} + - name: Sync fork with upstream + run: | + curl -L \ + -X POST \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ secrets.gh_token_for_fork }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + https://api.github.com/repos/${{ env.FORKED_OFFICIAL_IMAGES_REPO }}/merge-upstream \ + -d '{"branch":"master"}' + - name: Generate stackbrew library content env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -102,12 +112,14 @@ jobs: draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images - branch: redis-${{ steps.parse-release.outputs.release_version }} + branch: master + base: master + sign-commits: true commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR BOTHERING + PLEASE IGNORE AND SORRY FOR THE BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} From 061bc6b3c5567fda0b1d9d98bfe15cc5e01d64f2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:06:16 +0300 Subject: [PATCH 145/188] Revert "Try syncing fork and pushing to master" This reverts commit 8d99774d1c2c7b8d0f671353fda14207aa5da4f7. --- .github/workflows/release_publish.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index a3bd66969..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -76,16 +76,6 @@ jobs: path: official-images repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} - - name: Sync fork with upstream - run: | - curl -L \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.gh_token_for_fork }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{ env.FORKED_OFFICIAL_IMAGES_REPO }}/merge-upstream \ - -d '{"branch":"master"}' - - name: Generate stackbrew library content env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -112,14 +102,12 @@ jobs: draft: true push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} path: official-images - branch: master - base: master - sign-commits: true + branch: redis-${{ steps.parse-release.outputs.release_version }} commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR THE BOTHERING + PLEASE IGNORE AND SORRY FOR BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} From ad3f9b2a384ac79f4582f50754a402ec35a67df4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 14:14:14 +0300 Subject: [PATCH 146/188] Use official docker for testing --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..e551f8584 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 35a31805df9afa2d245eb3457520645209bd398c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 15:39:28 +0300 Subject: [PATCH 147/188] Collect image urls to release handle --- .../actions/build-and-tag-locally/action.yml | 24 +++++++++++- .github/workflows/pre-merge.yml | 37 +++++++++++++++++++ .github/workflows/release_build_and_test.yml | 25 ++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 060384d5b..a632dccff 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -1,3 +1,4 @@ + name: Build and Test inputs: @@ -231,11 +232,12 @@ runs: id: format-registry-tag shell: bash run: | - printf "tag=%s:%s%s-%s" \ + printf "tag=%s:%s%s-%s-%s" \ "${{ inputs.registry_repository }}" \ "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \ "${{ github.sha }}" \ "${{ inputs.distribution }}" \ + "${{ steps.platform.outputs.display_name }}" \ | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" - name: Push image @@ -247,3 +249,23 @@ runs: tags: ${{ steps.format-registry-tag.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max + + - name: Save image URL to artifact + shell: bash + run: | + if [[ "${{ inputs.publish_image }}" == "true" && "${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}" == "true" ]]; then + # Create a file with the image URL for this specific build + mkdir -p /tmp/image-urls + echo "${{ steps.format-registry-tag.outputs.tag }}" > "/tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt" + echo "Image URL saved: ${{ steps.format-registry-tag.outputs.tag }}" + else + echo "Image not published for this platform/distribution combination" + fi + + - name: Upload image URL artifact + uses: actions/upload-artifact@v4 + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + with: + name: image-url-${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + path: /tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt + retention-days: 1 diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 0924c7eb3..94b0246a9 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -10,6 +10,10 @@ on: description: 'Release tag to build' required: true type: string + outputs: + docker_image_urls: + description: 'Array of Docker image URLs that were published' + value: ${{ jobs.collect-image-urls.outputs.docker_image_urls }} jobs: build-and-test: @@ -59,3 +63,36 @@ jobs: registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} release_tag: ${{ inputs.release_tag }} + collect-image-urls: + runs-on: ubuntu-latest + needs: build-and-test + if: ${{ inputs.release_tag }} + outputs: + docker_image_urls: ${{ steps.collect-urls.outputs.urls }} + steps: + - name: Download all image URL artifacts + uses: actions/download-artifact@v4 + with: + pattern: image-url-* + path: ./image-urls + merge-multiple: true + + - name: Collect image URLs from artifacts + id: collect-urls + run: | + # Create JSON array from all URL files using jq + if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then + echo "Found image URL files:" + ls -la ./image-urls/ + + # Read all URL files and create JSON array with jq + urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') + + echo "Collected image URLs: $urls" + else + echo "No image URL artifacts found" + urls="[]" + fi + + echo "urls=$urls" >> "$GITHUB_OUTPUT" + diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 8c232ea8f..b4ac6a29a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -52,16 +52,11 @@ jobs: release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} build-and-test: - runs-on: ["ubuntu-latest"] - steps: - - shell: bash - run: | - echo Hi + uses: ./.github/workflows/pre-merge.yml needs: prepare-release - #uses: ./.github/workflows/pre-merge.yml - #secrets: inherit - #with: - # release_tag: ${{ github.event.inputs.release_tag }} + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} merge-back-to-release-branch: needs: [prepare-release, build-and-test] @@ -99,12 +94,22 @@ jobs: exit 1 fi + # Get docker image URLs from build-and-test job + DOCKER_IMAGE_URLS='${{ needs.build-and-test.outputs.docker_image_urls }}' + + # Validate that DOCKER_IMAGE_URLS is valid JSON + if ! echo "$DOCKER_IMAGE_URLS" | jq . > /dev/null 2>&1; then + echo "Warning: docker_image_urls is not valid JSON, using empty array" + DOCKER_IMAGE_URLS="[]" + fi + cat > release_handle.json << EOF { "release_commit_sha": "$RELEASE_COMMIT_SHA", "release_version": "${{ github.event.inputs.release_tag }}", "release_version_branch": "${{ steps.ensure-branch.outputs.release_version_branch }}", - "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}" + "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}", + "docker_image_urls": $DOCKER_IMAGE_URLS } EOF From 9c2abfe00e8c5875455fafba1a5213b8dee84428 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 15:41:28 +0300 Subject: [PATCH 148/188] Return to personal repo for testing --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index e551f8584..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From ab49c2b04bd86661c2fd2a0cc06f7044789aac5d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 17:36:31 +0300 Subject: [PATCH 149/188] Temporary disable some of the architectures --- .github/workflows/pre-merge.yml | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 94b0246a9..067302cfb 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -1,3 +1,4 @@ + name: Build and Test on: pull_request: @@ -26,15 +27,15 @@ jobs: - alpine platform: - linux/amd64 - - linux/i386 - - linux/arm/v5 - - linux/arm/v6 - - linux/arm/v7 - - linux/mips64le - - linux/ppc64le - - linux/s390x + # - linux/i386 + # - linux/arm/v5 + # - linux/arm/v6 + # - linux/arm/v7 + # - linux/mips64le + # - linux/ppc64le + # - linux/s390x - linux/arm64 - - linux/riscv64 + # - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -80,14 +81,9 @@ jobs: - name: Collect image URLs from artifacts id: collect-urls run: | - # Create JSON array from all URL files using jq if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then echo "Found image URL files:" - ls -la ./image-urls/ - - # Read all URL files and create JSON array with jq urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') - echo "Collected image URLs: $urls" else echo "No image URL artifacts found" @@ -95,4 +91,3 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" - From 688f9597dfa120a8799649006b04938d3b2b5115 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 17:43:23 +0300 Subject: [PATCH 150/188] Leave only 2 images --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 067302cfb..d9aa36b70 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -24,7 +24,7 @@ jobs: matrix: distribution: - debian - - alpine + # - alpine platform: - linux/amd64 # - linux/i386 From b997ae6ef2122eedc1b77a08f58ab42a21b88a8d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 18:25:45 +0300 Subject: [PATCH 151/188] Try self-hosted runners --- .github/workflows/pre-merge.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index d9aa36b70..376d965b9 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -18,7 +18,8 @@ on: jobs: build-and-test: - runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} + #runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} + runs-on: ["${{ contains(matrix.platform, 'arm64') && 'ARM64' || 'X64' }}", "self-hosted"] strategy: fail-fast: false matrix: From a7deb847c611575238e74d9efb89bead022f210c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 20:35:01 +0300 Subject: [PATCH 152/188] Fix image urls --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 376d965b9..6866e7c17 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -84,7 +84,7 @@ jobs: run: | if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then echo "Found image URL files:" - urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s 'split("\n") | map(select(length > 0))') + urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s -c 'split("\n") | map(select(length > 0))') echo "Collected image URLs: $urls" else echo "No image URL artifacts found" From c5fa85def853fe2a194e2c75a33013c514a2d5db Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 20:45:02 +0300 Subject: [PATCH 153/188] Used scoped layer cache --- .github/actions/build-and-tag-locally/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index a632dccff..d1e7a67ee 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -99,8 +99,8 @@ runs: load: true platforms: ${{ inputs.platform }} tags: ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + cache-to: type=gha,mode=max,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} - name: Save image shell: bash From a3a372be2757afa9a8bc8fb8e2c68e83e84fa624 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:24:55 +0300 Subject: [PATCH 154/188] Introduce wait for redis instead of always sleep --- test/run-entrypoint-tests.sh | 69 ++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 28d138ae0..5cfeff020 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -41,7 +41,6 @@ get_container_user_uid_gid_on_the_host() { container_user="$1" dir=$(mktemp -d -p .) docker run --rm -v "$(pwd)/$dir":/w -w /w --entrypoint=/bin/sh "$REDIS_IMG" -c "chown $container_user ." - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize stat -c "%u %g" "$dir" sudo rm -rf "$dir" } @@ -56,6 +55,42 @@ fi # Helper functions # +# Wait for Redis server or sentiel to be ready in a container by pinging it +# Arguments: +# $1 - container name/id +# Returns: +# 0 if Redis responds with PONG within timeout +# 1 if timeout CONTAINER_INIT_WAIT occurs +wait_for_redis_server_in_container() { + local container="$1" + local timeout="${CONTAINER_INIT_WAIT:-3}" + local elapsed=0 + local sleep_interval=0.1 + + if [ -z "$container" ]; then + return 1 + fi + + while [[ "$elapsed" < "$timeout" ]]; do + # Try to ping Redis server + if response=$(docker exec "$container" redis-cli ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + if response=$(docker exec "$container" redis-cli -p 26379 ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + # Sleep and increment elapsed time + sleep "$sleep_interval" + elapsed=$(awk "BEGIN {print $elapsed + $sleep_interval}") + done + + echo "Timeout: Redis server did not respond within ${timeout}s" + docker stop "$container" >/dev/null + return 1 +} + # creates one entry of directory structure # used in combination with iterate_dir_structure_with create_entry() { @@ -181,7 +216,6 @@ run_docker_and_test_ownership() { fi docker_output=$($docker_run 2>&1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize if [ "$TEST_VERBOSE" ]; then echo "After:" @@ -270,14 +304,9 @@ run_redis_docker_and_check_uid_gid() { docker_cmd="$*" # shellcheck disable=SC2086 container=$(docker run $docker_flags -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize ret=$? - assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - if [ $ret -gt 0 ]; then - echo "retarning" - return 1 - fi + wait_for_redis_server_in_container "$container" || return 1 cmdline=$(docker exec "$container" cat /proc/1/cmdline|tr -d \\0) assertContains "$docker_flags $docker_cmd, cmdline: $cmdline" "$cmdline" "$expected_cmd" @@ -302,7 +331,10 @@ run_redis_docker_and_check_modules() { docker_cmd="$1" # shellcheck disable=SC2086 container=$(docker run --rm -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + info=$(docker exec "$container" redis-cli info) [ "$PLATFORM" ] && [ "$PLATFORM" != "amd64" ] && startSkipping @@ -329,7 +361,6 @@ assert_redis_v8() { test_redis_version() { ret=$(docker run --rm "$REDIS_IMG" -v|tail -n 1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize assert_redis_v8 "$ret" } @@ -553,7 +584,9 @@ test_redis_server_persistence_with_bind_mount() { chmod 0444 "$dir" container=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -568,7 +601,10 @@ test_redis_server_persistence_with_bind_mount() { sudo chown -R "$HOST_OWNER" "$dir" container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -586,7 +622,9 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data' container=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -601,7 +639,10 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data && chmod 0000 -R /data' container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" From f5d5e905098266e46f105115faaae1cf9a3a8980 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:32:33 +0300 Subject: [PATCH 155/188] Increase timeout --- test/run-entrypoint-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 5cfeff020..264466aad 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -63,7 +63,7 @@ fi # 1 if timeout CONTAINER_INIT_WAIT occurs wait_for_redis_server_in_container() { local container="$1" - local timeout="${CONTAINER_INIT_WAIT:-3}" + local timeout="${CONTAINER_INIT_WAIT:-6}" local elapsed=0 local sleep_interval=0.1 From 9536d72f447e932cb460b50bc44664d27c72c7db Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:36:23 +0300 Subject: [PATCH 156/188] Really increase timeout --- test/run-entrypoint-tests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 264466aad..7bffbc4ba 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -21,7 +21,7 @@ ## # Container initialization wait time in seconds -CONTAINER_INIT_WAIT=3 +CONTAINER_INIT_WAIT=6 if [ -z "$REDIS_IMG" ]; then echo "REDIS_IMG may not be empty" @@ -63,7 +63,7 @@ fi # 1 if timeout CONTAINER_INIT_WAIT occurs wait_for_redis_server_in_container() { local container="$1" - local timeout="${CONTAINER_INIT_WAIT:-6}" + local timeout="${CONTAINER_INIT_WAIT:-3}" local elapsed=0 local sleep_interval=0.1 From 47d5c1bc4b8188ee5b0d66427ae9395f5b7ad8cc Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:44:22 +0300 Subject: [PATCH 157/188] Fix wait for redis: use correct container id --- test/run-entrypoint-tests.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 7bffbc4ba..4516c6c9d 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -21,7 +21,7 @@ ## # Container initialization wait time in seconds -CONTAINER_INIT_WAIT=6 +CONTAINER_INIT_WAIT=3 if [ -z "$REDIS_IMG" ]; then echo "REDIS_IMG may not be empty" @@ -603,7 +603,7 @@ test_redis_server_persistence_with_bind_mount() { container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") ret=$? assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - wait_for_redis_server_in_container "$container" || return 1 + wait_for_redis_server_in_container "$container2" || return 1 value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -641,7 +641,7 @@ test_redis_server_persistence_with_volume() { container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") ret=$? assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - wait_for_redis_server_in_container "$container" || return 1 + wait_for_redis_server_in_container "$container2" || return 1 value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" From c61a6ea646af20e0d961cef46494b49d63668d1f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Wed, 17 Sep 2025 22:53:23 +0300 Subject: [PATCH 158/188] Return official images for demo --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..e551f8584 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 5a34e94e026f4a91d33cd32793a2e2075301a7a7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 11:31:39 +0300 Subject: [PATCH 159/188] Add slack notification about image urls --- .github/workflows/pre-merge.yml | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6866e7c17..6e75785ea 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,3 +92,55 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" + + notify-slack: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && needs.collect-image-urls.outputs.docker_image_urls != '[]' }} + steps: + - name: Send Slack notification + run: | + # Parse the image URLs from JSON array + image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' + + # Create formatted list of image URLs + formatted_urls=$(echo "$image_urls" | jq -r '.[] | "• `" + . + "`"' | tr '\n' '\n') + + # Create Slack message payload + cat > slack_payload.json << EOF + { + "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}", + "blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "The following Docker images have been successfully published:\n\n$formatted_urls" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + } + ] + } + ] + } + EOF + + # Send to Slack + curl -X POST -H 'Content-type: application/json' \ + --data @slack_payload.json \ + "${{ secrets.SLACK_WEB_HOOK_URL }}" + + echo "Slack notification sent for release ${{ inputs.release_tag }}" From af9ca7f97e0af14b53c11ae5639047ad1526404c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 11:31:56 +0300 Subject: [PATCH 160/188] Revert "Return official images for demo" This reverts commit 0dd060d69308a6c683beb095838a699f5dcc7fa9. --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index e551f8584..7ca83d9b6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From c51ce0d8186da96133657019539b99aaad298e0f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:13:46 +0300 Subject: [PATCH 161/188] Reformat slack message --- .github/actions/common/func.sh | 43 ++++++++++++++++++++++++++++++ .github/workflows/pre-merge.yml | 46 ++++----------------------------- 2 files changed, 48 insertions(+), 41 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index efb4ccee3..edeea426f 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -330,4 +330,47 @@ prepare_releases_list() { increase_indent_level console_output 2 gray "$debug_output" decrease_indent_level +} + +slack_format_docker_image_urls_message() { + # Parse the image URLs from JSON array + local image_urls formatted_urls release_tag footer + image_urls=$(cat) + release_tag=$1 + footer=$2 + + # Create formatted list of image URLs + formatted_urls=$(echo "$image_urls" | jq -j '.[] | "\\n• \(.)"') + +# Create Slack message payload + cat << EOF +{ +"text": "🐳 Docker Images Published for Release $release_tag", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Images Published for Release $release_tag" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "The following Docker images have been successfully published:\n\n$formatted_urls" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "f$footer" + } + ] + } +] +} +EOF } \ No newline at end of file diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 6e75785ea..e49fbcb81 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -100,47 +100,11 @@ jobs: steps: - name: Send Slack notification run: | - # Parse the image URLs from JSON array image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - # Create formatted list of image URLs - formatted_urls=$(echo "$image_urls" | jq -r '.[] | "• `" + . + "`"' | tr '\n' '\n') + . .github/actions/common/func.sh - # Create Slack message payload - cat > slack_payload.json << EOF - { - "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}", - "blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "🐳 Docker Images Published for Release ${{ inputs.release_tag }}" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The following Docker images have been successfully published:\n\n$formatted_urls" - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" - } - ] - } - ] - } - EOF - - # Send to Slack - curl -X POST -H 'Content-type: application/json' \ - --data @slack_payload.json \ - "${{ secrets.SLACK_WEB_HOOK_URL }}" - - echo "Slack notification sent for release ${{ inputs.release_tag }}" + echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ + curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 18673cc69fc286c2c0f650ef80427ad34f13a03f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:26:42 +0300 Subject: [PATCH 162/188] Slack msg for PR creation --- .github/actions/common/func.sh | 42 +++++++++++++++++++++++++-- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 10 +++++++ 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index edeea426f..48a974c01 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -345,7 +345,7 @@ slack_format_docker_image_urls_message() { # Create Slack message payload cat << EOF { -"text": "🐳 Docker Images Published for Release $release_tag", +"text": "🐳 Docker Images Published for Redis: $release_tag", "blocks": [ { "type": "header", @@ -366,7 +366,45 @@ slack_format_docker_image_urls_message() { "elements": [ { "type": "mrkdwn", - "text": "f$footer" + "text": "$footer" + } + ] + } +] +} +EOF +} + +slack_format_docker_PR_message() { + release_tag=$1 + url=$2 + footer=$3 + +# Create Slack message payload + cat << EOF +{ +"text": "🐳 Docker Library PR created for Redis: $release_tag", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🐳 Docker Library PR created for Redis: $release_tag" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "$url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" } ] } diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index e49fbcb81..c2af8345f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -104,7 +104,7 @@ jobs: workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . .github/actions/common/func.sh + . ${{ github.workspace}}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 7ca83d9b6..2ac602468 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -136,3 +136,13 @@ jobs: name: release_info path: release_info.json retention-days: 400 + + - name: Send Slack notification + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + + . ${{ github.workspace }}/.github/actions/common/func.sh + + slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file From 13bacc07bb7d49c785a79b647a5aceefe7ee11c4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:39:26 +0300 Subject: [PATCH 163/188] Debug output --- .github/workflows/release_build_and_test.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b4ac6a29a..8a121b278 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,6 +24,14 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Test include + run: | + set -x + pwd + ls -la ${GITHUB_WORKSPACE} + find ${GITHUB_WORKSPACE} -name .github + exit 1 + - name: Log in to Container Registry uses: docker/login-action@v3 with: From b4675ca9f5c456b864e334174e2a380c8efb2e5d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:42:29 +0300 Subject: [PATCH 164/188] Further debug --- .github/workflows/release_build_and_test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 8a121b278..6cd8dafa7 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -30,6 +30,9 @@ jobs: pwd ls -la ${GITHUB_WORKSPACE} find ${GITHUB_WORKSPACE} -name .github + find ${GITHUB_WORKSPACE} -name func.sh + echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh exit 1 - name: Log in to Container Registry From 5db086416bbe82ef70749453ee762772c3f272ce Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:46:35 +0300 Subject: [PATCH 165/188] Try using GITHUB_WORKSPACE env --- .github/workflows/pre-merge.yml | 7 ++++++- .github/workflows/release_build_and_test.yml | 11 ----------- .github/workflows/release_publish.yml | 2 +- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index c2af8345f..5aa33d5ad 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -104,7 +104,12 @@ jobs: workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . ${{ github.workspace}}/.github/actions/common/func.sh + pwd + ls -la ${GITHUB_WORKSPACE} + find ${GITHUB_WORKSPACE} -name .github + find ${GITHUB_WORKSPACE} -name func.sh + echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index 6cd8dafa7..b4ac6a29a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -24,17 +24,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Test include - run: | - set -x - pwd - ls -la ${GITHUB_WORKSPACE} - find ${GITHUB_WORKSPACE} -name .github - find ${GITHUB_WORKSPACE} -name func.sh - echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - exit 1 - - name: Log in to Container Registry uses: docker/login-action@v3 with: diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 2ac602468..f5235941b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -142,7 +142,7 @@ jobs: workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - . ${{ github.workspace }}/.github/actions/common/func.sh + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file From fa7c8ebb3b1f7412d798fd307e366adc8b8931a4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 12:55:33 +0300 Subject: [PATCH 166/188] Use checkout action --- .github/workflows/pre-merge.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 5aa33d5ad..c8b354868 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -98,17 +98,15 @@ jobs: needs: collect-image-urls if: ${{ inputs.release_tag && needs.collect-image-urls.outputs.docker_image_urls != '[]' }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Send Slack notification run: | image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" - pwd - ls -la ${GITHUB_WORKSPACE} - find ${GITHUB_WORKSPACE} -name .github - find ${GITHUB_WORKSPACE} -name func.sh - echo ${GITHUB_WORKSPACE}/.github/actions/common/func.sh . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ From 26177aeb89d5b045b5af86c8957fde78cae4ffb5 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 13:04:53 +0300 Subject: [PATCH 167/188] Fix curl execution for slack --- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index c8b354868..4d2ebc712 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -110,4 +110,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ - curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f5235941b..2061aff9c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -145,4 +145,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}"} \ No newline at end of file + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From fa6360b3ba762165fbb53c69bf34864aca65f98c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 13:05:52 +0300 Subject: [PATCH 168/188] Use official repo --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 2061aff9c..c90cb76e3 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. From 164c1240dcc6b5530281d1d86e74a2de0554b19a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 20:45:31 +0300 Subject: [PATCH 169/188] Proper milestone filtering and tests --- release-automation/README.md | 16 +- .../src/stackbrew_generator/models.py | 10 + .../src/stackbrew_generator/version_filter.py | 34 +- .../tests/test_stackbrew_updater.py | 123 +++++++ .../tests/test_update_stackbrew_file.py | 297 ++++++++++++++++ .../tests/test_version_filter.py | 323 ++++++++++++++++++ 6 files changed, 787 insertions(+), 16 deletions(-) create mode 100644 release-automation/tests/test_stackbrew_updater.py create mode 100644 release-automation/tests/test_update_stackbrew_file.py create mode 100644 release-automation/tests/test_version_filter.py diff --git a/release-automation/README.md b/release-automation/README.md index e37e7189a..3e07a6d1e 100644 --- a/release-automation/README.md +++ b/release-automation/README.md @@ -16,13 +16,27 @@ Each version release is tagged with `vMajor.Minor.Patch` (e.g. `v8.2.1`) Milestone releases are tagged with `vMajor.Minor.Patch-Milestone` (e.g. `v8.2.1-m01`). Any suffix after patch version is considered a milestone. +Suffixes starting with `rc` are considered release candidates and are preferred over suffixes starting with `m` which in turn are preferred over any other suffix. + Tags without suffix are considered GA (General Availability) releases (e.g. `v8.2.1`). Internal releases are milestone releases containing `-int` in their name (e.g. `v8.2.1-m01-int1` or `8.4.0-int3`). They are not released to the public. Milestone releases never get latest or any other default tags, like `8`, `8.2`, `8.2.1`, `latest`, `bookworm`, etc. -For each mainline only one GA release and optionally one milestone release should be published in official-library. The most latest versions. +For each mainline only one GA release and optionally any number of milestone releases with higher versions than this GA may be published in official-library. + +Each patch version may have only one GA or milestone release, GA release is preferred over milestone release. + +For example for this list of tags the following rules will be applied + +* `v8.2.3-m01` - included because there is neither GA nor any higher milestone versions for 8.2.3 +* `v8.2.2-rc2` - included because it higher version among 8.2.2 +* `v8.2.2-rc1` - excluded because 8.2.2-rc2 is higher version +* `v8.2.2-m01` - excluded because 8.2.2-rc2 is higher version +* `v8.2.1-rc2` - excluded because there is 8.2.1 GA version +* `v8.2.1` - included because it is highest GA for 8.2 +* `v8.2.0` - exluded because 8.2.1 is higher version End of life versions are marked with `-eol` suffix (e.g. `v8.0.3-eol`). When there is a at least one minor version tagged with eol all versions in this minor series are considered EOL and are not included in the release file. diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py index 6d7c88991..8a9a1fff7 100644 --- a/release-automation/src/stackbrew_generator/models.py +++ b/release-automation/src/stackbrew_generator/models.py @@ -68,6 +68,16 @@ def mainline_version(self) -> str: """Get the mainline version string (major.minor).""" return f"{self.major}.{self.minor}" + @property + def sort_key(self) -> str: + suffix_weight = 0 + if self.suffix.startswith("rc"): + suffix_weight = 100 + elif self.suffix.startswith("m"): + suffix_weight = 50 + + return f"{self.major}.{self.minor}.{self.patch or 0}.{suffix_weight}.{self.suffix}" + def __str__(self) -> str: """String representation of the version.""" version = f"{self.major}.{self.minor}" diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index 23cb0ed3b..13db27443 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -2,6 +2,8 @@ from typing import Dict, List, Tuple +from collections import OrderedDict + from packaging.version import Version from rich.console import Console @@ -47,11 +49,12 @@ def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVe continue # Sort by version (newest first) - versions.sort(key=lambda x: x[0], reverse=True) + versions.sort(key=lambda x: x[0].sort_key, reverse=True) console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") return versions + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: """Filter out end-of-life versions. @@ -99,24 +102,25 @@ def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str, str]]) """ console.print("[blue]Filtering to actual versions (latest patch per minor/milestone)[/blue]") - seen_combinations = set() - filtered_versions = [] + patch_versions = OrderedDict() for version, commit, tag_ref in versions: - # Create a key for minor version + milestone status - combination_key = (version.mainline_version, version.is_milestone) + patch_key = (version.major, version.minor, version.patch) + if patch_key not in patch_versions: + patch_versions[patch_key] = (version, commit, tag_ref) + elif patch_versions[patch_key][0].is_milestone and not version.is_milestone: + # GA always takes precedence over milestone for the same major.minor.patch + patch_versions[patch_key] = (version, commit, tag_ref) + + print(patch_versions.values()) + filtered_versions = [] + mainlines_with_ga = set() - if combination_key not in seen_combinations: - seen_combinations.add(combination_key) + for version, commit, tag_ref in patch_versions.values(): + if version.mainline_version not in mainlines_with_ga: + if not version.is_milestone: + mainlines_with_ga.add(version.mainline_version) filtered_versions.append((version, commit, tag_ref)) - - milestone_str = "milestone" if version.is_milestone else "GA" - console.print(f"[dim]Selected [bold yellow]{version}[/bold yellow] ({milestone_str}) - {commit[:8]}[/dim]") - else: - milestone_str = "milestone" if version.is_milestone else "GA" - console.print(f"[dim]Skipping {version} ({milestone_str}) - already have this minor/milestone combination[/dim]") - - console.print(f"[dim]Selected {len(filtered_versions)} actual versions[/dim]") return filtered_versions def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: diff --git a/release-automation/tests/test_stackbrew_updater.py b/release-automation/tests/test_stackbrew_updater.py new file mode 100644 index 000000000..36b98590e --- /dev/null +++ b/release-automation/tests/test_stackbrew_updater.py @@ -0,0 +1,123 @@ +"""Tests for StackbrewUpdater class.""" + +import tempfile +from pathlib import Path + +from stackbrew_generator.stackbrew import StackbrewUpdater + +class TestStackbrewUpdater: + """Tests for StackbrewUpdater class.""" + + def setup_method(self): + """Set up test fixtures.""" + self.updater = StackbrewUpdater() + + def test_update_stackbrew_content_basic(self): + """Test basic stackbrew content update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + new_content = """Tags: 8.2.2, 8.2, 8, 8.2.2-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: debian + +Tags: 8.2.2-alpine, 8.2-alpine, 8-alpine, 8.2.2-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: alpine""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + # Update the content + updated_content = self.updater.update_stackbrew_content( + input_file, 8, new_content, verbose=False + ) + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_parse_stackbrew_entries(self): + """Test parsing stackbrew entries.""" + lines = [ + "Tags: 8.2.1, 8.2, 8", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: debian", + "", + "Tags: 8.2.1-alpine, 8.2-alpine", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: alpine" + ] + + entries = self.updater._parse_stackbrew_entries(lines) + + assert len(entries) == 2 + assert entries[0][0] == "Tags: 8.2.1, 8.2, 8" + assert entries[1][0] == "Tags: 8.2.1-alpine, 8.2-alpine" + + def test_entry_belongs_to_major_version(self): + """Test checking if entry belongs to major version.""" + entry_8x = [ + "Tags: 8.2.1, 8.2, 8, latest", + "Architectures: amd64", + "GitCommit: abc123", + "Directory: debian" + ] + + entry_7x = [ + "Tags: 7.4.0, 7.4, 7", + "Architectures: amd64", + "GitCommit: def456", + "Directory: debian" + ] + + assert self.updater._entry_belongs_to_major_version(entry_8x, 8) is True + assert self.updater._entry_belongs_to_major_version(entry_8x, 7) is False + assert self.updater._entry_belongs_to_major_version(entry_7x, 7) is True + assert self.updater._entry_belongs_to_major_version(entry_7x, 8) is False diff --git a/release-automation/tests/test_update_stackbrew_file.py b/release-automation/tests/test_update_stackbrew_file.py new file mode 100644 index 000000000..acdbd3ac1 --- /dev/null +++ b/release-automation/tests/test_update_stackbrew_file.py @@ -0,0 +1,297 @@ +"""Tests for update-stackbrew-file command.""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from typer.testing import CliRunner + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release + + +class TestUpdateStackbrewFile: + """Tests for update-stackbrew-file command.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_update_stackbrew_file_basic(self): + """Test basic stackbrew file update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Mock git client + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + # Mock distribution detector + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + # Mock version filter + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + + # Mock the version filter to return Redis 8.x versions + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.2"), "new123commit", "refs/tags/v8.2.2") + ] + + # Mock releases + mock_releases = [ + Release( + commit="new123commit", + version=RedisVersion.parse("8.2.2"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.2" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + # Run the command with output to file + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(input_file), + "--verbose" + ]) + + assert result.exit_code == 0 + + # Check that the file was updated + updated_content = input_file.read_text() + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_update_stackbrew_file_nonexistent_input(self): + """Test error handling for nonexistent input file.""" + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", "/nonexistent/file.txt" + ]) + + assert result.exit_code == 1 + assert "Input file does not exist" in result.stderr + + def test_update_stackbrew_file_no_versions_found(self): + """Test error handling when no versions are found.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [] + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "9", + "--input", str(input_file) + ]) + + assert result.exit_code == 1 + assert "No versions found for Redis 9.x" in result.stderr + + finally: + input_file.unlink() + + def test_update_stackbrew_file_with_output_option(self): + """Test using separate output file.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as input_f, \ + tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as output_f: + + input_f.write(sample_content) + input_file = Path(input_f.name) + output_file = Path(output_f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new456commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new456commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(output_file) + ]) + + assert result.exit_code == 0 + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + # Output file should have updated content + updated_content = output_file.read_text() + assert "new456commit" in updated_content + assert "8.2.0" in updated_content + assert "old123" not in updated_content + + finally: + input_file.unlink() + output_file.unlink() + + def test_update_stackbrew_file_stdout_output(self): + """Test outputting to stdout when no output file is specified.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new789commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new789commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file) + ]) + + assert result.exit_code == 0 + + # Should output to stdout + assert "new789commit" in result.stdout + assert "8.2.0" in result.stdout + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + finally: + input_file.unlink() diff --git a/release-automation/tests/test_version_filter.py b/release-automation/tests/test_version_filter.py new file mode 100644 index 000000000..7f704da28 --- /dev/null +++ b/release-automation/tests/test_version_filter.py @@ -0,0 +1,323 @@ +"""Tests for VersionFilter class.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.models import RedisVersion +from stackbrew_generator.version_filter import VersionFilter +from stackbrew_generator.git_operations import GitClient +from stackbrew_generator.exceptions import GitOperationError + + +class MockGitClient: + """Mock GitClient for testing.""" + + def __init__(self): + """Initialize mock git client.""" + self.remote_tags = [] + self.version_extraction_results = {} + + def set_remote_tags(self, tags): + """Set mock remote tags. + + Args: + tags: List of (commit, tag_ref) tuples + """ + self.remote_tags = tags + + def set_version_extraction_result(self, tag_ref, version_or_exception): + """Set mock version extraction result. + + Args: + tag_ref: Tag reference + version_or_exception: RedisVersion instance or Exception to raise + """ + self.version_extraction_results[tag_ref] = version_or_exception + + def list_remote_tags(self, major_version): + """Mock list_remote_tags method.""" + return self.remote_tags + + def extract_version_from_tag(self, tag_ref, major_version): + """Mock extract_version_from_tag method.""" + if tag_ref in self.version_extraction_results: + result = self.version_extraction_results[tag_ref] + if isinstance(result, Exception): + raise result + return result + # Default behavior - try to parse from tag_ref + return RedisVersion.parse(tag_ref.replace('refs/tags/', '')) + + +def create_version_tuples(version_strings): + """Helper to create version tuples from version strings. + + Args: + version_strings: List of version strings + + Returns: + List of (RedisVersion, commit, tag_ref) tuples + """ + tuples = [] + for i, version_str in enumerate(version_strings): + version = RedisVersion.parse(version_str) + commit = f"commit{i:03d}" + tag_ref = f"refs/tags/{version_str}" + tuples.append((version, commit, tag_ref)) + + tuples.sort(key=lambda x: x[0].sort_key, reverse=True) + return tuples + + +class TestVersionFilter: + """Tests for VersionFilter class.""" + + def test_init(self): + """Test VersionFilter initialization.""" + git_client = GitClient() + version_filter = VersionFilter(git_client) + assert version_filter.git_client is git_client + + def test_get_redis_versions_from_tags_success(self): + """Test successful version retrieval from tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should be sorted by version (newest first) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.1.0"] + assert version_strings == expected_versions + + # Check commits and tag refs + commits = [v[1] for v in result] + tag_refs = [v[2] for v in result] + expected_commits = ["commit001", "commit002", "commit003"] + expected_tag_refs = ["refs/tags/v8.2.1", "refs/tags/v8.2.0", "refs/tags/v8.1.0"] + assert commits == expected_commits + assert tag_refs == expected_tag_refs + + def test_get_redis_versions_from_tags_with_invalid_tags(self): + """Test version retrieval with some invalid tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/invalid-tag"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + # Set up invalid tag to raise exception + mock_git_client.set_version_extraction_result( + "refs/tags/invalid-tag", + ValueError("Invalid version format") + ) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should skip invalid tag and return only valid ones + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_get_redis_versions_from_tags_empty(self): + """Test version retrieval with no tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + assert result == [] + + def test_filter_eol_versions_basic(self): + """Test basic EOL version filtering.""" + version_filter = VersionFilter(MockGitClient()) + + # Create test versions with one EOL minor version + versions = create_version_tuples([ + "v8.2.1", + "v8.2.0", + "v8.1.0-eol", + "v8.1.2", + "v8.0.1", + "v8.0.0" + ]) + + result = version_filter.filter_eol_versions(versions) + + # Should filter out all 8.1.* versions (because 8.1.0-eol exists) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.0.1", "8.0.0"] + assert version_strings == expected_versions + + def test_filter_eol_versions_empty(self): + """Test EOL filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_eol_versions([]) + assert result == [] + + def test_filter_actual_versions_basic(self): + """Test basic actual version filtering (latest patch per minor/milestone).""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with multiple patches for same minor version + versions = create_version_tuples([ + "v8.2.2", # Latest patch for 8.2 GA + "v8.2.1", # Older patch for 8.2 GA + "v8.2.0", # Oldest patch for 8.2 GA + "v8.1.1", # Latest patch for 8.1 GA + "v8.1.0", # Older patch for 8.1 GA + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep only latest patch for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2", "8.1.1"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_same_patch(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version + "v8.2.1-m02", # Latest milestone for 8.2 + "v8.2.1-m01", # Older milestone for 8.2 + "v8.1.0", # GA version + "v8.1.0-m01", # Milestone for 8.1 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_mainline(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + "v8.1.1-m01", # Milestone for 8.1 + "v8.2.0-m03", # Older milestone for 8.2.0 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-m02", "8.2.1", "8.1.1-m01", "8.1.0"] + assert version_strings == expected_versions + + def test_when_filter_actual_versions_with_milestones_rc_is_preferred(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-rc01", # Latest milestone for 8.2.2 + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-rc01", "8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_milestone_only(self): + """Test actual version filtering with only milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + versions = create_version_tuples([ + "v8.2.1-m02", + "v8.2.1-m01", + "v8.1.0-m01", + ]) + + result = version_filter.filter_actual_versions(versions) + + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1-m02", "8.1.0-m01"] + assert version_strings == expected_versions + + def test_filter_actual_versions_empty(self): + """Test actual version filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_actual_versions([]) + assert result == [] + + def test_get_actual_major_redis_versions_success(self): + """Test the main entry point method with successful flow.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0-eol"), # Should be filtered out + ("commit004", "refs/tags/v8.0.1"), + ("commit005", "refs/tags/v8.0.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + # Should apply all filters: get tags -> filter EOL -> filter actual + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.0.1"] # Latest patches, no EOL + assert version_strings == expected_versions + + def test_get_actual_major_redis_versions_no_versions(self): + """Test main entry point with no versions found.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + assert result == [] + +class TestVersionFilterIntegration: + """Integration tests using real GitClient (mocked at subprocess level).""" + + @patch('stackbrew_generator.git_operations.subprocess.run') + def test_integration_with_real_git_client(self, mock_subprocess): + """Test VersionFilter with real GitClient (mocked subprocess).""" + # Mock git ls-remote output + mock_subprocess.return_value.stdout = ( + "commit001\trefs/tags/v8.2.1\n" + "commit002\trefs/tags/v8.2.0\n" + "commit003\trefs/tags/v8.1.0-eol\n" + ) + mock_subprocess.return_value.returncode = 0 + + git_client = GitClient() + version_filter = VersionFilter(git_client) + + result = version_filter.get_actual_major_redis_versions(8) + + # Should get filtered results + version_strings = [str(v[0]) for v in result] + commits = [v[1] for v in result] + expected_versions = ["8.2.1"] # Only 8.2.1 after all filtering + expected_commits = ["commit001"] + assert version_strings == expected_versions + assert commits == expected_commits \ No newline at end of file From 3468cecda9bd5aa2bc5760babaf72faaca37a73b Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Thu, 18 Sep 2025 20:48:33 +0300 Subject: [PATCH 170/188] Fix slack footer, return personal target --- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 4d2ebc712..b799f4e3f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -105,7 +105,7 @@ jobs: run: | image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index c90cb76e3..fab93201b 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,8 +13,8 @@ on: required: false env: - TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images # UUID is used to help automation to identify workflow run in the list of workflow runs. @@ -140,7 +140,7 @@ jobs: - name: Send Slack notification run: | workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | Workflow: [View]($workflow_url)" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh From f9da1d4f38211a8e1508762f58cd9aed62b50cb2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 09:51:10 +0300 Subject: [PATCH 171/188] Fix sort in eol filter, update tests --- release-automation/src/stackbrew_generator/version_filter.py | 2 +- release-automation/tests/test_version_filter.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py index 13db27443..bf0590795 100644 --- a/release-automation/src/stackbrew_generator/version_filter.py +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -86,7 +86,7 @@ def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> filtered_versions.extend(minor_group) # Sort again after filtering - filtered_versions.sort(key=lambda x: x[0], reverse=True) + filtered_versions.sort(key=lambda x: x[0].sort_key, reverse=True) console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") return filtered_versions diff --git a/release-automation/tests/test_version_filter.py b/release-automation/tests/test_version_filter.py index 7f704da28..7ebeb556d 100644 --- a/release-automation/tests/test_version_filter.py +++ b/release-automation/tests/test_version_filter.py @@ -145,6 +145,7 @@ def test_filter_eol_versions_basic(self): "v8.2.1", "v8.2.0", "v8.1.0-eol", + "v8.1.0-zoo1", "v8.1.2", "v8.0.1", "v8.0.0" From d44b5ea96b6be8d65278d0cde789e95934aafa57 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 10:54:01 +0300 Subject: [PATCH 172/188] Change slack msg format for images list --- .github/actions/common/func.sh | 78 +++++++++++++++++----------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 48a974c01..3220ce25d 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -334,45 +334,45 @@ prepare_releases_list() { slack_format_docker_image_urls_message() { # Parse the image URLs from JSON array - local image_urls formatted_urls release_tag footer - image_urls=$(cat) - release_tag=$1 - footer=$2 - - # Create formatted list of image URLs - formatted_urls=$(echo "$image_urls" | jq -j '.[] | "\\n• \(.)"') - -# Create Slack message payload - cat << EOF -{ -"text": "🐳 Docker Images Published for Redis: $release_tag", -"blocks": [ - { - "type": "header", - "text": { - "type": "plain_text", - "text": "🐳 Docker Images Published for Release $release_tag" - } - }, - { - "type": "section", - "text": { - "type": "mrkdwn", - "text": "The following Docker images have been successfully published:\n\n$formatted_urls" - } - }, - { - "type": "context", - "elements": [ - { - "type": "mrkdwn", - "text": "$footer" - } - ] - } -] -} -EOF + jq --arg release_tag "$1" --arg footer "$2" ' + map( + capture("(?(?[^:]+:)(?[^-]+)-(?[a-f0-9]+)-(?[^-]+)-(?[^-]+))$") + ) + as $items + | { + text: ("🐳 Docker Images Published for Redis: " + $release_tag), + blocks: [ + { + "type": "header", + "text": { "type": "plain_text", "text": ("🐳 Docker Images Published for Release " + $release_tag) } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ( + "The following Docker images have been published to Github Container Registry:\n\n" + + ( + $items + | map( + "Distribution: *" + .distro + "* " + + "Architecture: *" + .arch + "*" + + "\n```\n" + .url + "\n```" + ) + | join("\n\n") + ) + ) + } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": $footer } + ] + } + ] + } + ' } slack_format_docker_PR_message() { From 7757470497af1d5b54b7ac66a31fc99d626c65e4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:37:55 +0300 Subject: [PATCH 173/188] Add mentions and link to changes --- .github/workflows/release_publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index fab93201b..d8720de72 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -107,12 +107,15 @@ jobs: title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE AND SORRY FOR BOTHERING + PLEASE IGNORE IT AND SORRY FOR THE BOTHERING Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} + Compare: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + + @adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh - name: PR creation results run: | From 86b550b791dd436096f73e7854d0b546ae1d6ba9 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:47:42 +0300 Subject: [PATCH 174/188] New icon, move user mentions to env --- .github/actions/common/func.sh | 2 ++ .github/workflows/release_publish.yml | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 3220ce25d..13add08be 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -340,6 +340,7 @@ slack_format_docker_image_urls_message() { ) as $items | { + icon_url: ":redis-circle:", text: ("🐳 Docker Images Published for Redis: " + $release_tag), blocks: [ { @@ -383,6 +384,7 @@ slack_format_docker_PR_message() { # Create Slack message payload cat << EOF { +"icon_url": ":redis-circle:", "text": "🐳 Docker Library PR created for Redis: $release_tag", "blocks": [ { diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index d8720de72..8a97112e9 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -16,6 +16,8 @@ env: #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images + #PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" + PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" @@ -113,9 +115,9 @@ jobs: Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} - Compare: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + Compare: ${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} - @adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh + ${{ env.PR_USER_MENTIONS }} - name: PR creation results run: | From 800e079d561667b270a051a7d853d5da2a46172c Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 12:57:09 +0300 Subject: [PATCH 175/188] Fix icon and url --- .github/actions/common/func.sh | 4 ++-- .github/workflows/release_publish.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 13add08be..4c4837000 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -340,7 +340,7 @@ slack_format_docker_image_urls_message() { ) as $items | { - icon_url: ":redis-circle:", + icon_emoji: ":redis-circle:", text: ("🐳 Docker Images Published for Redis: " + $release_tag), blocks: [ { @@ -384,7 +384,7 @@ slack_format_docker_PR_message() { # Create Slack message payload cat << EOF { -"icon_url": ":redis-circle:", +"icon_emoji": ":redis-circle:", "text": "🐳 Docker Library PR created for Redis: $release_tag", "blocks": [ { diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 8a97112e9..293d94407 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -115,7 +115,7 @@ jobs: Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} Release tag: v${{ steps.parse-release.outputs.release_version }} - Compare: ${{ env.GITHUB_SERVER_URL }}/${{ env.GITHUB_REPOSITORY }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + Compare: ${{ github.server_url }}/${{ github.repository }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} ${{ env.PR_USER_MENTIONS }} From eedc99d994763500b8763904aad5eedd991d42c7 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 13:47:59 +0300 Subject: [PATCH 176/188] Test failure --- .github/actions/common/func.sh | 45 +++++++++++++++++++++++++++ .github/workflows/release_publish.yml | 12 +++++++ 2 files changed, 57 insertions(+) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 4c4837000..78f8e3885 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -413,4 +413,49 @@ slack_format_docker_PR_message() { ] } EOF +} + +slack_format_docker_PR_failed_message() { + header=$1 + workflow_url=$2 + footer=$3 + if [ -z "$header" ]; then + header=" " + fi + if [ -z "$footer" ]; then + footer=" " + fi + +# Create Slack message payload + cat << EOF +{ +"icon_emoji": ":redis-circle:", +"text": "$header", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "āŒ $header" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Workflow run: $workflow_url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" + } + ] + } +] +} +EOF } \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 293d94407..8cc2cd786 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -95,6 +95,7 @@ jobs: update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis cd official-images && git diff --color cd - + exit 1 - name: Create pull request to official-images id: create-pr @@ -150,4 +151,15 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ + | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + - name: Send Failure Slack notification + if: failure() + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 8a2aeb0e698d7e18ee89b058e39890a521efa9f4 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:01:24 +0300 Subject: [PATCH 177/188] Fix failure message failure --- .github/actions/common/func.sh | 2 +- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_publish.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 78f8e3885..82c4fed0c 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -415,7 +415,7 @@ slack_format_docker_PR_message() { EOF } -slack_format_docker_PR_failed_message() { +slack_format_failure_message() { header=$1 workflow_url=$2 footer=$3 diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index b799f4e3f..f4900fb56 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -110,4 +110,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 8cc2cd786..93a9f2f2c 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -151,7 +151,7 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" - name: Send Failure Slack notification if: failure() @@ -162,4 +162,4 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ - | curl -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 67a485b8bbacec3c7a4939e67644d64c2879316a Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:14:45 +0300 Subject: [PATCH 178/188] Remove bash stackbrew implementation --- .github/actions/common/func.sh | 288 --------------------------------- test/run-shell-func-tests.sh | 234 --------------------------- 2 files changed, 522 deletions(-) diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh index 82c4fed0c..36473aa4a 100644 --- a/.github/actions/common/func.sh +++ b/.github/actions/common/func.sh @@ -26,171 +26,6 @@ source_helper_file() { fi } -# Lists remote release branches for a specific major version (e.g., release/8.*) -git_ls_remote_major_release_version_branches() { - local remote="$1" - local major_version="$2" - execute_command --no-std -- git ls-remote --heads "$remote" "release/$major_version.*" - if [ -z "$last_cmd_stdout" ]; then - console_output 0 red "Error: No release branches found for major_version=$major_version" - return 1 - fi - echo "$last_cmd_stdout" -} - -# Lists remote tags for a specific major version (e.g., v8.*) -git_ls_remote_tags() { - local remote="$1" - local major_version="$2" - execute_command --no-std -- git ls-remote --refs --tags "$remote" "refs/tags/v$major_version.*" - if [ -z "$last_cmd_stdout" ]; then - console_output 0 red "Error: No tags found for major_version=$major_version" - return 1 - fi - echo "$last_cmd_stdout" -} - -# Filters and sorts release branches by major version in reverse version order -filter_major_release_version_branches() { - local major_version="$1" - while read -r line; do - local ref="$(echo "$line" | awk '{print $2}')" - local commit="$(echo "$line" | awk '{print $1}')" - if echo "$ref" | grep -q "release/$major_version\.[0-9][0-9]*$"; then - echo "$ref $commit" - fi - done | sort -Vr -} - -# Sorts version tags in reverse version order for a specific major version -# stdin: commit ref (git ls-remote) -# stdout: version commit (vX.X.X sha1) - sorted by version -sort_version_tags() { - local major_version="$1" - local version_tag commit ref - while read -r commit ref; do - version_tag="$(echo "$ref" | grep -o "v$major_version\.[0-9][0-9]*\.[0-9][0-9]*.*" || :)" - if [ -z "$version_tag" ]; then - console_output 2 red "Incorrect reference format: $ref" - return 1 - fi - printf "%s %s\n" "$version_tag" "$commit" - done | sort -Vr -} - -# Filters out end-of-life (EOL) versions by skipping entire minor version series marked with -eol suffix -# stdin: version commit (vX.X.X sha1) - must be sorted by version -# stdout: version commit (vX.X.X sha1) -filter_out_eol_versions() { - local major_version="$1" - local version_tag commit - local last_minor="" skip_minor="" minors="" - local major minor patch suffix - local versions - - mapfile -t versions - for line in "${versions[@]}"; do - read -r version_tag commit < <(echo "$line") - IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") - - if [ "$minor" != "$last_minor" ] && [ -n "$last_minor" ]; then - if [ -z "$skip_minor" ]; then - printf "%s" "$minors" - else - console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" - fi - minors="" - skip_minor="" - fi - last_minor="$minor" - - printf -v minors "%s%s\n" "$minors" "$version_tag $commit" - - if echo "$suffix" | grep -qi "-eol$"; then - skip_minor="$minor" - fi - done - if [ -z "$skip_minor" ]; then - printf "%s" "$minors" - else - console_output 2 gray "Skipping minor version $major_version.$last_minor.* due to EOL" - fi -} - -# Filters Redis versions to keep only the latest patch version (and optionally the latest milestone) for each minor version -# stdin: version commit (vX.X.X sha1) - must be sorted by version -# stdout: version commit (vX.X.X sha1) -filter_actual_major_redis_versions() { - local major_version="$1" - local last_minor="" last_is_milestone="" - local ref commit version_tag - console_output 2 gray "filter_actual_major_redis_versions" - while read -r version_tag commit; do - local major minor patch suffix is_milestone - IFS=: read -r major minor patch suffix < <(redis_version_split "$version_tag") - - if [ -n "$suffix" ]; then - is_milestone=1 - else - is_milestone="" - fi - - if [ "$last_minor" = "$minor" ] && [ "$last_is_milestone" = "$is_milestone" ]; then - console_output 2 gray "Skipping $version_tag, already have minor=$last_minor is_milestone=$last_is_milestone" - continue - fi - last_minor="$minor" - last_is_milestone="$is_milestone" - - console_output 2 gray "$version_tag $commit" - echo "$version_tag $commit" - done -} - -# Gets and filters actual Redis versions (tags) from a remote repository for a major version -get_actual_major_redis_versions() { - local remote="$1" - local major_version="$2" - execute_command git_ls_remote_tags "$remote" "$major_version" \ - | execute_command sort_version_tags "$major_version" \ - | execute_command filter_out_eol_versions "$major_version" \ - | execute_command filter_actual_major_redis_versions "$major_version" -} - -# Fetches unshallow refs from a remote repository for the provided list of references -git_fetch_unshallow_refs() { - local remote="$1" - local refs_to_fetch="" - while read -r line; do - local ref="$(echo "$line" | awk '{print $1}')" - refs_to_fetch="$refs_to_fetch $ref" - done - # shellcheck disable=SC2086 - execute_command --no-std -- git_fetch_unshallow "$remote" $refs_to_fetch -} - -# Extracts the distribution name from a Dockerfile's FROM statement (supports Alpine and Debian) -extract_distro_name_from_dockerfile() { - local base_img - base_img="$(grep -m1 -i '^from' | awk '{print $2}')" - - increase_indent_level - console_output 2 gray "Extracting distro from dockerfile" - - if echo "$base_img" | grep -q 'alpine:'; then - distro="$(echo "$base_img" | tr -d ':')" - elif echo "$base_img" | grep -q 'debian:'; then - distro="$(echo "${base_img//-slim/}" | awk -F: '{print $2}')" - else - console_output 0 red "Error: Unknown base image $base_img" - decrease_indent_level - return 1 - fi - console_output 2 gray "distro=$distro" - decrease_indent_level - echo "$distro" -} - # Splits a Redis version string into major:minor:patch:suffix components redis_version_split() { local version @@ -209,129 +44,6 @@ redis_version_split() { printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" } -# Shows a file from a specific git reference (commit/branch/tag) -git_show_file_from_ref() { - local ref=$1 - local file=$2 - execute_command git show "$ref:$file" -} - -# Generates a comma-separated list of Docker tags for a Redis version and distribution -# args: redis_version distro_names is_latest is_default -# is_latest empty for non-latest, otherwise latest -# is_default 1 for default distro, otherwise not default -generate_tags_list() { - local redis_version=$1 - local distro_names=$2 - local is_latest=$3 - local is_default=$4 - - local tags versions - - local major minor patch suffix - IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") - - local mainline_version - mainline_version="$major.$minor" - - versions=("$redis_version") - # generate mainline version tag only for GA releases, e.g 8.2 and 8.2-distro - # tags will be generated only for 8.2.1 but not for 8.2.1-m01 - if [ -z "$suffix" ]; then - versions+=("$mainline_version") - fi - if [ "$is_latest" != "" ]; then - versions+=("$major") - fi - - if [ "$is_default" = 1 ]; then - tags=("${versions[@]}") - fi - - for distro_name in $distro_names; do - for v in "${versions[@]}"; do - tags+=("$v-$distro_name") - done - done - - if [ "$is_latest" != "" ]; then - if [ "$is_default" = 1 ]; then - tags+=("latest") - fi - # shellcheck disable=SC2206 - tags+=($distro_names) - fi - # shellcheck disable=SC2001 - echo "$(IFS=, ; echo "${tags[*]}" | sed 's/,/, /g')" -} - -# Generates stackbrew library content (for specific major version) -# stdin: commit redis_version distro distro_version (sha1 vX.X.X alpine alpine3.21) -generate_stackbrew_library() { - local commit redis_version distro distro_version - local is_latest="" is_latest_unset=1 is_default - - local stackbrew_content="" - - mapfile -t releases - for line in "${releases[@]}"; do - read -r commit redis_version distro distro_version < <(echo "$line") - - local major minor patch suffix - IFS=: read -r major minor patch suffix < <(redis_version_split "$redis_version") - - # assigning latest to the first non milestone (empty suffix) version from top - if [ "$is_latest_unset" = 1 ]; then - if [ -z "$suffix" ]; then - is_latest="$minor" - is_latest_unset="" - fi - elif [ "$is_latest" != "$minor" ]; then - is_latest="" - fi - - if echo "$distro" | grep -q 'alpine'; then - is_default="" - distro_names="$distro $distro_version" - else - is_default=1 - distro_names="$distro_version" - fi - - local tags - tags=$(generate_tags_list "$redis_version" "$distro_names" "$is_latest" "$is_default") - printf -v stackbrew_content "%s%s\n" "$stackbrew_content" "$tags" - done - printf %s "$stackbrew_content" - console_output 2 gray "$stackbrew_content" -} - -# Prepares a list of releases with commit, Redis version, distro, and distro version information -# stdin: redis_version commit -prepare_releases_list() { - local redis_version commit - local debug_output="" version_line - while read -r redis_version commit; do - for distro in debian alpine; do - local dockerfile distro_version redis_version - dockerfile=$(git_show_file_from_ref "$commit" "$distro/Dockerfile") - console_output 3 gray "$dockerfile" - - distro_version=$(echo "$dockerfile" | extract_distro_name_from_dockerfile) - # validate version - redis_version_split "$redis_version" >/dev/null - - printf -v version_line "%s %s %s %s\n" "$commit" "$redis_version" "$distro" "$distro_version" - printf "%s" "$version_line" - printf -v debug_output "%s%s" "$debug_output" "$version_line" - done - done - console_output 2 gray "Final Releases list:" - increase_indent_level - console_output 2 gray "$debug_output" - decrease_indent_level -} - slack_format_docker_image_urls_message() { # Parse the image URLs from JSON array jq --arg release_tag "$1" --arg footer "$2" ' diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh index 1f4c98c94..f772ee9e4 100755 --- a/test/run-shell-func-tests.sh +++ b/test/run-shell-func-tests.sh @@ -10,60 +10,6 @@ set -u init_console_output -test_get_distro_name_from_dockerfile() { - distro_name=$(echo 'FROM alpine:3.22' | extract_distro_name_from_dockerfile) - assertEquals "alpine3.22" "$distro_name" - - distro_name=$(echo 'FROM debian:bookworm-slim' | extract_distro_name_from_dockerfile) - assertEquals "bookworm" "$distro_name" - - distro_name=$(echo 'FROM debian:bookworm' | extract_distro_name_from_dockerfile) - assertEquals "bookworm" "$distro_name" -} - -test_filter_major_release_version_branches() { - local input - input=$(cat < Date: Fri, 19 Sep 2025 14:15:15 +0300 Subject: [PATCH 179/188] Test build failure notification --- .github/workflows/pre-merge.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index f4900fb56..d3740d7e0 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,6 +92,7 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" + exit 1 notify-slack: runs-on: ubuntu-latest @@ -110,4 +111,19 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + notify-slack-when-failed: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && failure() }} + steps: + - name: Send Failure Slack notification + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker Build failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From e172434fcb969519d386edd0fbf58bdbbcd4d237 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:25:38 +0300 Subject: [PATCH 180/188] Checkout code for failure notification --- .github/workflows/pre-merge.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index d3740d7e0..b0b110261 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -118,6 +118,9 @@ jobs: needs: collect-image-urls if: ${{ inputs.release_tag && failure() }} steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Send Failure Slack notification run: | workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" From faa5f5ba509c605b4d695fa36584a41c851f684d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:38:24 +0300 Subject: [PATCH 181/188] Fix tag in build failure message --- .github/workflows/pre-merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index b0b110261..768fe5a1f 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -128,5 +128,5 @@ jobs: . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh - slack_format_failure_message "Docker Build failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ + slack_format_failure_message "Docker Build failed for Redis: ${{ inputs.release_tag || 'unknown'}}" "$workflow_url" "$footer" \ | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file From 9593d64c0cc36351a3799a7b61cee6ff6f29f46d Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:50:21 +0300 Subject: [PATCH 182/188] Return debian/Dockerfile to original state --- debian/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian/Dockerfile b/debian/Dockerfile index ad615c384..0e7d71718 100644 --- a/debian/Dockerfile +++ b/debian/Dockerfile @@ -14,8 +14,8 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV REDIS_DOWNLOAD_URL=https://github.com/redis/redis/archive/refs/tags/8.2.0.tar.gz -ENV REDIS_DOWNLOAD_SHA=c64219bdcba407d18c8dde1fb87b86945aebf75e60f5b44ff463785a962645ed +ENV REDIS_DOWNLOAD_URL=http://download.redis.io/releases/redis-8.2.1.tar.gz +ENV REDIS_DOWNLOAD_SHA=e2c1cb9dd4180a35b943b85dfc7dcdd42566cdbceca37d0d0b14c21731582d3e RUN set -eux; \ \ savedAptMark="$(apt-mark showmanual)"; \ From 67d28c77a53349c2b5e544ace13ed2a0b95715d2 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 14:44:54 +0300 Subject: [PATCH 183/188] Removed failure test --- .github/workflows/pre-merge.yml | 1 - .github/workflows/release_publish.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 768fe5a1f..9190d90ac 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -92,7 +92,6 @@ jobs: fi echo "urls=$urls" >> "$GITHUB_OUTPUT" - exit 1 notify-slack: runs-on: ubuntu-latest diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 93a9f2f2c..6f7e49e89 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -95,7 +95,6 @@ jobs: update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis cd official-images && git diff --color cd - - exit 1 - name: Create pull request to official-images id: create-pr From 7672cccf8b5d5237b05f57723715ac57824fbb26 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:10:14 +0300 Subject: [PATCH 184/188] Align with redis/docker-library-redis --- .../actions/build-and-tag-locally/action.yml | 1 - .../generate-stackbrew-library.sh | 55 ------------------- .github/workflows/pre-merge.yml | 27 +++++---- 3 files changed, 13 insertions(+), 70 deletions(-) delete mode 100755 .github/actions/create-library-pr/generate-stackbrew-library.sh diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index d1e7a67ee..4409fb294 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -1,4 +1,3 @@ - name: Build and Test inputs: diff --git a/.github/actions/create-library-pr/generate-stackbrew-library.sh b/.github/actions/create-library-pr/generate-stackbrew-library.sh deleted file mode 100755 index ed1fffdcc..000000000 --- a/.github/actions/create-library-pr/generate-stackbrew-library.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -# shellcheck disable=SC2034 -last_cmd_stdout="" -# shellcheck disable=SC2034 -last_cmd_stderr="" -# shellcheck disable=SC2034 -last_cmd_result=0 -# shellcheck disable=SC2034 -if [ -z "$VERBOSITY" ]; then - VERBOSITY=1 -fi - -SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" -# shellcheck disable=SC1091 -. "$SCRIPT_DIR/../common/func.sh" - -source_helper_file helpers.sh -source_helper_file github_helpers.sh - -init_console_output - -MAJOR_VERSION="" -REMOTE="origin" -while [[ $# -gt 0 ]]; do - case $1 in - --major-version) - MAJOR_VERSION=$2 - shift - shift - ;; - --remote) - REMOTE=$2 - shift - shift - ;; - *) - echo "Error: Unknown option $1" - exit 1 - ;; - esac -done - -if [ -z "$MAJOR_VERSION" ]; then - echo "Error: --major-version M is required as argument" - exit 1 -fi - -set -u -redis_versions=$(get_actual_major_redis_versions "$REMOTE" "$MAJOR_VERSION") -echo "$redis_versions" | git_fetch_unshallow_refs "$REMOTE" -echo "$redis_versions" | prepare_releases_list | generate_stackbrew_library - diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 9190d90ac..5b1b91cb6 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -18,25 +18,24 @@ on: jobs: build-and-test: - #runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} - runs-on: ["${{ contains(matrix.platform, 'arm64') && 'ARM64' || 'X64' }}", "self-hosted"] + runs-on: ${{ contains(matrix.platform, 'arm64') && 'ubuntu24-arm64-2-8' || 'ubuntu-latest' }} strategy: fail-fast: false matrix: distribution: - debian - # - alpine + - alpine platform: - linux/amd64 - # - linux/i386 - # - linux/arm/v5 - # - linux/arm/v6 - # - linux/arm/v7 - # - linux/mips64le - # - linux/ppc64le - # - linux/s390x + - linux/i386 + - linux/arm/v5 + - linux/arm/v6 + - linux/arm/v7 + - linux/mips64le + - linux/ppc64le + - linux/s390x - linux/arm64 - # - linux/riscv64 + - linux/riscv64 exclude: - distribution: alpine platform: linux/mips64le @@ -59,10 +58,10 @@ jobs: with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} - registry_username: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && github.actor || vars.REGISTRY_USERNAME }} - registry_password: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && secrets.GITHUB_TOKEN || secrets.REGISTRY_PASSWORD }} + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY == 'ghcr.io' && format('ghcr.io/{0}', github.repository) || vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ format('ghcr.io/{0}', github.repository) }} release_tag: ${{ inputs.release_tag }} collect-image-urls: From dc3a92dab5baf1dae666763cb5b9768233a5bb95 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:11:59 +0300 Subject: [PATCH 185/188] Gitignore for release-automation --- release-automation/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 release-automation/.gitignore diff --git a/release-automation/.gitignore b/release-automation/.gitignore new file mode 100644 index 000000000..932765aeb --- /dev/null +++ b/release-automation/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +venv From 326142f45cf9eeb1c98a0041931dd4893935bddd Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:17:15 +0300 Subject: [PATCH 186/188] Removed draft warning and returned official target and user mentions --- .github/workflows/release_publish.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 6f7e49e89..f21dba5ff 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -13,10 +13,10 @@ on: required: false env: - #TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images - #PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" + PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. @@ -108,9 +108,6 @@ jobs: commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" body: | - THIS IS A DRAFT PR FOR AUTOMATION TESTING - PLEASE IGNORE IT AND SORRY FOR THE BOTHERING - Automated update for Redis ${{ steps.parse-release.outputs.release_version }} Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} From 03cfe48b8f183fdc0bf235cbb9ccea59bc5edc8f Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 15:22:33 +0300 Subject: [PATCH 187/188] Comment out duplicate variables --- .github/workflows/release_publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index f21dba5ff..429dd7cd6 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -14,10 +14,10 @@ on: env: TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images - TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" - PR_USER_MENTIONS: "" + #PR_USER_MENTIONS: "" # UUID is used to help automation to identify workflow run in the list of workflow runs. run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" From 8a94c07f90ba9dad648fc2249fcad29ef0dbbdd0 Mon Sep 17 00:00:00 2001 From: Petar Shtuchkin Date: Fri, 19 Sep 2025 19:23:12 +0300 Subject: [PATCH 188/188] Fix url for redis-oss-release-automation --- .github/actions/apply-docker-version/action.yml | 2 +- .github/workflows/pre-merge.yml | 2 +- .github/workflows/release_build_and_test.yml | 8 ++++---- .github/workflows/release_publish.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml index 07cf7b3ec..97dd05fb0 100644 --- a/.github/actions/apply-docker-version/action.yml +++ b/.github/actions/apply-docker-version/action.yml @@ -17,7 +17,7 @@ runs: - name: Checkout common functions uses: actions/checkout@v4 with: - repository: redis/redis-oss-release-automation + repository: redis-developer/redis-oss-release-automation ref: main path: redis-oss-release-automation diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 5b1b91cb6..f4959d046 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -50,7 +50,7 @@ jobs: uses: actions/checkout@v4 - name: Ensure release branch if: ${{ inputs.release_tag }} - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ inputs.release_tag }} gh_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml index b4ac6a29a..bfbdfe31a 100644 --- a/.github/workflows/release_build_and_test.yml +++ b/.github/workflows/release_build_and_test.yml @@ -32,13 +32,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Validate Redis Release Archive - uses: redis/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main + uses: redis-developer/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main with: release_tag: ${{ github.event.inputs.release_tag }} - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: true @@ -68,7 +68,7 @@ jobs: - name: Ensure Release Branch id: ensure-branch - uses: redis/redis-oss-release-automation/.github/actions/ensure-release-branch@main + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main with: release_tag: ${{ github.event.inputs.release_tag }} allow_modify: false @@ -76,7 +76,7 @@ jobs: - name: Merge back to release branch id: merge-back - uses: redis/redis-oss-release-automation/.github/actions/merge-branches-verified@main + uses: redis-developer/redis-oss-release-automation/.github/actions/merge-branches-verified@main with: from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} to_branch: ${{ steps.ensure-branch.outputs.release_branch }} diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml index 429dd7cd6..53656054f 100644 --- a/.github/workflows/release_publish.yml +++ b/.github/workflows/release_publish.yml @@ -66,7 +66,7 @@ jobs: echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT - name: Create version tag - uses: redis/redis-oss-release-automation/.github/actions/create-tag-verified@main + uses: redis-developer/redis-oss-release-automation/.github/actions/create-tag-verified@main with: tag: v${{ steps.parse-release.outputs.release_version }} ref: ${{ steps.parse-release.outputs.release_commit_sha }}