diff --git a/.github/actions/apply-docker-version/action.yml b/.github/actions/apply-docker-version/action.yml new file mode 100644 index 000000000..97dd05fb0 --- /dev/null +++ b/.github/actions/apply-docker-version/action.yml @@ -0,0 +1,36 @@ +inputs: + release_tag: + description: 'Release tag to build' + required: true + release_version_branch: + description: 'Release version branch to commit to' + required: true + +outputs: + changed_files: + description: 'List of files that were modified' + value: ${{ steps.apply-version.outputs.changed_files }} + +runs: + using: "composite" + steps: + - name: Checkout common functions + uses: actions/checkout@v4 + with: + repository: redis-developer/redis-oss-release-automation + ref: main + path: redis-oss-release-automation + + - name: Apply docker version + id: apply-version + shell: bash + run: | + ${{ github.action_path }}/apply-docker-version.sh ${{ inputs.release_tag }} + + - name: Create verified commit + if: steps.apply-version.outputs.changed_files != '' + uses: iarekylew00t/verified-bot-commit@v1 + with: + message: ${{ inputs.release_tag }} + files: ${{ steps.apply-version.outputs.changed_files }} + ref: ${{ inputs.release_version_branch }} diff --git a/.github/actions/apply-docker-version/apply-docker-version.sh b/.github/actions/apply-docker-version/apply-docker-version.sh new file mode 100755 index 000000000..0078b05cb --- /dev/null +++ b/.github/actions/apply-docker-version/apply-docker-version.sh @@ -0,0 +1,104 @@ +#!/bin/bash +set -e + +# This script updates Redis version in Dockerfiles using environment variables +# REDIS_ARCHIVE_URL and REDIS_ARCHIVE_SHA, then commits changes if any were made. + +# shellcheck disable=SC2034 +last_cmd_stdout="" +# shellcheck disable=SC2034 +last_cmd_stderr="" +# shellcheck disable=SC2034 +last_cmd_result=0 +# shellcheck disable=SC2034 +VERBOSITY=1 + + + +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../common/func.sh" + +source_helper_file helpers.sh + +# Input TAG is expected in $1 +TAG="$1" + +if [ -z "$TAG" ]; then + echo "Error: TAG is required as first argument" + exit 1 +fi + +# Check if required environment variables are set +if [ -z "$REDIS_ARCHIVE_URL" ]; then + echo "Error: REDIS_ARCHIVE_URL environment variable is not set" + exit 1 +fi + +if [ -z "$REDIS_ARCHIVE_SHA" ]; then + echo "Error: REDIS_ARCHIVE_SHA environment variable is not set" + exit 1 +fi + +echo "TAG: $TAG" +echo "REDIS_ARCHIVE_URL: $REDIS_ARCHIVE_URL" +echo "REDIS_ARCHIVE_SHA: $REDIS_ARCHIVE_SHA" + +# Function to update Dockerfile +update_dockerfile() { + local dockerfile="$1" + local updated=false + + if [ ! -f "$dockerfile" ]; then + echo "Warning: $dockerfile not found, skipping" + return 1 + fi + + echo "Updating $dockerfile..." + + # Update REDIS_DOWNLOAD_URL + if grep -q "^ENV REDIS_DOWNLOAD_URL=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_URL=.*|ENV REDIS_DOWNLOAD_URL=$REDIS_ARCHIVE_URL|" "$dockerfile" + else + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_URL not found" + return 1 + fi + + + # Update REDIS_DOWNLOAD_SHA + if grep -q "^ENV REDIS_DOWNLOAD_SHA=" "$dockerfile"; then + sed -i "s|^ENV REDIS_DOWNLOAD_SHA=.*|ENV REDIS_DOWNLOAD_SHA=$REDIS_ARCHIVE_SHA|" "$dockerfile" + else + echo "Cannot update $dockerfile, ENV REDIS_DOWNLOAD_SHA not found" + return 1 + fi +} + +docker_files=("debian/Dockerfile" "alpine/Dockerfile") +# Track which files were modified +changed_files=() + +for dockerfile in "${docker_files[@]}"; do + update_dockerfile "$dockerfile" +done + +changed_files=($(git diff --name-only "${docker_files[@]}")) + +# Output the list of changed files for GitHub Actions +if [ ${#changed_files[@]} -gt 0 ]; then + echo "Files were modified:" + printf '%s\n' "${changed_files[@]}" + + # Set GitHub Actions output + changed_files_output=$(printf '%s\n' "${changed_files[@]}") + { + echo "changed_files<> "$GITHUB_OUTPUT" + + echo "Changed files output set for next step" +else + echo "No files were modified" + echo "changed_files=" >> "$GITHUB_OUTPUT" +fi \ No newline at end of file diff --git a/.github/actions/build-and-tag-locally/action.yml b/.github/actions/build-and-tag-locally/action.yml index 35f234b2f..4409fb294 100644 --- a/.github/actions/build-and-tag-locally/action.yml +++ b/.github/actions/build-and-tag-locally/action.yml @@ -19,6 +19,9 @@ inputs: registry_repository: description: 'Repository to push the image to' required: false + release_tag: + description: 'Release tag to build' + required: false runs: using: "composite" @@ -95,14 +98,14 @@ runs: load: true platforms: ${{ inputs.platform }} tags: ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=gha,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + cache-to: type=gha,mode=max,scope=${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} - name: Save image shell: bash run: | docker save -o /tmp/image-${{ steps.platform.outputs.display_name }}.tar ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Upload image uses: actions/upload-artifact@v4 with: @@ -115,7 +118,7 @@ runs: if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} run: | docker run -d --name sanity-test-${{ steps.platform.outputs.display_name }} ${{ github.sha }}:${{ steps.platform.outputs.display_name }} - + - name: Container Logs if: ${{ contains(fromJSON('["amd64", "i386", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -128,7 +131,7 @@ runs: run: | docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli ping docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli info server - + - name: Verify installed modules if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -148,7 +151,7 @@ runs: echo "The following modules are missing: ${missing_modules[*]}" exit 1 fi - + - name: Test RedisBloom if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -158,7 +161,7 @@ runs: [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:hash")" = "1" ] || { echo "RedisBloom test failed: 'redis:hash' not found"; exit 1; } [ "$(docker exec sanity-test-${{ steps.platform.outputs.display_name }} redis-cli BF.EXISTS popular_keys "redis:list")" = "0" ] || { echo "RedisBloom test failed: 'redis:list' found unexpectedly"; exit 1; } echo "RedisBloom test passed successfully" - + - name: Test RediSearch if: ${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} shell: bash @@ -224,12 +227,44 @@ runs: path: test/report-entrypoint.xml reporter: java-junit + - name: Format registry tag + id: format-registry-tag + shell: bash + run: | + printf "tag=%s:%s%s-%s-%s" \ + "${{ inputs.registry_repository }}" \ + "${{ inputs.release_tag != '' && format('{0}-', inputs.release_tag || '') }}" \ + "${{ github.sha }}" \ + "${{ inputs.distribution }}" \ + "${{ steps.platform.outputs.display_name }}" \ + | tr '[:upper:]' '[:lower:]' >> "$GITHUB_OUTPUT" + - name: Push image uses: docker/build-push-action@v6 if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} with: context: ${{ inputs.distribution }} push: true - tags: ${{ inputs.registry_repository }}:${{ github.sha }}-${{ inputs.distribution }} + tags: ${{ steps.format-registry-tag.outputs.tag }} cache-from: type=gha cache-to: type=gha,mode=max + + - name: Save image URL to artifact + shell: bash + run: | + if [[ "${{ inputs.publish_image }}" == "true" && "${{ contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }}" == "true" ]]; then + # Create a file with the image URL for this specific build + mkdir -p /tmp/image-urls + echo "${{ steps.format-registry-tag.outputs.tag }}" > "/tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt" + echo "Image URL saved: ${{ steps.format-registry-tag.outputs.tag }}" + else + echo "Image not published for this platform/distribution combination" + fi + + - name: Upload image URL artifact + uses: actions/upload-artifact@v4 + if: ${{ inputs.publish_image == 'true' && contains(fromJSON('["amd64", "arm64"]'), steps.platform.outputs.display_name) }} + with: + name: image-url-${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }} + path: /tmp/image-urls/${{ inputs.distribution }}-${{ steps.platform.outputs.display_name }}.txt + retention-days: 1 diff --git a/.github/actions/common/func.sh b/.github/actions/common/func.sh new file mode 100644 index 000000000..36473aa4a --- /dev/null +++ b/.github/actions/common/func.sh @@ -0,0 +1,173 @@ +#!/bin/bash + +# Sources a helper file from multiple possible locations (GITHUB_WORKSPACE, RELEASE_AUTOMATION_DIR, or relative path) +source_helper_file() { + local helper_file="$1" + local helper_errors="" + for dir in "GITHUB_WORKSPACE:$GITHUB_WORKSPACE/redis-oss-release-automation" "RELEASE_AUTOMATION_DIR:$RELEASE_AUTOMATION_DIR" ":../redis-oss-release-automation"; do + local var_name="${dir%%:*}" + local dir="${dir#*:}" + if [ -n "$var_name" ]; then + var_name="\$$var_name" + fi + local helper_path="$dir/.github/actions/common/$helper_file" + if [ -f "$helper_path" ]; then + helper_errors="" + # shellcheck disable=SC1090 + . "$helper_path" + break + else + helper_errors=$(printf "%s\n %s: %s" "$helper_errors" "$var_name" "$helper_path") + fi + done + if [ -n "$helper_errors" ]; then + echo "Error: $helper_file not found in any of the following locations: $helper_errors" >&2 + exit 1 + fi +} + +# Splits a Redis version string into major:minor:patch:suffix components +redis_version_split() { + local version + local numerics + # shellcheck disable=SC2001 + version=$(echo "$1" | sed 's/^v//') + + numerics=$(echo "$version" | grep -Po '^[1-9][0-9]*\.[0-9]+(\.[0-9]+|)' || :) + if [ -z "$numerics" ]; then + console_output 2 red "Cannot split version '$version', incorrect version format" + return 1 + fi + local major minor patch suffix + IFS=. read -r major minor patch < <(echo "$numerics") + suffix=${version:${#numerics}} + printf "%s:%s:%s:%s\n" "$major" "$minor" "$patch" "$suffix" +} + +slack_format_docker_image_urls_message() { + # Parse the image URLs from JSON array + jq --arg release_tag "$1" --arg footer "$2" ' + map( + capture("(?(?[^:]+:)(?[^-]+)-(?[a-f0-9]+)-(?[^-]+)-(?[^-]+))$") + ) + as $items + | { + icon_emoji: ":redis-circle:", + text: ("đŸŗ Docker Images Published for Redis: " + $release_tag), + blocks: [ + { + "type": "header", + "text": { "type": "plain_text", "text": ("đŸŗ Docker Images Published for Release " + $release_tag) } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": ( + "The following Docker images have been published to Github Container Registry:\n\n" + + ( + $items + | map( + "Distribution: *" + .distro + "* " + + "Architecture: *" + .arch + "*" + + "\n```\n" + .url + "\n```" + ) + | join("\n\n") + ) + ) + } + }, + { + "type": "context", + "elements": [ + { "type": "mrkdwn", "text": $footer } + ] + } + ] + } + ' +} + +slack_format_docker_PR_message() { + release_tag=$1 + url=$2 + footer=$3 + +# Create Slack message payload + cat << EOF +{ +"icon_emoji": ":redis-circle:", +"text": "đŸŗ Docker Library PR created for Redis: $release_tag", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "đŸŗ Docker Library PR created for Redis: $release_tag" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "$url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" + } + ] + } +] +} +EOF +} + +slack_format_failure_message() { + header=$1 + workflow_url=$2 + footer=$3 + if [ -z "$header" ]; then + header=" " + fi + if [ -z "$footer" ]; then + footer=" " + fi + +# Create Slack message payload + cat << EOF +{ +"icon_emoji": ":redis-circle:", +"text": "$header", +"blocks": [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "❌ $header" + } + }, + { + "type": "section", + "text": { + "type": "mrkdwn", + "text": "Workflow run: $workflow_url" + } + }, + { + "type": "context", + "elements": [ + { + "type": "mrkdwn", + "text": "$footer" + } + ] + } +] +} +EOF +} \ No newline at end of file diff --git a/.github/workflows/build_release_automation.yml b/.github/workflows/build_release_automation.yml new file mode 100644 index 000000000..a9c0ff005 --- /dev/null +++ b/.github/workflows/build_release_automation.yml @@ -0,0 +1,110 @@ +name: Build Release Automation Docker Image + +on: + workflow_dispatch: + inputs: + image_tag: + description: 'Docker image tag (default: latest)' + required: false + default: 'latest' + push_to_ghcr: + description: 'Push image to GHCR' + required: false + default: true + type: boolean + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}/release-automation + +jobs: + build-test-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=${{ github.event.inputs.image_tag }} + type=raw,value=latest,enable={{is_default_branch}} + type=sha,prefix={{branch}}- + + - name: Build Docker image (without pushing) + uses: docker/build-push-action@v5 + with: + context: ./release-automation + file: ./release-automation/docker/Dockerfile + push: false + tags: test-image:latest + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # Integration tests do need access to git repository + - name: Test the built image + run: | + # Start container and install dev dependencies for testing + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + --entrypoint /bin/bash \ + test-image:latest \ + -c " + cd release-automation + set -e + echo '=== Installing test dependencies ===' + pip install pytest pytest-cov + echo '=== Running tests ===' + pytest -v tests/ + " + + - name: Log in to Container Registry + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Tag and push image + if: ${{ github.event.inputs.push_to_ghcr == 'true' }} + run: | + # Tag the tested image with the proper tags + echo '${{ steps.meta.outputs.tags }}' | while read -r tag; do + docker tag test-image:latest "$tag" + docker push "$tag" + done + + - name: Output image details + run: | + echo "## Docker Image Built Successfully! đŸŗ" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "✅ **Tests passed**" >> $GITHUB_STEP_SUMMARY + echo "đŸ—ī¸ **Production image built** (without dev dependencies)" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "**Image:** \`${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}\`" >> $GITHUB_STEP_SUMMARY + echo "**Tags:**" >> $GITHUB_STEP_SUMMARY + echo '${{ steps.meta.outputs.tags }}' | sed 's/^/- `/' | sed 's/$/`/' >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + if [[ "${{ github.event.inputs.push_to_ghcr }}" == "true" ]]; then + echo "✅ **Image pushed to GHCR**" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo "To pull the image:" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`bash" >> $GITHUB_STEP_SUMMARY + echo "docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.event.inputs.image_tag }}" >> $GITHUB_STEP_SUMMARY + echo "\`\`\`" >> $GITHUB_STEP_SUMMARY + else + echo "â„šī¸ **Image built locally only (not pushed)**" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/pre-merge.yml b/.github/workflows/pre-merge.yml index 2b0a81525..f4959d046 100644 --- a/.github/workflows/pre-merge.yml +++ b/.github/workflows/pre-merge.yml @@ -1,12 +1,20 @@ + name: Build and Test on: pull_request: branches: - master - release/* - push: - branches: - - release/8.2 + workflow_call: + inputs: + release_tag: + description: 'Release tag to build' + required: true + type: string + outputs: + docker_image_urls: + description: 'Array of Docker image URLs that were published' + value: ${{ jobs.collect-image-urls.outputs.docker_image_urls }} jobs: build-and-test: @@ -40,11 +48,83 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 + - name: Ensure release branch + if: ${{ inputs.release_tag }} + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main + with: + release_tag: ${{ inputs.release_tag }} + gh_token: ${{ secrets.GITHUB_TOKEN }} - uses: ./.github/actions/build-and-tag-locally with: distribution: ${{ matrix.distribution }} platform: ${{ matrix.platform }} - registry_username: ${{ vars.REGISTRY_USERNAME }} - registry_password: ${{ secrets.REGISTRY_PASSWORD }} + registry_username: ${{ github.actor }} + registry_password: ${{ secrets.GITHUB_TOKEN }} publish_image: ${{ vars.PUBLISH_IMAGE }} - registry_repository: ${{ vars.REGISTRY_REPOSITORY }} + registry_repository: ${{ format('ghcr.io/{0}', github.repository) }} + release_tag: ${{ inputs.release_tag }} + + collect-image-urls: + runs-on: ubuntu-latest + needs: build-and-test + if: ${{ inputs.release_tag }} + outputs: + docker_image_urls: ${{ steps.collect-urls.outputs.urls }} + steps: + - name: Download all image URL artifacts + uses: actions/download-artifact@v4 + with: + pattern: image-url-* + path: ./image-urls + merge-multiple: true + + - name: Collect image URLs from artifacts + id: collect-urls + run: | + if [ -d "./image-urls" ] && [ "$(ls -A ./image-urls 2>/dev/null)" ]; then + echo "Found image URL files:" + urls=$(find ./image-urls -name "*.txt" -exec cat {} \; | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "Collected image URLs: $urls" + else + echo "No image URL artifacts found" + urls="[]" + fi + + echo "urls=$urls" >> "$GITHUB_OUTPUT" + + notify-slack: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && needs.collect-image-urls.outputs.docker_image_urls != '[]' }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Send Slack notification + run: | + image_urls='${{ needs.collect-image-urls.outputs.docker_image_urls }}' + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + echo "$image_urls" | slack_format_docker_image_urls_message "${{ inputs.release_tag }}" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + notify-slack-when-failed: + runs-on: ubuntu-latest + needs: collect-image-urls + if: ${{ inputs.release_tag && failure() }} + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Send Failure Slack notification + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker Build failed for Redis: ${{ inputs.release_tag || 'unknown'}}" "$workflow_url" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/.github/workflows/release_build_and_test.yml b/.github/workflows/release_build_and_test.yml new file mode 100644 index 000000000..bfbdfe31a --- /dev/null +++ b/.github/workflows/release_build_and_test.yml @@ -0,0 +1,124 @@ +# This workflow is a part of release automation process. +# It is intended to be run with workflow_dispatch event by the automation. + +# Warning: Workflow does switch branches and this may lead to confusion when changing workflow actions. +# The usual safety rule is to make changes to workflow or actions in base branch (e.g, release/8.X) +# Version branches (e.g, 8.0.10-rc5-int8) will merge changes from base branch automatically. +on: + workflow_dispatch: + inputs: + release_tag: + description: 'Release tag to build' + required: true + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false + +# UUID is used to help automation to identify workflow run in the list of workflow runs. +run-name: "Release Build and Test${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" + +jobs: + prepare-release: + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Log in to Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Validate Redis Release Archive + uses: redis-developer/redis-oss-release-automation/.github/actions/validate-redis-release-archive@main + with: + release_tag: ${{ github.event.inputs.release_tag }} + + - name: Ensure Release Branch + id: ensure-branch + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main + with: + release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: true + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Apply Docker Version + id: apply-version + uses: ./.github/actions/apply-docker-version + with: + release_tag: ${{ github.event.inputs.release_tag }} + release_version_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} + + build-and-test: + uses: ./.github/workflows/pre-merge.yml + needs: prepare-release + secrets: inherit + with: + release_tag: ${{ github.event.inputs.release_tag }} + + merge-back-to-release-branch: + needs: [prepare-release, build-and-test] + if: success() + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Ensure Release Branch + id: ensure-branch + uses: redis-developer/redis-oss-release-automation/.github/actions/ensure-release-branch@main + with: + release_tag: ${{ github.event.inputs.release_tag }} + allow_modify: false + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Merge back to release branch + id: merge-back + uses: redis-developer/redis-oss-release-automation/.github/actions/merge-branches-verified@main + with: + from_branch: ${{ steps.ensure-branch.outputs.release_version_branch }} + to_branch: ${{ steps.ensure-branch.outputs.release_branch }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Create release handle JSON + shell: bash + run: | + if [ -n "${{ steps.merge-back.outputs.merge_commit_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.merge_commit_sha }}" + elif [ -n "${{ steps.merge-back.outputs.target_before_merge_sha }}" ]; then + RELEASE_COMMIT_SHA="${{ steps.merge-back.outputs.target_before_merge_sha }}" + else + echo "Error: No commit SHA found, both merge_commit_sha and target_before_merge_sha are empty" >&2 + exit 1 + fi + + # Get docker image URLs from build-and-test job + DOCKER_IMAGE_URLS='${{ needs.build-and-test.outputs.docker_image_urls }}' + + # Validate that DOCKER_IMAGE_URLS is valid JSON + if ! echo "$DOCKER_IMAGE_URLS" | jq . > /dev/null 2>&1; then + echo "Warning: docker_image_urls is not valid JSON, using empty array" + DOCKER_IMAGE_URLS="[]" + fi + + cat > release_handle.json << EOF + { + "release_commit_sha": "$RELEASE_COMMIT_SHA", + "release_version": "${{ github.event.inputs.release_tag }}", + "release_version_branch": "${{ steps.ensure-branch.outputs.release_version_branch }}", + "release_branch": "${{ steps.ensure-branch.outputs.release_branch }}", + "docker_image_urls": $DOCKER_IMAGE_URLS + } + EOF + + echo "Created release_handle.json:" + cat release_handle.json + + - name: Upload release handle artifact + uses: actions/upload-artifact@v4 + with: + name: release_handle + path: release_handle.json + retention-days: 400 \ No newline at end of file diff --git a/.github/workflows/release_publish.yml b/.github/workflows/release_publish.yml new file mode 100644 index 000000000..53656054f --- /dev/null +++ b/.github/workflows/release_publish.yml @@ -0,0 +1,161 @@ +# This workflow publishes a release by creating a version tag. +# It is intended to be run with workflow_dispatch event by the automation. + +on: + workflow_dispatch: + inputs: + release_handle: + description: 'Release handle JSON string containing release information' + required: true + type: string + workflow_uuid: + description: 'Optional UUID to identify this workflow run' + required: false + +env: + TARGET_OFFICIAL_IMAGES_REPO: docker-library/official-images + #TARGET_OFFICIAL_IMAGES_REPO: Peter-Sh/official-images + FORKED_OFFICIAL_IMAGES_REPO: redis-developer/docker-library-official-images + PR_USER_MENTIONS: "@adamiBs @yossigo @adobrzhansky @maxb-io @dagansandler @Peter-Sh" + #PR_USER_MENTIONS: "" + +# UUID is used to help automation to identify workflow run in the list of workflow runs. +run-name: "Release Publish${{ github.event.inputs.workflow_uuid && format(': {0}', github.event.inputs.workflow_uuid) || '' }}" + +jobs: + publish-release: + runs-on: ["ubuntu-latest"] + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Parse release handle and validate + id: parse-release + shell: bash + run: | + # Parse the JSON input + RELEASE_HANDLE='${{ github.event.inputs.release_handle }}' + echo "Parsing release handle JSON:" + echo "$RELEASE_HANDLE" | jq . + + # Extract release_commit_sha + RELEASE_COMMIT_SHA=$(echo "$RELEASE_HANDLE" | jq -r '.release_commit_sha // empty') + + # Validate that release_commit_sha exists and is not empty + if [ -z "$RELEASE_COMMIT_SHA" ] || [ "$RELEASE_COMMIT_SHA" = "null" ]; then + echo "Error: release_commit_sha is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + # Extract release_version for tag creation + RELEASE_VERSION=$(echo "$RELEASE_HANDLE" | jq -r '.release_version // empty') + + if [ -z "$RELEASE_VERSION" ] || [ "$RELEASE_VERSION" = "null" ]; then + echo "Error: release_version is missing or empty in release_handle" + echo "Release handle content: $RELEASE_HANDLE" + exit 1 + fi + + echo "Successfully parsed release handle:" + echo " release_commit_sha: $RELEASE_COMMIT_SHA" + echo " release_version: $RELEASE_VERSION" + + # Set outputs for next steps + echo "release_commit_sha=$RELEASE_COMMIT_SHA" >> $GITHUB_OUTPUT + echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT + + - name: Create version tag + uses: redis-developer/redis-oss-release-automation/.github/actions/create-tag-verified@main + with: + tag: v${{ steps.parse-release.outputs.release_version }} + ref: ${{ steps.parse-release.outputs.release_commit_sha }} + gh_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout official-images repo + uses: actions/checkout@v4 + with: + path: official-images + repository: ${{ env.TARGET_OFFICIAL_IMAGES_REPO }} + + - name: Generate stackbrew library content + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Extract major version from release version (e.g., "8.2.1" -> "8") + MAJOR_VERSION=$(echo "${{ steps.parse-release.outputs.release_version }}" | cut -d. -f1) + echo "Major version: $MAJOR_VERSION" + + # Generate updated stackbrew content using the release automation Docker image + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + -e FORCE_COLOR=1 \ + $(echo "ghcr.io/${{ github.repository }}/release-automation:latest" | tr '[:upper:]' '[:lower:]') \ + update-stackbrew-file $MAJOR_VERSION --input official-images/library/redis --output official-images/library/redis + cd official-images && git diff --color + cd - + + - name: Create pull request to official-images + id: create-pr + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.GH_TOKEN_FOR_PR }} + draft: true + push-to-fork: ${{ env.FORKED_OFFICIAL_IMAGES_REPO }} + path: official-images + branch: redis-${{ steps.parse-release.outputs.release_version }} + commit-message: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + title: "Redis: Update to ${{ steps.parse-release.outputs.release_version }}" + body: | + Automated update for Redis ${{ steps.parse-release.outputs.release_version }} + + Release commit: ${{ steps.parse-release.outputs.release_commit_sha }} + Release tag: v${{ steps.parse-release.outputs.release_version }} + Compare: ${{ github.server_url }}/${{ github.repository }}/compare/v${{ steps.parse-release.outputs.release_version }}^1...v${{ steps.parse-release.outputs.release_version }} + + ${{ env.PR_USER_MENTIONS }} + + - name: PR creation results + run: | + echo "Pull Request Number: ${{ steps.create-pr.outputs.pull-request-number }}" + echo "Pull Request URL: ${{ steps.create-pr.outputs.pull-request-url }}" + + # Create release_info.json artifact + cat > release_info.json << EOF + { + "pull_request_number": "${{ steps.create-pr.outputs.pull-request-number }}", + "pull_request_url": "${{ steps.create-pr.outputs.pull-request-url }}" + } + EOF + + echo "Created release_info.json:" + cat release_info.json + + - name: Upload release info artifact + uses: actions/upload-artifact@v4 + with: + name: release_info + path: release_info.json + retention-days: 400 + + - name: Send Slack notification + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\` | View: <$workflow_url|workflow run>" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_docker_PR_message "${{ steps.parse-release.outputs.release_version }}" "${{ steps.create-pr.outputs.pull-request-url }}" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" + + - name: Send Failure Slack notification + if: failure() + run: | + workflow_url="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" + footer="Repository: ${{ github.repository }} | Commit: \`${{ github.sha }}\`" + + . ${GITHUB_WORKSPACE}/.github/actions/common/func.sh + + slack_format_failure_message "Docker PR failed for Redis: ${{ steps.parse-release.outputs.release_version || 'unknown'}}" "$workflow_url" "$footer" \ + | curl -s --fail-with-body -d@- "${{ secrets.SLACK_WEB_HOOK_URL }}" \ No newline at end of file diff --git a/release-automation/.gitignore b/release-automation/.gitignore new file mode 100644 index 000000000..932765aeb --- /dev/null +++ b/release-automation/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +venv diff --git a/release-automation/README.md b/release-automation/README.md new file mode 100644 index 000000000..3e07a6d1e --- /dev/null +++ b/release-automation/README.md @@ -0,0 +1,88 @@ +# Docker release process and release automation description + +This readme covers relase process for versions 8 and above. + +In version 8 the docker-library structure has changed. Static Dockerfiles are used instead of templating. Versions live in a different mainline branches and are marked with tags. + +The docker release process goal is to create a PR in official-docker library for library/redis file. + +library/redis stackbrew file should reflect the tags in redis/docker-library-redis repository. + +## Branches and tags + +Mainline branches are named `release/Major.Minor` (e.g. `release/8.2`) + +Each version release is tagged with `vMajor.Minor.Patch` (e.g. `v8.2.1`) + +Milestone releases are tagged with `vMajor.Minor.Patch-Milestone` (e.g. `v8.2.1-m01`). Any suffix after patch version is considered a milestone. + +Suffixes starting with `rc` are considered release candidates and are preferred over suffixes starting with `m` which in turn are preferred over any other suffix. + +Tags without suffix are considered GA (General Availability) releases (e.g. `v8.2.1`). + +Internal releases are milestone releases containing `-int` in their name (e.g. `v8.2.1-m01-int1` or `8.4.0-int3`). They are not released to the public. + +Milestone releases never get latest or any other default tags, like `8`, `8.2`, `8.2.1`, `latest`, `bookworm`, etc. + +For each mainline only one GA release and optionally any number of milestone releases with higher versions than this GA may be published in official-library. + +Each patch version may have only one GA or milestone release, GA release is preferred over milestone release. + +For example for this list of tags the following rules will be applied + +* `v8.2.3-m01` - included because there is neither GA nor any higher milestone versions for 8.2.3 +* `v8.2.2-rc2` - included because it higher version among 8.2.2 +* `v8.2.2-rc1` - excluded because 8.2.2-rc2 is higher version +* `v8.2.2-m01` - excluded because 8.2.2-rc2 is higher version +* `v8.2.1-rc2` - excluded because there is 8.2.1 GA version +* `v8.2.1` - included because it is highest GA for 8.2 +* `v8.2.0` - exluded because 8.2.1 is higher version + +End of life versions are marked with `-eol` suffix (e.g. `v8.0.3-eol`). When there is a at least one minor version tagged with eol all versions in this minor series are considered EOL and are not included in the release file. + +## Creating a release manually + +This process is automated using github workflows. However, it's useful to understand the manual process. + +Determine a mainline branch, e.g `release/8.2` for version `8.2.2`. + +Optionally create a release branch from the mainline branch, e.g. `8.2.2`. + +Modify dockerfiles. + +Test dockerfiles. + +If release branch was created, merge it back to mainline branch. + +Tag commit with `vMajor.Minor.Patch` (e.g. `v8.2.1`) in the mainline branch. + +Push your changes to redis/docker-library-redis repository. + +Create a PR to official-library refering the tag and commit you created. + + +# Release automation tool + +Release automation tool is used to generate library/redis file for official-library. It uses origin repository as a source of truth and follows the process described above. + +## Installation + +### From Source + +```bash +cd release-automation +pip install -e . +``` + +### Development Installation + +```bash +cd release-automation +pip install -e ".[dev]" +``` + +## Usage + +```bash +release-automation --help +``` diff --git a/release-automation/docker/Dockerfile b/release-automation/docker/Dockerfile new file mode 100644 index 000000000..2a6f061c2 --- /dev/null +++ b/release-automation/docker/Dockerfile @@ -0,0 +1,13 @@ +FROM python:3.11-slim-trixie + +RUN apt update && apt -y install git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* \ + # avoid dubious permissions problem in github CI + && git config --global --add safe.directory '*' + +COPY . /release-automation +RUN pip install -e /release-automation + +ENTRYPOINT ["release-automation"] +CMD ["--help"] diff --git a/release-automation/pyproject.toml b/release-automation/pyproject.toml new file mode 100644 index 000000000..50714fc96 --- /dev/null +++ b/release-automation/pyproject.toml @@ -0,0 +1,86 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "stackbrew-library-generator" +version = "0.1.0" +description = "Stackbrew Library Generator for Redis Docker Images" +authors = [ + {name = "Redis Team", email = "team@redis.io"}, +] +readme = "README.md" +requires-python = ">=3.8" +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", +] +dependencies = [ + "typer[all]>=0.9.0", + "rich>=13.0.0", + "pydantic>=2.0.0", + "packaging>=21.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "black>=23.0.0", + "isort>=5.12.0", + "mypy>=1.0.0", + "pre-commit>=3.0.0", +] + +[project.scripts] +release-automation = "stackbrew_generator.cli:app" + +[project.urls] +Homepage = "https://github.com/redis/docker-library-redis" +Repository = "https://github.com/redis/docker-library-redis" +Issues = "https://github.com/redis/docker-library-redis/issues" + +[tool.hatch.build.targets.wheel] +packages = ["src/stackbrew_generator"] + +[tool.hatch.build.targets.sdist] +include = [ + "/src", + "/README.md", + "/pyproject.toml", +] + +[tool.black] +line-length = 88 +target-version = ['py38'] +include = '\.pyi?$' + +[tool.isort] +profile = "black" +multi_line_output = 3 + +[tool.mypy] +python_version = "3.8" +warn_return_any = true +warn_unused_configs = true +disallow_untyped_defs = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py", "*_test.py"] +python_classes = ["Test*"] +python_functions = ["test_*"] +addopts = [ + "--strict-markers", + "--strict-config", + "--cov=stackbrew_generator", + "--cov-report=term-missing", + "--cov-report=html", + "--cov-report=xml", +] diff --git a/release-automation/src/stackbrew_generator/__init__.py b/release-automation/src/stackbrew_generator/__init__.py new file mode 100644 index 000000000..38a1b1c6e --- /dev/null +++ b/release-automation/src/stackbrew_generator/__init__.py @@ -0,0 +1,3 @@ +"""Stackbrew Library Generator for Redis Docker Images.""" + +__version__ = "0.1.0" diff --git a/release-automation/src/stackbrew_generator/cli.py b/release-automation/src/stackbrew_generator/cli.py new file mode 100644 index 000000000..f572bee7a --- /dev/null +++ b/release-automation/src/stackbrew_generator/cli.py @@ -0,0 +1,248 @@ +"""CLI interface for stackbrew library generator.""" + +import typer +from pathlib import Path +from rich.console import Console +from rich.traceback import install + +from .distribution import DistributionDetector +from .exceptions import StackbrewGeneratorError +from .git_operations import GitClient +from .logging_config import setup_logging +from .stackbrew import StackbrewGenerator, StackbrewUpdater +from .version_filter import VersionFilter + +# Install rich traceback handler +install(show_locals=True) + +app = typer.Typer( + name="release-automation", + help="Generate stackbrew library content for Redis Docker images", + add_completion=False, +) + +# Console for logging and user messages (stderr) +console = Console(stderr=True) + + +def _generate_stackbrew_content(major_version: int, remote: str, verbose: bool) -> str: + """Generate stackbrew content for a major version. + + This helper function contains the common logic for generating stackbrew content + that is used by both generate-stackbrew-content and update-stackbrew-file commands. + + Args: + major_version: Redis major version to process + remote: Git remote to use + verbose: Whether to enable verbose output + + Returns: + Generated stackbrew content as string + + Raises: + typer.Exit: If no versions found or other errors occur + """ + # Initialize components + git_client = GitClient(remote=remote) + version_filter = VersionFilter(git_client) + distribution_detector = DistributionDetector(git_client) + stackbrew_generator = StackbrewGenerator() + + # Get actual Redis versions to process + versions = version_filter.get_actual_major_redis_versions(major_version) + + if not versions: + console.print(f"[red]No versions found for Redis {major_version}.x[/red]") + raise typer.Exit(1) + + # Fetch required refs + refs_to_fetch = [commit for _, commit, _ in versions] + git_client.fetch_refs(refs_to_fetch) + + # Prepare releases list with distribution information + releases = distribution_detector.prepare_releases_list(versions) + + if not releases: + console.print("[red]No releases prepared[/red]") + raise typer.Exit(1) + + # Generate stackbrew library content + entries = stackbrew_generator.generate_stackbrew_library(releases) + output = stackbrew_generator.format_stackbrew_output(entries) + + if not output: + console.print("[yellow]No stackbrew content generated[/yellow]") + raise typer.Exit(1) + + if verbose: + console.print(f"[green]Generated stackbrew library with {len(entries)} entries[/green]") + + return output + + +@app.command(name="generate-stackbrew-content") +def generate_stackbrew_content( + major_version: int = typer.Argument( + ..., + help="Redis major version to process (e.g., 8 for Redis 8.x)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), +) -> None: + """Generate stackbrew library content for Redis Docker images. + + This command: + 1. Fetches Redis version tags from the specified remote + 2. Filters versions to remove EOL and select latest patches + 3. Extracts distribution information from Dockerfiles + 4. Generates appropriate Docker tags for each version/distribution + 5. Outputs stackbrew library content + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if verbose: + console.print(f"[bold blue]Stackbrew Library Generator[/bold blue]") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") + + try: + # Generate stackbrew content using the helper function + output = _generate_stackbrew_content(major_version, remote, verbose) + + # Output the stackbrew library content + print(output) + + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + + +@app.command() +def version() -> None: + """Show version information.""" + from . import __version__ + console.print(f"stackbrew-library-generator {__version__}") + + +@app.command() +def update_stackbrew_file( + major_version: int = typer.Argument( + ..., + help="Redis major version to update (e.g., 8 for Redis 8.x)" + ), + input_file: Path = typer.Option( + ..., + "--input", + "-i", + help="Path to the stackbrew library file to update" + ), + output_file: Path = typer.Option( + None, + "--output", + "-o", + help="Output file path (defaults to stdout)" + ), + remote: str = typer.Option( + "origin", + "--remote", + help="Git remote to use for fetching tags and branches" + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose output" + ), +) -> None: + """Update stackbrew library file by replacing entries for a specific major version. + + This command: + 1. Reads the existing stackbrew library file + 2. Generates new stackbrew content for the specified major version + 3. Replaces all entries related to that major version in their original position + 4. Preserves the header and entries for other major versions + 5. Outputs to stdout by default, or to specified output file + """ + # Set up logging + setup_logging(verbose=verbose, console=console) + + if not input_file.exists(): + console.print(f"[red]Input file does not exist: {input_file}[/red]") + raise typer.Exit(1) + + if verbose: + console.print(f"[bold blue]Stackbrew Library File Updater[/bold blue]") + console.print(f"Input file: {input_file}") + if output_file: + console.print(f"Output file: {output_file}") + else: + console.print("Output: stdout") + console.print(f"Major version: {major_version}") + console.print(f"Remote: {remote}") + + try: + # Generate new stackbrew content for the major version using helper function + new_content = _generate_stackbrew_content(major_version, remote, verbose) + + # Update the stackbrew file content + updater = StackbrewUpdater() + updated_content = updater.update_stackbrew_content( + input_file, major_version, new_content, verbose + ) + + # Write the updated content + if output_file: + output_file.write_text(updated_content, encoding='utf-8') + if verbose: + console.print(f"[green]Successfully updated {output_file} for Redis {major_version}.x[/green]") + else: + console.print(f"[green]Updated {output_file}[/green]") + else: + # Output to stdout + print(updated_content) + if verbose: + console.print(f"[green]Generated updated stackbrew content for Redis {major_version}.x[/green]") + + except StackbrewGeneratorError as e: + if verbose and hasattr(e, 'get_detailed_message'): + console.print(f"[red]{e.get_detailed_message()}[/red]") + else: + console.print(f"[red]Error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + except KeyboardInterrupt: + console.print("\n[yellow]Operation cancelled by user[/yellow]") + raise typer.Exit(130) + except Exception as e: + console.print(f"[red]Unexpected error: {e}[/red]") + if verbose: + console.print_exception() + raise typer.Exit(1) + + +if __name__ == "__main__": + app() diff --git a/release-automation/src/stackbrew_generator/distribution.py b/release-automation/src/stackbrew_generator/distribution.py new file mode 100644 index 000000000..5e2080e8e --- /dev/null +++ b/release-automation/src/stackbrew_generator/distribution.py @@ -0,0 +1,118 @@ +"""Distribution detection from Dockerfiles.""" + +from typing import List, Tuple + +from rich.console import Console + +from .exceptions import DistributionError +from .git_operations import GitClient +from .models import Distribution, RedisVersion, Release + +console = Console(stderr=True) + + +class DistributionDetector: + """Detects distribution information from Dockerfiles.""" + + def __init__(self, git_client: GitClient): + """Initialize distribution detector. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def extract_distribution_from_dockerfile(self, dockerfile_content: str) -> Distribution: + """Extract distribution information from Dockerfile content. + + Args: + dockerfile_content: Content of the Dockerfile + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + # Find the FROM line + from_line = None + for line in dockerfile_content.split('\n'): + line = line.strip() + if line.upper().startswith('FROM '): + from_line = line + break + + if not from_line: + raise DistributionError("No FROM line found in Dockerfile") + + try: + return Distribution.from_dockerfile_line(from_line) + except ValueError as e: + raise DistributionError(f"Failed to parse distribution from FROM line: {e}") from e + + def get_distribution_for_commit(self, commit: str, distro_type: str) -> Distribution: + """Get distribution information for a specific commit and distro type. + + Args: + commit: Git commit hash + distro_type: Distribution type ("debian" or "alpine") + + Returns: + Distribution instance + + Raises: + DistributionError: If distribution cannot be detected + """ + dockerfile_path = f"{distro_type}/Dockerfile" + + try: + dockerfile_content = self.git_client.show_file(commit, dockerfile_path) + console.print(f"[dim]Retrieved {dockerfile_path} from {commit[:8]}[/dim]") + + distribution = self.extract_distribution_from_dockerfile(dockerfile_content) + console.print(f"[dim]Detected distribution: {distribution.type.value} {distribution.name}[/dim]") + + return distribution + + except Exception as e: + raise DistributionError( + f"Failed to get distribution for {distro_type} from {commit}: {e}" + ) from e + + def prepare_releases_list(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Release]: + """Prepare list of releases with distribution information. + + Args: + versions: List of (RedisVersion, commit, tag_ref) tuples + + Returns: + List of Release objects with distribution information + """ + console.print("[blue]Preparing releases list with distribution information[/blue]") + + releases = [] + distro_types = ["debian", "alpine"] + + for version, commit, tag_ref in versions: + console.print(f"[dim]Processing [bold yellow]{version}[/bold yellow] - {commit[:8]}[/dim]") + + for distro_type in distro_types: + try: + distribution = self.get_distribution_for_commit(commit, distro_type) + + release = Release( + commit=commit, + version=version, + distribution=distribution, + git_fetch_ref=tag_ref + ) + + releases.append(release) + console.print(f"[dim] Added: {release.console_repr()}[/dim]", highlight=False) + + except DistributionError as e: + console.print(f"[yellow]Warning: Failed to process {distro_type} for {version}: {e}[/yellow]") + continue + + console.print(f"[green]Prepared {len(releases)} releases[/green]") + return releases diff --git a/release-automation/src/stackbrew_generator/exceptions.py b/release-automation/src/stackbrew_generator/exceptions.py new file mode 100644 index 000000000..f5b11b2a6 --- /dev/null +++ b/release-automation/src/stackbrew_generator/exceptions.py @@ -0,0 +1,138 @@ +"""Custom exceptions for stackbrew library generation.""" + +from typing import Optional, Any, Dict + + +class StackbrewGeneratorError(Exception): + """Base exception for stackbrew generator errors. + + Provides structured error information with context and suggestions. + """ + + def __init__( + self, + message: str, + context: Optional[Dict[str, Any]] = None, + suggestion: Optional[str] = None, + original_error: Optional[Exception] = None + ): + """Initialize error with context. + + Args: + message: Error message + context: Additional context information + suggestion: Suggested fix or next steps + original_error: Original exception that caused this error + """ + super().__init__(message) + self.context = context or {} + self.suggestion = suggestion + self.original_error = original_error + + def get_detailed_message(self) -> str: + """Get detailed error message with context and suggestions.""" + parts = [str(self)] + + if self.context: + parts.append("Context:") + for key, value in self.context.items(): + parts.append(f" {key}: {value}") + + if self.suggestion: + parts.append(f"Suggestion: {self.suggestion}") + + if self.original_error: + parts.append(f"Original error: {self.original_error}") + + return "\n".join(parts) + + +class GitOperationError(StackbrewGeneratorError): + """Exception raised for Git operation failures.""" + + def __init__( + self, + message: str, + command: Optional[str] = None, + exit_code: Optional[int] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if command: + context['command'] = command + if exit_code is not None: + context['exit_code'] = exit_code + + suggestion = kwargs.get('suggestion') + if not suggestion and command: + if 'ls-remote' in command: + suggestion = "Check that the remote repository exists and is accessible" + elif 'fetch' in command: + suggestion = "Ensure you have network access and proper Git credentials" + elif 'show' in command: + suggestion = "Verify that the commit exists and contains the requested file" + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class VersionParsingError(StackbrewGeneratorError): + """Exception raised for version parsing failures.""" + + def __init__(self, message: str, version_string: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if version_string: + context['version_string'] = version_string + + suggestion = kwargs.get('suggestion', + "Version should be in format 'X.Y.Z' or 'vX.Y.Z' with optional suffix") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class DistributionError(StackbrewGeneratorError): + """Exception raised for distribution detection failures.""" + + def __init__( + self, + message: str, + dockerfile_path: Optional[str] = None, + from_line: Optional[str] = None, + **kwargs + ): + context = kwargs.get('context', {}) + if dockerfile_path: + context['dockerfile_path'] = dockerfile_path + if from_line: + context['from_line'] = from_line + + suggestion = kwargs.get('suggestion', + "Dockerfile should have a FROM line with supported base image (alpine:* or debian:*)") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) + + +class ValidationError(StackbrewGeneratorError): + """Exception raised for validation failures.""" + + def __init__(self, message: str, field: Optional[str] = None, value: Optional[Any] = None, **kwargs): + context = kwargs.get('context', {}) + if field: + context['field'] = field + if value is not None: + context['value'] = value + + super().__init__(message, context=context, **kwargs) + + +class ConfigurationError(StackbrewGeneratorError): + """Exception raised for configuration errors.""" + + def __init__(self, message: str, config_key: Optional[str] = None, **kwargs): + context = kwargs.get('context', {}) + if config_key: + context['config_key'] = config_key + + suggestion = kwargs.get('suggestion', + "Check your configuration and environment variables") + + super().__init__(message, context=context, suggestion=suggestion, **kwargs) diff --git a/release-automation/src/stackbrew_generator/git_operations.py b/release-automation/src/stackbrew_generator/git_operations.py new file mode 100644 index 000000000..14dab035c --- /dev/null +++ b/release-automation/src/stackbrew_generator/git_operations.py @@ -0,0 +1,156 @@ +"""Git operations for stackbrew library generation.""" + +import re +import subprocess +from typing import Dict, List, Tuple + +from rich.console import Console + +from .exceptions import GitOperationError +from .models import RedisVersion + +console = Console(stderr=True) + + +class GitClient: + """Client for Git operations.""" + + def __init__(self, remote: str = "origin"): + """Initialize Git client. + + Args: + remote: Git remote name to use + """ + self.remote = remote + + def _run_command(self, cmd: List[str], capture_output: bool = True) -> subprocess.CompletedProcess: + """Run a git command with error handling. + + Args: + cmd: Command and arguments to run + capture_output: Whether to capture stdout/stderr + + Returns: + CompletedProcess result + + Raises: + GitOperationError: If command fails + """ + try: + result = subprocess.run( + cmd, + capture_output=capture_output, + text=True, + check=True, + ) + return result + except subprocess.CalledProcessError as e: + error_msg = f"Git command failed: {' '.join(cmd)}" + if e.stderr: + error_msg += f"\nError: {e.stderr.strip()}" + raise GitOperationError(error_msg) from e + except FileNotFoundError as e: + raise GitOperationError("Git command not found. Is git installed?") from e + + def list_remote_tags(self, major_version: int) -> List[Tuple[str, str]]: + """List remote tags for a specific major version. + + Args: + major_version: Major version to filter tags for + + Returns: + List of (commit, tag_ref) tuples + + Raises: + GitOperationError: If no tags found or git operation fails + """ + console.print(f"[dim]Listing remote tags for v{major_version}.*[/dim]") + + cmd = [ + "git", "ls-remote", "--refs", "--tags", + self.remote, f"refs/tags/v{major_version}.*" + ] + + result = self._run_command(cmd) + + if not result.stdout.strip(): + raise GitOperationError(f"No tags found for major version {major_version}") + + tags = [] + for line in result.stdout.strip().split('\n'): + if line: + commit, ref = line.split('\t', 1) + tags.append((commit, ref)) + + console.print(f"[dim]Found {len(tags)} tags[/dim]") + return tags + + def fetch_refs(self, refs: List[str]) -> None: + """Fetch specific refs from remote. + + Args: + refs: List of refs to fetch + + Raises: + GitOperationError: If fetch operation fails + """ + if not refs: + return + + console.print(f"[dim]Fetching {len(refs)} refs[/dim]") + + # Use git fetch with unshallow to ensure we have full history + cmd = ["git", "fetch", "--unshallow", self.remote] + refs + + try: + self._run_command(cmd, capture_output=False) + except GitOperationError: + # If --unshallow fails (repo already unshallow), try without it + cmd = ["git", "fetch", self.remote] + refs + self._run_command(cmd, capture_output=False) + + def show_file(self, commit: str, file_path: str) -> str: + """Show file content from a specific commit. + + Args: + commit: Git commit hash + file_path: Path to file in repository + + Returns: + File content as string + + Raises: + GitOperationError: If file cannot be retrieved + """ + cmd = ["git", "show", f"{commit}:{file_path}"] + + try: + result = self._run_command(cmd) + return result.stdout + except GitOperationError as e: + raise GitOperationError(f"Failed to get {file_path} from {commit}: {e}") from e + + def extract_version_from_tag(self, tag_ref: str, major_version: int) -> RedisVersion: + """Extract Redis version from tag reference. + + Args: + tag_ref: Git tag reference (e.g., refs/tags/v8.2.1) + major_version: Expected major version for validation + + Returns: + Parsed RedisVersion + + Raises: + GitOperationError: If tag format is invalid + """ + # Extract version from tag reference + match = re.search(rf"v{major_version}\.\d+(?:\.\d+)?.*", tag_ref) + if not match: + raise GitOperationError(f"Invalid tag format: {tag_ref}") + + version_str = match.group(0) + + try: + return RedisVersion.parse(version_str) + except ValueError as e: + raise GitOperationError(f"Failed to parse version from {tag_ref}: {e}") from e diff --git a/release-automation/src/stackbrew_generator/logging_config.py b/release-automation/src/stackbrew_generator/logging_config.py new file mode 100644 index 000000000..b261e7ae7 --- /dev/null +++ b/release-automation/src/stackbrew_generator/logging_config.py @@ -0,0 +1,95 @@ +"""Logging configuration for stackbrew generator.""" + +import logging +from typing import Optional + +from rich.console import Console +from rich.logging import RichHandler + + +def setup_logging( + level: str = "INFO", + verbose: bool = False, + console: Optional[Console] = None +) -> logging.Logger: + """Set up logging configuration. + + Args: + level: Logging level (DEBUG, INFO, WARNING, ERROR) + verbose: Enable verbose logging + console: Rich console instance to use (should use stderr) + + Returns: + Configured logger instance + """ + if console is None: + # Create console that outputs to stderr + console = Console(stderr=True) + + # Determine log level + if verbose: + log_level = logging.DEBUG + else: + log_level = getattr(logging, level.upper(), logging.INFO) + + # Configure root logger + logging.basicConfig( + level=log_level, + format="%(message)s", + datefmt="[%X]", + handlers=[ + RichHandler( + console=console, + show_path=verbose, + show_time=verbose, + rich_tracebacks=True, + tracebacks_show_locals=verbose, + ) + ], + ) + + # Get logger for our package + logger = logging.getLogger("stackbrew_generator") + logger.setLevel(log_level) + + return logger + + +def get_logger(name: str) -> logging.Logger: + """Get a logger instance for a specific module. + + Args: + name: Logger name (usually __name__) + + Returns: + Logger instance + """ + return logging.getLogger(f"stackbrew_generator.{name}") + + +class LoggingMixin: + """Mixin class to add logging capabilities to other classes.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = get_logger(self.__class__.__name__) + + def log_debug(self, message: str, *args, **kwargs) -> None: + """Log debug message.""" + self.logger.debug(message, *args, **kwargs) + + def log_info(self, message: str, *args, **kwargs) -> None: + """Log info message.""" + self.logger.info(message, *args, **kwargs) + + def log_warning(self, message: str, *args, **kwargs) -> None: + """Log warning message.""" + self.logger.warning(message, *args, **kwargs) + + def log_error(self, message: str, *args, **kwargs) -> None: + """Log error message.""" + self.logger.error(message, *args, **kwargs) + + def log_exception(self, message: str, *args, **kwargs) -> None: + """Log exception with traceback.""" + self.logger.exception(message, *args, **kwargs) diff --git a/release-automation/src/stackbrew_generator/models.py b/release-automation/src/stackbrew_generator/models.py new file mode 100644 index 000000000..8a9a1fff7 --- /dev/null +++ b/release-automation/src/stackbrew_generator/models.py @@ -0,0 +1,206 @@ +"""Data models for stackbrew library generation.""" + +import re +from enum import Enum +from typing import List, Optional + +from pydantic import BaseModel, Field, validator + + +class DistroType(str, Enum): + """Distribution type enumeration.""" + + ALPINE = "alpine" + DEBIAN = "debian" + + +class RedisVersion(BaseModel): + """Represents a parsed Redis version.""" + + major: int = Field(..., ge=1, description="Major version number") + minor: int = Field(..., ge=0, description="Minor version number") + patch: Optional[int] = Field(None, ge=0, description="Patch version number") + suffix: str = Field("", description="Version suffix (e.g., -m01, -rc1, -eol)") + + @classmethod + def parse(cls, version_str: str) -> "RedisVersion": + """Parse a version string into components. + + Args: + version_str: Version string (e.g., "v8.2.1-m01", "8.2", "7.4.0-eol") + + Returns: + RedisVersion instance + + Raises: + ValueError: If version string format is invalid + """ + # Remove 'v' prefix if present + version = version_str.lstrip("v") + + # Extract numeric part and suffix + match = re.match(r"^([1-9]\d*\.\d+(?:\.\d+)?)(.*)", version) + if not match: + raise ValueError(f"Invalid version format: {version_str}") + + numeric_part, suffix = match.groups() + + # Parse numeric components + parts = numeric_part.split(".") + major = int(parts[0]) + minor = int(parts[1]) + patch = int(parts[2]) if len(parts) > 2 else None + + return cls(major=major, minor=minor, patch=patch, suffix=suffix) + + @property + def is_milestone(self) -> bool: + """Check if this is a milestone version (has suffix).""" + return bool(self.suffix) + + @property + def is_eol(self) -> bool: + """Check if this version is end-of-life.""" + return self.suffix.lower().endswith("-eol") + + @property + def mainline_version(self) -> str: + """Get the mainline version string (major.minor).""" + return f"{self.major}.{self.minor}" + + @property + def sort_key(self) -> str: + suffix_weight = 0 + if self.suffix.startswith("rc"): + suffix_weight = 100 + elif self.suffix.startswith("m"): + suffix_weight = 50 + + return f"{self.major}.{self.minor}.{self.patch or 0}.{suffix_weight}.{self.suffix}" + + def __str__(self) -> str: + """String representation of the version.""" + version = f"{self.major}.{self.minor}" + if self.patch is not None: + version += f".{self.patch}" + return version + self.suffix + + def __lt__(self, other: "RedisVersion") -> bool: + """Compare versions for sorting.""" + if not isinstance(other, RedisVersion): + return NotImplemented + + # Compare major.minor.patch first + self_tuple = (self.major, self.minor, self.patch or 0) + other_tuple = (other.major, other.minor, other.patch or 0) + + if self_tuple != other_tuple: + return self_tuple < other_tuple + + # If numeric parts are equal, compare suffixes + # Empty suffix (GA) comes after suffixes (milestones) + if not self.suffix and other.suffix: + return False + if self.suffix and not other.suffix: + return True + + return self.suffix < other.suffix + + +class Distribution(BaseModel): + """Represents a Linux distribution.""" + + type: DistroType = Field(..., description="Distribution type") + name: str = Field(..., description="Distribution name/version") + + @classmethod + def from_dockerfile_line(cls, from_line: str) -> "Distribution": + """Parse distribution from Dockerfile FROM line. + + Args: + from_line: FROM line from Dockerfile (e.g., "FROM alpine:3.22") + + Returns: + Distribution instance + + Raises: + ValueError: If FROM line format is not supported + """ + # Extract base image from FROM line + parts = from_line.strip().split() + if len(parts) < 2 or parts[0].upper() != "FROM": + raise ValueError(f"Invalid FROM line: {from_line}") + + base_img = parts[1] + + if "alpine:" in base_img: + # Extract alpine version (e.g., alpine:3.22 -> alpine3.22) + version = base_img.split(":", 1)[1] + return cls(type=DistroType.ALPINE, name=f"alpine{version}") + elif "debian:" in base_img: + # Extract debian version, remove -slim suffix + version = base_img.split(":", 1)[1].replace("-slim", "") + return cls(type=DistroType.DEBIAN, name=version) + else: + raise ValueError(f"Unsupported base image: {base_img}") + + @property + def is_default(self) -> bool: + """Check if this is the default distribution (Debian).""" + return self.type == DistroType.DEBIAN + + @property + def tag_names(self) -> List[str]: + """Get tag name components for this distribution.""" + if self.type == DistroType.ALPINE: + return [self.type.value, self.name] + else: + return [self.name] + + +class Release(BaseModel): + """Represents a Redis release with distribution information.""" + + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") + + def __str__(self) -> str: + """String representation of the release.""" + return f"{self.commit[:8]} {self.version} {self.distribution.type.value} {self.distribution.name}" + + def console_repr(self) -> str: + """Rich console representation with markup.""" + return f"{self.commit[:8]} [bold yellow]{self.version}[/bold yellow] {self.distribution.type.value} [bold yellow]{self.distribution.name}[/bold yellow]" + + +class StackbrewEntry(BaseModel): + """Represents a stackbrew library entry with tags.""" + + tags: List[str] = Field(..., description="Docker tags for this entry") + commit: str = Field(..., description="Git commit hash") + version: RedisVersion = Field(..., description="Redis version") + distribution: Distribution = Field(..., description="Linux distribution") + git_fetch_ref: str = Field(..., description="Git fetch reference (e.g., refs/tags/v8.2.1)") + + @property + def architectures(self) -> List[str]: + """Get supported architectures based on distribution type.""" + if self.distribution.type == DistroType.DEBIAN: + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + elif self.distribution.type == DistroType.ALPINE: + return ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + else: + # Fallback to debian architectures for unknown distributions + return ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + + def __str__(self) -> str: + """String representation in stackbrew format.""" + lines = [] + lines.append(f"Tags: {', '.join(self.tags)}") + lines.append(f"Architectures: {', '.join(self.architectures)}") + lines.append(f"GitCommit: {self.commit}") + lines.append(f"GitFetch: {self.git_fetch_ref}") + lines.append(f"Directory: {self.distribution.type.value}") + return "\n".join(lines) diff --git a/release-automation/src/stackbrew_generator/stackbrew.py b/release-automation/src/stackbrew_generator/stackbrew.py new file mode 100644 index 000000000..faac43b94 --- /dev/null +++ b/release-automation/src/stackbrew_generator/stackbrew.py @@ -0,0 +1,295 @@ +"""Stackbrew library generation.""" + +import re +from pathlib import Path +from typing import List + +from rich.console import Console + +from .models import Release, StackbrewEntry + +console = Console(stderr=True) + + +class StackbrewGenerator: + """Generates stackbrew library content.""" + + def generate_tags_for_release( + self, + release: Release, + is_latest: bool = False + ) -> List[str]: + """Generate Docker tags for a release. + + Args: + release: Release to generate tags for + is_latest: Whether this is the latest version + + Returns: + List of Docker tags + """ + tags = [] + version = release.version + distribution = release.distribution + + # Base version tags + version_tags = [str(version)] + + # Add mainline version tag only for GA releases (no suffix) + if not version.is_milestone: + version_tags.append(version.mainline_version) + + # Add major version tag for latest versions + if is_latest: + version_tags.append(str(version.major)) + + # For default distribution (Debian), add version tags without distro suffix + if distribution.is_default: + tags.extend(version_tags) + + # Add distro-specific tags + for distro_name in distribution.tag_names: + for version_tag in version_tags: + tags.append(f"{version_tag}-{distro_name}") + + # Add special latest tags + if is_latest: + if distribution.is_default: + tags.append("latest") + # Add bare distro names as tags + tags.extend(distribution.tag_names) + + return tags + + def generate_stackbrew_library(self, releases: List[Release]) -> List[StackbrewEntry]: + """Generate stackbrew library entries from releases. + + Args: + releases: List of releases to process + + Returns: + List of StackbrewEntry objects + """ + console.print("[blue]Generating stackbrew library content[/blue]") + + if not releases: + console.print("[yellow]No releases to process[/yellow]") + return [] + + entries = [] + latest_minor = None + latest_minor_unset = True + + for release in releases: + # Determine latest version following bash logic: + # - Set latest_minor to the minor version of the first non-milestone version + # - Clear latest_minor if subsequent versions have different minor versions + if latest_minor_unset: + if not release.version.is_milestone: + latest_minor = release.version.minor + latest_minor_unset = False + console.print(f"[dim]Latest minor version set to: {latest_minor}[/dim]") + elif latest_minor != release.version.minor: + latest_minor = None + + # Check if this release should get latest tags + is_latest = latest_minor is not None + + # Generate tags for this release + tags = self.generate_tags_for_release(release, is_latest) + + if tags: + entry = StackbrewEntry( + tags=tags, + commit=release.commit, + version=release.version, + distribution=release.distribution, + git_fetch_ref=release.git_fetch_ref + ) + entries.append(entry) + + console.print(f"[dim]{release.console_repr()} -> {len(tags)} tags[/dim]") + else: + console.print(f"[yellow]No tags generated for {release}[/yellow]") + + console.print(f"[green]Generated {len(entries)} stackbrew entries[/green]") + console.print(f"[dim]{self.format_stackbrew_output(entries)}[/dim]") + return entries + + def format_stackbrew_output(self, entries: List[StackbrewEntry]) -> str: + """Format stackbrew entries as output string. + + Args: + entries: List of stackbrew entries + + Returns: + Formatted stackbrew library content + """ + if not entries: + return "" + + lines = [] + for i, entry in enumerate(entries): + if i > 0: + lines.append("") # Add blank line between entries + lines.append(str(entry)) + + return "\n".join(lines) + + +class StackbrewUpdater: + """Updates stackbrew library files by replacing entries for specific major versions.""" + + def __init__(self): + """Initialize the updater.""" + pass + + def update_stackbrew_content(self, input_file: Path, major_version: int, new_content: str, verbose: bool = False) -> str: + """Update stackbrew file content by replacing entries for a specific major version. + + Args: + input_file: Path to the input stackbrew file + major_version: Major version to replace entries for + new_content: New stackbrew content to insert + verbose: Whether to print verbose output + + Returns: + Updated stackbrew file content + """ + content = input_file.read_text(encoding='utf-8') + lines = content.split('\n') + + # Find header (everything before the first Tags: line) + header_lines = [] + content_start_idx = 0 + + for i, line in enumerate(lines): + if line.startswith('Tags:'): + content_start_idx = i + break + header_lines.append(line) + + if content_start_idx == 0 and not any(line.startswith('Tags:') for line in lines): + # No existing entries, just append new content + if verbose: + console.print("[dim]No existing entries found, appending new content[/dim]") + return content.rstrip() + '\n\n' + new_content + + # Parse entries and find where target major version entries start and end + entries = self._parse_stackbrew_entries(lines[content_start_idx:]) + target_entries = [] + other_entries_before = [] + other_entries_after = [] + target_start_found = False + target_end_found = False + removed_count = 0 + + for entry in entries: + if self._entry_belongs_to_major_version(entry, major_version): + target_entries.append(entry) + removed_count += 1 + if not target_start_found: + target_start_found = True + elif not target_start_found: + # Entries before target major version + other_entries_before.append(entry) + else: + # Entries after target major version + other_entries_after.append(entry) + if not target_end_found: + target_end_found = True + + if verbose: + if removed_count > 0: + console.print(f"[dim]Removed {removed_count} existing entries for Redis {major_version}.x[/dim]") + else: + console.print(f"[dim]No existing entries found for Redis {major_version}.x, placing at end[/dim]") + + # Reconstruct the file + result_lines = header_lines[:] + + # Add entries before target major version + for entry in other_entries_before: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + # Add new content for the target major version + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(new_content.split('\n')) + + # Add entries after target major version + for entry in other_entries_after: + if result_lines and result_lines[-1].strip(): # Add blank line if needed + result_lines.append('') + result_lines.extend(entry) + + return '\n'.join(result_lines) + + def _parse_stackbrew_entries(self, lines: List[str]) -> List[List[str]]: + """Parse stackbrew entries from lines, returning list of entry line groups. + + Args: + lines: Lines to parse + + Returns: + List of entry line groups + """ + entries = [] + current_entry = [] + + for line in lines: + line = line.rstrip() + + if line.startswith('Tags:') and current_entry: + # Start of new entry, save the previous one + entries.append(current_entry) + current_entry = [line] + elif line.startswith('Tags:'): + # First entry + current_entry = [line] + elif current_entry and (line.startswith(('Architectures:', 'GitCommit:', 'GitFetch:', 'Directory:')) or line.strip() == ''): + # Part of current entry + current_entry.append(line) + elif not line.strip() and not current_entry: + # Empty line before any entry starts, skip + continue + elif not line.strip() and current_entry: + # Empty line after entry content - end of entry + if current_entry: + entries.append(current_entry) + current_entry = [] + + # Don't forget the last entry + if current_entry: + entries.append(current_entry) + + return entries + + def _entry_belongs_to_major_version(self, entry_lines: List[str], major_version: int) -> bool: + """Check if a stackbrew entry belongs to the specified major version. + + Args: + entry_lines: Lines of the stackbrew entry + major_version: Major version to check for + + Returns: + True if the entry belongs to the major version + """ + for line in entry_lines: + if line.startswith('Tags:'): + tags_line = line[5:].strip() # Remove 'Tags:' prefix + tags = [tag.strip() for tag in tags_line.split(',')] + + # Check if any tag indicates this major version + for tag in tags: + # Look for patterns like "8", "8.2", "8.2.1", "8-alpine", etc. + if re.match(rf'^{major_version}(?:\.|$|-)', tag): + return True + # Also check for "latest" tag which typically belongs to the highest major version + # But we'll be conservative and not assume latest belongs to our major version + # unless we have other evidence + break + + return False diff --git a/release-automation/src/stackbrew_generator/version_filter.py b/release-automation/src/stackbrew_generator/version_filter.py new file mode 100644 index 000000000..bf0590795 --- /dev/null +++ b/release-automation/src/stackbrew_generator/version_filter.py @@ -0,0 +1,157 @@ +"""Version filtering and processing for Redis releases.""" + +from typing import Dict, List, Tuple + +from collections import OrderedDict + +from packaging.version import Version +from rich.console import Console + +from .git_operations import GitClient +from .models import RedisVersion + +console = Console(stderr=True) + + +class VersionFilter: + """Filters and processes Redis versions.""" + + def __init__(self, git_client: GitClient): + """Initialize version filter. + + Args: + git_client: Git client for operations + """ + self.git_client = git_client + + def get_redis_versions_from_tags(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: + """Get Redis versions from git tags. + + Args: + major_version: Major version to filter for + + Returns: + List of (RedisVersion, commit, tag_ref) tuples sorted by version (newest first) + """ + console.print(f"[blue]Getting Redis versions for major version {major_version}[/blue]") + + # Get remote tags + tags = self.git_client.list_remote_tags(major_version) + + # Parse versions from tags + versions = [] + for commit, tag_ref in tags: + try: + version = self.git_client.extract_version_from_tag(tag_ref, major_version) + versions.append((version, commit, tag_ref)) + except Exception as e: + console.print(f"[yellow]Warning: Skipping invalid tag {tag_ref}: {e}[/yellow]") + continue + + # Sort by version (newest first) + versions.sort(key=lambda x: x[0].sort_key, reverse=True) + + console.print(f"[dim]Parsed {len(versions)} valid versions[/dim]") + return versions + + + def filter_eol_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: + """Filter out end-of-life versions. + + Args: + versions: List of (RedisVersion, commit, tag_ref) tuples + + Returns: + Filtered list with EOL minor versions removed + """ + console.print("[blue]Filtering out EOL versions[/blue]") + + # Group versions by minor version + minor_versions: Dict[str, List[Tuple[RedisVersion, str, str]]] = {} + for version, commit, tag_ref in versions: + minor_key = version.mainline_version + if minor_key not in minor_versions: + minor_versions[minor_key] = [] + minor_versions[minor_key].append((version, commit, tag_ref)) + + # Check each minor version for EOL marker + filtered_versions = [] + for minor_key, minor_group in minor_versions.items(): + # Check if any version in this minor series is marked as EOL + has_eol = any(version.is_eol for version, _, _ in minor_group) + + if has_eol: + console.print(f"[yellow]Skipping minor version {minor_key}.* due to EOL[/yellow]") + else: + filtered_versions.extend(minor_group) + + # Sort again after filtering + filtered_versions.sort(key=lambda x: x[0].sort_key, reverse=True) + + console.print(f"[dim]Kept {len(filtered_versions)} versions after EOL filtering[/dim]") + return filtered_versions + + def filter_actual_versions(self, versions: List[Tuple[RedisVersion, str, str]]) -> List[Tuple[RedisVersion, str, str]]: + """Filter to keep only the latest patch version for each minor version and milestone status. + + Args: + versions: List of (RedisVersion, commit, tag_ref) tuples (should be sorted newest first) + + Returns: + Filtered list with only the latest versions for each minor/milestone combination + """ + console.print("[blue]Filtering to actual versions (latest patch per minor/milestone)[/blue]") + + patch_versions = OrderedDict() + + for version, commit, tag_ref in versions: + patch_key = (version.major, version.minor, version.patch) + if patch_key not in patch_versions: + patch_versions[patch_key] = (version, commit, tag_ref) + elif patch_versions[patch_key][0].is_milestone and not version.is_milestone: + # GA always takes precedence over milestone for the same major.minor.patch + patch_versions[patch_key] = (version, commit, tag_ref) + + print(patch_versions.values()) + filtered_versions = [] + mainlines_with_ga = set() + + for version, commit, tag_ref in patch_versions.values(): + if version.mainline_version not in mainlines_with_ga: + if not version.is_milestone: + mainlines_with_ga.add(version.mainline_version) + filtered_versions.append((version, commit, tag_ref)) + return filtered_versions + + def get_actual_major_redis_versions(self, major_version: int) -> List[Tuple[RedisVersion, str, str]]: + """Get the actual Redis versions to process for a major version. + + This is the main entry point that combines all filtering steps: + 1. Get versions from git tags + 2. Filter out EOL versions + 3. Filter to actual versions (latest patch per minor/milestone) + + Args: + major_version: Major version to process + + Returns: + List of (RedisVersion, commit, tag_ref) tuples for processing + """ + console.print(f"[bold blue]Processing Redis {major_version}.x versions[/bold blue]") + + # Get all versions from tags + versions = self.get_redis_versions_from_tags(major_version) + + if not versions: + console.print(f"[red]No versions found for major version {major_version}[/red]") + return [] + + # Apply filters + versions = self.filter_eol_versions(versions) + versions = self.filter_actual_versions(versions) + + console.print(f"[green]Final selection: {len(versions)} versions to process[/green]") + for version, commit, tag_ref in versions: + console.print(f"[green] [bold yellow]{version}[/bold yellow] - {commit[:8]}[/green]") + + return versions diff --git a/release-automation/tests/__init__.py b/release-automation/tests/__init__.py new file mode 100644 index 000000000..a60351fa0 --- /dev/null +++ b/release-automation/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for stackbrew library generator.""" diff --git a/release-automation/tests/test_integration.py b/release-automation/tests/test_integration.py new file mode 100644 index 000000000..2a11548a1 --- /dev/null +++ b/release-automation/tests/test_integration.py @@ -0,0 +1,57 @@ +"""Integration tests for the stackbrew generator.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType +from typer.testing import CliRunner + + +class TestIntegration: + """Integration tests for the complete workflow.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_version_command(self): + """Test version command.""" + result = self.runner.invoke(app, ["version"]) + assert result.exit_code == 0 + assert "stackbrew-library-generator" in result.stderr + + def test_invalid_major_version(self): + """Test handling of invalid major version.""" + result = self.runner.invoke(app, ["generate-stackbrew-content", "0"]) + assert result.exit_code != 0 + + @patch('stackbrew_generator.git_operations.GitClient') + def test_no_tags_found(self, mock_git_client_class): + """Test handling when no tags are found.""" + # Mock git client to return no tags + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + mock_git_client.list_remote_tags.return_value = [] + + result = self.runner.invoke(app, ["generate-stackbrew-content", "99"]) + assert result.exit_code == 1 + assert "No tags found" in result.stderr + + @patch('stackbrew_generator.version_filter.VersionFilter.get_actual_major_redis_versions') + def test_no_versions_found(self, mock_get_versions): + """Test handling when no versions are found.""" + # Mock git client to return no tags + mock_get_versions.return_value = [] + + result = self.runner.invoke(app, ["generate-stackbrew-content", "8"]) + #assert result.exit_code == 1 + assert "No versions found" in result.stderr + + def test_help_output(self): + """Test help output.""" + result = self.runner.invoke(app, ["generate-stackbrew-content", "--help"]) + assert result.exit_code == 0 + assert "Generate stackbrew library content" in result.stdout + assert "--remote" in result.stdout + assert "--verbose" in result.stdout diff --git a/release-automation/tests/test_models.py b/release-automation/tests/test_models.py new file mode 100644 index 000000000..0feef1f94 --- /dev/null +++ b/release-automation/tests/test_models.py @@ -0,0 +1,239 @@ +"""Tests for data models.""" + +import pytest + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release, StackbrewEntry + + +class TestRedisVersion: + """Tests for RedisVersion model.""" + + def test_parse_basic_version(self): + """Test parsing basic version strings.""" + version = RedisVersion.parse("8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_v_prefix(self): + """Test parsing version with 'v' prefix.""" + version = RedisVersion.parse("v8.2.1") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "" + + def test_parse_version_with_suffix(self): + """Test parsing version with suffix.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.major == 8 + assert version.minor == 2 + assert version.patch == 1 + assert version.suffix == "-m01" + + def test_parse_version_without_patch(self): + """Test parsing version without patch number.""" + version = RedisVersion.parse("8.2") + assert version.major == 8 + assert version.minor == 2 + assert version.patch is None + assert version.suffix == "" + + def test_parse_eol_version(self): + """Test parsing EOL version.""" + version = RedisVersion.parse("7.4.0-eol") + assert version.major == 7 + assert version.minor == 4 + assert version.patch == 0 + assert version.suffix == "-eol" + assert version.is_eol is True + + def test_parse_invalid_version(self): + """Test parsing invalid version strings.""" + with pytest.raises(ValueError): + RedisVersion.parse("invalid") + + with pytest.raises(ValueError): + RedisVersion.parse("0.1.0") # Major version must be >= 1 + + def test_is_milestone(self): + """Test milestone detection.""" + ga_version = RedisVersion.parse("8.2.1") + milestone_version = RedisVersion.parse("8.2.1-m01") + + assert ga_version.is_milestone is False + assert milestone_version.is_milestone is True + + def test_mainline_version(self): + """Test mainline version property.""" + version = RedisVersion.parse("8.2.1-m01") + assert version.mainline_version == "8.2" + + def test_string_representation(self): + """Test string representation.""" + version1 = RedisVersion.parse("8.2.1") + version2 = RedisVersion.parse("8.2.1-m01") + version3 = RedisVersion.parse("8.2") + + assert str(version1) == "8.2.1" + assert str(version2) == "8.2.1-m01" + assert str(version3) == "8.2" + + def test_version_comparison(self): + """Test version comparison for sorting.""" + v1 = RedisVersion.parse("8.2.1") + v2 = RedisVersion.parse("8.2.2") + v3 = RedisVersion.parse("8.2.1-m01") + v4 = RedisVersion.parse("8.3.0") + + # Test numeric comparison + assert v1 < v2 + assert v2 < v4 + + # Test milestone vs GA (GA comes after milestone) + assert v3 < v1 + + # Test sorting + versions = [v4, v1, v3, v2] + sorted_versions = sorted(versions) + assert sorted_versions == [v3, v1, v2, v4] + + +class TestDistribution: + """Tests for Distribution model.""" + + def test_from_dockerfile_alpine(self): + """Test parsing Alpine distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM alpine:3.22") + assert distro.type == DistroType.ALPINE + assert distro.name == "alpine3.22" + + def test_from_dockerfile_debian(self): + """Test parsing Debian distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_debian_slim(self): + """Test parsing Debian slim distribution from Dockerfile.""" + distro = Distribution.from_dockerfile_line("FROM debian:bookworm-slim") + assert distro.type == DistroType.DEBIAN + assert distro.name == "bookworm" + + def test_from_dockerfile_invalid(self): + """Test parsing invalid Dockerfile lines.""" + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("INVALID LINE") + + with pytest.raises(ValueError): + Distribution.from_dockerfile_line("FROM unsupported:latest") + + def test_is_default(self): + """Test default distribution detection.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.is_default is False + assert debian.is_default is True + + def test_tag_names(self): + """Test tag name generation.""" + alpine = Distribution(type=DistroType.ALPINE, name="alpine3.22") + debian = Distribution(type=DistroType.DEBIAN, name="bookworm") + + assert alpine.tag_names == ["alpine", "alpine3.22"] + assert debian.tag_names == ["bookworm"] + + +class TestRelease: + """Tests for Release model.""" + + def test_release_creation(self): + """Test creating a Release instance.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + assert release.commit == "abc123def456" + assert release.version == version + assert release.distribution == distribution + + def test_release_string_representation(self): + """Test Release string representation.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + release = Release( + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected = "abc123de 8.2.1 debian bookworm" + assert str(release) == expected + + +class TestStackbrewEntry: + """Tests for StackbrewEntry model.""" + + def test_debian_architectures(self): + """Test that Debian distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + + entry = StackbrewEntry( + tags=["8.2.1", "latest"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v5", "arm32v7", "arm64v8", "i386", "mips64le", "ppc64le", "s390x"] + assert entry.architectures == expected_architectures + + def test_alpine_architectures(self): + """Test that Alpine distributions get the correct architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + expected_architectures = ["amd64", "arm32v6", "arm32v7", "arm64v8", "i386", "ppc64le", "riscv64", "s390x"] + assert entry.architectures == expected_architectures + + def test_stackbrew_entry_string_format(self): + """Test that StackbrewEntry formats correctly with architectures.""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + + entry = StackbrewEntry( + tags=["8.2.1-alpine", "alpine"], + commit="abc123def456", + version=version, + distribution=distribution, + git_fetch_ref="refs/tags/v8.2.1" + ) + + output = str(entry) + + # Check that it contains the expected Alpine architectures + assert "amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x" in output + assert "Tags: 8.2.1-alpine, alpine" in output + assert "GitCommit: abc123def456" in output + assert "GitFetch: refs/tags/v8.2.1" in output + assert "Directory: alpine" in output diff --git a/release-automation/tests/test_stackbrew.py b/release-automation/tests/test_stackbrew.py new file mode 100644 index 000000000..e0864a41f --- /dev/null +++ b/release-automation/tests/test_stackbrew.py @@ -0,0 +1,203 @@ +"""Tests for stackbrew library generation.""" + +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release +from stackbrew_generator.stackbrew import StackbrewGenerator + + +class TestStackbrewGenerator: + """Tests for StackbrewGenerator.""" + + def setup_method(self): + """Set up test fixtures.""" + self.generator = StackbrewGenerator() + + def test_generate_tags_debian_ga_latest(self): + """Test tag generation for Debian GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1", # Full version + "8.2", # Mainline version (GA only) + "8", # Major version (latest only) + "8.2.1-bookworm", # Version with distro + "8.2-bookworm", # Mainline with distro + "8-bookworm", # Major with distro + "latest", # Latest tag (default distro only) + "bookworm" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_debian_ga_not_latest(self): + """Test tag generation for Debian GA version (not latest).""" + version = RedisVersion.parse("7.4.1") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v7.4.1") + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + expected_tags = [ + "7.4.1", # Full version + "7.4", # Mainline version (GA only) + "7.4.1-bookworm", # Version with distro + "7.4-bookworm" # Mainline with distro + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_alpine_ga_latest(self): + """Test tag generation for Alpine GA version (latest).""" + version = RedisVersion.parse("8.2.1") + distribution = Distribution(type=DistroType.ALPINE, name="alpine3.22") + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1") + + tags = self.generator.generate_tags_for_release(release, is_latest=True) + + expected_tags = [ + "8.2.1-alpine", # Version with distro type + "8.2.1-alpine3.22", # Version with full distro name + "8.2-alpine", # Mainline with distro type + "8.2-alpine3.22", # Mainline with full distro name + "8-alpine", # Major with distro type + "8-alpine3.22", # Major with full distro name + "alpine", # Bare distro type (latest only) + "alpine3.22" # Bare distro name (latest only) + ] + + assert set(tags) == set(expected_tags) + + def test_generate_tags_milestone_version(self): + """Test tag generation for milestone version.""" + version = RedisVersion.parse("8.2.1-m01") + distribution = Distribution(type=DistroType.DEBIAN, name="bookworm") + release = Release(commit="abc123", version=version, distribution=distribution, git_fetch_ref="refs/tags/v8.2.1-m01") + + tags = self.generator.generate_tags_for_release(release, is_latest=False) + + # Milestone versions should not get mainline version tags or major version tags + expected_tags = [ + "8.2.1-m01", # Full version only + "8.2.1-m01-bookworm", # Version with distro + ] + + assert set(tags) == set(expected_tags) + + + + def test_generate_stackbrew_library(self): + """Test complete stackbrew library generation.""" + releases = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" + ), + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.ALPINE, name="alpine3.22"), + git_fetch_ref="refs/tags/v8.2.1" + ), + Release( + commit="def456", + version=RedisVersion.parse("8.1.5"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.1.5" + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + assert len(entries) == 3 + + # Check that the 8.2.1 versions are marked as latest + debian_8_2_1 = next(e for e in entries if e.version.patch == 1 and e.distribution.type == DistroType.DEBIAN) + assert "latest" in debian_8_2_1.tags + assert "8" in debian_8_2_1.tags + + # Check that 8.1.5 is not marked as latest + debian_8_1_5 = next(e for e in entries if e.version.minor == 1) + assert "latest" not in debian_8_1_5.tags + assert "8" not in debian_8_1_5.tags + + def test_format_stackbrew_output(self): + """Test stackbrew output formatting.""" + entries = [ + Release( + commit="abc123", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.1" + ) + ] + + stackbrew_entries = self.generator.generate_stackbrew_library(entries) + output = self.generator.format_stackbrew_output(stackbrew_entries) + + assert isinstance(output, str) + assert len(output) > 0 + # Should contain comma-separated tags + assert "," in output + + def test_generate_stackbrew_library_with_head_milestone(self): + """Test stackbrew generation with milestone at head (matches bash test).""" + # This matches the bash test case: test_generate_stackbrew_library_with_head_milestone + releases = [ + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" + ), + Release( + commit="8d4437bdd0443189f9b3ba5943fdf793f821e8e2", + version=RedisVersion.parse("8.2.2-m01-int1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.2-m01-int1" + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.2.1" + ), + Release( + commit="a13b78815d980881e57f15b9cf13cd2f26f3fab6", + version=RedisVersion.parse("8.2.1"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.22"), + git_fetch_ref="refs/tags/v8.2.1" + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM debian:bookworm"), + git_fetch_ref="refs/tags/v8.0.3" + ), + Release( + commit="101262a8cf05b98137d88bc17e77db90c24cc783", + version=RedisVersion.parse("8.0.3"), + distribution=Distribution.from_dockerfile_line("FROM alpine:3.21"), + git_fetch_ref="refs/tags/v8.0.3" + ) + ] + + entries = self.generator.generate_stackbrew_library(releases) + + # Expected tags based on bash test + expected_tags = [ + ["8.2.2-m01-int1", "8.2.2-m01-int1-bookworm"], # milestone - no major/mainline tags + ["8.2.2-m01-int1-alpine", "8.2.2-m01-int1-alpine3.22"], # milestone - no major/mainline tags + ["8.2.1", "8.2", "8", "8.2.1-bookworm", "8.2-bookworm", "8-bookworm", "latest", "bookworm"], # GA - gets all tags + ["8.2.1-alpine", "8.2-alpine", "8-alpine", "8.2.1-alpine3.22", "8.2-alpine3.22", "8-alpine3.22", "alpine", "alpine3.22"], # GA - gets all tags + ["8.0.3", "8.0", "8.0.3-bookworm", "8.0-bookworm"], # different minor - no major tags + ["8.0.3-alpine", "8.0-alpine", "8.0.3-alpine3.21", "8.0-alpine3.21"] # different minor - no major tags + ] + + assert len(entries) == 6 + for i, entry in enumerate(entries): + assert set(entry.tags) == set(expected_tags[i]), f"Tags mismatch for entry {i}: {entry.tags} != {expected_tags[i]}" diff --git a/release-automation/tests/test_stackbrew_updater.py b/release-automation/tests/test_stackbrew_updater.py new file mode 100644 index 000000000..36b98590e --- /dev/null +++ b/release-automation/tests/test_stackbrew_updater.py @@ -0,0 +1,123 @@ +"""Tests for StackbrewUpdater class.""" + +import tempfile +from pathlib import Path + +from stackbrew_generator.stackbrew import StackbrewUpdater + +class TestStackbrewUpdater: + """Tests for StackbrewUpdater class.""" + + def setup_method(self): + """Set up test fixtures.""" + self.updater = StackbrewUpdater() + + def test_update_stackbrew_content_basic(self): + """Test basic stackbrew content update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + new_content = """Tags: 8.2.2, 8.2, 8, 8.2.2-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: debian + +Tags: 8.2.2-alpine, 8.2-alpine, 8-alpine, 8.2.2-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v6, arm32v7, arm64v8, i386, ppc64le, riscv64, s390x +GitCommit: new123commit +GitFetch: refs/tags/v8.2.2 +Directory: alpine""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + # Update the content + updated_content = self.updater.update_stackbrew_content( + input_file, 8, new_content, verbose=False + ) + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_parse_stackbrew_entries(self): + """Test parsing stackbrew entries.""" + lines = [ + "Tags: 8.2.1, 8.2, 8", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: debian", + "", + "Tags: 8.2.1-alpine, 8.2-alpine", + "Architectures: amd64, arm64v8", + "GitCommit: abc123", + "Directory: alpine" + ] + + entries = self.updater._parse_stackbrew_entries(lines) + + assert len(entries) == 2 + assert entries[0][0] == "Tags: 8.2.1, 8.2, 8" + assert entries[1][0] == "Tags: 8.2.1-alpine, 8.2-alpine" + + def test_entry_belongs_to_major_version(self): + """Test checking if entry belongs to major version.""" + entry_8x = [ + "Tags: 8.2.1, 8.2, 8, latest", + "Architectures: amd64", + "GitCommit: abc123", + "Directory: debian" + ] + + entry_7x = [ + "Tags: 7.4.0, 7.4, 7", + "Architectures: amd64", + "GitCommit: def456", + "Directory: debian" + ] + + assert self.updater._entry_belongs_to_major_version(entry_8x, 8) is True + assert self.updater._entry_belongs_to_major_version(entry_8x, 7) is False + assert self.updater._entry_belongs_to_major_version(entry_7x, 7) is True + assert self.updater._entry_belongs_to_major_version(entry_7x, 8) is False diff --git a/release-automation/tests/test_update_stackbrew_file.py b/release-automation/tests/test_update_stackbrew_file.py new file mode 100644 index 000000000..acdbd3ac1 --- /dev/null +++ b/release-automation/tests/test_update_stackbrew_file.py @@ -0,0 +1,297 @@ +"""Tests for update-stackbrew-file command.""" + +import tempfile +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +from typer.testing import CliRunner + +from stackbrew_generator.cli import app +from stackbrew_generator.models import RedisVersion, Distribution, DistroType, Release + + +class TestUpdateStackbrewFile: + """Tests for update-stackbrew-file command.""" + + def setup_method(self): + """Set up test fixtures.""" + self.runner = CliRunner() + + def test_update_stackbrew_file_basic(self): + """Test basic stackbrew file update functionality.""" + # Create a sample stackbrew file + sample_content = """# This file was generated via https://github.com/redis/docker-library-redis/blob/abc123/generate-stackbrew-library.sh + +Maintainers: David Maier (@dmaier-redislabs), + Yossi Gottlieb (@yossigo) +GitRepo: https://github.com/redis/docker-library-redis.git + +Tags: 8.2.1, 8.2, 8, 8.2.1-bookworm, 8.2-bookworm, 8-bookworm, latest, bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: debian + +Tags: 8.2.1-alpine, 8.2-alpine, 8-alpine, 8.2.1-alpine3.22, 8.2-alpine3.22, 8-alpine3.22, alpine, alpine3.22 +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old123commit +GitFetch: refs/tags/v8.2.1 +Directory: alpine + +Tags: 7.4.0, 7.4, 7, 7.4.0-bookworm, 7.4-bookworm, 7-bookworm +Architectures: amd64, arm32v5, arm32v7, arm64v8, i386, mips64le, ppc64le, s390x +GitCommit: old456commit +GitFetch: refs/tags/v7.4.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Mock git client + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + # Mock distribution detector + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + # Mock version filter + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + + # Mock the version filter to return Redis 8.x versions + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.2"), "new123commit", "refs/tags/v8.2.2") + ] + + # Mock releases + mock_releases = [ + Release( + commit="new123commit", + version=RedisVersion.parse("8.2.2"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.2" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + # Run the command with output to file + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(input_file), + "--verbose" + ]) + + assert result.exit_code == 0 + + # Check that the file was updated + updated_content = input_file.read_text() + + # Should still have the header + assert "Maintainers: David Maier" in updated_content + assert "GitRepo: https://github.com/redis/docker-library-redis.git" in updated_content + + # Should have new Redis 8.x content + assert "new123commit" in updated_content + assert "8.2.2" in updated_content + + # Should still have Redis 7.x content (unchanged) + assert "7.4.0" in updated_content + assert "old456commit" in updated_content + + # Should not have old Redis 8.x content + assert "old123commit" not in updated_content + assert "8.2.1" not in updated_content + + finally: + input_file.unlink() + + def test_update_stackbrew_file_nonexistent_input(self): + """Test error handling for nonexistent input file.""" + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", "/nonexistent/file.txt" + ]) + + assert result.exit_code == 1 + assert "Input file does not exist" in result.stderr + + def test_update_stackbrew_file_no_versions_found(self): + """Test error handling when no versions are found.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [] + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "9", + "--input", str(input_file) + ]) + + assert result.exit_code == 1 + assert "No versions found for Redis 9.x" in result.stderr + + finally: + input_file.unlink() + + def test_update_stackbrew_file_with_output_option(self): + """Test using separate output file.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as input_f, \ + tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as output_f: + + input_f.write(sample_content) + input_file = Path(input_f.name) + output_file = Path(output_f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new456commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new456commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file), + "--output", str(output_file) + ]) + + assert result.exit_code == 0 + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + # Output file should have updated content + updated_content = output_file.read_text() + assert "new456commit" in updated_content + assert "8.2.0" in updated_content + assert "old123" not in updated_content + + finally: + input_file.unlink() + output_file.unlink() + + def test_update_stackbrew_file_stdout_output(self): + """Test outputting to stdout when no output file is specified.""" + sample_content = """# Header +Maintainers: Test +GitRepo: https://example.com + +Tags: 8.1.0, 8.1, 8.1.0-bookworm, 8.1-bookworm +Architectures: amd64 +GitCommit: old123 +GitFetch: refs/tags/v8.1.0 +Directory: debian +""" + + with tempfile.NamedTemporaryFile(mode='w', suffix='.txt', delete=False) as f: + f.write(sample_content) + input_file = Path(f.name) + + try: + with patch('stackbrew_generator.cli.DistributionDetector') as mock_detector_class, \ + patch('stackbrew_generator.cli.GitClient') as mock_git_client_class, \ + patch('stackbrew_generator.cli.VersionFilter') as mock_version_filter_class: + + # Setup mocks + mock_git_client = Mock() + mock_git_client_class.return_value = mock_git_client + + mock_distribution_detector = Mock() + mock_detector_class.return_value = mock_distribution_detector + + mock_version_filter = Mock() + mock_version_filter_class.return_value = mock_version_filter + mock_version_filter.get_actual_major_redis_versions.return_value = [ + (RedisVersion.parse("8.2.0"), "new789commit", "refs/tags/v8.2.0") + ] + + mock_releases = [ + Release( + commit="new789commit", + version=RedisVersion.parse("8.2.0"), + distribution=Distribution(type=DistroType.DEBIAN, name="bookworm"), + git_fetch_ref="refs/tags/v8.2.0" + ) + ] + mock_distribution_detector.prepare_releases_list.return_value = mock_releases + + result = self.runner.invoke(app, [ + "update-stackbrew-file", + "8", + "--input", str(input_file) + ]) + + assert result.exit_code == 0 + + # Should output to stdout + assert "new789commit" in result.stdout + assert "8.2.0" in result.stdout + + # Original file should be unchanged + original_content = input_file.read_text() + assert "old123" in original_content + + finally: + input_file.unlink() diff --git a/release-automation/tests/test_version_filter.py b/release-automation/tests/test_version_filter.py new file mode 100644 index 000000000..7ebeb556d --- /dev/null +++ b/release-automation/tests/test_version_filter.py @@ -0,0 +1,324 @@ +"""Tests for VersionFilter class.""" + +import pytest +from unittest.mock import Mock, patch + +from stackbrew_generator.models import RedisVersion +from stackbrew_generator.version_filter import VersionFilter +from stackbrew_generator.git_operations import GitClient +from stackbrew_generator.exceptions import GitOperationError + + +class MockGitClient: + """Mock GitClient for testing.""" + + def __init__(self): + """Initialize mock git client.""" + self.remote_tags = [] + self.version_extraction_results = {} + + def set_remote_tags(self, tags): + """Set mock remote tags. + + Args: + tags: List of (commit, tag_ref) tuples + """ + self.remote_tags = tags + + def set_version_extraction_result(self, tag_ref, version_or_exception): + """Set mock version extraction result. + + Args: + tag_ref: Tag reference + version_or_exception: RedisVersion instance or Exception to raise + """ + self.version_extraction_results[tag_ref] = version_or_exception + + def list_remote_tags(self, major_version): + """Mock list_remote_tags method.""" + return self.remote_tags + + def extract_version_from_tag(self, tag_ref, major_version): + """Mock extract_version_from_tag method.""" + if tag_ref in self.version_extraction_results: + result = self.version_extraction_results[tag_ref] + if isinstance(result, Exception): + raise result + return result + # Default behavior - try to parse from tag_ref + return RedisVersion.parse(tag_ref.replace('refs/tags/', '')) + + +def create_version_tuples(version_strings): + """Helper to create version tuples from version strings. + + Args: + version_strings: List of version strings + + Returns: + List of (RedisVersion, commit, tag_ref) tuples + """ + tuples = [] + for i, version_str in enumerate(version_strings): + version = RedisVersion.parse(version_str) + commit = f"commit{i:03d}" + tag_ref = f"refs/tags/{version_str}" + tuples.append((version, commit, tag_ref)) + + tuples.sort(key=lambda x: x[0].sort_key, reverse=True) + return tuples + + +class TestVersionFilter: + """Tests for VersionFilter class.""" + + def test_init(self): + """Test VersionFilter initialization.""" + git_client = GitClient() + version_filter = VersionFilter(git_client) + assert version_filter.git_client is git_client + + def test_get_redis_versions_from_tags_success(self): + """Test successful version retrieval from tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should be sorted by version (newest first) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.1.0"] + assert version_strings == expected_versions + + # Check commits and tag refs + commits = [v[1] for v in result] + tag_refs = [v[2] for v in result] + expected_commits = ["commit001", "commit002", "commit003"] + expected_tag_refs = ["refs/tags/v8.2.1", "refs/tags/v8.2.0", "refs/tags/v8.1.0"] + assert commits == expected_commits + assert tag_refs == expected_tag_refs + + def test_get_redis_versions_from_tags_with_invalid_tags(self): + """Test version retrieval with some invalid tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/invalid-tag"), + ("commit003", "refs/tags/v8.1.0"), + ]) + + # Set up invalid tag to raise exception + mock_git_client.set_version_extraction_result( + "refs/tags/invalid-tag", + ValueError("Invalid version format") + ) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + # Should skip invalid tag and return only valid ones + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_get_redis_versions_from_tags_empty(self): + """Test version retrieval with no tags.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_redis_versions_from_tags(8) + + assert result == [] + + def test_filter_eol_versions_basic(self): + """Test basic EOL version filtering.""" + version_filter = VersionFilter(MockGitClient()) + + # Create test versions with one EOL minor version + versions = create_version_tuples([ + "v8.2.1", + "v8.2.0", + "v8.1.0-eol", + "v8.1.0-zoo1", + "v8.1.2", + "v8.0.1", + "v8.0.0" + ]) + + result = version_filter.filter_eol_versions(versions) + + # Should filter out all 8.1.* versions (because 8.1.0-eol exists) + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.2.0", "8.0.1", "8.0.0"] + assert version_strings == expected_versions + + def test_filter_eol_versions_empty(self): + """Test EOL filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_eol_versions([]) + assert result == [] + + def test_filter_actual_versions_basic(self): + """Test basic actual version filtering (latest patch per minor/milestone).""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with multiple patches for same minor version + versions = create_version_tuples([ + "v8.2.2", # Latest patch for 8.2 GA + "v8.2.1", # Older patch for 8.2 GA + "v8.2.0", # Oldest patch for 8.2 GA + "v8.1.1", # Latest patch for 8.1 GA + "v8.1.0", # Older patch for 8.1 GA + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep only latest patch for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2", "8.1.1"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_same_patch(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version + "v8.2.1-m02", # Latest milestone for 8.2 + "v8.2.1-m01", # Older milestone for 8.2 + "v8.1.0", # GA version + "v8.1.0-m01", # Milestone for 8.1 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_with_milestones_in_mainline(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + "v8.1.1-m01", # Milestone for 8.1 + "v8.2.0-m03", # Older milestone for 8.2.0 + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-m02", "8.2.1", "8.1.1-m01", "8.1.0"] + assert version_strings == expected_versions + + def test_when_filter_actual_versions_with_milestones_rc_is_preferred(self): + """Test actual version filtering with milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + # Create versions with both GA and milestone versions + versions = create_version_tuples([ + "v8.2.1", # GA version for 8.2 mainline + "v8.2.2-rc01", # Latest milestone for 8.2.2 + "v8.2.2-m02", # Latest milestone for 8.2.2 + "v8.2.2-m01", # Older milestone for 8.2.2 + "v8.1.0", # GA version + ]) + + result = version_filter.filter_actual_versions(versions) + + # Should keep latest GA and latest milestone for each minor version + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.2-rc01", "8.2.1", "8.1.0"] + assert version_strings == expected_versions + + def test_filter_actual_versions_milestone_only(self): + """Test actual version filtering with only milestone versions.""" + version_filter = VersionFilter(MockGitClient()) + + versions = create_version_tuples([ + "v8.2.1-m02", + "v8.2.1-m01", + "v8.1.0-m01", + ]) + + result = version_filter.filter_actual_versions(versions) + + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1-m02", "8.1.0-m01"] + assert version_strings == expected_versions + + def test_filter_actual_versions_empty(self): + """Test actual version filtering with empty input.""" + version_filter = VersionFilter(MockGitClient()) + result = version_filter.filter_actual_versions([]) + assert result == [] + + def test_get_actual_major_redis_versions_success(self): + """Test the main entry point method with successful flow.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([ + ("commit001", "refs/tags/v8.2.1"), + ("commit002", "refs/tags/v8.2.0"), + ("commit003", "refs/tags/v8.1.0-eol"), # Should be filtered out + ("commit004", "refs/tags/v8.0.1"), + ("commit005", "refs/tags/v8.0.0"), + ]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + # Should apply all filters: get tags -> filter EOL -> filter actual + version_strings = [str(v[0]) for v in result] + expected_versions = ["8.2.1", "8.0.1"] # Latest patches, no EOL + assert version_strings == expected_versions + + def test_get_actual_major_redis_versions_no_versions(self): + """Test main entry point with no versions found.""" + mock_git_client = MockGitClient() + mock_git_client.set_remote_tags([]) + + version_filter = VersionFilter(mock_git_client) + result = version_filter.get_actual_major_redis_versions(8) + + assert result == [] + +class TestVersionFilterIntegration: + """Integration tests using real GitClient (mocked at subprocess level).""" + + @patch('stackbrew_generator.git_operations.subprocess.run') + def test_integration_with_real_git_client(self, mock_subprocess): + """Test VersionFilter with real GitClient (mocked subprocess).""" + # Mock git ls-remote output + mock_subprocess.return_value.stdout = ( + "commit001\trefs/tags/v8.2.1\n" + "commit002\trefs/tags/v8.2.0\n" + "commit003\trefs/tags/v8.1.0-eol\n" + ) + mock_subprocess.return_value.returncode = 0 + + git_client = GitClient() + version_filter = VersionFilter(git_client) + + result = version_filter.get_actual_major_redis_versions(8) + + # Should get filtered results + version_strings = [str(v[0]) for v in result] + commits = [v[1] for v in result] + expected_versions = ["8.2.1"] # Only 8.2.1 after all filtering + expected_commits = ["commit001"] + assert version_strings == expected_versions + assert commits == expected_commits \ No newline at end of file diff --git a/test/run-entrypoint-tests.sh b/test/run-entrypoint-tests.sh index 28d138ae0..4516c6c9d 100755 --- a/test/run-entrypoint-tests.sh +++ b/test/run-entrypoint-tests.sh @@ -41,7 +41,6 @@ get_container_user_uid_gid_on_the_host() { container_user="$1" dir=$(mktemp -d -p .) docker run --rm -v "$(pwd)/$dir":/w -w /w --entrypoint=/bin/sh "$REDIS_IMG" -c "chown $container_user ." - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize stat -c "%u %g" "$dir" sudo rm -rf "$dir" } @@ -56,6 +55,42 @@ fi # Helper functions # +# Wait for Redis server or sentiel to be ready in a container by pinging it +# Arguments: +# $1 - container name/id +# Returns: +# 0 if Redis responds with PONG within timeout +# 1 if timeout CONTAINER_INIT_WAIT occurs +wait_for_redis_server_in_container() { + local container="$1" + local timeout="${CONTAINER_INIT_WAIT:-3}" + local elapsed=0 + local sleep_interval=0.1 + + if [ -z "$container" ]; then + return 1 + fi + + while [[ "$elapsed" < "$timeout" ]]; do + # Try to ping Redis server + if response=$(docker exec "$container" redis-cli ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + if response=$(docker exec "$container" redis-cli -p 26379 ping 2>/dev/null) && [ "$response" = "PONG" ]; then + return 0 + fi + + # Sleep and increment elapsed time + sleep "$sleep_interval" + elapsed=$(awk "BEGIN {print $elapsed + $sleep_interval}") + done + + echo "Timeout: Redis server did not respond within ${timeout}s" + docker stop "$container" >/dev/null + return 1 +} + # creates one entry of directory structure # used in combination with iterate_dir_structure_with create_entry() { @@ -181,7 +216,6 @@ run_docker_and_test_ownership() { fi docker_output=$($docker_run 2>&1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize if [ "$TEST_VERBOSE" ]; then echo "After:" @@ -270,14 +304,9 @@ run_redis_docker_and_check_uid_gid() { docker_cmd="$*" # shellcheck disable=SC2086 container=$(docker run $docker_flags -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize ret=$? - assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" - if [ $ret -gt 0 ]; then - echo "retarning" - return 1 - fi + wait_for_redis_server_in_container "$container" || return 1 cmdline=$(docker exec "$container" cat /proc/1/cmdline|tr -d \\0) assertContains "$docker_flags $docker_cmd, cmdline: $cmdline" "$cmdline" "$expected_cmd" @@ -302,7 +331,10 @@ run_redis_docker_and_check_modules() { docker_cmd="$1" # shellcheck disable=SC2086 container=$(docker run --rm -d "$REDIS_IMG" $docker_cmd) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 + info=$(docker exec "$container" redis-cli info) [ "$PLATFORM" ] && [ "$PLATFORM" != "amd64" ] && startSkipping @@ -329,7 +361,6 @@ assert_redis_v8() { test_redis_version() { ret=$(docker run --rm "$REDIS_IMG" -v|tail -n 1) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize assert_redis_v8 "$ret" } @@ -553,7 +584,9 @@ test_redis_server_persistence_with_bind_mount() { chmod 0444 "$dir" container=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -568,7 +601,10 @@ test_redis_server_persistence_with_bind_mount() { sudo chown -R "$HOST_OWNER" "$dir" container2=$(docker run --rm -d -v "$(pwd)/$dir":/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container2" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" @@ -586,7 +622,9 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data' container=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG" --appendonly yes) - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container" || return 1 result=$(echo save | docker exec -i "$container" redis-cli) assertEquals "OK" "$result" @@ -601,7 +639,10 @@ test_redis_server_persistence_with_volume() { docker run --rm -v test_redis:/data --entrypoint=/bin/sh "$REDIS_IMG" -c 'chown -R 0:0 /data && chmod 0000 -R /data' container2=$(docker run --rm -d -v test_redis:/data "$REDIS_IMG") - sleep $CONTAINER_INIT_WAIT # Wait for container to fully initialize + ret=$? + assertTrue "Container '$docker_flags $REDIS_IMG $docker_cmd' created" "[ $ret -eq 0 ]" + wait_for_redis_server_in_container "$container2" || return 1 + value=$(echo "GET FOO" | docker exec -i "$container2" redis-cli) assertEquals "$container" "$value" diff --git a/test/run-shell-func-tests.sh b/test/run-shell-func-tests.sh new file mode 100755 index 000000000..f772ee9e4 --- /dev/null +++ b/test/run-shell-func-tests.sh @@ -0,0 +1,73 @@ +#!/bin/bash +set -e -o pipefail +SCRIPT_DIR="$(dirname -- "$( readlink -f -- "$0"; )")" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/../.github/actions/common/func.sh" + +source_helper_file "helpers.sh" + +set -u + +init_console_output + +test_redis_version_split() { + local major minor patch suffix + local version + + version="8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="v8.2.1" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "2" "$minor" + assertEquals "patch of $version" "1" "$patch" + assertEquals "suffix of $version" "" "$suffix" + + version="8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="v8.0-m01" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "" "$patch" + assertEquals "suffix of $version" "-m01" "$suffix" + + version="8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" + + version="v8.0.3-m03-int" + IFS=: read -r major minor patch suffix < <(redis_version_split "$version") + assertEquals "return code for $version" "0" "$?" + assertEquals "major of $version" "8" "$major" + assertEquals "minor of $version" "0" "$minor" + assertEquals "patch of $version" "3" "$patch" + assertEquals "suffix of $version" "-m03-int" "$suffix" +} + +test_redis_version_split_fail() { + IFS=: read -r major minor patch suffix < <(redis_version_split 8.x.x) + assertNotEquals "return code" "0" "$?" +} + + +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/shunit2" \ No newline at end of file