From 3fc64a104e5f2e13e5ef786cb49c4d77a8913777 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Thu, 26 Jun 2025 09:29:51 +0200 Subject: [PATCH 1/7] Skip tests based on git changes on GitLab --- .gitlab-ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9afc147a22e..5645851599d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -223,7 +223,11 @@ build_tests: MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on script: - - ./gradlew clean $GRADLE_TARGET -PskipTests $GRADLE_ARGS + - | + if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + fi + - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS populate_dep_cache: extends: build_tests @@ -327,7 +331,11 @@ test_published_artifacts: variables: CACHE_TYPE: lib script: - - ./gradlew $GRADLE_TARGET -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS + - | + if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + fi + - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS after_script: - *cgroup_info - source .gitlab/gitlab-utils.sh @@ -460,6 +468,10 @@ muzzle-dep-report: - if: $CI_COMMIT_BRANCH == "master" when: on_success script: + - | + if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + fi - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; then From 283207e2868c147809ff9f67ffe4b4b0a51d159c Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Fri, 27 Jun 2025 17:51:17 +0200 Subject: [PATCH 2/7] Use DDCI_PULL_REQUEST_TARGET_SHA --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5645851599d..79d6ca49fa0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -225,7 +225,7 @@ build_tests: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" fi - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS @@ -333,7 +333,7 @@ test_published_artifacts: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" fi - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS after_script: @@ -470,7 +470,7 @@ muzzle-dep-report: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$CI_DEFAULT_BRANCH" + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" fi - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; From 15a1ad8e1dda88e913f913b64c531d059dc383b5 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Fri, 27 Jun 2025 19:11:37 +0200 Subject: [PATCH 3/7] Add script to infer base reference --- .gitlab/find-gh-base-ref.sh | 67 +++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100755 .gitlab/find-gh-base-ref.sh diff --git a/.gitlab/find-gh-base-ref.sh b/.gitlab/find-gh-base-ref.sh new file mode 100755 index 00000000000..5e47d5bc65e --- /dev/null +++ b/.gitlab/find-gh-base-ref.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Determines the base branch for the current PR (if we are running in a PR). +set -euo pipefail + +# Happy path: if we're just one commit away from master, base ref is master. +if [[ $(git log --pretty=oneline origin/master..HEAD | wc -l) -eq 1 ]]; then + echo "We are just one commit away from master, base ref is master" >&2 + echo "master" + exit 0 +fi + +# In GitLab: we have no reference to the base branch or even the PR number. +# We have to find it from the current branch name, which is defined in +# CI_COMMIT_REF_NAME. +if [[ -z "${CI_COMMIT_REF_NAME}" ]]; then + echo "CI_COMMIT_REF_NAME is not set, not running in GitLab CI?" >&2 + exit 1 +fi + +if [[ -z "${GITHUB_TOKEN:-}" ]]; then + echo "GITHUB_TOKEN is not set, please set it" >&2 + exit 1 +fi + +if ! command -v curl >/dev/null 2>&1; then + echo "curl is not installed, please install it" >&2 + exit 1 +fi + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is not installed, please install it" >&2 + exit 1 +fi + +while true; do + set +e + PR_DATA=$(curl \ + -XGET \ + --silent \ + --include \ + --fail-with-body \ + -H 'Accept: application/vnd.github+json' \ + -H "Authorization: Bearer ${GITHUB_TOKEN}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/datadog/dd-trace-java/pulls?head=DataDog:${CI_COMMIT_REF_NAME}&sort=updated&direction=desc") + exit_code=$? + set -e + if [[ ${exit_code} -eq 0 ]]; then + PR_NUMBER=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].number') + PR_BASE_REF=$(echo "$PR_DATA" | sed '1,/^[[:space:]]*$/d' | jq -r '.[].base.ref') + echo "PR is https://github.com/datadog/dd-trace-java/pull/${PR_NUMBER} and base ref is ${PR_BASE_REF}">&2 + echo "${PR_BASE_REF}" + exit 0 + fi + if echo "$PR_DATA" | grep -q "^x-ratelimit-reset:"; then + reset_timestamp=$(echo -n "$PR_DATA" | grep "^x-ratelimit-reset:" | sed -e 's/^x-ratelimit-reset: //' -e 's/\r//') + now=$(date +%s) + sleep_time=$((reset_timestamp - now + 1)) + echo "GitHub rate limit exceeded, sleeping for ${sleep_time} seconds" >&2 + sleep "${sleep_time}" + continue + fi + echo -e "GitHub request failed for an unknown reason:\n$(echo "$PR_DATA" | sed '/^$/q')" >&2 + echo "Assuming base ref is master" >&2 + echo "master" + exit 0 +done From a1ec019ae056912aba9880de92cb3f5ae8d57a5f Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Fri, 27 Jun 2025 19:12:48 +0200 Subject: [PATCH 4/7] Infer base branch --- .gitlab-ci.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 79d6ca49fa0..077ed86f02a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -225,7 +225,8 @@ build_tests: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" + export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" fi - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS @@ -333,7 +334,8 @@ test_published_artifacts: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" + export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" fi - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS after_script: @@ -470,7 +472,8 @@ muzzle-dep-report: script: - | if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=$DDCI_PULL_REQUEST_TARGET_SHA" + export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" fi - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; From 0dccb3d026e32eab4ee7ac1b8f398904f917cb0a Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 30 Jun 2025 08:15:19 +0200 Subject: [PATCH 5/7] Fetch GitHub token from AWS SSM --- .gitlab/find-gh-base-ref.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.gitlab/find-gh-base-ref.sh b/.gitlab/find-gh-base-ref.sh index 5e47d5bc65e..cf06c1ec8fe 100755 --- a/.gitlab/find-gh-base-ref.sh +++ b/.gitlab/find-gh-base-ref.sh @@ -18,8 +18,17 @@ if [[ -z "${CI_COMMIT_REF_NAME}" ]]; then fi if [[ -z "${GITHUB_TOKEN:-}" ]]; then - echo "GITHUB_TOKEN is not set, please set it" >&2 - exit 1 + echo "GITHUB_TOKEN is not set, fetching from AWS SSM" >&2 + if ! command -v aws >/dev/null 2>&1; then + echo "aws is not installed, please install it" >&2 + exit 1 + fi + GITHUB_TOKEN=$(aws ssm get-parameter --name "ci.$CI_PROJECT_NAME.gh_token" --with-decryption --query "Parameter.Value" --output text) + if [[ -z "${GITHUB_TOKEN}" ]]; then + echo "Failed to fetch GITHUB_TOKEN from AWS SSM" >&2 + exit 1 + fi + export GITHUB_TOKEN fi if ! command -v curl >/dev/null 2>&1; then From b06a47b9fd6422389d94e990ec51044b1c98422b Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 30 Jun 2025 08:32:55 +0200 Subject: [PATCH 6/7] Fix AWS SSM token name --- .gitlab/find-gh-base-ref.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab/find-gh-base-ref.sh b/.gitlab/find-gh-base-ref.sh index cf06c1ec8fe..2e20a4ea31c 100755 --- a/.gitlab/find-gh-base-ref.sh +++ b/.gitlab/find-gh-base-ref.sh @@ -23,7 +23,7 @@ if [[ -z "${GITHUB_TOKEN:-}" ]]; then echo "aws is not installed, please install it" >&2 exit 1 fi - GITHUB_TOKEN=$(aws ssm get-parameter --name "ci.$CI_PROJECT_NAME.gh_token" --with-decryption --query "Parameter.Value" --output text) + GITHUB_TOKEN=$(aws ssm get-parameter --name "ci.$CI_PROJECT_NAME.gh_release_token" --with-decryption --query "Parameter.Value" --output text) if [[ -z "${GITHUB_TOKEN}" ]]; then echo "Failed to fetch GITHUB_TOKEN from AWS SSM" >&2 exit 1 From bf4cb576d0f10b4ba950ca86bbaa70466faa2316 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 30 Jun 2025 09:18:29 +0200 Subject: [PATCH 7/7] Extract gitlab_base_ref_params --- .gitlab-ci.yml | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 077ed86f02a..ee98e27458f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,13 @@ default: - .gitlab/cgroup-info.sh - gitlab_section_end "cgroup-info" +.gitlab_base_ref_params: &gitlab_base_ref_params + - | + if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then + export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) + export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" + fi + .gradle_build: &gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}base stage: build @@ -223,11 +230,7 @@ build_tests: MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on script: - - | - if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" - fi + - *gitlab_base_ref_params - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS populate_dep_cache: @@ -332,11 +335,7 @@ test_published_artifacts: variables: CACHE_TYPE: lib script: - - | - if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" - fi + - *gitlab_base_ref_params - ./gradlew $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PrunBuildSrcTests -PskipSpotless -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS after_script: - *cgroup_info @@ -470,11 +469,7 @@ muzzle-dep-report: - if: $CI_COMMIT_BRANCH == "master" when: on_success script: - - | - if [[ ! $CI_COMMIT_BRANCH =~ ^(master|release/.*)$ ]]; then - export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) - export GRADLE_PARAMS="$GRADLE_PARAMS -PgitBaseRef=origin/$GIT_BASE_REF" - fi + - *gitlab_base_ref_params - > if [ "$PROFILE_TESTS" == "true" ] && [ "$testJvm" != "ibm8" ] && [ "$testJvm" != "oracle8" ]; then