From 2bc8dba425755051e5a6523e7b18290ec9bbf5d0 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 15 May 2025 17:13:44 +0200 Subject: [PATCH 1/6] fix: CircleCI instrumentation test by running them on 2xlarge --- .circleci/config.continue.yml.j2 | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index ca4d78856e7..ad541004a5d 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -179,9 +179,14 @@ commands: display_memory_usage: steps: - run: - name: Max Memory Used + name: Runner Memory Usage # The file does not seem to exist when DLC is disabled - command: cat /sys/fs/cgroup/memory/memory.max_usage_in_bytes || true + command: | + # Show cgroupv2 memory usage + printf "cgroup memory.peak: $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 'not found')" + printf "cgroup memory.max : $(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo 'not found')" + printf "cgroup memory.high: $(cat /sys/fs/cgroup/memory.high 2>/dev/null || echo 'not found')" + printf "ram memory : $(grep MemTotal /proc/meminfo | tr -s ' ' | cut -d ' ' -f 2)%s\n" when: always @@ -282,7 +287,7 @@ commands: jobs: build: <<: *defaults - resource_class: xlarge + resource_class: 2xlarge parameters: gradleTarget: @@ -304,15 +309,15 @@ jobs: - restore_dependency_cache: cacheType: << parameters.cacheType >> - + - run: name: Build Project command: >- {% if is_nightly %} - ./gradlew resolveAndLockAll --write-locks && + ./gradlew resolveAndLockAll --write-locks --no-daemon && {% endif %} MAVEN_OPTS="-Xms64M -Xmx256M" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2560M -Xms2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" + GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2560M -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'" ./gradlew clean << parameters.gradleTarget >> -PskipTests From 0a53e5f409b33d99767551332d1555626733e3ed Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 15 May 2025 17:15:54 +0200 Subject: [PATCH 2/6] fix: parameterize resource_class on build jobs --- .circleci/config.continue.yml.j2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index ad541004a5d..a4192ca4a95 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -287,9 +287,11 @@ commands: jobs: build: <<: *defaults - resource_class: 2xlarge parameters: + resourceClass: + type: string + default: xlarge gradleTarget: type: string cacheType: @@ -301,6 +303,7 @@ jobs: type: string default: ".*" + resource_class: << parameters.resourceClass >> steps: - setup_code @@ -974,6 +977,7 @@ build_test_jobs: &build_test_jobs cacheType: base - build: name: build_inst + resourceClass: 2xlarge gradleTarget: :instrumentationTest cacheType: inst triggeredBy: *instrumentation_modules From 467efea88deee148ba3e6fc7f5245d5dbf33a421 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Thu, 15 May 2025 17:46:36 +0200 Subject: [PATCH 3/6] fix: Use 2xlarge on build_latestdep, and build_smoke --- .circleci/config.continue.yml.j2 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index a4192ca4a95..d9bb2b4b501 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -183,9 +183,9 @@ commands: # The file does not seem to exist when DLC is disabled command: | # Show cgroupv2 memory usage - printf "cgroup memory.peak: $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 'not found')" - printf "cgroup memory.max : $(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo 'not found')" - printf "cgroup memory.high: $(cat /sys/fs/cgroup/memory.high 2>/dev/null || echo 'not found')" + printf "cgroup memory.peak: $(cat /sys/fs/cgroup/memory.peak 2>/dev/null || echo 'not found')%s\n" + printf "cgroup memory.max : $(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo 'not found')%s\n" + printf "cgroup memory.high: $(cat /sys/fs/cgroup/memory.high 2>/dev/null || echo 'not found')%s\n" printf "ram memory : $(grep MemTotal /proc/meminfo | tr -s ' ' | cut -d ' ' -f 2)%s\n" when: always @@ -983,10 +983,12 @@ build_test_jobs: &build_test_jobs triggeredBy: *instrumentation_modules - build: name: build_latestdep + resourceClass: 2xlarge gradleTarget: :instrumentationLatestDepTest cacheType: latestdep - build: name: build_smoke + resourceClass: 2xlarge gradleTarget: :smokeTest cacheType: smoke - build: From 7755bb97bfacac2323b428c258d8b7766760fbcb Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Fri, 16 May 2025 11:28:01 +0200 Subject: [PATCH 4/6] fix: Bump resource class for various test and check jobs Z Test jobs 1. z_test_21_inst 2. z_test_17_inst 3. z_test_8_smoke Checks 1. check_smoke 2. check_base 3. check_inst Test instrumentation latest 1. test_8_inst_latest 2. test_17_inst_latest --- .circleci/config.continue.yml.j2 | 30 ++++++++++++++++++++++++------ 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index d9bb2b4b501..d142e2ada5f 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -392,6 +392,9 @@ jobs: <<: *defaults parameters: + resourceClass: + type: string + default: medium+ parallelism: type: integer default: 1 @@ -403,7 +406,7 @@ jobs: type: string default: ".*" - resource_class: medium+ + resource_class: << parameters.resourceClass >> parallelism: << parameters.parallelism >> @@ -453,6 +456,8 @@ jobs: when: on_fail command: .circleci/cancel_workflow.sh + - display_memory_usage + build_clean_cache: <<: *defaults @@ -501,13 +506,13 @@ jobs: tests: &tests <<: *defaults - # since tests use test containers, they will use a Linux VM / Remote Docker executor, so there is no medium+ size - resource_class: large - docker: - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}<< parameters.testJvm >> parameters: + resourceClass: + type: string + default: large environment: type: string default: "" @@ -543,6 +548,9 @@ jobs: cacheType: type: string + # since tests use test containers, they will use a Linux VM / Remote Docker executor, so there is no medium+ size + resource_class: << parameters.resourceClass >> + parallelism: << parameters.parallelism >> steps: @@ -660,13 +668,18 @@ jobs: exit 1 fi + # TODO: merge xlarge_tests and tests? or rename this? xlarge_tests: <<: *tests docker: + # The first image listed in your configuration for a job is referred to as the primary container image + # and this is where all steps in the job will run. Secondary containers can also be specified to run + # alongside for running services, such as, databases. - image: << pipeline.parameters.docker_image >>:{{ docker_image_prefix }}<< parameters.testJvm >> environment: - CI_USE_TEST_AGENT=true + - image: ghcr.io/datadog/dd-apm-test-agent/ddapm-test-agent:v1.11.0 environment: - LOG_LEVEL=DEBUG @@ -675,8 +688,6 @@ jobs: - DD_POOL_TRACE_CHECK_FAILURES=true - DD_DISABLE_ERROR_RESPONSES=true - ENABLED_CHECKS=trace_content_length,trace_stall,meta_tracer_version_header,trace_count_header,trace_peer_service,trace_dd_service - # TODO: merge xlarge_tests and tests? or rename this? - resource_class: large # The only way to do fan-in in CircleCI seems to have a proper job, so let's have one that @@ -1012,6 +1023,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: check_base + resourceClass: large gradleTarget: ":baseCheck" cacheType: base @@ -1019,6 +1031,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: check_inst + resourceClass: large parallelism: 5 gradleTarget: ":instrumentationCheck" cacheType: inst @@ -1028,6 +1041,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: check_smoke + resourceClass: large gradleTarget: ":smokeCheck" cacheType: smoke @@ -1086,6 +1100,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: z_test_<< matrix.testJvm >>_inst + resourceClass: xlarge gradleTarget: ":instrumentationTest" gradleParameters: "-PskipFlakyTests" triggeredBy: *instrumentation_modules @@ -1114,6 +1129,7 @@ build_test_jobs: &build_test_jobs - ok_to_test - build_latestdep name: test_8_inst_latest + resourceClass: xlarge gradleTarget: ":instrumentationLatestDepTest" gradleParameters: "-PskipFlakyTests" triggeredBy: *instrumentation_modules @@ -1128,6 +1144,7 @@ build_test_jobs: &build_test_jobs - ok_to_test - build_latestdep name: test_17_inst_latest + resourceClass: xlarge gradleTarget: ":instrumentationLatestDepTest" gradleParameters: "-PskipFlakyTests" triggeredBy: *instrumentation_modules @@ -1315,6 +1332,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: z_test_8_smoke + resourceClass: xlarge gradleTarget: "stageMainDist :smokeTest" gradleParameters: "-PskipFlakyTests" stage: smoke From 05bddaf6195f8995853c35d7ac8211f9b0dc31be Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Fri, 16 May 2025 12:37:59 +0200 Subject: [PATCH 5/6] fix: Bump resource class to xlarge check_inst --- .circleci/config.continue.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index d142e2ada5f..2806e14fae4 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -1031,7 +1031,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: check_inst - resourceClass: large + resourceClass: xlarge parallelism: 5 gradleTarget: ":instrumentationCheck" cacheType: inst From 681b6fd09543f83b375f0f94855914d092d72885 Mon Sep 17 00:00:00 2001 From: Brice Dutheil Date: Fri, 16 May 2025 13:09:30 +0200 Subject: [PATCH 6/6] fix: Bump resource class to 2xlarge check_inst --- .circleci/config.continue.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 2806e14fae4..87fd6886ad1 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -1031,7 +1031,7 @@ build_test_jobs: &build_test_jobs requires: - ok_to_test name: check_inst - resourceClass: xlarge + resourceClass: 2xlarge parallelism: 5 gradleTarget: ":instrumentationCheck" cacheType: inst