Run CI only for changed modules in pull requests (#6610)

Author: smola

.circleci/collect_results.sh

@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;

.circleci/config.continue.yml.j2

@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -301,6 +306,10 @@ jobs:
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew clean
             << parameters.gradleTarget >>
+{% if use_git_changes %}
+            -Paffected_module_detector.enable
+            -PbaseBranch=origin/{{ pr_base_ref }}
+{% endif %}
             -PskipTests
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
@@ -393,6 +402,10 @@ jobs:
             GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx2G -Xms2G -XX:ErrorFile=/tmp/hs_err_pid%p.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp'"
             ./gradlew
             << parameters.gradleTarget >>
+{% if use_git_changes %}
+            -Paffected_module_detector.enable
+            -PbaseBranch=origin/{{ pr_base_ref }}
+{% endif %}
             -PskipTests
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
@@ -539,6 +552,10 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -Paffected_module_detector.enable
+            -PbaseBranch=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>

.circleci/render_config.py

@@ -28,7 +28,7 @@
 }
 # Version to use for all the base Docker images, see
 # https://github.com/DataDog/dd-trace-java-docker-build/pkgs/container/dd-trace-java-docker-build
-DOCKER_IMAGE_VERSION="v24.01"
+DOCKER_IMAGE_VERSION = "v24.01"
 
 # Get labels from pull requests to override some defaults for jobs to run.
 # `run-tests: all` will run all tests.
@@ -52,7 +52,7 @@
             )
             resp.raise_for_status()
         except Exception as e:
-            print(f"Request filed: {e}")
+            print(f"Request failed: {e}")
             time.sleep(1)
             continue
         data = resp.json()
@@ -63,12 +63,16 @@
     labels = {
         l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
     }
+    # get the base reference (e.g. `master`), commit hash is also available at the `sha` field.
+    pr_base_ref = data.get("base", {}).get("ref")
 else:
     labels = set()
 
-
 branch = os.environ.get("CIRCLE_BRANCH", "")
-if branch == "master" or branch.startswith("release/v") or "all" in labels:
+run_all = "all" in labels
+is_master_or_release = branch == "master" or branch.startswith("release/v")
+
+if is_master_or_release or run_all:
     all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
 else:
     all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
@@ -83,6 +87,9 @@
 is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true"
 is_regular = not is_nightly and not is_weekly
 
+# Use git changes detection on PRs
+use_git_changes = not run_all and not is_master_or_release and is_regular
+
 vars = {
     "is_nightly": is_nightly,
     "is_weekly": is_weekly,
@@ -92,12 +99,14 @@
     "nocov_jdks": nocov_jdks,
     "flaky": branch == "master" or "flaky" in labels or "all" in labels,
     "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-",
+    "use_git_changes": use_git_changes,
+    "pr_base_ref": pr_base_ref,
 }
 
 print(f"Variables for this build: {vars}")
 
 loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
-env = jinja2.Environment(loader=loader)
+env = jinja2.Environment(loader=loader, trim_blocks=True)
 tpl = env.get_template(TPL_FILENAME)
 out = tpl.render(**vars)
 

build.gradle

@@ -1,6 +1,7 @@
 buildscript {
   dependencies {
     classpath "pl.allegro.tech.build:axion-release-plugin:1.14.4"
+    classpath "com.dropbox.affectedmoduledetector:affectedmoduledetector:0.2.2"
   }
 
   configurations.all {
@@ -24,6 +25,8 @@ plugins {
   id "me.champeau.jmh" version "0.7.0" apply false
   id 'org.gradle.playframework' version '0.13' apply false
   id 'info.solidsoft.pitest' version '1.9.11'  apply false
+
+  id 'com.dropbox.affectedmoduledetector' version '0.2.2'
 }
 
 description = 'dd-trace-java'
@@ -160,15 +163,35 @@ allprojects { project ->
   }
 }
 
+affectedModuleDetector {
+  baseDir = "${project.rootDir}"
+  pathsAffectingAllModules = [".circleci/", "buildSrc/"]
+  specifiedBranch = project.hasProperty("baseBranch") ? project.property("baseBranch") : "origin/master"
+  compareFrom = "SpecifiedBranchCommit"
+  excludedModules = []
+  includeUncommitted = true
+}
+
+subprojects { subproject ->
+  subproject.tasks.withType(Test) { testTask ->
+    com.dropbox.affectedmoduledetector.AffectedModuleDetector.configureTaskGuard(testTask)
+  }
+}
 
 def testAggregate(String baseTaskName, includePrefixes, excludePrefixes) {
+
   def createRootTask = { rootTaskName, subProjTaskName ->
-    tasks.register(rootTaskName) { aggTest ->
+    tasks.register(rootTaskName) { aggTask ->
       subprojects { subproject ->
         if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
+          def affected = com.dropbox.affectedmoduledetector.AffectedModuleDetector.isProjectAffected(subproject)
           def testTask = subproject.tasks.findByName(subProjTaskName)
           if (testTask != null) {
-            aggTest.dependsOn(testTask)
+            if (affected) {
+              aggTask.dependsOn(testTask)
+            } else {
+              logger.info("Skipped based on git changes: ${subproject.path}")
+            }
           }
         }
       }