[TEST] Testing changes for #7464

.circleci/collect_results.sh

@@ -10,8 +10,17 @@ shopt -s globstar
 TEST_RESULTS_DIR=./results
 mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1
 
-echo "saving test results"
 mkdir -p $TEST_RESULTS_DIR
-find workspace/**/build/test-results -name \*.xml -exec sh -c '
+
+mkdir -p workspace
+mapfile -t test_result_dirs < <(find workspace -name test-results -type d)
+
+if [[ ${#test_result_dirs[@]} -eq 0 ]]; then
+  echo "No test results found"
+  exit 0
+fi
+
+echo "saving test results"
+find "${test_result_dirs[@]}" -name \*.xml -exec sh -c '
   file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \;

.circleci/config.continue.yml.j2

@@ -32,7 +32,7 @@ system_test_matrix: &system_test_matrix
 
 agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils"
 core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot"
-instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
+instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication"
 debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core"
 profiling_modules: &profiling_modules "dd-java-agent/agent-profiling"
 
@@ -99,6 +99,11 @@ commands:
   setup_code:
     steps:
       - checkout
+{% if use_git_changes %}
+      - run:
+          name: Fetch base branch
+          command: git fetch origin {{ pr_base_ref }}
+{% endif %}
       - run:
           name: Checkout merge commit
           command: .circleci/checkout_merge_commit.sh
@@ -312,6 +317,9 @@ jobs:
             ./gradlew clean
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             << pipeline.parameters.gradle_flags >>
             --max-workers=8
             --rerun-tasks
@@ -411,6 +419,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             -PskipTests
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PrunBuildSrcTests
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             << pipeline.parameters.gradle_flags >>
@@ -556,6 +567,9 @@ jobs:
             ./gradlew
             << parameters.gradleTarget >>
             << parameters.gradleParameters >>
+{% if use_git_changes %}
+            -PgitBaseRef=origin/{{ pr_base_ref }}
+{% endif %}
             -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX}
             <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><</ parameters.testJvm >>
             << pipeline.parameters.gradle_flags >>

.circleci/render_config.py

@@ -52,7 +52,7 @@
             )
             resp.raise_for_status()
         except Exception as e:
-            print(f"Request filed: {e}")
+            print(f"Request failed: {e}")
             time.sleep(1)
             continue
         data = resp.json()
@@ -63,12 +63,18 @@
     labels = {
         l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ")
     }
+    # get the base reference (e.g. `master`), commit hash is also available at the `sha` field.
+    pr_base_ref = data.get("base", {}).get("ref")
 else:
     labels = set()
+    pr_base_ref = ""
 
 
 branch = os.environ.get("CIRCLE_BRANCH", "")
-if branch == "master" or branch.startswith("release/v") or "all" in labels:
+run_all = "all" in labels
+is_master_or_release = branch == "master" or branch.startswith("release/v")
+
+if is_master_or_release or run_all:
     all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS
 else:
     all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels)
@@ -83,6 +89,9 @@
 is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true"
 is_regular = not is_nightly and not is_weekly
 
+# Use git changes detection on PRs
+use_git_changes = not run_all and not is_master_or_release and is_regular
+
 vars = {
     "is_nightly": is_nightly,
     "is_weekly": is_weekly,
@@ -92,12 +101,14 @@
     "nocov_jdks": nocov_jdks,
     "flaky": branch == "master" or "flaky" in labels or "all" in labels,
     "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-",
+    "use_git_changes": use_git_changes,
+    "pr_base_ref": pr_base_ref,
 }
 
 print(f"Variables for this build: {vars}")
 
 loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR)
-env = jinja2.Environment(loader=loader)
+env = jinja2.Environment(loader=loader, trim_blocks=True)
 tpl = env.get_template(TPL_FILENAME)
 out = tpl.render(**vars)
 

build.gradle

@@ -151,57 +151,4 @@ allprojects {
   }
 }
 
-allprojects { project ->
-  project.ext {
-    activePartition = true
-  }
-  final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition")
-  if (shouldUseTaskPartitions) {
-    final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int
-    final int taskPartition = project.rootProject.property("taskPartition") as int
-    final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount)
-    project.setProperty("activePartition", currentTaskPartition == taskPartition)
-  }
-}
-
-
-def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {
-  def createRootTask = { rootTaskName, subProjTaskName ->
-    def coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
-    tasks.register(rootTaskName) { aggTest ->
-      subprojects { subproject ->
-        if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
-          def testTask = subproject.tasks.findByName(subProjTaskName)
-          if (testTask != null) {
-            aggTest.dependsOn(testTask)
-          }
-          if (coverage) {
-            def coverageTask = subproject.tasks.findByName("jacocoTestReport")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-            coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
-            if (coverageTask != null) {
-              aggTest.dependsOn(coverageTask)
-            }
-          }
-        }
-      }
-    }
-  }
-
-  createRootTask "${baseTaskName}Test", 'allTests'
-  createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests'
-  createRootTask "${baseTaskName}Check", 'check'
-}
-
-testAggregate("smoke", [":dd-smoke-tests"], [])
-testAggregate("instrumentation", [":dd-java-agent:instrumentation"], [])
-testAggregate("profiling", [":dd-java-agent:agent-profiling"], [])
-testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true)
-testAggregate("base", [":"], [
-  ":dd-java-agent:instrumentation",
-  ":dd-smoke-tests",
-  ":dd-java-agent:agent-profiling",
-  ":dd-java-agent:agent-debugger"
-])
+apply from: "$rootDir/gradle/ci_jobs.gradle"

dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy

@@ -82,7 +82,7 @@ class IastKafka2SmokeTest extends AbstractIastServerSmokeTest {
     endpoint << ['json', 'string', 'byteArray', 'byteBuffer']
   }
 
-  void 'test kafka #endpoint value source'() {
+  void 'test xxx kafka #endpoint value source'() {
     setup:
     final type = "${endpoint}_source_value"
     final url = "http://localhost:${httpPort}/iast/kafka/$endpoint?type=${type}"

gradle/ci_jobs.gradle

@@ -0,0 +1,190 @@
+/**
+ * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize
+ * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes
+ * with -PgitBaseRef.
+ */
+import java.nio.file.Paths
+
+allprojects { project ->
+  project.ext {
+    activePartition = true
+  }
+  final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition")
+  if (shouldUseTaskPartitions) {
+    final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int
+    final int taskPartition = project.rootProject.property("taskPartition") as int
+    final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount)
+    project.setProperty("activePartition", currentTaskPartition == taskPartition)
+  }
+}
+
+File relativeToGitRoot(File f) {
+  return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile()
+}
+
+String isAffectedBy(Task baseTask, Map<Project, Set<String>> affectedProjects) {
+  HashSet<Task> visited = []
+  LinkedList<Task> queue = [baseTask]
+  while (!queue.isEmpty()) {
+    Task t = queue.poll()
+    if (visited.contains(t)) {
+      continue
+    }
+    visited.add(t)
+
+    final Set<String> affectedTasks = affectedProjects.get(t.project)
+    if (affectedTasks != null) {
+      if (affectedTasks.contains("all")) {
+        return "${t.project.path}:${t.name}"
+      }
+      if (affectedTasks.contains(t.name)) {
+        return "${t.project.path}:${t.name}"
+      }
+    }
+
+    t.taskDependencies.each { queue.addAll(it.getDependencies(t)) }
+  }
+  return null
+}
+
+List<File> getChangedFiles(String baseRef, String newRef) {
+  final stdout = new StringBuilder()
+  final stderr = new StringBuilder()
+  final proc = "git diff --name-only ${baseRef}..${newRef}".execute()
+  proc.consumeProcessOutput(stdout, stderr)
+  proc.waitForOrKill(1000)
+  assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}"
+  def out = stdout.toString().trim()
+  if (out.isEmpty()) {
+    return []
+  }
+  logger.debug("git diff output: ${out}")
+  return out.split("\n").collect {
+    new File(rootProject.projectDir, it.trim())
+  }
+}
+
+rootProject.ext {
+  useGitChanges = false
+}
+
+if (rootProject.hasProperty("gitBaseRef")) {
+  final String baseRef = rootProject.property("gitBaseRef")
+  final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD"
+
+  rootProject.ext {
+    it.changedFiles = getChangedFiles(baseRef, newRef)
+    useGitChanges = true
+  }
+
+  final ignoredFiles = fileTree(rootProject.projectDir) {
+    include '.gitignore', '.editorconfig'
+    include '*.md', '**/*.md'
+    include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd'
+    include 'NOTICE'
+    include 'static-analysis.datadog.yml'
+  }
+  rootProject.changedFiles.each { File f ->
+    if (ignoredFiles.contains(f)) {
+      logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}")
+    }
+  }
+  rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) }
+
+  final globalEffectFiles = fileTree(rootProject.projectDir) {
+    include '.circleci/**'
+    include 'build.gradle'
+    include 'gradle/**'
+  }
+
+  for (File f in rootProject.changedFiles) {
+    if (globalEffectFiles.contains(f)) {
+      logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)")
+      rootProject.useGitChanges = false
+      break
+    }
+  }
+
+  if (rootProject.useGitChanges) {
+    logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}")
+
+    final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() }
+    Map<Project, Set<String>> _affectedProjects = [:]
+    // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in
+    // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used.
+    final List<Map<String, String>> matchers = [
+      [prefix: 'src/testFixtures/', task: 'testFixturesClasses'],
+      [prefix: 'src/test/', task: 'testClasses'],
+      [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses']
+    ]
+    for (File f in rootProject.changedFiles) {
+      Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") }
+      if (p == null) {
+        logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)")
+        rootProject.useGitChanges = false
+        break
+      }
+      // Make sure path separator is /
+      final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/')
+      final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all"
+      logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})")
+      _affectedProjects.computeIfAbsent(p, { new HashSet<String>() }).add(task)
+    }
+    rootProject.ext {
+      it.affectedProjects = _affectedProjects
+    }
+  }
+}
+
+def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) {
+  def createRootTask = { String rootTaskName, String subProjTaskName ->
+    def coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
+    tasks.register(rootTaskName) { aggTest ->
+      subprojects { subproject ->
+        if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) {
+          Task testTask = subproject.tasks.findByName(subProjTaskName)
+          boolean isAffected = true
+          if (testTask != null) {
+            if (rootProject.useGitChanges) {
+              final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects"))
+              if (fileTrigger != null) {
+                logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})")
+              } else {
+                logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)")
+                isAffected = false
+              }
+            }
+            if (isAffected) {
+              aggTest.dependsOn(testTask)
+            }
+          }
+          if (isAffected && coverage) {
+            def coverageTask = subproject.tasks.findByName("jacocoTestReport")
+            if (coverageTask != null) {
+              aggTest.dependsOn(coverageTask)
+            }
+            coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
+            if (coverageTask != null) {
+              aggTest.dependsOn(coverageTask)
+            }
+          }
+        }
+      }
+    }
+  }
+
+  createRootTask "${baseTaskName}Test", 'allTests'
+  createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests'
+  createRootTask "${baseTaskName}Check", 'check'
+}
+
+testAggregate("smoke", [":dd-smoke-tests"], [])
+testAggregate("instrumentation", [":dd-java-agent:instrumentation"], [])
+testAggregate("profiling", [":dd-java-agent:agent-profiling"], [])
+testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true)
+testAggregate("base", [":"], [
+  ":dd-java-agent:instrumentation",
+  ":dd-smoke-tests",
+  ":dd-java-agent:agent-profiling",
+  ":dd-java-agent:agent-debugger"
+])