From ee5dc89b7461434f9659f73bab49adc988b009ba Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 12 Aug 2024 09:52:48 +0200 Subject: [PATCH 1/8] Run CI only for tasks affected by git changes --- .circleci/collect_results.sh | 13 ++- .circleci/config.continue.yml.j2 | 16 ++- .circleci/render_config.py | 17 +++- build.gradle | 161 ++++++++++++++++++++++++++++++- 4 files changed, 197 insertions(+), 10 deletions(-) diff --git a/.circleci/collect_results.sh b/.circleci/collect_results.sh index 4c580e9b08b..44bc59f7f54 100755 --- a/.circleci/collect_results.sh +++ b/.circleci/collect_results.sh @@ -10,8 +10,17 @@ shopt -s globstar TEST_RESULTS_DIR=./results mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1 -echo "saving test results" mkdir -p $TEST_RESULTS_DIR -find workspace/**/build/test-results -name \*.xml -exec sh -c ' + +mkdir -p workspace +mapfile -t test_result_dirs < <(find workspace -name test-results -type d) + +if [[ ${#test_result_dirs[@]} -eq 0 ]]; then + echo "No test results found" + exit 0 +fi + +echo "saving test results" +find "${test_result_dirs[@]}" -name \*.xml -exec sh -c ' file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_") cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \; diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 5c6984eee88..ab0aa317e79 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -32,7 +32,7 @@ system_test_matrix: &system_test_matrix agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils" core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot" -instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication" +instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication" debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core" profiling_modules: &profiling_modules "dd-java-agent/agent-profiling" @@ -99,6 +99,11 @@ commands: setup_code: steps: - checkout +{% if use_git_changes %} + - run: + name: Fetch base branch + command: git fetch origin {{ pr_base_ref }} +{% endif %} - run: name: Checkout merge commit command: .circleci/checkout_merge_commit.sh @@ -312,6 +317,9 @@ jobs: ./gradlew clean << parameters.gradleTarget >> -PskipTests +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} << pipeline.parameters.gradle_flags >> --max-workers=8 --rerun-tasks @@ -411,6 +419,9 @@ jobs: ./gradlew << parameters.gradleTarget >> -PskipTests +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} -PrunBuildSrcTests -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} << pipeline.parameters.gradle_flags >> @@ -556,6 +567,9 @@ jobs: ./gradlew << parameters.gradleTarget >> << parameters.gradleParameters >> +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><> << pipeline.parameters.gradle_flags >> diff --git a/.circleci/render_config.py b/.circleci/render_config.py index 170ef594e81..f0222d19d90 100755 --- a/.circleci/render_config.py +++ b/.circleci/render_config.py @@ -52,7 +52,7 @@ ) resp.raise_for_status() except Exception as e: - print(f"Request filed: {e}") + print(f"Request failed: {e}") time.sleep(1) continue data = resp.json() @@ -63,12 +63,18 @@ labels = { l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ") } + # get the base reference (e.g. `master`), commit hash is also available at the `sha` field. + pr_base_ref = data.get("base", {}).get("ref") else: labels = set() + pr_base_ref = "" branch = os.environ.get("CIRCLE_BRANCH", "") -if branch == "master" or branch.startswith("release/v") or "all" in labels: +run_all = "all" in labels +is_master_or_release = branch == "master" or branch.startswith("release/v") + +if is_master_or_release or run_all: all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS else: all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels) @@ -83,6 +89,9 @@ is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true" is_regular = not is_nightly and not is_weekly +# Use git changes detection on PRs +use_git_changes = not run_all and not is_master_or_release and is_regular + vars = { "is_nightly": is_nightly, "is_weekly": is_weekly, @@ -92,12 +101,14 @@ "nocov_jdks": nocov_jdks, "flaky": branch == "master" or "flaky" in labels or "all" in labels, "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-", + "use_git_changes": use_git_changes, + "pr_base_ref": pr_base_ref, } print(f"Variables for this build: {vars}") loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR) -env = jinja2.Environment(loader=loader) +env = jinja2.Environment(loader=loader, trim_blocks=True) tpl = env.get_template(TPL_FILENAME) out = tpl.render(**vars) diff --git a/build.gradle b/build.gradle index 1b36756ca25..da5d587ceeb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,3 +1,5 @@ +import java.nio.file.Paths + buildscript { dependencies { classpath "pl.allegro.tech.build:axion-release-plugin:1.14.4" @@ -164,18 +166,169 @@ allprojects { project -> } } +Set getTaskDependenciesRecursive(Task baseTask, Set visited = []) { + if (visited.contains(baseTask)) { + return [] + } + Set dependencies = [baseTask] + visited.add(baseTask) + for (td in baseTask.taskDependencies) { + for (t in td.getDependencies(baseTask)) { + dependencies.add(t) + dependencies.addAll(getTaskDependenciesRecursive(t, visited)) + } + } + return dependencies +} + +File relativeToGitRoot(File f) { + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +} + +String isAffectedBy(Task baseTask, Map> affectedProjects) { + for (Task t in getTaskDependenciesRecursive(baseTask)) { + if (!affectedProjects.containsKey(t.project)) { + continue + } + final Set affectedTasks = affectedProjects.get(t.project) + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } + } + return null +} + +List getChangedFiles(String baseRef, String newRef) { + final stdout = new StringBuilder() + final stderr = new StringBuilder() + final proc = "git diff --name-only ${baseRef}..${newRef}".execute() + proc.consumeProcessOutput(stdout, stderr) + proc.waitForOrKill(1000) + assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" + def out = stdout.toString().trim() + if (out.isEmpty()) { + return [] + } + logger.debug("git diff output: ${out}") + return out.split("\n").collect { + new File(rootProject.projectDir, it.trim()) + } +} + +rootProject.ext { + useGitChanges = false +} + +if (rootProject.hasProperty("gitBaseRef")) { + // -PgitBaseRef sets the base git reference to compare changes to. In CI, this should generally be set to the target + // branch, usually master. + final String baseRef = rootProject.property("gitBaseRef") + // -PgitNewRef sets the new git new reference to compare changes to. This is useful for testing the test selection method + // itself. Otherwise, comparing against current HEAD is what makes sense for CI. + final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" + + rootProject.ext { + it.changedFiles = getChangedFiles(baseRef, newRef) + useGitChanges = true + } + + // The ignoredFiles FileTree selects any file that should not trigger any tasks. + final ignoredFiles = fileTree(rootProject.projectDir) { + include '.gitingore', '.editorconfig' + include '*.md', '**/*.md' + include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' + include 'NOTICE' + include 'static-analysis.datadog.yml' + } + rootProject.changedFiles.each { File f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") + } + } + rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } + + // The globalEffectsFile FileTree selects any file that should trigger all tasks, regardless of gradle dependency + // tracking. + final globalEffectFiles = fileTree(rootProject.projectDir) { + include '.circleci/**' + include 'build.gradle' + include 'gradle/**' + } + + for (File f in rootProject.changedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.useGitChanges = false + break + } + } + + if (rootProject.useGitChanges) { + logger.warn("Git change tracking is enabled, base: ${baseRef}") + + // Get all projects, sorted by descending path length. + final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } + for (File f in rootProject.changedFiles) { + Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no tasks will be skipped)") + rootProject.useGitChanges = false + break + } + final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()) + final pathComponents = relPath.collect({ it.toString() }).toList() + Map> _affectedProjects = [:] + if (pathComponents.size() < 3) { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") + } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixturesClasses") { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testFixturesClasses") + } else if (pathComponents[0] == "src" && pathComponents[1] == "testClasses") { + // TODO: We could include other variants here such as latestTest, etc. + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testClasses") + } else if (pathComponents[0] == "src" && pathComponents[1] == "jmhCompileGeneratedClasses") { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("jmhCompileGeneratedClasses") + } else { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") + } + rootProject.ext { + it.affectedProjects = _affectedProjects + } + } + } + +} def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { - def createRootTask = { rootTaskName, subProjTaskName -> + def createRootTask = { String rootTaskName, String subProjTaskName -> def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") tasks.register(rootTaskName) { aggTest -> subprojects { subproject -> if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { - def testTask = subproject.tasks.findByName(subProjTaskName) + Task testTask = subproject.tasks.findByName(subProjTaskName) + boolean isAffected = true if (testTask != null) { - aggTest.dependsOn(testTask) + if (rootProject.useGitChanges) { + final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) + if (fileTrigger != null) { + logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") + } else { + logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") + isAffected = false + } + } + if (isAffected) { + aggTest.dependsOn(testTask) + } } - if (coverage) { + if (isAffected && coverage) { def coverageTask = subproject.tasks.findByName("jacocoTestReport") if (coverageTask != null) { aggTest.dependsOn(coverageTask) From d7c8db1f0da5c1d28a92613631b0233bb47449c3 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 19 Aug 2024 13:05:40 +0200 Subject: [PATCH 2/8] Move code for CI-oriented tasks to gradle/ci_jobs.gradle --- build.gradle | 208 +---------------------------------------- gradle/ci_jobs.gradle | 212 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 207 deletions(-) create mode 100644 gradle/ci_jobs.gradle diff --git a/build.gradle b/build.gradle index da5d587ceeb..92441f04f0e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,3 @@ -import java.nio.file.Paths - buildscript { dependencies { classpath "pl.allegro.tech.build:axion-release-plugin:1.14.4" @@ -153,208 +151,4 @@ allprojects { } } -allprojects { project -> - project.ext { - activePartition = true - } - final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") - if (shouldUseTaskPartitions) { - final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int - final int taskPartition = project.rootProject.property("taskPartition") as int - final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) - project.setProperty("activePartition", currentTaskPartition == taskPartition) - } -} - -Set getTaskDependenciesRecursive(Task baseTask, Set visited = []) { - if (visited.contains(baseTask)) { - return [] - } - Set dependencies = [baseTask] - visited.add(baseTask) - for (td in baseTask.taskDependencies) { - for (t in td.getDependencies(baseTask)) { - dependencies.add(t) - dependencies.addAll(getTaskDependenciesRecursive(t, visited)) - } - } - return dependencies -} - -File relativeToGitRoot(File f) { - return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() -} - -String isAffectedBy(Task baseTask, Map> affectedProjects) { - for (Task t in getTaskDependenciesRecursive(baseTask)) { - if (!affectedProjects.containsKey(t.project)) { - continue - } - final Set affectedTasks = affectedProjects.get(t.project) - if (affectedTasks.contains("all")) { - return "${t.project.path}:${t.name}" - } - if (affectedTasks.contains(t.name)) { - return "${t.project.path}:${t.name}" - } - } - return null -} - -List getChangedFiles(String baseRef, String newRef) { - final stdout = new StringBuilder() - final stderr = new StringBuilder() - final proc = "git diff --name-only ${baseRef}..${newRef}".execute() - proc.consumeProcessOutput(stdout, stderr) - proc.waitForOrKill(1000) - assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" - def out = stdout.toString().trim() - if (out.isEmpty()) { - return [] - } - logger.debug("git diff output: ${out}") - return out.split("\n").collect { - new File(rootProject.projectDir, it.trim()) - } -} - -rootProject.ext { - useGitChanges = false -} - -if (rootProject.hasProperty("gitBaseRef")) { - // -PgitBaseRef sets the base git reference to compare changes to. In CI, this should generally be set to the target - // branch, usually master. - final String baseRef = rootProject.property("gitBaseRef") - // -PgitNewRef sets the new git new reference to compare changes to. This is useful for testing the test selection method - // itself. Otherwise, comparing against current HEAD is what makes sense for CI. - final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" - - rootProject.ext { - it.changedFiles = getChangedFiles(baseRef, newRef) - useGitChanges = true - } - - // The ignoredFiles FileTree selects any file that should not trigger any tasks. - final ignoredFiles = fileTree(rootProject.projectDir) { - include '.gitingore', '.editorconfig' - include '*.md', '**/*.md' - include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' - include 'NOTICE' - include 'static-analysis.datadog.yml' - } - rootProject.changedFiles.each { File f -> - if (ignoredFiles.contains(f)) { - logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") - } - } - rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } - - // The globalEffectsFile FileTree selects any file that should trigger all tasks, regardless of gradle dependency - // tracking. - final globalEffectFiles = fileTree(rootProject.projectDir) { - include '.circleci/**' - include 'build.gradle' - include 'gradle/**' - } - - for (File f in rootProject.changedFiles) { - if (globalEffectFiles.contains(f)) { - logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") - rootProject.useGitChanges = false - break - } - } - - if (rootProject.useGitChanges) { - logger.warn("Git change tracking is enabled, base: ${baseRef}") - - // Get all projects, sorted by descending path length. - final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } - for (File f in rootProject.changedFiles) { - Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } - if (p == null) { - logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no tasks will be skipped)") - rootProject.useGitChanges = false - break - } - final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()) - final pathComponents = relPath.collect({ it.toString() }).toList() - Map> _affectedProjects = [:] - if (pathComponents.size() < 3) { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") - } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixturesClasses") { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testFixturesClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "testClasses") { - // TODO: We could include other variants here such as latestTest, etc. - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "jmhCompileGeneratedClasses") { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("jmhCompileGeneratedClasses") - } else { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") - } - rootProject.ext { - it.affectedProjects = _affectedProjects - } - } - } - -} - -def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { - def createRootTask = { String rootTaskName, String subProjTaskName -> - def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - tasks.register(rootTaskName) { aggTest -> - subprojects { subproject -> - if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { - Task testTask = subproject.tasks.findByName(subProjTaskName) - boolean isAffected = true - if (testTask != null) { - if (rootProject.useGitChanges) { - final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) - if (fileTrigger != null) { - logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") - } else { - logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") - isAffected = false - } - } - if (isAffected) { - aggTest.dependsOn(testTask) - } - } - if (isAffected && coverage) { - def coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - } - } - } - } - } - - createRootTask "${baseTaskName}Test", 'allTests' - createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' - createRootTask "${baseTaskName}Check", 'check' -} - -testAggregate("smoke", [":dd-smoke-tests"], []) -testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) -testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) -testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) -testAggregate("base", [":"], [ - ":dd-java-agent:instrumentation", - ":dd-smoke-tests", - ":dd-java-agent:agent-profiling", - ":dd-java-agent:agent-debugger" -]) +apply from: "$rootDir/gradle/ci_jobs.gradle" diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle new file mode 100644 index 00000000000..3605ca346c6 --- /dev/null +++ b/gradle/ci_jobs.gradle @@ -0,0 +1,212 @@ +/** + * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize + * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes + * with -PgitBaseRef. + */ +import java.nio.file.Paths + +allprojects { project -> + project.ext { + activePartition = true + } + final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") + if (shouldUseTaskPartitions) { + final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int + final int taskPartition = project.rootProject.property("taskPartition") as int + final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) + project.setProperty("activePartition", currentTaskPartition == taskPartition) + } +} + +Set getTaskDependenciesRecursive(Task baseTask, Set visited = []) { + if (visited.contains(baseTask)) { + return [] + } + Set dependencies = [baseTask] + visited.add(baseTask) + for (td in baseTask.taskDependencies) { + for (t in td.getDependencies(baseTask)) { + dependencies.add(t) + dependencies.addAll(getTaskDependenciesRecursive(t, visited)) + } + } + return dependencies +} + +File relativeToGitRoot(File f) { + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +} + +String isAffectedBy(Task baseTask, Map> affectedProjects) { + for (Task t in getTaskDependenciesRecursive(baseTask)) { + if (!affectedProjects.containsKey(t.project)) { + continue + } + final Set affectedTasks = affectedProjects.get(t.project) + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } + } + return null +} + +List getChangedFiles(String baseRef, String newRef) { + final stdout = new StringBuilder() + final stderr = new StringBuilder() + final proc = "git diff --name-only ${baseRef}..${newRef}".execute() + proc.consumeProcessOutput(stdout, stderr) + proc.waitForOrKill(1000) + assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" + def out = stdout.toString().trim() + if (out.isEmpty()) { + return [] + } + logger.debug("git diff output: ${out}") + return out.split("\n").collect { + new File(rootProject.projectDir, it.trim()) + } +} + +rootProject.ext { + useGitChanges = false +} + +if (rootProject.hasProperty("gitBaseRef")) { + // -PgitBaseRef sets the base git reference to compare changes to. In CI, this should generally be set to the target + // branch, usually master. + final String baseRef = rootProject.property("gitBaseRef") + // -PgitNewRef sets the new git new reference to compare changes to. This is useful for testing the test selection method + // itself. Otherwise, comparing against current HEAD is what makes sense for CI. + final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" + + rootProject.ext { + it.changedFiles = getChangedFiles(baseRef, newRef) + useGitChanges = true + } + + // The ignoredFiles FileTree selects any file that should not trigger any tasks. + final ignoredFiles = fileTree(rootProject.projectDir) { + include '.gitingore', '.editorconfig' + include '*.md', '**/*.md' + include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' + include 'NOTICE' + include 'static-analysis.datadog.yml' + } + rootProject.changedFiles.each { File f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") + } + } + rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } + + // The globalEffectsFile FileTree selects any file that should trigger all tasks, regardless of gradle dependency + // tracking. + final globalEffectFiles = fileTree(rootProject.projectDir) { + include '.circleci/**' + include 'build.gradle' + include 'gradle/**' + } + + for (File f in rootProject.changedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.useGitChanges = false + break + } + } + + if (rootProject.useGitChanges) { + logger.warn("Git change tracking is enabled, base: ${baseRef}") + + // Get all projects, sorted by descending path length. + final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } + for (File f in rootProject.changedFiles) { + Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no tasks will be skipped)") + rootProject.useGitChanges = false + break + } + final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()) + final pathComponents = relPath.collect({ it.toString() }).toList() + Map> _affectedProjects = [:] + if (pathComponents.size() < 3) { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") + } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixturesClasses") { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testFixturesClasses") + } else if (pathComponents[0] == "src" && pathComponents[1] == "testClasses") { + // TODO: We could include other variants here such as latestTest, etc. + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testClasses") + } else if (pathComponents[0] == "src" && pathComponents[1] == "jmhCompileGeneratedClasses") { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("jmhCompileGeneratedClasses") + } else { + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") + } + rootProject.ext { + it.affectedProjects = _affectedProjects + } + } + } + +} + +def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { + def createRootTask = { String rootTaskName, String subProjTaskName -> + def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") + tasks.register(rootTaskName) { aggTest -> + subprojects { subproject -> + if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { + Task testTask = subproject.tasks.findByName(subProjTaskName) + boolean isAffected = true + if (testTask != null) { + if (rootProject.useGitChanges) { + final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) + if (fileTrigger != null) { + logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") + } else { + logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") + isAffected = false + } + } + if (isAffected) { + aggTest.dependsOn(testTask) + } + } + if (isAffected && coverage) { + def coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + aggTest.dependsOn(coverageTask) + } + coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (coverageTask != null) { + aggTest.dependsOn(coverageTask) + } + } + } + } + } + } + + createRootTask "${baseTaskName}Test", 'allTests' + createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' + createRootTask "${baseTaskName}Check", 'check' +} + +testAggregate("smoke", [":dd-smoke-tests"], []) +testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) +testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) +testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) +testAggregate("base", [":"], [ + ":dd-java-agent:instrumentation", + ":dd-smoke-tests", + ":dd-java-agent:agent-profiling", + ":dd-java-agent:agent-debugger" +]) From 45e5205350f1b75c888cd02dce8dfa90299dac88 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 19 Aug 2024 14:01:32 +0200 Subject: [PATCH 3/8] Fix changes path patterns --- gradle/ci_jobs.gradle | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index 3605ca346c6..822d11d174f 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -136,14 +136,13 @@ if (rootProject.hasProperty("gitBaseRef")) { if (pathComponents.size() < 3) { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") - } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixturesClasses") { + } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixtures") { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testFixturesClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "testClasses") { - // TODO: We could include other variants here such as latestTest, etc. + } else if (pathComponents[0] == "src" && pathComponents[1] == "test") { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "jmhCompileGeneratedClasses") { + } else if (pathComponents[0] == "src" && pathComponents[1] == "jmh") { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("jmhCompileGeneratedClasses") } else { From 9af97597f0df7bd779126af8826875f9ad385833 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Mon, 19 Aug 2024 14:17:18 +0200 Subject: [PATCH 4/8] Fix affected projects for multiple changed files --- gradle/ci_jobs.gradle | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index 822d11d174f..8fa1951a4d0 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -123,6 +123,7 @@ if (rootProject.hasProperty("gitBaseRef")) { // Get all projects, sorted by descending path length. final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } + Map> _affectedProjects = [:] for (File f in rootProject.changedFiles) { Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } if (p == null) { @@ -132,7 +133,6 @@ if (rootProject.hasProperty("gitBaseRef")) { } final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()) final pathComponents = relPath.collect({ it.toString() }).toList() - Map> _affectedProjects = [:] if (pathComponents.size() < 3) { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") @@ -149,9 +149,9 @@ if (rootProject.hasProperty("gitBaseRef")) { logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") } - rootProject.ext { - it.affectedProjects = _affectedProjects - } + } + rootProject.ext { + it.affectedProjects = _affectedProjects } } From e55adae2d759b4ff0ee1e0b8270a35f3d3f8dde9 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Tue, 20 Aug 2024 12:54:20 +0200 Subject: [PATCH 5/8] Simplify code --- gradle/ci_jobs.gradle | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index 8fa1951a4d0..3c41cb0facd 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -89,7 +89,7 @@ if (rootProject.hasProperty("gitBaseRef")) { // The ignoredFiles FileTree selects any file that should not trigger any tasks. final ignoredFiles = fileTree(rootProject.projectDir) { - include '.gitingore', '.editorconfig' + include '.gitignore', '.editorconfig' include '*.md', '**/*.md' include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' include 'NOTICE' @@ -124,31 +124,23 @@ if (rootProject.hasProperty("gitBaseRef")) { // Get all projects, sorted by descending path length. final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } Map> _affectedProjects = [:] + final List> matchers = [ + [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], + [prefix: 'src/test/', task: 'testClasses'], + [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] + ] for (File f in rootProject.changedFiles) { Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } if (p == null) { - logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no tasks will be skipped)") + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") rootProject.useGitChanges = false break } - final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()) - final pathComponents = relPath.collect({ it.toString() }).toList() - if (pathComponents.size() < 3) { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") - } else if (pathComponents[0] == "src" && pathComponents[1] == "testFixtures") { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testFixturesClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testFixturesClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "test") { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (testClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("testClasses") - } else if (pathComponents[0] == "src" && pathComponents[1] == "jmh") { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (jmhCompileGeneratedClasses)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("jmhCompileGeneratedClasses") - } else { - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (all)") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add("all") - } + // Make sure path separator is / + final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') + final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) } rootProject.ext { it.affectedProjects = _affectedProjects From d8d3066ef1e28dce9519530865140744ca387480 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Tue, 20 Aug 2024 16:24:04 +0200 Subject: [PATCH 6/8] Simplify code --- gradle/ci_jobs.gradle | 49 +++++++++++++++---------------------------- 1 file changed, 17 insertions(+), 32 deletions(-) diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index 3c41cb0facd..fe85cdfb57b 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -18,37 +18,31 @@ allprojects { project -> } } -Set getTaskDependenciesRecursive(Task baseTask, Set visited = []) { - if (visited.contains(baseTask)) { - return [] - } - Set dependencies = [baseTask] - visited.add(baseTask) - for (td in baseTask.taskDependencies) { - for (t in td.getDependencies(baseTask)) { - dependencies.add(t) - dependencies.addAll(getTaskDependenciesRecursive(t, visited)) - } - } - return dependencies -} - File relativeToGitRoot(File f) { return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() } String isAffectedBy(Task baseTask, Map> affectedProjects) { - for (Task t in getTaskDependenciesRecursive(baseTask)) { - if (!affectedProjects.containsKey(t.project)) { + HashSet visited = [] + LinkedList queue = [baseTask] + while (!queue.isEmpty()) { + Task t = queue.poll() + if (visited.contains(t)) { continue } + visited.add(t) + final Set affectedTasks = affectedProjects.get(t.project) - if (affectedTasks.contains("all")) { - return "${t.project.path}:${t.name}" - } - if (affectedTasks.contains(t.name)) { - return "${t.project.path}:${t.name}" + if (affectedTasks != null) { + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } } + + t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } } return null } @@ -75,11 +69,7 @@ rootProject.ext { } if (rootProject.hasProperty("gitBaseRef")) { - // -PgitBaseRef sets the base git reference to compare changes to. In CI, this should generally be set to the target - // branch, usually master. final String baseRef = rootProject.property("gitBaseRef") - // -PgitNewRef sets the new git new reference to compare changes to. This is useful for testing the test selection method - // itself. Otherwise, comparing against current HEAD is what makes sense for CI. final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" rootProject.ext { @@ -87,7 +77,6 @@ if (rootProject.hasProperty("gitBaseRef")) { useGitChanges = true } - // The ignoredFiles FileTree selects any file that should not trigger any tasks. final ignoredFiles = fileTree(rootProject.projectDir) { include '.gitignore', '.editorconfig' include '*.md', '**/*.md' @@ -102,8 +91,6 @@ if (rootProject.hasProperty("gitBaseRef")) { } rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } - // The globalEffectsFile FileTree selects any file that should trigger all tasks, regardless of gradle dependency - // tracking. final globalEffectFiles = fileTree(rootProject.projectDir) { include '.circleci/**' include 'build.gradle' @@ -119,9 +106,8 @@ if (rootProject.hasProperty("gitBaseRef")) { } if (rootProject.useGitChanges) { - logger.warn("Git change tracking is enabled, base: ${baseRef}") + logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") - // Get all projects, sorted by descending path length. final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } Map> _affectedProjects = [:] final List> matchers = [ @@ -146,7 +132,6 @@ if (rootProject.hasProperty("gitBaseRef")) { it.affectedProjects = _affectedProjects } } - } def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { From a89f5471dc7920294fc069b7eadb16b0fe1a51de Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Tue, 20 Aug 2024 18:18:08 +0200 Subject: [PATCH 7/8] Clarify prefix matching logic --- gradle/ci_jobs.gradle | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle index fe85cdfb57b..f535eebd9af 100644 --- a/gradle/ci_jobs.gradle +++ b/gradle/ci_jobs.gradle @@ -110,6 +110,8 @@ if (rootProject.hasProperty("gitBaseRef")) { final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } Map> _affectedProjects = [:] + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. final List> matchers = [ [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], [prefix: 'src/test/', task: 'testClasses'], From e420db6f76460c0e55618e49f7a43a353e2a1556 Mon Sep 17 00:00:00 2001 From: Santiago Mola Date: Wed, 21 Aug 2024 11:29:36 +0200 Subject: [PATCH 8/8] [TEST] edit dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy --- .../kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy index 9c3b9a46685..987e333b29d 100644 --- a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy +++ b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy @@ -82,7 +82,7 @@ class IastKafka2SmokeTest extends AbstractIastServerSmokeTest { endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] } - void 'test kafka #endpoint value source'() { + void 'test xxx kafka #endpoint value source'() { setup: final type = "${endpoint}_source_value" final url = "http://localhost:${httpPort}/iast/kafka/$endpoint?type=${type}"