diff --git a/.circleci/collect_results.sh b/.circleci/collect_results.sh index 4c580e9b08b..44bc59f7f54 100755 --- a/.circleci/collect_results.sh +++ b/.circleci/collect_results.sh @@ -10,8 +10,17 @@ shopt -s globstar TEST_RESULTS_DIR=./results mkdir -p $TEST_RESULTS_DIR >/dev/null 2>&1 -echo "saving test results" mkdir -p $TEST_RESULTS_DIR -find workspace/**/build/test-results -name \*.xml -exec sh -c ' + +mkdir -p workspace +mapfile -t test_result_dirs < <(find workspace -name test-results -type d) + +if [[ ${#test_result_dirs[@]} -eq 0 ]]; then + echo "No test results found" + exit 0 +fi + +echo "saving test results" +find "${test_result_dirs[@]}" -name \*.xml -exec sh -c ' file=$(echo "$0" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_") cp "$0" "$1/$file"' {} $TEST_RESULTS_DIR \; diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 5c6984eee88..ab0aa317e79 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -32,7 +32,7 @@ system_test_matrix: &system_test_matrix agent_integration_tests_modules: &agent_integration_tests_modules "dd-trace-core|communication|internal-api|utils" core_modules: &core_modules "dd-java-agent|dd-trace-core|communication|internal-api|telemetry|utils|dd-java-agent/agent-bootstrap|dd-java-agent/agent-installer|dd-java-agent/agent-tooling|dd-java-agent/agent-builder|dd-java-agent/appsec|dd-java-agent/agent-crashtracking|dd-trace-api|dd-trace-ot" -instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication" +instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation|dd-java-agent/agent-tooling|dd-java-agent/agent-iast|dd-java-agent/agent-installer|dd-java-agent/agent-builder|dd-java-agent/agent-bootstrap|dd-java-agent/appsec|dd-java-agent/testing|dd-trace-core|dd-trace-api|internal-api|communication" debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core" profiling_modules: &profiling_modules "dd-java-agent/agent-profiling" @@ -99,6 +99,11 @@ commands: setup_code: steps: - checkout +{% if use_git_changes %} + - run: + name: Fetch base branch + command: git fetch origin {{ pr_base_ref }} +{% endif %} - run: name: Checkout merge commit command: .circleci/checkout_merge_commit.sh @@ -312,6 +317,9 @@ jobs: ./gradlew clean << parameters.gradleTarget >> -PskipTests +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} << pipeline.parameters.gradle_flags >> --max-workers=8 --rerun-tasks @@ -411,6 +419,9 @@ jobs: ./gradlew << parameters.gradleTarget >> -PskipTests +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} -PrunBuildSrcTests -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} << pipeline.parameters.gradle_flags >> @@ -556,6 +567,9 @@ jobs: ./gradlew << parameters.gradleTarget >> << parameters.gradleParameters >> +{% if use_git_changes %} + -PgitBaseRef=origin/{{ pr_base_ref }} +{% endif %} -PtaskPartitionCount=${CIRCLE_NODE_TOTAL} -PtaskPartition=${CIRCLE_NODE_INDEX} <<# parameters.testJvm >>-PtestJvm=<< parameters.testJvm >><> << pipeline.parameters.gradle_flags >> diff --git a/.circleci/render_config.py b/.circleci/render_config.py index 170ef594e81..f0222d19d90 100755 --- a/.circleci/render_config.py +++ b/.circleci/render_config.py @@ -52,7 +52,7 @@ ) resp.raise_for_status() except Exception as e: - print(f"Request filed: {e}") + print(f"Request failed: {e}") time.sleep(1) continue data = resp.json() @@ -63,12 +63,18 @@ labels = { l.replace("run-tests: ", "") for l in labels if l.startswith("run-tests: ") } + # get the base reference (e.g. `master`), commit hash is also available at the `sha` field. + pr_base_ref = data.get("base", {}).get("ref") else: labels = set() + pr_base_ref = "" branch = os.environ.get("CIRCLE_BRANCH", "") -if branch == "master" or branch.startswith("release/v") or "all" in labels: +run_all = "all" in labels +is_master_or_release = branch == "master" or branch.startswith("release/v") + +if is_master_or_release or run_all: all_jdks = ALWAYS_ON_JDKS | MASTER_ONLY_JDKS else: all_jdks = ALWAYS_ON_JDKS | (MASTER_ONLY_JDKS & labels) @@ -83,6 +89,9 @@ is_weekly = os.environ.get("CIRCLE_IS_WEEKLY", "false") == "true" is_regular = not is_nightly and not is_weekly +# Use git changes detection on PRs +use_git_changes = not run_all and not is_master_or_release and is_regular + vars = { "is_nightly": is_nightly, "is_weekly": is_weekly, @@ -92,12 +101,14 @@ "nocov_jdks": nocov_jdks, "flaky": branch == "master" or "flaky" in labels or "all" in labels, "docker_image_prefix": "" if is_nightly else f"{DOCKER_IMAGE_VERSION}-", + "use_git_changes": use_git_changes, + "pr_base_ref": pr_base_ref, } print(f"Variables for this build: {vars}") loader = jinja2.FileSystemLoader(searchpath=SCRIPT_DIR) -env = jinja2.Environment(loader=loader) +env = jinja2.Environment(loader=loader, trim_blocks=True) tpl = env.get_template(TPL_FILENAME) out = tpl.render(**vars) diff --git a/build.gradle b/build.gradle index 1b36756ca25..92441f04f0e 100644 --- a/build.gradle +++ b/build.gradle @@ -151,57 +151,4 @@ allprojects { } } -allprojects { project -> - project.ext { - activePartition = true - } - final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") - if (shouldUseTaskPartitions) { - final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int - final int taskPartition = project.rootProject.property("taskPartition") as int - final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) - project.setProperty("activePartition", currentTaskPartition == taskPartition) - } -} - - -def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { - def createRootTask = { rootTaskName, subProjTaskName -> - def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - tasks.register(rootTaskName) { aggTest -> - subprojects { subproject -> - if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { - def testTask = subproject.tasks.findByName(subProjTaskName) - if (testTask != null) { - aggTest.dependsOn(testTask) - } - if (coverage) { - def coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - } - } - } - } - } - - createRootTask "${baseTaskName}Test", 'allTests' - createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' - createRootTask "${baseTaskName}Check", 'check' -} - -testAggregate("smoke", [":dd-smoke-tests"], []) -testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) -testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) -testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) -testAggregate("base", [":"], [ - ":dd-java-agent:instrumentation", - ":dd-smoke-tests", - ":dd-java-agent:agent-profiling", - ":dd-java-agent:agent-debugger" -]) +apply from: "$rootDir/gradle/ci_jobs.gradle" diff --git a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy index 9c3b9a46685..987e333b29d 100644 --- a/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy +++ b/dd-smoke-tests/kafka-2/src/test/groovy/IastKafka2SmokeTest.groovy @@ -82,7 +82,7 @@ class IastKafka2SmokeTest extends AbstractIastServerSmokeTest { endpoint << ['json', 'string', 'byteArray', 'byteBuffer'] } - void 'test kafka #endpoint value source'() { + void 'test xxx kafka #endpoint value source'() { setup: final type = "${endpoint}_source_value" final url = "http://localhost:${httpPort}/iast/kafka/$endpoint?type=${type}" diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle new file mode 100644 index 00000000000..f535eebd9af --- /dev/null +++ b/gradle/ci_jobs.gradle @@ -0,0 +1,190 @@ +/** + * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize + * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes + * with -PgitBaseRef. + */ +import java.nio.file.Paths + +allprojects { project -> + project.ext { + activePartition = true + } + final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") + if (shouldUseTaskPartitions) { + final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int + final int taskPartition = project.rootProject.property("taskPartition") as int + final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) + project.setProperty("activePartition", currentTaskPartition == taskPartition) + } +} + +File relativeToGitRoot(File f) { + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +} + +String isAffectedBy(Task baseTask, Map> affectedProjects) { + HashSet visited = [] + LinkedList queue = [baseTask] + while (!queue.isEmpty()) { + Task t = queue.poll() + if (visited.contains(t)) { + continue + } + visited.add(t) + + final Set affectedTasks = affectedProjects.get(t.project) + if (affectedTasks != null) { + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } + } + + t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } + } + return null +} + +List getChangedFiles(String baseRef, String newRef) { + final stdout = new StringBuilder() + final stderr = new StringBuilder() + final proc = "git diff --name-only ${baseRef}..${newRef}".execute() + proc.consumeProcessOutput(stdout, stderr) + proc.waitForOrKill(1000) + assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" + def out = stdout.toString().trim() + if (out.isEmpty()) { + return [] + } + logger.debug("git diff output: ${out}") + return out.split("\n").collect { + new File(rootProject.projectDir, it.trim()) + } +} + +rootProject.ext { + useGitChanges = false +} + +if (rootProject.hasProperty("gitBaseRef")) { + final String baseRef = rootProject.property("gitBaseRef") + final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" + + rootProject.ext { + it.changedFiles = getChangedFiles(baseRef, newRef) + useGitChanges = true + } + + final ignoredFiles = fileTree(rootProject.projectDir) { + include '.gitignore', '.editorconfig' + include '*.md', '**/*.md' + include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' + include 'NOTICE' + include 'static-analysis.datadog.yml' + } + rootProject.changedFiles.each { File f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") + } + } + rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } + + final globalEffectFiles = fileTree(rootProject.projectDir) { + include '.circleci/**' + include 'build.gradle' + include 'gradle/**' + } + + for (File f in rootProject.changedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.useGitChanges = false + break + } + } + + if (rootProject.useGitChanges) { + logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") + + final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } + Map> _affectedProjects = [:] + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. + final List> matchers = [ + [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], + [prefix: 'src/test/', task: 'testClasses'], + [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] + ] + for (File f in rootProject.changedFiles) { + Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") + rootProject.useGitChanges = false + break + } + // Make sure path separator is / + final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') + final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") + _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) + } + rootProject.ext { + it.affectedProjects = _affectedProjects + } + } +} + +def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { + def createRootTask = { String rootTaskName, String subProjTaskName -> + def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") + tasks.register(rootTaskName) { aggTest -> + subprojects { subproject -> + if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { + Task testTask = subproject.tasks.findByName(subProjTaskName) + boolean isAffected = true + if (testTask != null) { + if (rootProject.useGitChanges) { + final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) + if (fileTrigger != null) { + logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") + } else { + logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") + isAffected = false + } + } + if (isAffected) { + aggTest.dependsOn(testTask) + } + } + if (isAffected && coverage) { + def coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + aggTest.dependsOn(coverageTask) + } + coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (coverageTask != null) { + aggTest.dependsOn(coverageTask) + } + } + } + } + } + } + + createRootTask "${baseTaskName}Test", 'allTests' + createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' + createRootTask "${baseTaskName}Check", 'check' +} + +testAggregate("smoke", [":dd-smoke-tests"], []) +testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) +testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) +testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) +testAggregate("base", [":"], [ + ":dd-java-agent:instrumentation", + ":dd-smoke-tests", + ":dd-java-agent:agent-profiling", + ":dd-java-agent:agent-debugger" +])