diff --git a/build.gradle.kts b/build.gradle.kts index eb7d87b4c43..abcd900cd3f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,10 +1,12 @@ import com.diffplug.gradle.spotless.SpotlessExtension +import datadog.gradle.plugin.ci.testAggregate plugins { id("datadog.gradle-debug") id("datadog.dependency-locking") id("datadog.tracer-version") id("datadog.dump-hanged-test") + id("datadog.ci-jobs") id("com.diffplug.spotless") version "6.13.0" id("com.github.spotbugs") version "5.0.14" @@ -137,4 +139,16 @@ allprojects { } } -apply(from = "$rootDir/gradle/ci_jobs.gradle") +testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList()) +testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList()) +testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList()) +testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), forceCoverage = true) +testAggregate( + "base", listOf(":"), + listOf( + ":dd-java-agent:instrumentation", + ":dd-smoke-tests", + ":dd-java-agent:agent-profiling", + ":dd-java-agent:agent-debugger" + ) +) diff --git a/buildSrc/src/main/kotlin/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/CIJobsExtensions.kt new file mode 100644 index 00000000000..1e284329e4c --- /dev/null +++ b/buildSrc/src/main/kotlin/CIJobsExtensions.kt @@ -0,0 +1,108 @@ +package datadog.gradle.plugin.ci + +import org.gradle.api.Project +import org.gradle.api.Task +import org.gradle.kotlin.dsl.extra + +/** + * Checks if a task is affected by git changes + */ +internal fun isAffectedBy(baseTask: Task, affectedProjects: Map>): String? { + val visited = mutableSetOf() + val queue = mutableListOf(baseTask) + + while (queue.isNotEmpty()) { + val t = queue.removeAt(0) + if (visited.contains(t)) { + continue + } + visited.add(t) + + val affectedTasks = affectedProjects[t.project] + if (affectedTasks != null) { + if (affectedTasks.contains("all")) { + return "${t.project.path}:${t.name}" + } + if (affectedTasks.contains(t.name)) { + return "${t.project.path}:${t.name}" + } + } + + t.taskDependencies.getDependencies(t).forEach { queue.add(it) } + } + return null +} + +/** + * Creates a single aggregate root task that depends on matching subproject tasks + */ +private fun Project.createRootTask( + rootTaskName: String, + subProjTaskName: String, + includePrefixes: List, + excludePrefixes: List, + forceCoverage: Boolean +) { + val coverage = forceCoverage || rootProject.hasProperty("checkCoverage") + tasks.register(rootTaskName) { + subprojects.forEach { subproject -> + val activePartition = subproject.extra.get("activePartition") as Boolean + if (activePartition && + includePrefixes.any { subproject.path.startsWith(it) } && + !excludePrefixes.any { subproject.path.startsWith(it) }) { + + val testTask = subproject.tasks.findByName(subProjTaskName) + var isAffected = true + + if (testTask != null) { + val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean + if (useGitChanges) { + @Suppress("UNCHECKED_CAST") + val affectedProjects = rootProject.extra.get("affectedProjects") as Map> + val fileTrigger = isAffectedBy(testTask, affectedProjects) + if (fileTrigger != null) { + logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)") + } else { + logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)") + isAffected = false + } + } + if (isAffected) { + dependsOn(testTask) + } + } + + if (isAffected && coverage) { + val coverageTask = subproject.tasks.findByName("jacocoTestReport") + if (coverageTask != null) { + dependsOn(coverageTask) + } + val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification") + if (verificationTask != null) { + dependsOn(verificationTask) + } + } + } + } + } +} + +/** + * Creates aggregate test tasks for CI using createRootTask() above + * + * Creates three subtasks for the given base task name: + * - ${baseTaskName}Test - runs allTests + * - ${baseTaskName}LatestDepTest - runs allLatestDepTests + * - ${baseTaskName}Check - runs check + */ +fun Project.testAggregate( + baseTaskName: String, + includePrefixes: List, + excludePrefixes: List = emptyList(), + forceCoverage: Boolean = false +) { + createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage) + createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage) +} + diff --git a/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts new file mode 100644 index 00000000000..a7b495c9096 --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts @@ -0,0 +1,132 @@ +/* + * This plugin defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize + * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes + * with -PgitBaseRef. + */ + +import datadog.gradle.plugin.ci.isAffectedBy +import java.io.File +import kotlin.math.abs + +// Set up activePartition property on all projects +allprojects { + extra.set("activePartition", true) + + val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition") + if (shouldUseTaskPartitions) { + val taskPartitionCount = rootProject.property("taskPartitionCount") as String + val taskPartition = rootProject.property("taskPartition") as String + val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt()) + extra.set("activePartition", currentTaskPartition == taskPartition.toInt()) + } +} + +fun relativeToGitRoot(f: File): File { + return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() +} + +fun getChangedFiles(baseRef: String, newRef: String): List { + val stdout = StringBuilder() + val stderr = StringBuilder() + + val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef")) + proc.inputStream.bufferedReader().use { stdout.append(it.readText()) } + proc.errorStream.bufferedReader().use { stderr.append(it.readText()) } + proc.waitFor() + require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" } + + val out = stdout.toString().trim() + if (out.isEmpty()) { + return emptyList() + } + + logger.debug("git diff output: $out") + return out.split("\n").map { File(rootProject.projectDir, it.trim()) } +} + +// Initialize git change tracking +rootProject.extra.set("useGitChanges", false) + +if (rootProject.hasProperty("gitBaseRef")) { + val baseRef = rootProject.property("gitBaseRef") as String + val newRef = if (rootProject.hasProperty("gitNewRef")) { + rootProject.property("gitNewRef") as String + } else { + "HEAD" + } + + val changedFiles = getChangedFiles(baseRef, newRef) + rootProject.extra.set("changedFiles", changedFiles) + rootProject.extra.set("useGitChanges", true) + + val ignoredFiles = fileTree(rootProject.projectDir) { + include(".gitignore", ".editorconfig") + include("*.md", "**/*.md") + include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd") + include("NOTICE") + include("static-analysis.datadog.yml") + } + + changedFiles.forEach { f -> + if (ignoredFiles.contains(f)) { + logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") + } + } + + val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) } + rootProject.extra.set("changedFiles", filteredChangedFiles) + + val globalEffectFiles = fileTree(rootProject.projectDir) { + include(".gitlab/**") + include("build.gradle") + include("gradle/**") + } + + for (f in filteredChangedFiles) { + if (globalEffectFiles.contains(f)) { + logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") + rootProject.extra.set("useGitChanges", false) + break + } + } + + if (rootProject.extra.get("useGitChanges") as Boolean) { + logger.warn("Git change tracking is enabled: $baseRef..$newRef") + + val projects = subprojects.sortedByDescending { it.projectDir.path.length } + val affectedProjects = mutableMapOf>() + + // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in + // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. + val matchers = listOf( + mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"), + mapOf("prefix" to "src/test/", "task" to "testClasses"), + mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses") + ) + + for (f in filteredChangedFiles) { + val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } + if (p == null) { + logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") + rootProject.extra.set("useGitChanges", false) + break + } + + // Make sure path separator is / + val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/") + val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all" + logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)") + affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task) + } + + rootProject.extra.set("affectedProjects", affectedProjects) + } +} + +tasks.register("runMuzzle") { + val muzzleSubprojects = subprojects.filter { p -> + val activePartition = p.extra.get("activePartition") as Boolean + activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle") + } + dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" }) +} diff --git a/gradle/ci_jobs.gradle b/gradle/ci_jobs.gradle deleted file mode 100644 index 4fd8a2350db..00000000000 --- a/gradle/ci_jobs.gradle +++ /dev/null @@ -1,197 +0,0 @@ -/** - * This script defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize - * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes - * with -PgitBaseRef. - */ -import java.nio.file.Paths - -allprojects { project -> - project.ext { - activePartition = true - } - final boolean shouldUseTaskPartitions = project.rootProject.hasProperty("taskPartitionCount") && project.rootProject.hasProperty("taskPartition") - if (shouldUseTaskPartitions) { - final int taskPartitionCount = project.rootProject.property("taskPartitionCount") as int - final int taskPartition = project.rootProject.property("taskPartition") as int - final currentTaskPartition = Math.abs(project.path.hashCode() % taskPartitionCount) - project.setProperty("activePartition", currentTaskPartition == taskPartition) - } -} - -File relativeToGitRoot(File f) { - return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() -} - -String isAffectedBy(Task baseTask, Map> affectedProjects) { - HashSet visited = [] - LinkedList queue = [baseTask] - while (!queue.isEmpty()) { - Task t = queue.poll() - if (visited.contains(t)) { - continue - } - visited.add(t) - - final Set affectedTasks = affectedProjects.get(t.project) - if (affectedTasks != null) { - if (affectedTasks.contains("all")) { - return "${t.project.path}:${t.name}" - } - if (affectedTasks.contains(t.name)) { - return "${t.project.path}:${t.name}" - } - } - - t.taskDependencies.each { queue.addAll(it.getDependencies(t)) } - } - return null -} - -List getChangedFiles(String baseRef, String newRef) { - final stdout = new StringBuilder() - final stderr = new StringBuilder() - final proc = "git diff --name-only ${baseRef}..${newRef}".execute() - proc.consumeProcessOutput(stdout, stderr) - proc.waitForOrKill(1000) - assert proc.exitValue() == 0, "git diff command failed, stderr: ${stderr}" - def out = stdout.toString().trim() - if (out.isEmpty()) { - return [] - } - logger.debug("git diff output: ${out}") - return out.split("\n").collect { - new File(rootProject.projectDir, it.trim()) - } -} - -rootProject.ext { - useGitChanges = false -} - -if (rootProject.hasProperty("gitBaseRef")) { - final String baseRef = rootProject.property("gitBaseRef") - final String newRef = rootProject.hasProperty("gitNewRef") ? rootProject.property("gitNewRef") : "HEAD" - - rootProject.ext { - it.changedFiles = getChangedFiles(baseRef, newRef) - useGitChanges = true - } - - final ignoredFiles = fileTree(rootProject.projectDir) { - include '.gitignore', '.editorconfig' - include '*.md', '**/*.md' - include 'gradlew', 'gradlew.bat', 'mvnw', 'mvnw.cmd' - include 'NOTICE' - include 'static-analysis.datadog.yml' - } - rootProject.changedFiles.each { File f -> - if (ignoredFiles.contains(f)) { - logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}") - } - } - rootProject.changedFiles = rootProject.changedFiles.findAll { !ignoredFiles.contains(it) } - - final globalEffectFiles = fileTree(rootProject.projectDir) { - include '.gitlab/**' - include 'build.gradle' - include 'gradle/**' - } - - for (File f in rootProject.changedFiles) { - if (globalEffectFiles.contains(f)) { - logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)") - rootProject.useGitChanges = false - break - } - } - - if (rootProject.useGitChanges) { - logger.warn("Git change tracking is enabled: ${baseRef}..${newRef}") - - final projects = subprojects.sort { a, b -> b.projectDir.path.length() <=> a.projectDir.path.length() } - Map> _affectedProjects = [:] - // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in - // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used. - final List> matchers = [ - [prefix: 'src/testFixtures/', task: 'testFixturesClasses'], - [prefix: 'src/test/', task: 'testClasses'], - [prefix: 'src/jmh/', task: 'jmhCompileGeneratedClasses'] - ] - for (File f in rootProject.changedFiles) { - Project p = projects.find { f.toString().startsWith(it.projectDir.path + "/") } - if (p == null) { - logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)") - rootProject.useGitChanges = false - break - } - // Make sure path separator is / - final relPath = Paths.get(p.projectDir.path).relativize(f.toPath()).collect { it.toString() }.join('/') - final String task = matchers.find { relPath.startsWith(it.prefix) }?.task ?: "all" - logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} (${task})") - _affectedProjects.computeIfAbsent(p, { new HashSet() }).add(task) - } - rootProject.ext { - it.affectedProjects = _affectedProjects - } - } -} - -def testAggregate(String baseTaskName, includePrefixes, excludePrefixes, boolean forceCoverage = false) { - def createRootTask = { String rootTaskName, String subProjTaskName -> - def coverage = forceCoverage || rootProject.hasProperty("checkCoverage") - tasks.register(rootTaskName) { aggTest -> - subprojects { subproject -> - if (subproject.property("activePartition") && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) }) { - Task testTask = subproject.tasks.findByName(subProjTaskName) - boolean isAffected = true - if (testTask != null) { - if (rootProject.useGitChanges) { - final fileTrigger = isAffectedBy(testTask, rootProject.property("affectedProjects")) - if (fileTrigger != null) { - logger.warn("Selecting ${subproject.path}:${subProjTaskName} (triggered by ${fileTrigger})") - } else { - logger.warn("Skipping ${subproject.path}:${subProjTaskName} (not affected by changed files)") - isAffected = false - } - } - if (isAffected) { - aggTest.dependsOn(testTask) - } - } - if (isAffected && coverage) { - def coverageTask = subproject.tasks.findByName("jacocoTestReport") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - coverageTask = subproject.tasks.findByName("jacocoTestCoverageVerification") - if (coverageTask != null) { - aggTest.dependsOn(coverageTask) - } - } - } - } - } - } - - createRootTask "${baseTaskName}Test", 'allTests' - createRootTask "${baseTaskName}LatestDepTest", 'allLatestDepTests' - createRootTask "${baseTaskName}Check", 'check' -} - -testAggregate("smoke", [":dd-smoke-tests"], []) -testAggregate("instrumentation", [":dd-java-agent:instrumentation"], []) -testAggregate("profiling", [":dd-java-agent:agent-profiling"], []) -testAggregate("debugger", [":dd-java-agent:agent-debugger"], [], true) -testAggregate("base", [":"], [ - ":dd-java-agent:instrumentation", - ":dd-smoke-tests", - ":dd-java-agent:agent-profiling", - ":dd-java-agent:agent-debugger" -]) - - -tasks.register('runMuzzle') { - dependsOn subprojects.findAll { p -> - p.activePartition && p.plugins.hasPlugin('java') && p.plugins.hasPlugin('muzzle') - }.collect { p -> p.path + ":muzzle" } -}