Convert ci_jobs.gradle to a convention plugin and extension

build.gradle.kts

@@ -1,10 +1,12 @@
 import com.diffplug.gradle.spotless.SpotlessExtension
+import datadog.gradle.plugin.ci.testAggregate
 
 plugins {
   id("datadog.gradle-debug")
   id("datadog.dependency-locking")
   id("datadog.tracer-version")
   id("datadog.dump-hanged-test")
+  id("datadog.ci-jobs")
 
   id("com.diffplug.spotless") version "6.13.0"
   id("com.github.spotbugs") version "5.0.14"
@@ -137,4 +139,16 @@ allprojects {
   }
 }
 
-apply(from = "$rootDir/gradle/ci_jobs.gradle")
+testAggregate("smoke", listOf(":dd-smoke-tests"), emptyList())
+testAggregate("instrumentation", listOf(":dd-java-agent:instrumentation"), emptyList())
+testAggregate("profiling", listOf(":dd-java-agent:agent-profiling"), emptyList())
+testAggregate("debugger", listOf(":dd-java-agent:agent-debugger"), forceCoverage = true)
+testAggregate(
+  "base", listOf(":"),
+  listOf(
+    ":dd-java-agent:instrumentation",
+    ":dd-smoke-tests",
+    ":dd-java-agent:agent-profiling",
+    ":dd-java-agent:agent-debugger"
+  )
+)

buildSrc/src/main/kotlin/CIJobsExtensions.kt

@@ -0,0 +1,108 @@
+package datadog.gradle.plugin.ci
+
+import org.gradle.api.Project
+import org.gradle.api.Task
+import org.gradle.kotlin.dsl.extra
+
+/**
+ * Checks if a task is affected by git changes
+ */
+internal fun isAffectedBy(baseTask: Task, affectedProjects: Map<Project, Set<String>>): String? {
+    val visited = mutableSetOf<Task>()
+    val queue = mutableListOf(baseTask)
+
+    while (queue.isNotEmpty()) {
+        val t = queue.removeAt(0)
+        if (visited.contains(t)) {
+            continue
+        }
+        visited.add(t)
+
+        val affectedTasks = affectedProjects[t.project]
+        if (affectedTasks != null) {
+            if (affectedTasks.contains("all")) {
+                return "${t.project.path}:${t.name}"
+            }
+            if (affectedTasks.contains(t.name)) {
+                return "${t.project.path}:${t.name}"
+            }
+        }
+
+        t.taskDependencies.getDependencies(t).forEach { queue.add(it) }
+    }
+    return null
+}
+
+/**
+ * Creates a single aggregate root task that depends on matching subproject tasks
+ */
+private fun Project.createRootTask(
+    rootTaskName: String,
+    subProjTaskName: String,
+    includePrefixes: List<String>,
+    excludePrefixes: List<String>,
+    forceCoverage: Boolean
+) {
+    val coverage = forceCoverage || rootProject.hasProperty("checkCoverage")
+    tasks.register(rootTaskName) {
+        subprojects.forEach { subproject ->
+            val activePartition = subproject.extra.get("activePartition") as Boolean
+            if (activePartition &&
+                includePrefixes.any { subproject.path.startsWith(it) } &&
+                !excludePrefixes.any { subproject.path.startsWith(it) }) {
+
+                val testTask = subproject.tasks.findByName(subProjTaskName)
+                var isAffected = true
+
+                if (testTask != null) {
+                    val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean
+                    if (useGitChanges) {
+                        @Suppress("UNCHECKED_CAST")
+                        val affectedProjects = rootProject.extra.get("affectedProjects") as Map<Project, Set<String>>
+                        val fileTrigger = isAffectedBy(testTask, affectedProjects)
+                        if (fileTrigger != null) {
+                            logger.warn("Selecting ${subproject.path}:$subProjTaskName (triggered by $fileTrigger)")
+                        } else {
+                            logger.warn("Skipping ${subproject.path}:$subProjTaskName (not affected by changed files)")
+                            isAffected = false
+                        }
+                    }
+                    if (isAffected) {
+                        dependsOn(testTask)
+                    }
+                }
+
+                if (isAffected && coverage) {
+                    val coverageTask = subproject.tasks.findByName("jacocoTestReport")
+                    if (coverageTask != null) {
+                        dependsOn(coverageTask)
+                    }
+                    val verificationTask = subproject.tasks.findByName("jacocoTestCoverageVerification")
+                    if (verificationTask != null) {
+                        dependsOn(verificationTask)
+                    }
+                }
+            }
+        }
+    }
+}
+
+/**
+ * Creates aggregate test tasks for CI using createRootTask() above
+ * 
+ * Creates three subtasks for the given base task name:
+ * - ${baseTaskName}Test - runs allTests
+ * - ${baseTaskName}LatestDepTest - runs allLatestDepTests
+ * - ${baseTaskName}Check - runs check
+ */
+fun Project.testAggregate(
+    baseTaskName: String,
+    includePrefixes: List<String>,
+    excludePrefixes: List<String> = emptyList(),
+    forceCoverage: Boolean = false
+) {
+    createRootTask("${baseTaskName}Test", "allTests", includePrefixes, excludePrefixes, forceCoverage)
+    createRootTask("${baseTaskName}LatestDepTest", "allLatestDepTests", includePrefixes, excludePrefixes, forceCoverage)
+    createRootTask("${baseTaskName}Check", "check", includePrefixes, excludePrefixes, forceCoverage)
+}
+

buildSrc/src/main/kotlin/datadog.ci-jobs.gradle.kts

@@ -0,0 +1,132 @@
+/*
+ * This plugin defines a set of tasks to be used in CI. These aggregate tasks support partitioning (to parallelize
+ * jobs) with -PtaskPartitionCount and -PtaskPartition, and limiting tasks to those affected by git changes
+ * with -PgitBaseRef.
+ */
+
+import datadog.gradle.plugin.ci.isAffectedBy
+import java.io.File
+import kotlin.math.abs
+
+// Set up activePartition property on all projects
+allprojects {
+    extra.set("activePartition", true)
+
+    val shouldUseTaskPartitions = rootProject.hasProperty("taskPartitionCount") && rootProject.hasProperty("taskPartition")
+    if (shouldUseTaskPartitions) {
+        val taskPartitionCount = rootProject.property("taskPartitionCount") as String
+        val taskPartition = rootProject.property("taskPartition") as String
+        val currentTaskPartition = abs(project.path.hashCode() % taskPartitionCount.toInt())
+        extra.set("activePartition", currentTaskPartition == taskPartition.toInt())
+    }
+}
+
+fun relativeToGitRoot(f: File): File {
+    return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile()
+}
+
+fun getChangedFiles(baseRef: String, newRef: String): List<File> {
+    val stdout = StringBuilder()
+    val stderr = StringBuilder()
+
+    val proc = Runtime.getRuntime().exec(arrayOf("git", "diff", "--name-only", "$baseRef..$newRef"))
+    proc.inputStream.bufferedReader().use { stdout.append(it.readText()) }
+    proc.errorStream.bufferedReader().use { stderr.append(it.readText()) }
+    proc.waitFor()
+    require(proc.exitValue() == 0) { "git diff command failed, stderr: $stderr" }
+
+    val out = stdout.toString().trim()
+    if (out.isEmpty()) {
+        return emptyList()
+    }
+
+    logger.debug("git diff output: $out")
+    return out.split("\n").map { File(rootProject.projectDir, it.trim()) }
+}
+
+// Initialize git change tracking
+rootProject.extra.set("useGitChanges", false)
+
+if (rootProject.hasProperty("gitBaseRef")) {
+    val baseRef = rootProject.property("gitBaseRef") as String
+    val newRef = if (rootProject.hasProperty("gitNewRef")) {
+        rootProject.property("gitNewRef") as String
+    } else {
+        "HEAD"
+    }
+
+    val changedFiles = getChangedFiles(baseRef, newRef)
+    rootProject.extra.set("changedFiles", changedFiles)
+    rootProject.extra.set("useGitChanges", true)
+
+    val ignoredFiles = fileTree(rootProject.projectDir) {
+        include(".gitignore", ".editorconfig")
+        include("*.md", "**/*.md")
+        include("gradlew", "gradlew.bat", "mvnw", "mvnw.cmd")
+        include("NOTICE")
+        include("static-analysis.datadog.yml")
+    }
+
+    changedFiles.forEach { f ->
+        if (ignoredFiles.contains(f)) {
+            logger.warn("Ignoring changed file: ${relativeToGitRoot(f)}")
+        }
+    }
+
+    val filteredChangedFiles = changedFiles.filter { !ignoredFiles.contains(it) }
+    rootProject.extra.set("changedFiles", filteredChangedFiles)
+
+    val globalEffectFiles = fileTree(rootProject.projectDir) {
+        include(".gitlab/**")
+        include("build.gradle")
+        include("gradle/**")
+    }
+
+    for (f in filteredChangedFiles) {
+        if (globalEffectFiles.contains(f)) {
+            logger.warn("Global effect change: ${relativeToGitRoot(f)} (no tasks will be skipped)")
+            rootProject.extra.set("useGitChanges", false)
+            break
+        }
+    }
+
+    if (rootProject.extra.get("useGitChanges") as Boolean) {
+        logger.warn("Git change tracking is enabled: $baseRef..$newRef")
+
+        val projects = subprojects.sortedByDescending { it.projectDir.path.length }
+        val affectedProjects = mutableMapOf<Project, MutableSet<String>>()
+
+        // Path prefixes mapped to affected task names. A file not matching any of these prefixes will affect all tasks in
+        // the project ("all" can be used a task name to explicitly state the same). Only the first matching prefix is used.
+        val matchers = listOf(
+            mapOf("prefix" to "src/testFixtures/", "task" to "testFixturesClasses"),
+            mapOf("prefix" to "src/test/", "task" to "testClasses"),
+            mapOf("prefix" to "src/jmh/", "task" to "jmhCompileGeneratedClasses")
+        )
+
+        for (f in filteredChangedFiles) {
+            val p = projects.find { f.toString().startsWith(it.projectDir.path + "/") }
+            if (p == null) {
+                logger.warn("Changed file: ${relativeToGitRoot(f)} at root project (no task will be skipped)")
+                rootProject.extra.set("useGitChanges", false)
+                break
+            }
+
+            // Make sure path separator is /
+            val relPath = p.projectDir.toPath().relativize(f.toPath()).joinToString("/")
+            val task = matchers.find { relPath.startsWith(it["prefix"]!!) }?.get("task") ?: "all"
+            logger.warn("Changed file: ${relativeToGitRoot(f)} in project ${p.path} ($task)")
+            affectedProjects.computeIfAbsent(p) { mutableSetOf() }.add(task)
+        }
+
+        rootProject.extra.set("affectedProjects", affectedProjects)
+    }
+}
+
+tasks.register("runMuzzle") {
+    val muzzleSubprojects = subprojects.filter { p ->
+        val activePartition = p.extra.get("activePartition") as Boolean
+        activePartition && p.plugins.hasPlugin("java") && p.plugins.hasPlugin("muzzle")
+    }
+    dependsOn(muzzleSubprojects.map { p -> "${p.path}:muzzle" })
+}