From b4a5988c22abacf7e69d1cbd3075bcc9da5289cb Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 16 Apr 2018 15:31:12 -0700 Subject: [PATCH 1/3] Build: Move java home checks to pre-execution phase This commit moves the checks on JAVAX_HOME (where X is the java version number) existing to the end of gradle's configuration phase, and based on whether the tasks needing the java home are configured to execute. relates #29519 --- .../elasticsearch/gradle/BuildPlugin.groovy | 27 ++++++++++--------- .../gradle/test/ClusterFormationTasks.groovy | 4 +++ .../elasticsearch/gradle/test/NodeInfo.groovy | 21 +++++++++++---- distribution/bwc/build.gradle | 4 +-- qa/reindex-from-old/build.gradle | 2 +- 5 files changed, 37 insertions(+), 21 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 3103f23472ed7..00a99d41fc112 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -38,6 +38,7 @@ import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.ProjectDependency import org.gradle.api.artifacts.ResolvedArtifact import org.gradle.api.artifacts.dsl.RepositoryHandler +import org.gradle.api.execution.TaskExecutionGraph import org.gradle.api.plugins.JavaPlugin import org.gradle.api.publish.maven.MavenPublication import org.gradle.api.publish.maven.plugins.MavenPublishPlugin @@ -221,20 +222,20 @@ class BuildPlugin implements Plugin { return System.getenv('JAVA' + version + '_HOME') } - /** - * Get Java home for the project for the specified version. If the specified version is not configured, an exception with the specified - * message is thrown. - * - * @param project the project - * @param version the version of Java home to obtain - * @param message the exception message if Java home for the specified version is not configured - * @return Java home for the specified version - * @throws GradleException if Java home for the specified version is not configured - */ - static String getJavaHome(final Project project, final int version, final String message) { - if (project.javaVersions.get(version) == null) { - throw new GradleException(message) + /** Add a check before gradle execution phase which ensures java home for the given java version is set. */ + static void requireJavaHome(Project project, Task task, int version) { + project.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> + if (taskGraph.hasTask(task)) { + if (project.javaVersions.get(version) == null) { + throw new GradleException("JAVA${version}_HOME required to run task ${task.path}") + } + } } + } + + /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ + static String getJavaHome(final Project project, final Task task, final int version) { + requireJavaHome(project, task, version) return project.javaVersions.get(version) } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 5f9e4c49b34e9..7b1088a57fd3b 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -20,6 +20,7 @@ package org.elasticsearch.gradle.test import org.apache.tools.ant.DefaultLogger import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.BuildPlugin import org.elasticsearch.gradle.LoggedExec import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties @@ -607,6 +608,9 @@ class ClusterFormationTasks { } Task start = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) + if (node.javaVersion != null) { + BuildPlugin.requireJavaHome(project, start, node.javaVersion) + } start.doLast(elasticsearchRunner) return start } diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index 1fc944eeec6eb..d319ca68a254a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -36,6 +36,9 @@ import static org.elasticsearch.gradle.BuildPlugin.getJavaHome * A container for the files and configuration associated with a single node in a test cluster. */ class NodeInfo { + /** Gradle project this node is part of */ + Project project + /** common configuration for all nodes, including this one */ ClusterConfiguration config @@ -84,6 +87,9 @@ class NodeInfo { /** directory to install plugins from */ File pluginsTmpDir + /** Major version of java this node runs with, or {@code null} if using the runtime java version */ + Integer javaVersion + /** environment variables to start the node with */ Map env @@ -165,12 +171,11 @@ class NodeInfo { args.add("${esScript}") } + if (nodeVersion.before("6.2.0")) { - env = ['JAVA_HOME': "${-> getJavaHome(project, 8, "JAVA8_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"] + javaVersion = 8 } else if (nodeVersion.onOrAfter("6.2.0") && nodeVersion.before("6.3.0")) { - env = ['JAVA_HOME': "${-> getJavaHome(project, 9, "JAVA9_HOME must be set to run BWC tests against [" + nodeVersion + "]")}"] - } else { - env = ['JAVA_HOME': (String) project.runtimeJavaHome] + javaVersion = 9 } args.addAll("-E", "node.portsfile=true") @@ -182,7 +187,7 @@ class NodeInfo { // in the cluster-specific options esJavaOpts = String.join(" ", "-ea", "-esa", esJavaOpts) } - env.put('ES_JAVA_OPTS', esJavaOpts) + env = ['ES_JAVA_OPTS': esJavaOpts] for (Map.Entry property : System.properties.entrySet()) { if (property.key.startsWith('tests.es.')) { args.add("-E") @@ -242,6 +247,11 @@ class NodeInfo { return Native.toString(shortPath).substring(4) } + /** Return the java home used by this node. */ + String getJavaHome() { + return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion) + } + /** Returns debug string for the command that started this node. */ String getCommandString() { String esCommandString = "\nNode ${nodeNum} configuration:\n" @@ -249,6 +259,7 @@ class NodeInfo { esCommandString += "| cwd: ${cwd}\n" esCommandString += "| command: ${executable} ${args.join(' ')}\n" esCommandString += '| environment:\n' + esCommandString += "| JAVA_HOME: ${javaHome}\n" env.each { k, v -> esCommandString += "| ${k}: ${v}\n" } if (config.daemonize) { esCommandString += "|\n| [${wrapperScript.name}]\n" diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 48b84b4036240..3bad09f926f49 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -147,9 +147,9 @@ subprojects { workingDir = checkoutDir if (["5.6", "6.0", "6.1"].contains(bwcBranch)) { // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds - environment('JAVA_HOME', "${-> getJavaHome(project, 8, "JAVA8_HOME is required to build BWC versions for BWC branch [" + bwcBranch + "]")}") + environment('JAVA_HOME', getJavaHome(project, it, 8)) } else if ("6.2".equals(bwcBranch)) { - environment('JAVA_HOME', "${-> getJavaHome(project, 9, "JAVA9_HOME is required to build BWC versions for BWC branch [" + bwcBranch + "]")}") + environment('JAVA_HOME', getJavaHome(project, it, 9)) } else { environment('JAVA_HOME', project.compilerJavaHome) } diff --git a/qa/reindex-from-old/build.gradle b/qa/reindex-from-old/build.gradle index c4b4927a4a2b1..617a98f59c0cb 100644 --- a/qa/reindex-from-old/build.gradle +++ b/qa/reindex-from-old/build.gradle @@ -77,7 +77,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn unzip executable = new File(project.runtimeJavaHome, 'bin/java') env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" - env 'JAVA_HOME', "${-> getJavaHome(project, 7, "JAVA7_HOME must be set to run reindex-from-old")}" + env 'JAVA_HOME', getJavaHome(project, it, 7) args 'oldes.OldElasticsearch', baseDir, unzip.temporaryDir, From d82b740c7df839515f3acd31da67112f4a2a3415 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Mon, 16 Apr 2018 15:54:40 -0700 Subject: [PATCH 2/3] forgot to set project in node info --- .../main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy index d319ca68a254a..d5f77a8d52112 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy @@ -115,6 +115,7 @@ class NodeInfo { NodeInfo(ClusterConfiguration config, int nodeNum, Project project, String prefix, Version nodeVersion, File sharedDir) { this.config = config this.nodeNum = nodeNum + this.project = project this.sharedDir = sharedDir if (config.clusterName != null) { clusterName = config.clusterName From a98747a3d4addde8b3c7670b1936951086a4f021 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 17 Apr 2018 09:50:01 -0700 Subject: [PATCH 3/3] iter --- .../elasticsearch/gradle/BuildPlugin.groovy | 29 ++++++++++++++----- .../gradle/test/ClusterFormationTasks.groovy | 2 +- distribution/bwc/build.gradle | 4 +-- qa/reindex-from-old/build.gradle | 2 +- 4 files changed, 25 insertions(+), 12 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy index 00a99d41fc112..d0ae4fd470312 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy @@ -223,20 +223,33 @@ class BuildPlugin implements Plugin { } /** Add a check before gradle execution phase which ensures java home for the given java version is set. */ - static void requireJavaHome(Project project, Task task, int version) { - project.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> - if (taskGraph.hasTask(task)) { - if (project.javaVersions.get(version) == null) { - throw new GradleException("JAVA${version}_HOME required to run task ${task.path}") + static void requireJavaHome(Task task, int version) { + Project rootProject = task.project.rootProject // use root project for global accounting + if (rootProject.hasProperty('requiredJavaVersions') == false) { + rootProject.rootProject.ext.requiredJavaVersions = [:].withDefault{key -> return []} + rootProject.gradle.taskGraph.whenReady { TaskExecutionGraph taskGraph -> + List messages = [] + for (entry in rootProject.requiredJavaVersions) { + if (rootProject.javaVersions.get(entry.key) != null) { + continue + } + List tasks = entry.value.findAll { taskGraph.hasTask(it) }.collect { " ${it.path}" } + if (tasks.isEmpty() == false) { + messages.add("JAVA${entry.key}_HOME required to run tasks:\n${tasks.join('\n')}") + } + } + if (messages.isEmpty() == false) { + throw new GradleException(messages.join('\n')) } } } + rootProject.requiredJavaVersions.get(version).add(task) } /** A convenience method for getting java home for a version of java and requiring that version for the given task to execute */ - static String getJavaHome(final Project project, final Task task, final int version) { - requireJavaHome(project, task, version) - return project.javaVersions.get(version) + static String getJavaHome(final Task task, final int version) { + requireJavaHome(task, version) + return task.project.javaVersions.get(version) } private static String findRuntimeJavaHome(final String compilerJavaHome) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy index 7b1088a57fd3b..b26320b400cc9 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy @@ -609,7 +609,7 @@ class ClusterFormationTasks { Task start = project.tasks.create(name: name, type: DefaultTask, dependsOn: setup) if (node.javaVersion != null) { - BuildPlugin.requireJavaHome(project, start, node.javaVersion) + BuildPlugin.requireJavaHome(start, node.javaVersion) } start.doLast(elasticsearchRunner) return start diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 3bad09f926f49..8e29ff6011006 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -147,9 +147,9 @@ subprojects { workingDir = checkoutDir if (["5.6", "6.0", "6.1"].contains(bwcBranch)) { // we are building branches that are officially built with JDK 8, push JAVA8_HOME to JAVA_HOME for these builds - environment('JAVA_HOME', getJavaHome(project, it, 8)) + environment('JAVA_HOME', getJavaHome(it, 8)) } else if ("6.2".equals(bwcBranch)) { - environment('JAVA_HOME', getJavaHome(project, it, 9)) + environment('JAVA_HOME', getJavaHome(it, 9)) } else { environment('JAVA_HOME', project.compilerJavaHome) } diff --git a/qa/reindex-from-old/build.gradle b/qa/reindex-from-old/build.gradle index 617a98f59c0cb..8da714dd6278a 100644 --- a/qa/reindex-from-old/build.gradle +++ b/qa/reindex-from-old/build.gradle @@ -77,7 +77,7 @@ if (Os.isFamily(Os.FAMILY_WINDOWS)) { dependsOn unzip executable = new File(project.runtimeJavaHome, 'bin/java') env 'CLASSPATH', "${ -> project.configurations.oldesFixture.asPath }" - env 'JAVA_HOME', getJavaHome(project, it, 7) + env 'JAVA_HOME', getJavaHome(it, 7) args 'oldes.OldElasticsearch', baseDir, unzip.temporaryDir,