Skip to content

Commit fef2153

Browse files
rjernstrjernst
authored
Bundle java in distributions (#38013)
* Bundle java in distributions Setting up a jdk is currently a required external step when installing elasticsearch. This is particularly problematic for the rpm/deb packages as installing a jdk in the same package installation command does not guarantee any order, so must be done in separate steps. Additionally, JAVA_HOME must be set and often causes problems in selecting a correct jdk when, for example, the system java is an older unsupported version. This commit bundles platform specific openjdks into each distribution. In addition to eliminating the issues above, it also presents future possible improvements like using jlink to build jdk images only containing modules that elasticsearch uses. closes #31845
1 parent 0619e6e commit fef2153

File tree

24 files changed

+238
-135
lines changed

24 files changed

+238
-135
lines changed

‎Vagrantfile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,8 @@ def sh_install_deps(config,
350350
if [ -z "\\\$JAVA_HOME" ]; then
351351
export JAVA_HOME=$(dirname $(dirname $(readlink -f $(which java))))
352352
fi
353+
export SYSTEM_JAVA_HOME=\\\$JAVA_HOME
354+
unset JAVA_HOME
353355
JAVA
354356
ensure tar
355357
ensure curl
@@ -388,6 +390,7 @@ Defaults env_keep += "BATS_TESTS"
388390
Defaults env_keep += "PACKAGING_ARCHIVES"
389391
Defaults env_keep += "PACKAGING_TESTS"
390392
Defaults env_keep += "JAVA_HOME"
393+
Defaults env_keep += "SYSTEM_JAVA_HOME"
391394
SUDOERS_VARS
392395
chmod 0440 /etc/sudoers.d/elasticsearch_vars
393396
SHELL
@@ -408,6 +411,9 @@ def windows_common(config, name)
408411
config.vm.provision 'set env variables', type: 'shell', inline: <<-SHELL
409412
$ErrorActionPreference = "Stop"
410413
[Environment]::SetEnvironmentVariable("PACKAGING_ARCHIVES", "C:/project/build/packaging/archives", "Machine")
414+
$javaHome = [Environment]::GetEnvironmentVariable("JAVA_HOME", "Machine")
415+
[Environment]::SetEnvironmentVariable("SYSTEM_JAVA_HOME", $javaHome, "Machine")
411416
[Environment]::SetEnvironmentVariable("PACKAGING_TESTS", "C:/project/build/packaging/tests", "Machine")
417+
[Environment]::SetEnvironmentVariable("JAVA_HOME", $null, "Machine")
412418
SHELL
413419
end

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -213,6 +213,7 @@ class BuildPlugin implements Plugin<Project> {
213213
project.rootProject.ext.runtimeJavaHome = runtimeJavaHome
214214
project.rootProject.ext.compilerJavaVersion = compilerJavaVersionEnum
215215
project.rootProject.ext.runtimeJavaVersion = runtimeJavaVersionEnum
216+
project.rootProject.ext.isRuntimeJavaHomeSet = compilerJavaHome.equals(runtimeJavaHome) == false
216217
project.rootProject.ext.javaVersions = javaVersions
217218
project.rootProject.ext.buildChecksDone = true
218219
project.rootProject.ext.minimumCompilerVersion = minimumCompilerVersion
@@ -231,6 +232,7 @@ class BuildPlugin implements Plugin<Project> {
231232
project.ext.runtimeJavaHome = project.rootProject.ext.runtimeJavaHome
232233
project.ext.compilerJavaVersion = project.rootProject.ext.compilerJavaVersion
233234
project.ext.runtimeJavaVersion = project.rootProject.ext.runtimeJavaVersion
235+
project.ext.isRuntimeJavaHomeSet = project.rootProject.ext.isRuntimeJavaHomeSet
234236
project.ext.javaVersions = project.rootProject.ext.javaVersions
235237
project.ext.inFipsJvm = project.rootProject.ext.inFipsJvm
236238
project.ext.gradleJavaVersion = project.rootProject.ext.gradleJavaVersion

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -641,7 +641,13 @@ class ClusterFormationTasks {
641641
static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
642642
return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
643643
exec.workingDir node.cwd
644-
exec.environment 'JAVA_HOME', node.getJavaHome()
644+
// TODO: this must change to 7.0.0 after bundling java has been backported
645+
if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) {
646+
exec.environment.put('JAVA_HOME', project.runtimeJavaHome)
647+
} else {
648+
// force JAVA_HOME to *not* be set
649+
exec.environment.remove('JAVA_HOME')
650+
}
645651
if (Os.isFamily(Os.FAMILY_WINDOWS)) {
646652
exec.executable 'cmd'
647653
exec.args '/C', 'call'
@@ -658,8 +664,12 @@ class ClusterFormationTasks {
658664
static Task configureStartTask(String name, Project project, Task setup, NodeInfo node) {
659665
// this closure is converted into ant nodes by groovy's AntBuilder
660666
Closure antRunner = { AntBuilder ant ->
661-
ant.exec(executable: node.executable, spawn: node.config.daemonize, dir: node.cwd, taskname: 'elasticsearch') {
667+
ant.exec(executable: node.executable, spawn: node.config.daemonize, newenvironment: true,
668+
dir: node.cwd, taskname: 'elasticsearch') {
662669
node.env.each { key, value -> env(key: key, value: value) }
670+
if (project.isRuntimeJavaHomeSet || node.nodeVersion.before(Version.fromString("8.0.0"))) {
671+
env(key: 'JAVA_HOME', value: project.runtimeJavaHome)
672+
}
663673
node.args.each { arg(value: it) }
664674
}
665675
}

‎buildSrc/src/main/groovy/org/elasticsearch/gradle/test/NodeInfo.groovy‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import com.sun.jna.Native
2323
import com.sun.jna.WString
2424
import org.apache.tools.ant.taskdefs.condition.Os
2525
import org.elasticsearch.gradle.Version
26-
import org.gradle.api.InvalidUserDataException
2726
import org.gradle.api.Project
2827

2928
import java.nio.file.Files
@@ -240,19 +239,13 @@ class NodeInfo {
240239
return Native.toString(shortPath).substring(4)
241240
}
242241

243-
/** Return the java home used by this node. */
244-
String getJavaHome() {
245-
return javaVersion == null ? project.runtimeJavaHome : project.javaVersions.get(javaVersion)
246-
}
247-
248242
/** Returns debug string for the command that started this node. */
249243
String getCommandString() {
250244
String esCommandString = "\nNode ${nodeNum} configuration:\n"
251245
esCommandString += "|-----------------------------------------\n"
252246
esCommandString += "| cwd: ${cwd}\n"
253247
esCommandString += "| command: ${executable} ${args.join(' ')}\n"
254248
esCommandString += '| environment:\n'
255-
esCommandString += "| JAVA_HOME: ${javaHome}\n"
256249
env.each { k, v -> esCommandString += "| ${k}: ${v}\n" }
257250
if (config.daemonize) {
258251
esCommandString += "|\n| [${wrapperScript.name}]\n"

‎buildSrc/src/main/minimumRuntime/org/elasticsearch/gradle/VersionProperties.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
* Accessor for shared dependency versions used by elasticsearch, namely the elasticsearch and lucene versions.
1111
*/
1212
public class VersionProperties {
13+
1314
public static String getElasticsearch() {
1415
return elasticsearch;
1516
}
@@ -18,17 +19,25 @@ public static String getLucene() {
1819
return lucene;
1920
}
2021

22+
public static String getBundledJdk() {
23+
return bundledJdk;
24+
}
25+
2126
public static Map<String, String> getVersions() {
2227
return versions;
2328
}
2429

2530
private static final String elasticsearch;
2631
private static final String lucene;
32+
private static final String bundledJdk;
2733
private static final Map<String, String> versions = new HashMap<String, String>();
34+
2835
static {
2936
Properties props = getVersionProperties();
3037
elasticsearch = props.getProperty("elasticsearch");
3138
lucene = props.getProperty("lucene");
39+
bundledJdk = props.getProperty("bundled_jdk");
40+
3241
for (String property : props.stringPropertyNames()) {
3342
versions.put(property, props.getProperty(property));
3443
}

‎buildSrc/version.properties‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
elasticsearch = 8.0.0
22
lucene = 8.0.0-snapshot-ff9509a8df
33

4+
bundled_jdk = 11.0.2+9
5+
46
# optional dependencies
57
spatial4j = 0.7
68
jts = 1.15.0

‎distribution/archives/build.gradle‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ task createPluginsDir(type: EmptyDirTask) {
4545
dirMode 0755
4646
}
4747

48-
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean oss) {
48+
CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, String platform, boolean oss) {
4949
return copySpec {
5050
into("elasticsearch-${version}") {
5151
into('lib') {
@@ -59,6 +59,11 @@ CopySpec archiveFiles(CopySpec modulesFiles, String distributionType, boolean os
5959
into('bin') {
6060
with binFiles(distributionType, oss)
6161
}
62+
if (platform != null) {
63+
into('jdk') {
64+
with jdkFiles(platform)
65+
}
66+
}
6267
into('') {
6368
from {
6469
dirMode 0755
@@ -102,19 +107,19 @@ Closure commonZipConfig = {
102107

103108
task buildIntegTestZip(type: Zip) {
104109
configure(commonZipConfig)
105-
with archiveFiles(transportModulesFiles, 'zip', true)
110+
with archiveFiles(transportModulesFiles, 'zip', null, true)
106111
}
107112

108113
task buildWindowsZip(type: Zip) {
109114
configure(commonZipConfig)
110115
archiveClassifier = 'windows-x86_64'
111-
with archiveFiles(modulesFiles(false), 'zip', false)
116+
with archiveFiles(modulesFiles(false), 'zip', 'windows', false)
112117
}
113118

114119
task buildOssWindowsZip(type: Zip) {
115120
configure(commonZipConfig)
116121
archiveClassifier = 'windows-x86_64'
117-
with archiveFiles(modulesFiles(true), 'zip', true)
122+
with archiveFiles(modulesFiles(true), 'zip', 'windows', true)
118123
}
119124

120125
Closure commonTarConfig = {
@@ -127,25 +132,25 @@ Closure commonTarConfig = {
127132
task buildDarwinTar(type: Tar) {
128133
configure(commonTarConfig)
129134
archiveClassifier = 'darwin-x86_64'
130-
with archiveFiles(modulesFiles(false), 'tar', false)
135+
with archiveFiles(modulesFiles(false), 'tar', 'darwin', false)
131136
}
132137

133138
task buildOssDarwinTar(type: Tar) {
134139
configure(commonTarConfig)
135140
archiveClassifier = 'darwin-x86_64'
136-
with archiveFiles(modulesFiles(true), 'tar', true)
141+
with archiveFiles(modulesFiles(true), 'tar', 'darwin', true)
137142
}
138143

139144
task buildLinuxTar(type: Tar) {
140145
configure(commonTarConfig)
141146
archiveClassifier = 'linux-x86_64'
142-
with archiveFiles(modulesFiles(false), 'tar', false)
147+
with archiveFiles(modulesFiles(false), 'tar', 'linux', false)
143148
}
144149

145150
task buildOssLinuxTar(type: Tar) {
146151
configure(commonTarConfig)
147152
archiveClassifier = 'linux-x86_64'
148-
with archiveFiles(modulesFiles(true), 'tar', true)
153+
with archiveFiles(modulesFiles(true), 'tar', 'linux', true)
149154
}
150155

151156
Closure tarExists = { it -> new File('/bin/tar').exists() || new File('/usr/bin/tar').exists() || new File('/usr/local/bin/tar').exists() }

‎distribution/build.gradle‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,11 +20,14 @@
2020
import org.elasticsearch.gradle.ConcatFilesTask
2121
import org.elasticsearch.gradle.MavenFilteringHack
2222
import org.elasticsearch.gradle.NoticeTask
23+
import org.elasticsearch.gradle.VersionProperties
2324
import org.elasticsearch.gradle.test.RunTask
2425
import org.apache.tools.ant.filters.FixCrLfFilter
2526

2627
import java.nio.file.Files
2728
import java.nio.file.Path
29+
import java.util.regex.Matcher
30+
import java.util.regex.Pattern
2831

2932
/*****************************************************************************
3033
* Third party dependencies report *
@@ -210,6 +213,51 @@ xpack.subprojects.findAll { it.parent == xpack }.each { Project xpackModule ->
210213
copyLog4jProperties(buildDefaultLog4jConfig, xpackModule)
211214
}
212215

216+
/*****************************************************************************
217+
* JDKs *
218+
*****************************************************************************/
219+
// extract the bundled jdk version, broken into elements as: [feature, interim, update, build]
220+
// Note the "patch" version is not yet handled here, as it has not yet been used by java.
221+
Pattern JDK_VERSION = Pattern.compile("(\\d+)\\.(\\d+)\\.(\\d+)\\+(\\d+)")
222+
Matcher jdkVersionMatcher = JDK_VERSION.matcher(VersionProperties.bundledJdk)
223+
if (jdkVersionMatcher.matches() == false) {
224+
throw new IllegalArgumentException("Malformed jdk version [" + VersionProperties.bundledJdk + "]")
225+
}
226+
String jdkVersion = jdkVersionMatcher.group(1) + '.' + jdkVersionMatcher.group(2) + '.' + jdkVersionMatcher.group(3)
227+
String jdkMajor = jdkVersionMatcher.group(1)
228+
String jdkBuild = jdkVersionMatcher.group(4)
229+
230+
repositories {
231+
ivy {
232+
url "https://download.java.net"
233+
patternLayout {
234+
artifact "java/GA/jdk${jdkMajor}/${jdkBuild}/GPL/openjdk-[revision]_[module]-x64_bin.[ext]"
235+
}
236+
}
237+
}
238+
for (String platform : ['linux', 'darwin', 'windows']) {
239+
String jdkConfigName = "jdk_${platform}"
240+
Configuration jdkConfig = configurations.create(jdkConfigName)
241+
String extension = platform.equals('windows') ? 'zip' : 'tar.gz'
242+
dependencies.add(jdkConfigName, "jdk:${platform.equals('darwin') ? 'osx' : platform}:${jdkVersion}@${extension}")
243+
244+
int rootNdx = platform.equals('darwin') ? 2 : 1
245+
Closure removeRootDir = {
246+
it.eachFile { FileCopyDetails details ->
247+
details.relativePath = new RelativePath(true, details.relativePath.segments[rootNdx..-1] as String[])
248+
}
249+
it.includeEmptyDirs false
250+
}
251+
project.task("extract${platform.capitalize()}Jdk", type: Copy) {
252+
into "${buildDir}/jdks/openjdk-${jdkVersion}_${platform}"
253+
if (extension.equals('zip')) {
254+
from({ zipTree(jdkConfig.singleFile) }, removeRootDir)
255+
} else {
256+
from({ tarTree(resources.gzip(jdkConfig.singleFile)) }, removeRootDir)
257+
}
258+
}
259+
}
260+
213261
// make sure we have a clean task since we aren't a java project, but we have tasks that
214262
// put stuff in the build dir
215263
task clean(type: Delete) {
@@ -332,6 +380,17 @@ configure(subprojects.findAll { ['archives', 'packages'].contains(it.name) }) {
332380
from buildDefaultNotice
333381
}
334382
}
383+
384+
jdkFiles = { platform ->
385+
copySpec {
386+
from project(':distribution').tasks.getByName("extract${platform.capitalize()}Jdk")
387+
eachFile { FileCopyDetails details ->
388+
if (details.relativePath.segments[-2] == 'bin') {
389+
details.mode = 0755
390+
}
391+
}
392+
}
393+
}
335394
}
336395

337396
}

‎distribution/packages/build.gradle‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,9 @@ Closure commonPackageConfig(String type, boolean oss) {
135135
into('modules') {
136136
with modulesFiles(oss)
137137
}
138+
into('jdk') {
139+
with jdkFiles('linux')
140+
}
138141
// we need to specify every intermediate directory in these paths so the package managers know they are explicitly
139142
// intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same
140143
// directory are fine

‎distribution/packages/src/common/scripts/preinst‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,6 @@ err_exit() {
1515
exit 1
1616
}
1717

18-
# Check for these at preinst time due to failures in postinst if they do not exist
19-
if [ -x "$JAVA_HOME/bin/java" ]; then
20-
JAVA="$JAVA_HOME/bin/java"
21-
elif command -v java; then
22-
JAVA=`command -v java`
23-
else
24-
JAVA=""
25-
fi
26-
27-
if [ -z "$JAVA" ]; then
28-
err_exit "could not find java; set JAVA_HOME"
29-
fi
30-
3118
case "$1" in
3219

3320
# Debian ####################################################

0 commit comments

Comments
 (0)