diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java index cab250e3417b4..2a60a873079a4 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DockerBase.java @@ -25,7 +25,9 @@ public enum DockerBase { CENTOS("centos:8"), // "latest" here is intentional, since the image name specifies "8" - UBI("docker.elastic.co/ubi8/ubi-minimal:latest"); + UBI("docker.elastic.co/ubi8/ubi-minimal:latest"), + // The Iron Bank base image is UBI (albeit hardened), but we are required to parameterize the Docker build + IRON_BANK("${BASE_REGISTRY}/${BASE_IMAGE}:${BASE_TAG}"); private final String image; diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 7763140123ad3..a0c4f12b5169f 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -6,6 +6,9 @@ import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.docker.DockerBuildTask import org.elasticsearch.gradle.info.BuildParams import org.elasticsearch.gradle.testfixtures.TestFixturesPlugin + +import java.nio.file.Path + apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.test.fixtures' apply plugin: 'elasticsearch.internal-distribution-download' @@ -46,6 +49,15 @@ ext.expansions = { Architecture architecture, boolean oss, DockerBase base, bool final String elasticsearch = "elasticsearch-${oss ? 'oss-' : ''}${VersionProperties.elasticsearch}-${classifier}.tar.gz" + String buildArgs = '' + if (base == DockerBase.IRON_BANK) { + buildArgs = """ +ARG BASE_REGISTRY=nexus-docker-secure.levelup-nexus.svc.cluster.local:18082 +ARG BASE_IMAGE=redhat/ubi/ubi8 +ARG BASE_TAG=8.2 +""" + } + /* Both the following Dockerfile commands put the resulting artifact at * the same location, regardless of classifier, so that the commands that * follow in the Dockerfile don't have to know about the runtime @@ -61,23 +73,40 @@ RUN curl --retry 8 -S -L \\ """ } + def (major,minor) = VersionProperties.elasticsearch.split("\\.") + return [ 'base_image' : base.getImage(), + 'bin_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'bin', + 'build_args' : buildArgs, 'build_date' : BuildParams.buildDate, + 'config_dir' : base == DockerBase.IRON_BANK ? 'scripts' : 'config', 'git_revision' : BuildParams.gitRevision, 'license' : oss ? 'Apache-2.0' : 'Elastic-License', 'package_manager' : base == DockerBase.UBI ? 'microdnf' : 'yum', 'source_elasticsearch': sourceElasticsearch, 'docker_base' : base.name().toLowerCase(), - 'version' : VersionProperties.elasticsearch + 'version' : VersionProperties.elasticsearch, + 'major_minor_version' : "${major}.${minor}" ] } +/** + * This filter squashes long runs of newlines so that the output + * is a little more aesthetically pleasing. + */ +class SquashNewlinesFilter extends FilterReader { + SquashNewlinesFilter(Reader input) { + super(new StringReader(input.text.replaceAll("\n{2,}", "\n\n"))) + } +} + private static String buildPath(Architecture architecture, boolean oss, DockerBase base) { return 'build/' + (architecture == Architecture.AARCH64 ? 'aarch64-' : '') + (oss ? 'oss-' : '') + (base == DockerBase.UBI ? 'ubi-' : '') + + (base == DockerBase.UBI ? 'ubi-' : (base == DockerBase.IRON_BANK ? 'ironbank-' : '')) + 'docker' } @@ -85,34 +114,57 @@ private static String taskName(String prefix, Architecture architecture, boolean return prefix + (architecture == Architecture.AARCH64 ? 'Aarch64' : '') + (oss ? 'Oss' : '') + - (base == DockerBase.UBI ? 'Ubi' : '') + + (base == DockerBase.UBI ? 'Ubi' : (base == DockerBase.IRON_BANK ? 'IronBank' : '')) + suffix } project.ext { dockerBuildContext = { Architecture architecture, boolean oss, DockerBase base, boolean local -> copySpec { - into('bin') { - from project.projectDir.toPath().resolve("src/docker/bin") - } - - into('config') { - /* - * The OSS and default distributions have different configurations, therefore we want to allow overriding the default configuration - * from files in the 'oss' sub-directory. We don't want the 'oss' sub-directory to appear in the final build context, however. - */ - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - from(project.projectDir.toPath().resolve("src/docker/config")) { - exclude 'oss' + final Map varExpansions = expansions(architecture, oss, base, local) + final Path projectDir = project.projectDir.toPath() + + if (base == DockerBase.IRON_BANK) { + into('scripts') { + from(projectDir.resolve("src/docker/bin")) { + // We need to use an entrypoint that doesn't expect to run as `root` for Iron Bank, + // so don't copy this one. It's actually the version from v8.0, so this won't be + // necessary in future versions. + exclude 'docker-entrypoint.sh' + } + from(projectDir.resolve("src/docker/config")) { + exclude '**/oss' + } + from(projectDir.resolve("src/docker/iron_bank")) { + expand(varExpansions) + // Exclude this script so that no expansions are performed + exclude 'docker-entrypoint.sh' + } + from(projectDir.resolve("src/docker/iron_bank")) { + // ...and now copy the entrypoint without expansions + include 'docker-entrypoint.sh' + } } - if (oss) { - // Overlay the config file - from project.projectDir.toPath().resolve("src/docker/config/oss") + } else { + into('bin') { + from projectDir.resolve("src/docker/bin") + } + + into('config') { + // The OSS and default distribution can have different configuration, therefore we want to + // allow overriding the default configuration by creating config files in oss or default + // build-context sub-modules. + duplicatesStrategy = DuplicatesStrategy.INCLUDE + from projectDir.resolve("src/docker/config") + if (oss) { + from projectDir.resolve("src/docker/config/oss") + } } } from(project.projectDir.toPath().resolve("src/docker/Dockerfile")) { - expand(expansions(architecture, oss, base, local)) + expand(varExpansions) + filter SquashNewlinesFilter } } } @@ -324,6 +376,8 @@ subprojects { Project subProject -> final Architecture architecture = subProject.name.contains('aarch64-') ? Architecture.AARCH64 : Architecture.X64 final boolean oss = subProject.name.contains('oss-') + // We can ignore Iron Bank at the moment as we don't + // build those images ourselves. final DockerBase base = subProject.name.contains('ubi-') ? DockerBase.UBI : DockerBase.CENTOS final String arch = architecture == Architecture.AARCH64 ? '-aarch64' : '' diff --git a/distribution/docker/ironbank-docker-build-context/build.gradle b/distribution/docker/ironbank-docker-build-context/build.gradle new file mode 100644 index 0000000000000..00ddbe6ac374f --- /dev/null +++ b/distribution/docker/ironbank-docker-build-context/build.gradle @@ -0,0 +1,14 @@ +import org.elasticsearch.gradle.Architecture +import org.elasticsearch.gradle.DockerBase + +apply plugin: 'base' + +tasks.register("buildIronBankDockerBuildContext", Tar) { + archiveExtension = 'tar.gz' + compression = Compression.GZIP + archiveClassifier = "docker-build-context" + archiveBaseName = "elasticsearch-ironbank" + // We always treat Iron Bank builds as local, because that is how they + // are built + with dockerBuildContext(Architecture.X64, false, DockerBase.IRON_BANK, true) +} diff --git a/distribution/docker/src/docker/Dockerfile b/distribution/docker/src/docker/Dockerfile index ab7a745afe592..9a1911fcbdfbf 100644 --- a/distribution/docker/src/docker/Dockerfile +++ b/distribution/docker/src/docker/Dockerfile @@ -3,6 +3,7 @@ # # Beginning of multi stage Dockerfile ################################################################################ + <% /* This file is passed through Groovy's SimpleTemplateEngine, so dollars and backslashes have to be escaped in order for them to appear in the final Dockerfile. You @@ -13,17 +14,31 @@ We use control-flow tags in this file to conditionally render the content. The layout/presentation here has been adjusted so that it looks reasonable when rendered, at the slight expense of how it looks here. + + Note that this file is also filtered to squash together newlines, so we can + add as many newlines here as necessary to improve legibility. */ %> ################################################################################ # Build stage 0 `builder`: # Extract Elasticsearch artifact ################################################################################ +${build_args} + FROM ${base_image} AS builder + <% if (docker_base == 'ubi') { %> # Install required packages to extract the Elasticsearch distribution RUN ${package_manager} install -y tar gzip <% } %> + +<% if (docker_base == 'iron_bank') { %> +# `tini` is a tiny but valid init for containers. This is used to cleanly +# control how ES and any child processes are shut down. +COPY tini /bin/tini +RUN chmod 0755 /bin/tini +<% } else { %> + # `tini` is a tiny but valid init for containers. This is used to cleanly # control how ES and any child processes are shut down. # @@ -41,19 +56,23 @@ RUN set -eux ; \\ curl --retry 8 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\ sha256sum -c \${tini_bin}.sha256sum ; \\ rm \${tini_bin}.sha256sum ; \\ - mv \${tini_bin} /tini ; \\ - chmod +x /tini + mv \${tini_bin} /bin/tini ; \\ + chmod +x /bin/tini + +<% } %> RUN mkdir /usr/share/elasticsearch WORKDIR /usr/share/elasticsearch ${source_elasticsearch} -RUN tar zxf /opt/elasticsearch.tar.gz --strip-components=1 +RUN tar -zxf /opt/elasticsearch.tar.gz --strip-components=1 + +# Configure the distribution for Docker RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' /usr/share/elasticsearch/bin/elasticsearch-env -RUN mkdir -p config config/jvm.options.d data logs +RUN mkdir -p config config/jvm.options.d data logs plugins RUN chmod 0775 config config/jvm.options.d data logs plugins -COPY config/elasticsearch.yml config/log4j2.properties config/ +COPY ${config_dir}/elasticsearch.yml ${config_dir}/log4j2.properties config/ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties ################################################################################ @@ -65,7 +84,17 @@ RUN chmod 0660 config/elasticsearch.yml config/log4j2.properties FROM ${base_image} -ENV ELASTIC_CONTAINER true +<% if (docker_base == "iron_bank") { %> +<% +/* Reviews of the Iron Bank Dockerfile said that they preferred simpler */ +/* scripting so this version doesn't have the retry loop featured below. */ +%> +RUN ${package_manager} update --setopt=tsflags=nodocs -y && \\ + ${package_manager} install --setopt=tsflags=nodocs -y \\ + nc shadow-utils zip unzip && \\ + ${package_manager} clean all + +<% } else { %> RUN for iter in {1..10}; do \\ ${package_manager} update --setopt=tsflags=nodocs -y && \\ @@ -76,14 +105,18 @@ RUN for iter in {1..10}; do \\ done; \\ (exit \$exit_code) +<% } %> + RUN groupadd -g 1000 elasticsearch && \\ adduser -u 1000 -g 1000 -G 0 -d /usr/share/elasticsearch elasticsearch && \\ chmod 0775 /usr/share/elasticsearch && \\ chown -R 1000:0 /usr/share/elasticsearch +ENV ELASTIC_CONTAINER true + WORKDIR /usr/share/elasticsearch COPY --from=builder --chown=1000:0 /usr/share/elasticsearch /usr/share/elasticsearch -COPY --from=builder --chown=0:0 /tini /tini +COPY --from=builder --chown=0:0 /bin/tini /bin/tini # Replace OpenJDK's built-in CA certificate keystore with the one from the OS # vendor. The latter is superior in several ways. @@ -92,7 +125,7 @@ RUN ln -sf /etc/pki/ca-trust/extracted/java/cacerts /usr/share/elasticsearch/jdk ENV PATH /usr/share/elasticsearch/bin:\$PATH -COPY bin/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +COPY ${bin_dir}/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh # 1. The JDK's directories' permissions don't allow `java` to be executed under a different # group to the default. Fix this. @@ -128,7 +161,8 @@ LABEL org.label-schema.build-date="${build_date}" \\ org.opencontainers.image.url="https://www.elastic.co/products/elasticsearch" \\ org.opencontainers.image.vendor="Elastic" \\ org.opencontainers.image.version="${version}" -<% if (docker_base == 'ubi') { %> + +<% if (docker_base == 'ubi' || docker_base == 'iron_bank') { %> LABEL name="Elasticsearch" \\ maintainer="infra@elastic.co" \\ vendor="Elastic" \\ @@ -141,10 +175,14 @@ RUN mkdir /licenses && \\ cp LICENSE.txt /licenses/LICENSE <% } %> -ENTRYPOINT ["/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] +ENTRYPOINT ["/bin/tini", "--", "/usr/local/bin/docker-entrypoint.sh"] # Dummy overridable parameter parsed by entrypoint CMD ["eswrapper"] +<% if (docker_base == 'iron_bank') { %> +HEALTHCHECK --interval=10s --timeout=5s --start-period=1m --retries=5 CMD curl -I -f --max-time 5 http://localhost:9200 || exit 1 +<% } %> + ################################################################################ # End of multi-stage Dockerfile ################################################################################ diff --git a/distribution/docker/src/docker/iron_bank/.gitignore b/distribution/docker/src/docker/iron_bank/.gitignore new file mode 100644 index 0000000000000..cda31c3986d13 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/.gitignore @@ -0,0 +1,2 @@ +# Ignore any locally downloaded or dropped releases +*.tar.gz diff --git a/distribution/docker/src/docker/iron_bank/Jenkinsfile b/distribution/docker/src/docker/iron_bank/Jenkinsfile new file mode 100644 index 0000000000000..7422f1f7a2a9c --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/Jenkinsfile @@ -0,0 +1,2 @@ +@Library('DCCSCR@master') _ +dccscrPipeline(version: '${version}') diff --git a/distribution/docker/src/docker/iron_bank/README.md b/distribution/docker/src/docker/iron_bank/README.md new file mode 100644 index 0000000000000..6767a6faa66f8 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/README.md @@ -0,0 +1,37 @@ +# Elasticsearch + +**Elasticsearch** is a distributed, RESTful search and analytics engine capable of +solving a growing number of use cases. As the heart of the Elastic Stack, it +centrally stores your data so you can discover the expected and uncover the +unexpected. + +For more information about Elasticsearch, please visit +https://www.elastic.co/products/elasticsearch. + +### Installation instructions + +Please follow the documentation on [how to install Elasticsearch with Docker](https://www.elastic.co/guide/en/elasticsearch/reference/current/docker.html). + +### Where to file issues and PRs + +- [Issues](https://github.com/elastic/elasticsearch/issues) +- [PRs](https://github.com/elastic/elasticsearch/pulls) + +### Where to get help + +- [Elasticsearch Discuss Forums](https://discuss.elastic.co/c/elasticsearch) +- [Elasticsearch Documentation](https://www.elastic.co/guide/en/elasticsearch/reference/master/index.html) + +### Still need help? + +You can learn more about the Elastic Community and also understand how to get more help +visiting [Elastic Community](https://www.elastic.co/community). + + +This software is governed by the [Elastic +License](https://github.com/elastic/elasticsearch/blob/${major_minor_version}/licenses/ELASTIC-LICENSE.txt), +and includes the full set of [free +features](https://www.elastic.co/subscriptions). + +View the detailed release notes +[here](https://www.elastic.co/guide/en/elasticsearch/reference/${major_minor_version}/es-release-notes.html). diff --git a/distribution/docker/src/docker/iron_bank/docker-entrypoint.sh b/distribution/docker/src/docker/iron_bank/docker-entrypoint.sh new file mode 100644 index 0000000000000..76f23e1589512 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/docker-entrypoint.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +# Files created by Elasticsearch should always be group writable too +umask 0002 + +# Allow user specify custom CMD, maybe bin/elasticsearch itself +# for example to directly specify `-E` style parameters for elasticsearch on k8s +# or simply to run /bin/bash to check the image +if [[ "$1" == "eswrapper" || $(basename "$1") == "elasticsearch" ]]; then + # Rewrite CMD args to remove the explicit command, + # so that we are backwards compatible with the docs + # from the previous Elasticsearch versions < 6 + # and configuration option: + # https://www.elastic.co/guide/en/elasticsearch/reference/5.6/docker.html#_d_override_the_image_8217_s_default_ulink_url_https_docs_docker_com_engine_reference_run_cmd_default_command_or_options_cmd_ulink + # Without this, user could specify `elasticsearch -E x.y=z` but + # `bin/elasticsearch -E x.y=z` would not work. In any case, + # we want to continue through this script, and not exec early. + set -- "${@:2}" +else + # Run whatever command the user wanted + exec "$@" +fi + +# Allow environment variables to be set by creating a file with the +# contents, and setting an environment variable with the suffix _FILE to +# point to it. This can be used to provide secrets to a container, without +# the values being specified explicitly when running the container. +# +# This is also sourced in elasticsearch-env, and is only needed here +# as well because we use ELASTIC_PASSWORD below. Sourcing this script +# is idempotent. +source /usr/share/elasticsearch/bin/elasticsearch-env-from-file + +if [[ -f bin/elasticsearch-users ]]; then + # Check for the ELASTIC_PASSWORD environment variable to set the + # bootstrap password for Security. + # + # This is only required for the first node in a cluster with Security + # enabled, but we have no way of knowing which node we are yet. We'll just + # honor the variable if it's present. + if [[ -n "$ELASTIC_PASSWORD" ]]; then + [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (elasticsearch-keystore create) + if ! (elasticsearch-keystore has-passwd --silent) ; then + # keystore is unencrypted + if ! (elasticsearch-keystore list | grep -q '^bootstrap.password$'); then + (echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password') + fi + else + # keystore requires password + if ! (echo "$KEYSTORE_PASSWORD" \ + | elasticsearch-keystore list | grep -q '^bootstrap.password$') ; then + COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$ELASTIC_PASSWORD")" + (echo "$COMMANDS" | elasticsearch-keystore add -x 'bootstrap.password') + fi + fi + fi +fi + +# Signal forwarding and child reaping is handled by `tini`, which is the +# actual entrypoint of the container +exec /usr/share/elasticsearch/bin/elasticsearch <<<"$KEYSTORE_PASSWORD" diff --git a/distribution/docker/src/docker/iron_bank/download.json b/distribution/docker/src/docker/iron_bank/download.json new file mode 100644 index 0000000000000..63f61a289b0e7 --- /dev/null +++ b/distribution/docker/src/docker/iron_bank/download.json @@ -0,0 +1,20 @@ +{ + "resources": [ + { + "url": "https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${version}-linux-x86_64.tar.gz", + "filename": "elasticsearch-${version}-linux-x86_64.tar.gz", + "validation": { + "type": "sha512", + "value": "" + } + }, + { + "url": "https://github.com/krallin/tini/releases/download/v0.19.0/tini-amd64", + "filename": "tini", + "validation": { + "type": "sha256", + "value": "93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c" + } + } + ] +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 42ad3839b4c24..8ca2e6cbecec9 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -682,7 +682,7 @@ public void test131InitProcessHasCorrectPID() { assertThat("Incorrect PID", fields[0], equalTo("1")); assertThat("Incorrect UID", fields[1], equalTo("0")); assertThat("Incorrect GID", fields[2], equalTo("0")); - assertThat("Incorrect init command", fields[3], startsWith("/tini")); + assertThat("Incorrect init command", fields[3], startsWith("/bin/tini")); } /** diff --git a/settings.gradle b/settings.gradle index b9321b5863871..66fda08e566c4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -38,6 +38,7 @@ List projects = [ 'distribution:docker:docker-aarch64-export', 'distribution:docker:docker-build-context', 'distribution:docker:docker-export', + 'distribution:docker:ironbank-docker-build-context', 'distribution:docker:oss-docker-aarch64-build-context', 'distribution:docker:oss-docker-aarch64-export', 'distribution:docker:oss-docker-build-context',