From 6a1877df949d28d4b47eafbfa78be46a714cdffa Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Wed, 11 Sep 2019 15:33:51 +0100 Subject: [PATCH 01/11] Introduce packaging tests for Docker Add packaging tests for our Docker images, similar to what we have for RPMs or Debian packages. This works by running a container and probing it e.g. via `docker exec`. Test can also be run in Vagrant, by exporting the Docker images to disk and loading them again in VMs. Docker is installed via `Vagrantfile` in a selection of boxes. --- Vagrantfile | 71 +++- .../gradle/test/DistroTestPlugin.java | 20 +- .../gradle/DistributionDownloadPlugin.java | 28 +- .../gradle/ElasticsearchDistribution.java | 18 +- distribution/docker/build.gradle | 56 +++ .../docker/docker-export/build.gradle | 2 + .../docker/oss-docker-export/build.gradle | 2 + .../packaging/test/DockerTests.java | 224 +++++++++++ .../packaging/test/PackagingTestCase.java | 7 +- .../packaging/util/Distribution.java | 20 +- .../elasticsearch/packaging/util/Docker.java | 365 ++++++++++++++++++ .../packaging/util/FileMatcher.java | 1 + .../packaging/util/Installation.java | 14 + .../packaging/util/Platforms.java | 4 + .../elasticsearch/packaging/util/Shell.java | 8 +- settings.gradle | 2 + 16 files changed, 809 insertions(+), 33 deletions(-) create mode 100644 distribution/docker/docker-export/build.gradle create mode 100644 distribution/docker/oss-docker-export/build.gradle create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java create mode 100644 qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java diff --git a/Vagrantfile b/Vagrantfile index 4d1c4e92b7a84..8e21b80fbd41f 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,5 +1,5 @@ # -*- mode: ruby -*- -# vi: set ft=ruby : +# vi:ft=ruby ts=2 sw=2 sts=2 et: # This Vagrantfile exists to test packaging. Read more about its use in the # vagrant section in TESTING.asciidoc. @@ -183,6 +183,35 @@ def deb_common(config, name, extra: '') install_command: 'apt-get install -y', extra: extra_with_lintian ) + deb_docker(config) +end + +def deb_docker(config) + config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + # Install packages to allow apt to use a repository over HTTPS + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common + + # Add Docker’s official GPG key + curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - + + # Set up the stable Docker repository + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" + + # Install Docker. Unlike Fedora and CentOS, this also start the daemon. + apt-get update + apt-get install -y docker-ce docker-ce-cli containerd.io + + # Add vagrant to the Docker group, so that it can run commands + usermod -aG docker vagrant + SHELL end def rpm_common(config, name) @@ -193,6 +222,26 @@ def rpm_common(config, name) update_tracking_file: '/var/cache/yum/last_update', install_command: 'yum install -y' ) + rpm_docker(config) +end + +def rpm_docker(config) + config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + # Install prerequisites + yum install -y yum-utils device-mapper-persistent-data lvm2 + + # Add repository + yum-config-manager -y --add-repo https://download.docker.com/linux/centos/docker-ce.repo + + # Install Docker + yum install -y docker-ce docker-ce-cli containerd.io + + # Start Docker + systemctl start docker + + # Add vagrant to the Docker group, so that it can run commands + usermod -aG docker vagrant + SHELL end def dnf_common(config, name) @@ -209,6 +258,26 @@ def dnf_common(config, name) install_command: 'dnf install -y', install_command_retries: 5 ) + dnf_docker(config) +end + +def dnf_docker(config) + config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + # Install prerequisites + dnf -y install dnf-plugins-core + + # Add repository + dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo + + # Install Docker + dnf install -y docker-ce docker-ce-cli containerd.io + + # Start Docker + systemctl start docker + + # Add vagrant to the Docker group, so that it can run commands + usermod -aG docker vagrant + SHELL end def suse_common(config, name, extra: '') diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java index b80fa78e5c87d..d8c292256c346 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -172,7 +172,7 @@ private static List configureVM(Project project) { String box = project.getName(); // setup jdks used by the distro tests, and by gradle executing - + NamedDomainObjectContainer jdksContainer = JdkDownloadPlugin.getContainer(project); String platform = box.contains("windows") ? "windows" : "linux"; Jdk systemJdk = createJdk(jdksContainer, "system", SYSTEM_JDK_VERSION, platform); @@ -309,13 +309,13 @@ private static TaskProvider configureBatsTest(Project project, Str } }); } - + private List configureDistributions(Project project, Version upgradeVersion) { NamedDomainObjectContainer distributions = DistributionDownloadPlugin.getContainer(project); List currentDistros = new ArrayList<>(); List upgradeDistros = new ArrayList<>(); - for (Type type : Arrays.asList(Type.DEB, Type.RPM)) { + for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); @@ -366,7 +366,8 @@ private static void addDistro(NamedDomainObjectContainer + if (subProject.name.contains('docker-export')) { + apply plugin: 'distribution' + + def oss = subProject.name.startsWith('oss') + + def exportTaskName = taskName("export", oss, "DockerImage") + def buildTaskName = taskName("build", oss, "DockerImage") + def tarFile = "${parent.buildDir}/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" + + final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { + executable 'docker' + args "save", + "-o", + tarFile, + "elasticsearch${oss ? '-oss' : ''}:test" + } + + exportDockerImageTask.dependsOn(parent.tasks.getByName(buildTaskName)) + + artifacts.add('default', file(tarFile)) { + type 'tar' + name "elasticsearch${oss ? '-oss' : ''}" + builtBy exportTaskName + } + + assemble.dependsOn exportTaskName + } +} diff --git a/distribution/docker/docker-export/build.gradle b/distribution/docker/docker-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/docker-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/distribution/docker/oss-docker-export/build.gradle b/distribution/docker/oss-docker-export/build.gradle new file mode 100644 index 0000000000000..537b5a093683e --- /dev/null +++ b/distribution/docker/oss-docker-export/build.gradle @@ -0,0 +1,2 @@ +// This file is intentionally blank. All configuration of the +// export is done in the parent project. diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java new file mode 100644 index 0000000000000..8c7ad7a8fd5d3 --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -0,0 +1,224 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Docker.DockerShell; +import org.elasticsearch.packaging.util.Installation; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell.Result; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; + +import static java.nio.file.attribute.PosixFilePermissions.fromString; +import static org.elasticsearch.packaging.util.Docker.assertPermissionsAndOwnership; +import static org.elasticsearch.packaging.util.Docker.copyFromContainer; +import static org.elasticsearch.packaging.util.Docker.ensureImageIsLoaded; +import static org.elasticsearch.packaging.util.Docker.existsInContainer; +import static org.elasticsearch.packaging.util.Docker.removeContainer; +import static org.elasticsearch.packaging.util.Docker.runContainer; +import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation; +import static org.elasticsearch.packaging.util.Docker.waitForElasticsearchToBecomeAvailable; +import static org.elasticsearch.packaging.util.Docker.waitForPathToExist; +import static org.elasticsearch.packaging.util.FileMatcher.p660; +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.FileUtils.getTempDir; +import static org.elasticsearch.packaging.util.FileUtils.mkdir; +import static org.elasticsearch.packaging.util.FileUtils.rm; +import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.emptyString; +import static org.junit.Assume.assumeTrue; + +public class DockerTests extends PackagingTestCase { + protected DockerShell sh; + + @BeforeClass + public static void filterDistros() { + assumeTrue("only Docker", distribution.isDocker()); + + ensureImageIsLoaded(distribution); + } + + @AfterClass + public static void cleanup() { + // runContainer also calls this, so we don't need this method to be annotated as `@After` + removeContainer(); + } + + @Before + public void setupTest() throws Exception { + sh = new DockerShell(); + installation = runContainer(distribution()); + } + + /** + * Checks that the Docker image can be run, and that it passes various checks. + */ + public void test10Install() { + verifyContainerInstallation(installation, distribution()); + } + + /** + * Checks that no plugins are initially active. + */ + public void test20PluginsListWithNoPlugins() { + final Installation.Executables bin = installation.executables(); + final Result r = sh.run(bin.elasticsearchPlugin + " list"); + + assertThat("Expected no plugins to be listed", r.stdout, emptyString()); + } + + /** + * Check that a keystore can be manually created using the provided CLI tool. + */ + public void test40CreateKeystoreManually() throws InterruptedException { + final Installation.Executables bin = installation.executables(); + + final Path keystorePath = installation.config("elasticsearch.keystore"); + + waitForPathToExist(keystorePath); + + // Move the auto-created one out of the way, or else the CLI prompts asks us to confirm + sh.run("mv " + keystorePath + " " + keystorePath + ".bak"); + + sh.run(bin.elasticsearchKeystore + " create"); + + final Result r = sh.run(bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + } + + /** + * Send some basic index, count and delete requests, in order to check that the installation + * is minimally functional. + */ + public void test50BasicApiTests() throws Exception { + waitForElasticsearchToBecomeAvailable(distribution); + + assertTrue(existsInContainer(installation.logs.resolve("gc.log"))); + + ServerUtils.runElasticsearchTests(); + } + + /** + * Check that the default keystore is automatically created + */ + public void test60AutoCreateKeystore() throws Exception { + final Path keystorePath = installation.config("elasticsearch.keystore"); + + waitForPathToExist(keystorePath); + + assertPermissionsAndOwnership(keystorePath, p660); + + final Installation.Executables bin = installation.executables(); + final Result result = sh.run(bin.elasticsearchKeystore + " list"); + assertThat(result.stdout, containsString("keystore.seed")); + } + + /** + * Check that the default config can be overridden using a bind mount, and that env vars are respected + */ + public void test70BindMountCustomPathConfAndJvmOptions() throws Exception { + final Path tempConf = getTempDir().resolve("esconf-alternate"); + + try { + mkdir(tempConf); + copyFromContainer(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml")); + copyFromContainer(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties")); + + // we have to disable Log4j from using JMX lest it will hit a security + // manager exception before we have configured logging; this will fail + // startup since we detect usages of logging before it is configured + final String jvmOptions = + "-Xms512m\n" + + "-Xmx512m\n" + + "-Dlog4j2.disable.jmx=true\n"; + append(tempConf.resolve("jvm.options"), jvmOptions); + + // Make the temp directory and contents accessible when bind-mounted + Files.setPosixFilePermissions(tempConf, fromString("rwxrwxrwx")); + + // Restart the container + removeContainer(); + runContainer(distribution(), tempConf, Map.of( + "ES_JAVA_OPTS", "-XX:-UseCompressedOops" + )); + + waitForElasticsearchToBecomeAvailable(distribution); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); + } finally { + rm(tempConf); + } + } + + /** + * Check whether the elasticsearch-certutil tool has been shipped correctly, + * and if present then it can execute. + */ + public void test90SecurityCliPackaging() { + final Installation.Executables bin = installation.executables(); + + final Path securityCli = installation.lib.resolve("tools").resolve("security-cli"); + + if (distribution().isDefault()) { + assertTrue(existsInContainer(securityCli)); + + Result result = sh.run(bin.elasticsearchCertutil + " --help"); + assertThat(result.stdout, containsString("Simplifies certificate creation for use with the Elastic Stack")); + + // Ensure that the exit code from the java command is passed back up through the shell script + result = sh.runIgnoreExitCode(bin.elasticsearchCertutil + " invalid-command"); + assertThat(result.isSuccess(), is(false)); + assertThat(result.stdout, containsString("Unknown command [invalid-command]")); + } else { + assertFalse(existsInContainer(securityCli)); + } + } + + /** + * Check that the elasticsearch-shard tool is shipped in the Docker image and is executable. + */ + public void test91ElasticsearchShardCliPackaging() { + final Installation.Executables bin = installation.executables(); + + final Result result = sh.run(bin.elasticsearchShard + " -h"); + assertThat(result.stdout, containsString("A CLI tool to remove corrupted parts of unrecoverable shards")); + } + + /** + * Check that the elasticsearch-node tool is shipped in the Docker image and is executable. + */ + public void test92ElasticsearchNodeCliPackaging() { + final Installation.Executables bin = installation.executables(); + + final Result result = sh.run(bin.elasticsearchNode + " -h"); + assertThat(result.stdout, + containsString("A CLI tool to do unsafe cluster and index manipulations on current node")); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 6d7534c8bb455..6716319bf40c4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -67,11 +67,10 @@ public abstract class PackagingTestCase extends Assert { protected static final String systemJavaHome; static { Shell sh = new Shell(); - if (Platforms.LINUX) { - systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); - } else { - assert Platforms.WINDOWS; + if (Platforms.WINDOWS) { systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim(); + } else { + systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index aa040fb15fcd9..eaa23018fe428 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -33,9 +33,16 @@ public class Distribution { public Distribution(Path path) { this.path = path; String filename = path.getFileName().toString(); - int lastDot = filename.lastIndexOf('.'); - String extension = filename.substring(lastDot + 1); - this.packaging = Packaging.valueOf(extension.equals("gz") ? "TAR" : extension.toUpperCase(Locale.ROOT)); + + if (filename.endsWith(".gz")) { + this.packaging = Packaging.TAR; + } else if (filename.endsWith(".docker.tar")) { + this.packaging = Packaging.DOCKER; + } else { + int lastDot = filename.lastIndexOf('.'); + this.packaging = Packaging.valueOf(filename.substring(lastDot + 1)); + } + this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX; this.flavor = filename.contains("oss") ? Flavor.OSS : Flavor.DEFAULT; this.hasJdk = filename.contains("no-jdk") == false; @@ -57,12 +64,17 @@ public boolean isPackage() { return packaging == Packaging.RPM || packaging == Packaging.DEB; } + public boolean isDocker() { + return packaging == Packaging.DOCKER; + } + public enum Packaging { TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN), ZIP(".zip", Platforms.WINDOWS), DEB(".deb", Platforms.isDPKG()), - RPM(".rpm", Platforms.isRPM()); + RPM(".rpm", Platforms.isRPM()), + DOCKER(".docker.tar", Platforms.isDocker()); /** The extension of this distribution's file */ public final String extension; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java new file mode 100644 index 0000000000000..93e95041d6f5f --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -0,0 +1,365 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermission; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static java.nio.file.attribute.PosixFilePermissions.fromString; +import static org.elasticsearch.packaging.util.FileMatcher.p644; +import static org.elasticsearch.packaging.util.FileMatcher.p660; +import static org.elasticsearch.packaging.util.FileMatcher.p755; +import static org.elasticsearch.packaging.util.FileMatcher.p775; +import static org.elasticsearch.packaging.util.FileUtils.getCurrentVersion; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * Utilities for running packaging tests against the Elasticsearch Docker images. + */ +public class Docker { + private static final Log logger = LogFactory.getLog(Docker.class); + + private static final Shell sh = new Shell(); + private static final DockerShell dockerShell = new DockerShell(); + + /** + * Tracks the currently running Docker image. An earlier implementation used a fixed container name, + * but that appeared to cause problems with repeatedly destroying and recreating containers with + * the same name. + */ + private static String containerId = null; + + /** + * Checks whether the required Docker image exists. If not, the image is loaded from disk. No check is made + * to see whether the image is up-to-date. + * @param distribution details about the docker image to potentially load. + */ + public static void ensureImageIsLoaded(Distribution distribution) { + final long count = sh.run("docker image ls --format '{{.Repository}}' " + distribution.flavor.name).stdout.lines().count(); + + if (count != 0) { + return; + } + + logger.info("Loading Docker image: " + distribution.path); + sh.run("docker load -i " + distribution.path); + } + + /** + * Runs an Elasticsearch Docker container. + * @param distribution details about the docker image being tested. + */ + public static Installation runContainer(Distribution distribution) throws Exception { + return runContainer(distribution, null, Collections.emptyMap()); + } + + /** + * Runs an Elasticsearch Docker container, with options for overriding the config directory + * through a bind mount, and passing additional environment variables. + * + * @param distribution details about the docker image being tested. + * @param configPath the path to the config to bind mount, or null + * @param envVars environment variables to set when running the container + */ + public static Installation runContainer(Distribution distribution, Path configPath, Map envVars) throws Exception { + removeContainer(); + + final List args = new ArrayList<>(); + + args.add("docker run"); + + // Remove the container once it exits + args.add("--rm"); + + // Run the container in the background + args.add("--detach"); + + envVars.forEach((key, value) -> args.add("--env " + key + "=\"" + value + "\"")); + + // The container won't run without configuring discovery + args.add("--env discovery.type=single-node"); + + // Map ports in the container to the host, so that we can send requests + args.add("--publish 9200:9200"); + args.add("--publish 9300:9300"); + + if (configPath != null) { + // Bind-mount the config dir, if specified + args.add("--volume \"" + configPath + ":/usr/share/elasticsearch/config\""); + } + + args.add(distribution.flavor.name + ":test"); + + final String command = String.join(" ", args); + logger.debug("Running command: " + command); + containerId = sh.run(command).stdout.trim(); + + waitForElasticsearchToStart(); + + return Installation.ofContainer(); + } + + /** + * Waits for the Elasticsearch process to start executing in the container. + * This is called every time a container is started. + */ + private static void waitForElasticsearchToStart() throws InterruptedException { + boolean isElasticsearchRunning = false; + int attempt = 0; + + do { + String psOutput = dockerShell.run("ps ax").stdout; + + if (psOutput.contains("/usr/share/elasticsearch/jdk/bin/java -X")) { + isElasticsearchRunning = true; + break; + } + + Thread.sleep(1000); + } while (attempt++ < 5); + + if (!isElasticsearchRunning) { + final String logs = sh.run("docker logs " + containerId).stdout; + fail("Elasticsearch container did start successfully.\n\n" + logs); + } + } + + /** + * Waits for the Elasticsearch process to finish booting up. This takes a few seconds, so this action + * is not taken every time a container is started, since many tests check the Docker image in + * some way, and not Elasticsearch specifically. + */ + public static void waitForElasticsearchToBecomeAvailable(Distribution distribution) throws InterruptedException { + int attempt = 0; + + // Give Elasticsearch time to become available. I measured roughly 10s on my laptop + Thread.sleep(10 * 1000); + + do { + String dockerLogs = sh.run("docker logs " + containerId).stdout; + + if (distribution.isOSS() && dockerLogs.contains("recovered [0] indices into cluster_state")) { + return; + } else if (dockerLogs.contains("Active license is now")) { + return; + } + + Thread.sleep(1000); + } while (attempt++ < 5); + + String logs = sh.run("docker logs " + containerId).stdout; + fail("Elasticsearch did not become ready to accept connections.\n\n" + logs); + } + + /** + * Removes the currently running container. + */ + public static void removeContainer() { + if (containerId != null) { + // Remove the container, forcibly killing it if necessary + logger.debug("Removing container " + containerId); + sh.run("docker rm -f " + containerId); + + containerId = null; + } + } + + /** + * Copies a file from the container into the local filesystem + * @param from the file to copy in the container + * @param to the location to place the copy + */ + public static void copyFromContainer(Path from, Path to) { + final String script = "docker cp " + containerId + ":" + from + " " + to; + logger.debug("Copying file from container with: " + script); + sh.run(script); + } + + /** + * Extends {@link Shell} so that executed commands happen in the currently running Docker container. + */ + public static class DockerShell extends Shell { + @Override + protected String[] getScriptCommand(String script) { + assert containerId != null; + + return super.getScriptCommand("docker exec " + + "--user elasticsearch " + + "--tty " + + containerId + " " + + script); + } + } + + /** + * Checks whether a path exist in the Docker container. + */ + public static boolean existsInContainer(Path path) { + logger.debug("Checking whether file " + path + " exists in container"); + final Shell.Result result = dockerShell.runIgnoreExitCode("test -e " + path); + + return result.isSuccess(); + } + + /** + * Checks that the specified path's permissions and ownership match those specified. + */ + public static void assertPermissionsAndOwnership(Path path, Set expectedPermissions) { + logger.debug("Checking permissions and ownership of [" + path + "]"); + final Shell.Result result = dockerShell.run("ls -ld " + path); + + final String[] components = result.stdout.split("\\s+"); + + // The final substring() is because we don't check the directory bit, and we + // also don't want any SELinux security context indicator. + Set actualPermissions = fromString(components[0].substring(1, 10)); + + assertEquals("Permissions of " + path + " are wrong", actualPermissions, expectedPermissions); + assertThat("File owner of " + path + " is wrong", components[2], equalTo("elasticsearch")); + assertThat("File group of " + path + " is wrong", components[3], equalTo("root")); + } + + /** + * Waits for up to 5 seconds for a path to exist in the container. + */ + public static void waitForPathToExist(Path path) throws InterruptedException { + int attempt = 0; + + do { + if (existsInContainer(path)) { + return; + } + + Thread.sleep(500); + } while (attempt++ < 10); + + fail(path + " failed to exist after 5000ms"); + } + + /** + * Perform a variety of checks on an installation. If the current distribution is not OSS, additional checks are carried out. + */ + public static void verifyContainerInstallation(Installation installation, Distribution distribution) { + verifyOssInstallation(installation); + if (distribution.flavor == Distribution.Flavor.DEFAULT) { + verifyDefaultInstallation(installation); + } + } + + private static void verifyOssInstallation(Installation es) { + dockerShell.run("id elasticsearch"); + dockerShell.run("getent group elasticsearch"); + + final Shell.Result passwdResult = dockerShell.run("getent passwd elasticsearch"); + final String homeDir = passwdResult.stdout.trim().split(":")[5]; + assertThat(homeDir, equalTo("/usr/share/elasticsearch")); + + Stream.of( + es.home, + es.data, + es.logs, + es.config + ).forEach(dir -> assertPermissionsAndOwnership(dir, p775)); + + Stream.of( + es.plugins, + es.modules + ).forEach(dir -> assertPermissionsAndOwnership(dir, p755)); + + // FIXME these files should all have the same permissions + Stream.of( + "elasticsearch.keystore", +// "elasticsearch.yml", + "jvm.options" +// "log4j2.properties" + ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660)); + + Stream.of( + "elasticsearch.yml", + "log4j2.properties" + ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p644)); + + assertThat( + dockerShell.run(es.bin("elasticsearch-keystore") + " list").stdout, + containsString("keystore.seed")); + + Stream.of( + es.bin, + es.lib + ).forEach(dir -> assertPermissionsAndOwnership(dir, p755)); + + Stream.of( + "elasticsearch", + "elasticsearch-cli", + "elasticsearch-env", + "elasticsearch-enve", + "elasticsearch-keystore", + "elasticsearch-node", + "elasticsearch-plugin", + "elasticsearch-shard" + ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755)); + + Stream.of( + "LICENSE.txt", + "NOTICE.txt", + "README.textile" + ).forEach(doc -> assertPermissionsAndOwnership(es.home.resolve(doc), p644)); + } + + private static void verifyDefaultInstallation(Installation es) { + Stream.of( + "elasticsearch-certgen", + "elasticsearch-certutil", + "elasticsearch-croneval", + "elasticsearch-saml-metadata", + "elasticsearch-setup-passwords", + "elasticsearch-sql-cli", + "elasticsearch-syskeygen", + "elasticsearch-users", + "x-pack-env", + "x-pack-security-env", + "x-pack-watcher-env" + ).forEach(executable -> assertPermissionsAndOwnership(es.bin(executable), p755)); + + // at this time we only install the current version of archive distributions, but if that changes we'll need to pass + // the version through here + assertPermissionsAndOwnership(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), p755); + + Stream.of( + "role_mapping.yml", + "roles.yml", + "users", + "users_roles" + ).forEach(configFile -> assertPermissionsAndOwnership(es.config(configFile), p660)); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java index f6e598b5a0d55..89113ae098ea2 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileMatcher.java @@ -45,6 +45,7 @@ public class FileMatcher extends TypeSafeMatcher { public enum Fileness { File, Directory } + public static final Set p775 = fromString("rwxrwxr-x"); public static final Set p755 = fromString("rwxr-xr-x"); public static final Set p750 = fromString("rwxr-x---"); public static final Set p660 = fromString("rw-rw----"); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java index 9e3ba5b52e284..c5fdf0106df29 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Installation.java @@ -84,6 +84,20 @@ public static Installation ofPackage(Distribution.Packaging packaging) { ); } + public static Installation ofContainer() { + String root = "/usr/share/elasticsearch"; + return new Installation( + Paths.get(root), + Paths.get(root + "/config"), + Paths.get(root + "/data"), + Paths.get(root + "/logs"), + Paths.get(root + "/plugins"), + Paths.get(root + "/modules"), + null, + null + ); + } + public Path bin(String executableName) { return bin.resolve(executableName); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java index 6258c1336b2fc..b0778bf460ee6 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java @@ -65,6 +65,10 @@ public static boolean isSysVInit() { return new Shell().runIgnoreExitCode("which service").isSuccess(); } + public static boolean isDocker() { + return new Shell().runIgnoreExitCode("which docker").isSuccess(); + } + public static void onWindows(PlatformAction action) throws Exception { if (WINDOWS) { action.run(); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java index dc490de05b9c8..316c478acf7e4 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Shell.java @@ -30,7 +30,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.stream.Stream; /** * Wrapper to run shell commands and collect their outputs in a less verbose way @@ -72,7 +71,8 @@ public Result run( String command, Object... args) { String formattedCommand = String.format(Locale.ROOT, command, args); return run(formattedCommand); } - private String[] getScriptCommand(String script) { + + protected String[] getScriptCommand(String script) { if (Platforms.WINDOWS) { return powershellCommand(script); } else { @@ -81,11 +81,11 @@ private String[] getScriptCommand(String script) { } private static String[] bashCommand(String script) { - return Stream.concat(Stream.of("bash", "-c"), Stream.of(script)).toArray(String[]::new); + return new String[] { "bash", "-c", script }; } private static String[] powershellCommand(String script) { - return Stream.concat(Stream.of("powershell.exe", "-Command"), Stream.of(script)).toArray(String[]::new); + return new String[] { "powershell.exe", "-Command", script }; } private Result runScript(String[] command) { diff --git a/settings.gradle b/settings.gradle index 1118ee7714c99..36732fc239ecd 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,6 +32,8 @@ List projects = [ 'distribution:docker', 'distribution:docker:oss-docker-build-context', 'distribution:docker:docker-build-context', + 'distribution:docker:oss-docker-export', + 'distribution:docker:docker-export', 'distribution:packages:oss-deb', 'distribution:packages:deb', 'distribution:packages:oss-no-jdk-deb', From 519342f5d43202905be7441c48da6f4c4467d093 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 12 Sep 2019 13:55:23 +0100 Subject: [PATCH 02/11] Install Docker in fewer OSs via Vagrantfile --- Vagrantfile | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 8e21b80fbd41f..d117dd309fb8d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,5 +1,5 @@ # -*- mode: ruby -*- -# vi:ft=ruby ts=2 sw=2 sts=2 et: +# vim: ft=ruby ts=2 sw=2 sts=2 et: # This Vagrantfile exists to test packaging. Read more about its use in the # vagrant section in TESTING.asciidoc. @@ -63,6 +63,7 @@ Vagrant.configure(2) do |config| # Install Jayatana so we can work around it being present. [ -f /usr/share/java/jayatanaag.jar ] || install jayatana SHELL + deb_docker config end end 'ubuntu-1804'.tap do |box| @@ -72,6 +73,7 @@ Vagrant.configure(2) do |config| # Install Jayatana so we can work around it being present. [ -f /usr/share/java/jayatanaag.jar ] || install jayatana SHELL + deb_docker config end end 'debian-8'.tap do |box| @@ -88,6 +90,7 @@ Vagrant.configure(2) do |config| config.vm.box = 'elastic/debian-9-x86_64' deb_common config, box end + deb_docker config end 'centos-6'.tap do |box| config.vm.define box, define_opts do |config| @@ -99,6 +102,7 @@ Vagrant.configure(2) do |config| config.vm.define box, define_opts do |config| config.vm.box = 'elastic/centos-7-x86_64' rpm_common config, box + rpm_docker config end end 'oel-6'.tap do |box| @@ -117,12 +121,14 @@ Vagrant.configure(2) do |config| config.vm.define box, define_opts do |config| config.vm.box = 'elastic/fedora-28-x86_64' dnf_common config, box + dnf_docker config end end 'fedora-29'.tap do |box| config.vm.define box, define_opts do |config| config.vm.box = 'elastic/fedora-28-x86_64' dnf_common config, box + dnf_docker config end end 'opensuse-42'.tap do |box| @@ -183,7 +189,6 @@ def deb_common(config, name, extra: '') install_command: 'apt-get install -y', extra: extra_with_lintian ) - deb_docker(config) end def deb_docker(config) @@ -222,7 +227,6 @@ def rpm_common(config, name) update_tracking_file: '/var/cache/yum/last_update', install_command: 'yum install -y' ) - rpm_docker(config) end def rpm_docker(config) @@ -237,7 +241,7 @@ def rpm_docker(config) yum install -y docker-ce docker-ce-cli containerd.io # Start Docker - systemctl start docker + systemctl enable --now docker # Add vagrant to the Docker group, so that it can run commands usermod -aG docker vagrant @@ -258,7 +262,6 @@ def dnf_common(config, name) install_command: 'dnf install -y', install_command_retries: 5 ) - dnf_docker(config) end def dnf_docker(config) @@ -273,7 +276,7 @@ def dnf_docker(config) dnf install -y docker-ce docker-ce-cli containerd.io # Start Docker - systemctl start docker + systemctl enable --now docker # Add vagrant to the Docker group, so that it can run commands usermod -aG docker vagrant From 1980219c70da7b9dba73a99b43ae2b59cb004b87 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 12 Sep 2019 14:35:46 +0100 Subject: [PATCH 03/11] Address review feedback --- .../gradle/DistributionDownloadPlugin.java | 3 +- .../gradle/ElasticsearchDistribution.java | 12 +++++++ distribution/docker/build.gradle | 32 +++---------------- .../packaging/test/PackagingTestCase.java | 1 + .../elasticsearch/packaging/util/Docker.java | 17 ++++++---- 5 files changed, 29 insertions(+), 36 deletions(-) diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java index 6f388a2161949..c0807f8ad7d5c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java @@ -94,8 +94,7 @@ void setupDistributions(Project project) { dependencies.add(distribution.configuration.getName(), dependencyNotation(project, distribution)); // no extraction allowed for rpm, deb or docker - Type distroType = distribution.getType(); - if (distroType != Type.RPM && distroType != Type.DEB && distroType != Type.DOCKER) { + if (distribution.getType().shouldExtract()) { // for the distribution extracted, add a root level task that does the extraction, and depend on that // extracted configuration as an artifact consisting of the extracted distribution directory dependencies.add(distribution.getExtracted().configuration.getName(), diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java index 48fa1ba360792..83fcf0b722b4c 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/ElasticsearchDistribution.java @@ -53,6 +53,18 @@ public enum Type { public String toString() { return super.toString().toLowerCase(Locale.ROOT); } + + public boolean shouldExtract() { + switch (this) { + case DEB: + case DOCKER: + case RPM: + return false; + + default: + return true; + } + } } public enum Flavor { diff --git a/distribution/docker/build.gradle b/distribution/docker/build.gradle index 24e348ba69544..a89620d427765 100644 --- a/distribution/docker/build.gradle +++ b/distribution/docker/build.gradle @@ -170,32 +170,6 @@ void addBuildDockerImage(final boolean oss) { BuildPlugin.requireDocker(buildDockerImageTask) } -/** - * Exports the generated Docker image to disk, so that it can be easily - * reloaded, for example into a VM. Although this involves writing out - * the entire image, it's still quicker than rebuilding the main archive - * in the VM. - */ -void addExportDockerImage(final boolean oss) { - def exportTaskName = taskName("export", oss, "DockerImage") - def tarFile = "${buildDir}/elasticsearch${oss ? '-oss' : ''}_test.docker.tar" - - task(exportTaskName, type: LoggedExec) { - executable 'docker' - args "save", - "-o", - tarFile, - "elasticsearch${oss ? '-oss' : ''}:test" - } - - artifacts.add(oss ? 'archives' : 'default', file(tarFile)) { - type 'tar' - name "elasticsearch${oss ? '-oss' : ''}" - builtBy exportTaskName - } -} - - for (final boolean oss : [false, true]) { addCopyDockerContextTask(oss) addBuildDockerImage(oss) @@ -211,6 +185,10 @@ if (tasks.findByName("composePull")) { tasks.composePull.enabled = false } +/* + * The export subprojects write out the generated Docker images to disk, so + * that they can be easily reloaded, for example into a VM. + */ subprojects { Project subProject -> if (subProject.name.contains('docker-export')) { apply plugin: 'distribution' @@ -219,7 +197,7 @@ subprojects { Project subProject -> def exportTaskName = taskName("export", oss, "DockerImage") def buildTaskName = taskName("build", oss, "DockerImage") - def tarFile = "${parent.buildDir}/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" + def tarFile = "${parent.projectDir}/build/elasticsearch${oss ? '-oss' : ''}_test.${VersionProperties.elasticsearch}.docker.tar" final Task exportDockerImageTask = task(exportTaskName, type: LoggedExec) { executable 'docker' diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 6716319bf40c4..56fcd17599b56 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -70,6 +70,7 @@ public abstract class PackagingTestCase extends Assert { if (Platforms.WINDOWS) { systemJavaHome = sh.run("$Env:SYSTEM_JAVA_HOME").stdout.trim(); } else { + assert Platforms.LINUX || Platforms.DARWIN; systemJavaHome = sh.run("echo $SYSTEM_JAVA_HOME").stdout.trim(); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index 93e95041d6f5f..90f7abc3ae80d 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -214,7 +214,7 @@ protected String[] getScriptCommand(String script) { assert containerId != null; return super.getScriptCommand("docker exec " + - "--user elasticsearch " + + "--user elasticsearch:root " + "--tty " + containerId + " " + script); @@ -222,7 +222,7 @@ protected String[] getScriptCommand(String script) { } /** - * Checks whether a path exist in the Docker container. + * Checks whether a path exists in the Docker container. */ public static boolean existsInContainer(Path path) { logger.debug("Checking whether file " + path + " exists in container"); @@ -236,17 +236,20 @@ public static boolean existsInContainer(Path path) { */ public static void assertPermissionsAndOwnership(Path path, Set expectedPermissions) { logger.debug("Checking permissions and ownership of [" + path + "]"); - final Shell.Result result = dockerShell.run("ls -ld " + path); - final String[] components = result.stdout.split("\\s+"); + final String[] components = dockerShell.run("stat --format=\"%U %G %A\" " + path).stdout.split("\\s+"); + + final String username = components[0]; + final String group = components[1]; + final String permissions = components[2]; // The final substring() is because we don't check the directory bit, and we // also don't want any SELinux security context indicator. - Set actualPermissions = fromString(components[0].substring(1, 10)); + Set actualPermissions = fromString(permissions.substring(1, 10)); assertEquals("Permissions of " + path + " are wrong", actualPermissions, expectedPermissions); - assertThat("File owner of " + path + " is wrong", components[2], equalTo("elasticsearch")); - assertThat("File group of " + path + " is wrong", components[3], equalTo("root")); + assertThat("File owner of " + path + " is wrong", username, equalTo("elasticsearch")); + assertThat("File group of " + path + " is wrong", group, equalTo("root")); } /** From dcf27b38a575ca04f3bdd7c3f35a54cc2f3b052c Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 19 Sep 2019 12:42:46 +0100 Subject: [PATCH 04/11] Vagrantfile tweaks --- Vagrantfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index d117dd309fb8d..c2c4f1f84b12b 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -192,7 +192,7 @@ def deb_common(config, name, extra: '') end def deb_docker(config) - config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using apt', run: 'always', type: 'shell', inline: <<-SHELL # Install packages to allow apt to use a repository over HTTPS apt-get install -y \ apt-transport-https \ @@ -230,7 +230,7 @@ def rpm_common(config, name) end def rpm_docker(config) - config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using yum', run: 'always', type: 'shell', inline: <<-SHELL # Install prerequisites yum install -y yum-utils device-mapper-persistent-data lvm2 @@ -265,7 +265,7 @@ def dnf_common(config, name) end def dnf_docker(config) - config.vm.provision 'install Docker', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using dnf', run: 'always', type: 'shell', inline: <<-SHELL # Install prerequisites dnf -y install dnf-plugins-core From 3cb1bfe2593fbe7977e0a7a9c1b2bfd2003f5f13 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 20 Sep 2019 12:29:04 +0100 Subject: [PATCH 05/11] Vagrantfile fixes --- Vagrantfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index c2c4f1f84b12b..82ffe446ed3d5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -89,8 +89,8 @@ Vagrant.configure(2) do |config| config.vm.define box, define_opts do |config| config.vm.box = 'elastic/debian-9-x86_64' deb_common config, box + deb_docker config end - deb_docker config end 'centos-6'.tap do |box| config.vm.define box, define_opts do |config| @@ -230,7 +230,7 @@ def rpm_common(config, name) end def rpm_docker(config) - config.vm.provision 'install Docker using yum', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using yum', type: 'shell', inline: <<-SHELL # Install prerequisites yum install -y yum-utils device-mapper-persistent-data lvm2 @@ -265,7 +265,7 @@ def dnf_common(config, name) end def dnf_docker(config) - config.vm.provision 'install Docker using dnf', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using dnf', type: 'shell', inline: <<-SHELL # Install prerequisites dnf -y install dnf-plugins-core @@ -340,7 +340,7 @@ def linux_common(config, # This prevents leftovers from previous tests using the # same VM from messing up the current test - config.vm.provision 'clean es installs in tmp', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'clean es installs in tmp', type: 'shell', inline: <<-SHELL rm -rf /tmp/elasticsearch* SHELL From 563882eae957b4b81fafd5e16d9af7c4542aaba1 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Mon, 23 Sep 2019 14:11:39 +0100 Subject: [PATCH 06/11] Fix gaffe with mapping file extension to Packaging value --- .../java/org/elasticsearch/packaging/util/Distribution.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index eaa23018fe428..72bd79ff7b466 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -40,7 +40,7 @@ public Distribution(Path path) { this.packaging = Packaging.DOCKER; } else { int lastDot = filename.lastIndexOf('.'); - this.packaging = Packaging.valueOf(filename.substring(lastDot + 1)); + this.packaging = Packaging.valueOf(filename.substring(lastDot + 1).toUpperCase(Locale.ROOT)); } this.platform = filename.contains("windows") ? Platform.WINDOWS : Platform.LINUX; From 3797a3e1e2430ca482d3b7a3cb7619c4cf2f3efa Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Thu, 26 Sep 2019 11:19:52 +0100 Subject: [PATCH 07/11] Attempt to make Docker packaging tests more robust --- .../elasticsearch/packaging/util/Docker.java | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index 90f7abc3ae80d..8e48421dcfeca 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -186,11 +186,25 @@ public static void waitForElasticsearchToBecomeAvailable(Distribution distributi */ public static void removeContainer() { if (containerId != null) { - // Remove the container, forcibly killing it if necessary - logger.debug("Removing container " + containerId); - sh.run("docker rm -f " + containerId); - - containerId = null; + try { + // Remove the container, forcibly killing it if necessary + logger.debug("Removing container " + containerId); + final String command = "docker rm -f " + containerId; + final Shell.Result result = sh.runIgnoreExitCode(command); + + if (result.isSuccess() == false) { + // I'm not sure why we're already removing this container, but that's OK. + if (result.stderr.contains("removal of container " + " is already in progress") == false) { + throw new RuntimeException( + "Command was not successful: [" + command + "] result: " + result.toString()); + } + } + } finally { + // Null out the containerId under all circumstances, so that even if the remove command fails + // for some reason, the other tests will still proceed. Otherwise they can get stuck, continually + // trying to remove a non-existent container ID. + containerId = null; + } } } From 62d78ac99a15778b0b9226a2ba5cb6bed65b5b26 Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 27 Sep 2019 16:38:07 +0100 Subject: [PATCH 08/11] Fix docker install on Ubuntu It turns out that it's not the same as Debian. --- Vagrantfile | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 82ffe446ed3d5..93b60b46872fd 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -63,7 +63,7 @@ Vagrant.configure(2) do |config| # Install Jayatana so we can work around it being present. [ -f /usr/share/java/jayatanaag.jar ] || install jayatana SHELL - deb_docker config + ubuntu_docker config end end 'ubuntu-1804'.tap do |box| @@ -73,7 +73,7 @@ Vagrant.configure(2) do |config| # Install Jayatana so we can work around it being present. [ -f /usr/share/java/jayatanaag.jar ] || install jayatana SHELL - deb_docker config + ubuntu_docker config end end 'debian-8'.tap do |box| @@ -191,8 +191,37 @@ def deb_common(config, name, extra: '') ) end +def ubuntu_docker(config) + config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL + # Install packages to allow apt to use a repository over HTTPS + apt-get install -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + software-properties-common + + # Add Docker’s official GPG key + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - + + # Set up the stable Docker repository + add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/ubuntu \ + $(lsb_release -cs) \ + stable" + + # Install Docker. Unlike Fedora and CentOS, this also start the daemon. + apt-get update + apt-get install -y docker-ce docker-ce-cli containerd.io + + # Add vagrant to the Docker group, so that it can run commands + usermod -aG docker vagrant + SHELL +end + + def deb_docker(config) - config.vm.provision 'install Docker using apt', run: 'always', type: 'shell', inline: <<-SHELL + config.vm.provision 'install Docker using apt', type: 'shell', inline: <<-SHELL # Install packages to allow apt to use a repository over HTTPS apt-get install -y \ apt-transport-https \ From 2139539de5d328f6c695b779d8e066964fbf3a9f Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 1 Oct 2019 11:04:24 +0100 Subject: [PATCH 09/11] Use standard method to check if ES is ready --- .../packaging/test/DockerTests.java | 6 ++--- .../elasticsearch/packaging/util/Docker.java | 27 ------------------- .../packaging/util/ServerUtils.java | 3 ++- 3 files changed, 5 insertions(+), 31 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index 8c7ad7a8fd5d3..f93d874f02e66 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -40,7 +40,6 @@ import static org.elasticsearch.packaging.util.Docker.removeContainer; import static org.elasticsearch.packaging.util.Docker.runContainer; import static org.elasticsearch.packaging.util.Docker.verifyContainerInstallation; -import static org.elasticsearch.packaging.util.Docker.waitForElasticsearchToBecomeAvailable; import static org.elasticsearch.packaging.util.Docker.waitForPathToExist; import static org.elasticsearch.packaging.util.FileMatcher.p660; import static org.elasticsearch.packaging.util.FileUtils.append; @@ -48,6 +47,7 @@ import static org.elasticsearch.packaging.util.FileUtils.mkdir; import static org.elasticsearch.packaging.util.FileUtils.rm; import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; +import static org.elasticsearch.packaging.util.ServerUtils.waitForElasticsearch; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.emptyString; @@ -116,7 +116,7 @@ public void test40CreateKeystoreManually() throws InterruptedException { * is minimally functional. */ public void test50BasicApiTests() throws Exception { - waitForElasticsearchToBecomeAvailable(distribution); + waitForElasticsearch(); assertTrue(existsInContainer(installation.logs.resolve("gc.log"))); @@ -167,7 +167,7 @@ public void test70BindMountCustomPathConfAndJvmOptions() throws Exception { "ES_JAVA_OPTS", "-XX:-UseCompressedOops" )); - waitForElasticsearchToBecomeAvailable(distribution); + waitForElasticsearch(); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index 8e48421dcfeca..d78b60236bc4a 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -154,33 +154,6 @@ private static void waitForElasticsearchToStart() throws InterruptedException { } } - /** - * Waits for the Elasticsearch process to finish booting up. This takes a few seconds, so this action - * is not taken every time a container is started, since many tests check the Docker image in - * some way, and not Elasticsearch specifically. - */ - public static void waitForElasticsearchToBecomeAvailable(Distribution distribution) throws InterruptedException { - int attempt = 0; - - // Give Elasticsearch time to become available. I measured roughly 10s on my laptop - Thread.sleep(10 * 1000); - - do { - String dockerLogs = sh.run("docker logs " + containerId).stdout; - - if (distribution.isOSS() && dockerLogs.contains("recovered [0] indices into cluster_state")) { - return; - } else if (dockerLogs.contains("Active license is now")) { - return; - } - - Thread.sleep(1000); - } while (attempt++ < 5); - - String logs = sh.run("docker logs " + containerId).stdout; - fail("Elasticsearch did not become ready to accept connections.\n\n" + logs); - } - /** * Removes the currently running container. */ diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 6331b4bf46e9a..d62bfe1e70b99 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -22,6 +22,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.http.HttpResponse; +import org.apache.http.NoHttpResponseException; import org.apache.http.client.fluent.Request; import org.apache.http.conn.HttpHostConnectException; import org.apache.http.entity.ContentType; @@ -70,7 +71,7 @@ public static void waitForElasticsearch(String status, String index) throws IOEx started = true; - } catch (HttpHostConnectException e) { + } catch (HttpHostConnectException | NoHttpResponseException e) { // we want to retry if the connection is refused LOG.debug("Got connection refused when waiting for cluster health", e); } From f5278083b9d3cd242f46d24bfb005c0dec9e0c2e Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Tue, 1 Oct 2019 14:47:13 +0100 Subject: [PATCH 10/11] Tweak exception handing in waitForElasticsearch(...) --- .../java/org/elasticsearch/packaging/util/ServerUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index d62bfe1e70b99..d0ec26877ee3e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -24,11 +24,11 @@ import org.apache.http.HttpResponse; import org.apache.http.NoHttpResponseException; import org.apache.http.client.fluent.Request; -import org.apache.http.conn.HttpHostConnectException; import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; import java.io.IOException; +import java.net.SocketException; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -71,7 +71,7 @@ public static void waitForElasticsearch(String status, String index) throws IOEx started = true; - } catch (HttpHostConnectException | NoHttpResponseException e) { + } catch (SocketException | NoHttpResponseException e) { // we want to retry if the connection is refused LOG.debug("Got connection refused when waiting for cluster health", e); } From e7aca8850869a44a51b5ff9920b65417a8f859fa Mon Sep 17 00:00:00 2001 From: Rory Hunter Date: Fri, 4 Oct 2019 09:46:15 +0100 Subject: [PATCH 11/11] Address review feedback --- .../gradle/test/DistroTestPlugin.java | 15 ++++++++------- distribution/docker/build.gradle | 2 +- .../elasticsearch/packaging/test/DockerTests.java | 7 ++++--- .../packaging/util/Distribution.java | 4 ---- .../elasticsearch/packaging/util/ServerUtils.java | 2 -- 5 files changed, 13 insertions(+), 17 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java index 0a01639644607..6203d0c9b12e5 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/test/DistroTestPlugin.java @@ -322,7 +322,11 @@ private List configureDistributions(Project project, for (Type type : Arrays.asList(Type.DEB, Type.RPM, Type.DOCKER)) { for (Flavor flavor : Flavor.values()) { for (boolean bundledJdk : Arrays.asList(true, false)) { - addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); + // We should never add a Docker distro with bundledJdk == false + boolean skip = type == Type.DOCKER && bundledJdk == false; + if (skip == false) { + addDistro(distributions, type, null, flavor, bundledJdk, VersionProperties.getElasticsearch(), currentDistros); + } } } // upgrade version is always bundled jdk @@ -370,8 +374,7 @@ private static void addDistro(NamedDomainObjectContainer if (subProject.name.contains('docker-export')) { apply plugin: 'distribution' - def oss = subProject.name.startsWith('oss') + final boolean oss = subProject.name.startsWith('oss') def exportTaskName = taskName("export", oss, "DockerImage") def buildTaskName = taskName("build", oss, "DockerImage") diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index f93d874f02e66..52205263d3ed3 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -20,6 +20,7 @@ package org.elasticsearch.packaging.test; import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Docker.DockerShell; import org.elasticsearch.packaging.util.Installation; import org.elasticsearch.packaging.util.ServerUtils; @@ -58,7 +59,7 @@ public class DockerTests extends PackagingTestCase { @BeforeClass public static void filterDistros() { - assumeTrue("only Docker", distribution.isDocker()); + assumeTrue("only Docker", distribution.packaging == Distribution.Packaging.DOCKER); ensureImageIsLoaded(distribution); } @@ -116,7 +117,7 @@ public void test40CreateKeystoreManually() throws InterruptedException { * is minimally functional. */ public void test50BasicApiTests() throws Exception { - waitForElasticsearch(); + waitForElasticsearch(installation); assertTrue(existsInContainer(installation.logs.resolve("gc.log"))); @@ -167,7 +168,7 @@ public void test70BindMountCustomPathConfAndJvmOptions() throws Exception { "ES_JAVA_OPTS", "-XX:-UseCompressedOops" )); - waitForElasticsearch(); + waitForElasticsearch(installation); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 72bd79ff7b466..13b2f31c7e4fd 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -64,10 +64,6 @@ public boolean isPackage() { return packaging == Packaging.RPM || packaging == Packaging.DEB; } - public boolean isDocker() { - return packaging == Packaging.DOCKER; - } - public enum Packaging { TAR(".tar.gz", Platforms.LINUX || Platforms.DARWIN), diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 24fd04a774619..9d91b2a15bdfb 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -20,7 +20,6 @@ package org.elasticsearch.packaging.util; import org.apache.http.HttpResponse; -import org.apache.http.NoHttpResponseException; import org.apache.http.client.fluent.Request; import org.apache.http.entity.ContentType; import org.apache.http.util.EntityUtils; @@ -28,7 +27,6 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; -import java.net.SocketException; import java.util.Objects; import java.util.concurrent.TimeUnit;