diff --git a/Vagrantfile b/Vagrantfile index 93b60b46872fd..273f2a6751a08 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -471,6 +471,7 @@ JAVA ensure curl ensure unzip ensure rsync + ensure expect installed bats || { # Bats lives in a git repository.... diff --git a/distribution/docker/src/docker/bin/docker-entrypoint.sh b/distribution/docker/src/docker/bin/docker-entrypoint.sh index 0fb5aa61e7cee..65134fa3f7985 100644 --- a/distribution/docker/src/docker/bin/docker-entrypoint.sh +++ b/distribution/docker/src/docker/bin/docker-entrypoint.sh @@ -84,8 +84,18 @@ if [[ -f bin/elasticsearch-users ]]; then # honor the variable if it's present. if [[ -n "$ELASTIC_PASSWORD" ]]; then [[ -f /usr/share/elasticsearch/config/elasticsearch.keystore ]] || (run_as_other_user_if_needed elasticsearch-keystore create) - if ! (run_as_other_user_if_needed elasticsearch-keystore list | grep -q '^bootstrap.password$'); then - (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password') + if ! (run_as_other_user_if_needed elasticsearch-keystore has-passwd --silent) ; then + # keystore is unencrypted + if ! (run_as_other_user_if_needed elasticsearch-keystore list | grep -q '^bootstrap.password$'); then + (run_as_other_user_if_needed echo "$ELASTIC_PASSWORD" | elasticsearch-keystore add -x 'bootstrap.password') + fi + else + # keystore requires password + if ! (run_as_other_user_if_needed echo "$KEYSTORE_PASSWORD" \ + | elasticsearch-keystore list | grep -q '^bootstrap.password$') ; then + COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$ELASTIC_PASSWORD")" + (run_as_other_user_if_needed echo "$COMMANDS" | elasticsearch-keystore add -x 'bootstrap.password') + fi fi fi fi @@ -97,4 +107,4 @@ if [[ "$(id -u)" == "0" ]]; then fi fi -run_as_other_user_if_needed /usr/share/elasticsearch/bin/elasticsearch "${es_opts[@]}" +run_as_other_user_if_needed /usr/share/elasticsearch/bin/elasticsearch "${es_opts[@]}" <<<"$KEYSTORE_PASSWORD" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index 25c50c0fa7392..0a797a0338ea2 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -234,6 +234,10 @@ Closure commonPackageConfig(String type, boolean oss, boolean jdk) { from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" fileMode 0644 } + into('/usr/share/elasticsearch/bin') { + from "${packagingFiles}/systemd/systemd-entrypoint" + fileMode 0755 + } // ========= sysV init ========= configurationFile '/etc/init.d/elasticsearch' diff --git a/distribution/packages/src/common/scripts/posttrans b/distribution/packages/src/common/scripts/posttrans index fdb9aafba38f6..c5614aa0a3c71 100644 --- a/distribution/packages/src/common/scripts/posttrans +++ b/distribution/packages/src/common/scripts/posttrans @@ -4,7 +4,12 @@ if [ ! -f /etc/elasticsearch/elasticsearch.keystore ]; then chmod 660 /etc/elasticsearch/elasticsearch.keystore md5sum /etc/elasticsearch/elasticsearch.keystore > /etc/elasticsearch/.elasticsearch.keystore.initial_md5sum else - /usr/share/elasticsearch/bin/elasticsearch-keystore upgrade + if /usr/share/elasticsearch/bin/elasticsearch-keystore has-passwd --silent ; then + echo "### Warning: unable to upgrade encrypted keystore" 1>&2 + echo " Please run elasticsearch-keystore upgrade and enter password" 1>&2 + else + /usr/share/elasticsearch/bin/elasticsearch-keystore upgrade + fi fi ${scripts.footer} diff --git a/distribution/packages/src/common/systemd/elasticsearch.service b/distribution/packages/src/common/systemd/elasticsearch.service index ed32b0708adff..acdc77ca99408 100644 --- a/distribution/packages/src/common/systemd/elasticsearch.service +++ b/distribution/packages/src/common/systemd/elasticsearch.service @@ -19,7 +19,7 @@ WorkingDirectory=/usr/share/elasticsearch User=elasticsearch Group=elasticsearch -ExecStart=/usr/share/elasticsearch/bin/elasticsearch -p ${PID_DIR}/elasticsearch.pid --quiet +ExecStart=/usr/share/elasticsearch/bin/systemd-entrypoint -p ${PID_DIR}/elasticsearch.pid --quiet # StandardOutput is configured to redirect to journalctl since # some error messages may be logged in standard output before diff --git a/distribution/packages/src/common/systemd/systemd-entrypoint b/distribution/packages/src/common/systemd/systemd-entrypoint new file mode 100644 index 0000000000000..e3c3f1eab00a1 --- /dev/null +++ b/distribution/packages/src/common/systemd/systemd-entrypoint @@ -0,0 +1,10 @@ +#!/bin/sh + +# This wrapper script allows SystemD to feed a file containing a passphrase into +# the main Elasticsearch startup script + +if [ -n "$ES_KEYSTORE_PASSPHRASE_FILE" ] ; then + exec /usr/share/elasticsearch/bin/elasticsearch "$@" < "$ES_KEYSTORE_PASSPHRASE_FILE" +else + exec /usr/share/elasticsearch/bin/elasticsearch "$@" +fi diff --git a/distribution/src/bin/elasticsearch b/distribution/src/bin/elasticsearch index 53329cc6bad41..8d460a7a7bbfc 100755 --- a/distribution/src/bin/elasticsearch +++ b/distribution/src/bin/elasticsearch @@ -20,6 +20,19 @@ if [ -z "$ES_TMPDIR" ]; then ES_TMPDIR=`"$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.TempDirectory` fi +# get keystore password before setting java options to avoid +# conflicting GC configurations for the keystore tools +unset KEYSTORE_PASSWORD +KEYSTORE_PASSWORD= +if ! echo $* | grep -E -q '(^-h |-h$| -h |--help$|--help |^-V |-V$| -V |--version$|--version )' \ + && "`dirname "$0"`"/elasticsearch-keystore has-passwd --silent +then + if ! read -s -r -p "Elasticsearch keystore password: " KEYSTORE_PASSWORD ; then + echo "Failed to read keystore password on console" 1>&2 + exit 1 + fi +fi + ES_JVM_OPTIONS="$ES_PATH_CONF"/jvm.options ES_JAVA_OPTS=`export ES_TMPDIR; "$JAVA" -cp "$ES_CLASSPATH" org.elasticsearch.tools.launchers.JvmOptionsParser "$ES_JVM_OPTIONS"` @@ -35,7 +48,7 @@ if ! echo $* | grep -E '(^-d |-d$| -d |--daemonize$|--daemonize )' > /dev/null; -Des.bundled_jdk="$ES_BUNDLED_JDK" \ -cp "$ES_CLASSPATH" \ org.elasticsearch.bootstrap.Elasticsearch \ - "$@" + "$@" <<<"$KEYSTORE_PASSWORD" else exec \ "$JAVA" \ @@ -48,7 +61,7 @@ else -cp "$ES_CLASSPATH" \ org.elasticsearch.bootstrap.Elasticsearch \ "$@" \ - <&- & + <<<"$KEYSTORE_PASSWORD" & retval=$? pid=$! [ $retval -eq 0 ] || exit $retval diff --git a/distribution/src/bin/elasticsearch-cli.bat b/distribution/src/bin/elasticsearch-cli.bat index 80b488c66e98c..866e8efc6689b 100644 --- a/distribution/src/bin/elasticsearch-cli.bat +++ b/distribution/src/bin/elasticsearch-cli.bat @@ -25,5 +25,5 @@ set ES_JAVA_OPTS=-Xms4m -Xmx64m -XX:+UseSerialGC %ES_JAVA_OPTS% -cp "%ES_CLASSPATH%" ^ "%ES_MAIN_CLASS%" ^ %* - + exit /b %ERRORLEVEL% diff --git a/distribution/src/bin/elasticsearch.bat b/distribution/src/bin/elasticsearch.bat index 9460554f81f41..48a34fdd332db 100644 --- a/distribution/src/bin/elasticsearch.bat +++ b/distribution/src/bin/elasticsearch.bat @@ -4,6 +4,7 @@ setlocal enabledelayedexpansion setlocal enableextensions SET params='%*' +SET checkpassword=Y :loop FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO ( @@ -18,6 +19,20 @@ FOR /F "usebackq tokens=1* delims= " %%A IN (!params!) DO ( SET silent=Y ) + IF "!current!" == "-h" ( + SET checkpassword=N + ) + IF "!current!" == "--help" ( + SET checkpassword=N + ) + + IF "!current!" == "-V" ( + SET checkpassword=N + ) + IF "!current!" == "--version" ( + SET checkpassword=N + ) + IF "!silent!" == "Y" ( SET nopauseonerror=Y ) ELSE ( @@ -41,6 +56,18 @@ IF ERRORLEVEL 1 ( EXIT /B %ERRORLEVEL% ) +SET KEYSTORE_PASSWORD= +IF "%checkpassword%"=="Y" ( + CALL "%~dp0elasticsearch-keystore.bat" has-passwd --silent + IF !ERRORLEVEL! EQU 0 ( + SET /P KEYSTORE_PASSWORD=Elasticsearch keystore password: + IF !ERRORLEVEL! NEQ 0 ( + ECHO Failed to read keystore password on standard input + EXIT /B !ERRORLEVEL! + ) + ) +) + if not defined ES_TMPDIR ( for /f "tokens=* usebackq" %%a in (`CALL %JAVA% -cp "!ES_CLASSPATH!" "org.elasticsearch.tools.launchers.TempDirectory"`) do set ES_TMPDIR=%%a ) @@ -54,7 +81,20 @@ if "%MAYBE_JVM_OPTIONS_PARSER_FAILED%" == "jvm_options_parser_failed" ( exit /b 1 ) -%JAVA% %ES_JAVA_OPTS% -Delasticsearch -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" -Des.bundled_jdk="%ES_BUNDLED_JDK%" -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams! +rem windows batch pipe will choke on special characters in strings +SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^^=^^^^! +SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^&=^^^&! +SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^|=^^^|! +SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^<=^^^=^^^>! +SET KEYSTORE_PASSWORD=!KEYSTORE_PASSWORD:^\=^^^\! + +ECHO.!KEYSTORE_PASSWORD!| %JAVA% %ES_JAVA_OPTS% -Delasticsearch ^ + -Des.path.home="%ES_HOME%" -Des.path.conf="%ES_PATH_CONF%" ^ + -Des.distribution.flavor="%ES_DISTRIBUTION_FLAVOR%" ^ + -Des.distribution.type="%ES_DISTRIBUTION_TYPE%" ^ + -Des.bundled_jdk="%ES_BUNDLED_JDK%" ^ + -cp "%ES_CLASSPATH%" "org.elasticsearch.bootstrap.Elasticsearch" !newparams! endlocal endlocal diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommand.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommand.java new file mode 100644 index 0000000000000..d3cc85299710e --- /dev/null +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommand.java @@ -0,0 +1,57 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import joptsimple.OptionSet; +import org.elasticsearch.cli.KeyStoreAwareCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.env.Environment; + +import java.nio.file.Path; + +public class HasPasswordKeyStoreCommand extends KeyStoreAwareCommand { + + static final int NO_PASSWORD_EXIT_CODE = 1; + + HasPasswordKeyStoreCommand() { + super("Succeeds if the keystore exists and is password-protected, " + + "fails with exit code " + NO_PASSWORD_EXIT_CODE + " otherwise."); + } + + @Override + protected void execute(Terminal terminal, OptionSet options, Environment env) throws Exception { + final Path configFile = env.configFile(); + final KeyStoreWrapper keyStore = KeyStoreWrapper.load(configFile); + + // We handle error printing here so we can respect the "--silent" flag + // We have to throw an exception to get a nonzero exit code + if (keyStore == null) { + terminal.errorPrintln(Terminal.Verbosity.NORMAL, "ERROR: Elasticsearch keystore not found"); + throw new UserException(NO_PASSWORD_EXIT_CODE, null); + } + if (keyStore.hasPassword() == false) { + terminal.errorPrintln(Terminal.Verbosity.NORMAL, "ERROR: Keystore is not password-protected"); + throw new UserException(NO_PASSWORD_EXIT_CODE, null); + } + + terminal.println(Terminal.Verbosity.NORMAL, "Keystore is password-protected"); + } +} diff --git a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java index 151c13ad68ebe..f08c83432de3f 100644 --- a/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java +++ b/distribution/tools/keystore-cli/src/main/java/org/elasticsearch/common/settings/KeyStoreCli.java @@ -36,6 +36,7 @@ private KeyStoreCli() { subcommands.put("remove", new RemoveSettingKeyStoreCommand()); subcommands.put("upgrade", new UpgradeKeyStoreCommand()); subcommands.put("passwd", new ChangeKeyStorePasswordCommand()); + subcommands.put("has-passwd", new HasPasswordKeyStoreCommand()); } public static void main(String[] args) throws Exception { diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java index 6b336fdf2b78c..df2056e8c6b18 100644 --- a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/bootstrap/BootstrapTests.java @@ -29,17 +29,24 @@ import org.junit.After; import org.junit.Before; +import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import static org.hamcrest.Matchers.equalTo; + public class BootstrapTests extends ESTestCase { Environment env; List fileSystems = new ArrayList<>(); + private static final int MAX_PASSPHRASE_LENGTH = 10; + @After public void closeMockFileSystems() throws IOException { IOUtils.close(fileSystems); @@ -66,4 +73,43 @@ public void testLoadSecureSettings() throws Exception { assertTrue(Files.exists(configPath.resolve("elasticsearch.keystore"))); } } + + public void testReadCharsFromStdin() throws Exception { + assertPassphraseRead("hello", "hello"); + assertPassphraseRead("hello\n", "hello"); + assertPassphraseRead("hello\r\n", "hello"); + + assertPassphraseRead("hellohello", "hellohello"); + assertPassphraseRead("hellohello\n", "hellohello"); + assertPassphraseRead("hellohello\r\n", "hellohello"); + + assertPassphraseRead("hello\nhi\n", "hello"); + assertPassphraseRead("hello\r\nhi\r\n", "hello"); + } + + public void testPassphraseTooLong() throws Exception { + byte[] source = "hellohello!\n".getBytes(StandardCharsets.UTF_8); + try (InputStream stream = new ByteArrayInputStream(source)) { + expectThrows(RuntimeException.class, "Password exceeded maximum length of 10", + () -> Bootstrap.readPassphrase(stream, MAX_PASSPHRASE_LENGTH)); + } + } + + public void testNoPassPhraseProvided() throws Exception { + byte[] source = "\r\n".getBytes(StandardCharsets.UTF_8); + try (InputStream stream = new ByteArrayInputStream(source)) { + expectThrows(RuntimeException.class, "Keystore passphrase required but none provided.", + () -> Bootstrap.readPassphrase(stream, MAX_PASSPHRASE_LENGTH)); + } + } + + private void assertPassphraseRead(String source, String expected) { + try (InputStream stream = new ByteArrayInputStream(source.getBytes(StandardCharsets.UTF_8))) { + SecureString result = Bootstrap.readPassphrase(stream, MAX_PASSPHRASE_LENGTH); + assertThat(result, equalTo(expected)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + } diff --git a/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommandTests.java b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommandTests.java new file mode 100644 index 0000000000000..32f5feae225a6 --- /dev/null +++ b/distribution/tools/keystore-cli/src/test/java/org/elasticsearch/common/settings/HasPasswordKeyStoreCommandTests.java @@ -0,0 +1,68 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.settings; + +import org.elasticsearch.cli.Command; +import org.elasticsearch.cli.UserException; +import org.elasticsearch.env.Environment; + +import java.util.Map; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.nullValue; + +public class HasPasswordKeyStoreCommandTests extends KeyStoreCommandTestCase { + @Override + protected Command newCommand() { + return new HasPasswordKeyStoreCommand() { + @Override + protected Environment createEnv(Map settings) throws UserException { + return env; + } + }; + } + + public void testFailsWithNoKeystore() throws Exception { + UserException e = expectThrows(UserException.class, this::execute); + assertEquals("Exit code should be 1", HasPasswordKeyStoreCommand.NO_PASSWORD_EXIT_CODE, e.exitCode); + assertThat("Exception should have null message", e.getMessage(), is(nullValue())); + } + + public void testFailsWhenKeystoreLacksPassword() throws Exception { + createKeystore(""); + UserException e = expectThrows(UserException.class, this::execute); + assertEquals("Exit code should be 1", HasPasswordKeyStoreCommand.NO_PASSWORD_EXIT_CODE, e.exitCode); + assertThat("Exception should have null message", e.getMessage(), is(nullValue())); + } + + public void testSucceedsWhenKeystoreHasPassword() throws Exception { + createKeystore("password"); + String output = execute(); + assertThat(output, containsString("Keystore is password-protected")); + } + + public void testSilentSucceedsWhenKeystoreHasPassword() throws Exception { + createKeystore("password"); + String output = execute("--silent"); + assertThat(output, is(emptyString())); + } +} diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java index 2a270153f474c..7d09f722141af 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Command.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Command.java @@ -97,7 +97,9 @@ public final int main(String[] args, Terminal terminal) throws Exception { if (e.exitCode == ExitCodes.USAGE) { printHelp(terminal, true); } - terminal.errorPrintln(Terminal.Verbosity.SILENT, "ERROR: " + e.getMessage()); + if (e.getMessage() != null) { + terminal.errorPrintln(Terminal.Verbosity.SILENT, "ERROR: " + e.getMessage()); + } return e.exitCode; } return ExitCodes.OK; diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java index 718b4796c0209..70f42c6f9f8db 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/Terminal.java @@ -24,7 +24,9 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintWriter; +import java.io.Reader; import java.nio.charset.Charset; +import java.util.Arrays; import java.util.Locale; /** @@ -78,6 +80,16 @@ public void setVerbosity(Verbosity verbosity) { /** Reads password text from the terminal input. See {@link Console#readPassword()}}. */ public abstract char[] readSecret(String prompt); + /** Read password text form terminal input up to a maximum length. */ + public char[] readSecret(String prompt, int maxLength) { + char[] result = readSecret(prompt); + if (result.length > maxLength) { + Arrays.fill(result, '\0'); + throw new IllegalStateException("Secret exceeded maximum length of " + maxLength); + } + return result; + } + /** Returns a Writer which can be used to write to the terminal directly using standard output. */ public abstract PrintWriter getWriter(); @@ -151,6 +163,45 @@ public final boolean promptYesNo(String prompt, boolean defaultYes) { } } + /** + * Read from the reader until we find a newline. If that newline + * character is immediately preceded by a carriage return, we have + * a Windows-style newline, so we discard the carriage return as well + * as the newline. + */ + public static char[] readLineToCharArray(Reader reader, int maxLength) { + char[] buf = new char[maxLength + 2]; + try { + int len = 0; + int next; + while ((next = reader.read()) != -1) { + char nextChar = (char) next; + if (nextChar == '\n') { + break; + } + if (len < buf.length) { + buf[len] = nextChar; + } + len++; + } + + if (len > 0 && len < buf.length && buf[len-1] == '\r') { + len--; + } + + if (len > maxLength) { + Arrays.fill(buf, '\0'); + throw new RuntimeException("Input exceeded maximum length of " + maxLength); + } + + char[] shortResult = Arrays.copyOf(buf, len); + Arrays.fill(buf, '\0'); + return shortResult; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private static class ConsoleTerminal extends Terminal { private static final Console CONSOLE = System.console(); @@ -179,10 +230,13 @@ public char[] readSecret(String prompt) { } } - private static class SystemTerminal extends Terminal { + /** visible for testing */ + static class SystemTerminal extends Terminal { private static final PrintWriter WRITER = newWriter(); + private BufferedReader reader; + SystemTerminal() { super(System.lineSeparator()); } @@ -192,6 +246,14 @@ private static PrintWriter newWriter() { return new PrintWriter(System.out); } + /** visible for testing */ + BufferedReader getReader() { + if (reader == null) { + reader = new BufferedReader(new InputStreamReader(System.in, Charset.defaultCharset())); + } + return reader; + } + @Override public PrintWriter getWriter() { return WRITER; @@ -200,9 +262,8 @@ public PrintWriter getWriter() { @Override public String readText(String text) { getErrorWriter().print(text); // prompts should go to standard error to avoid mixing with list output - BufferedReader reader = new BufferedReader(new InputStreamReader(System.in, Charset.defaultCharset())); try { - final String line = reader.readLine(); + final String line = getReader().readLine(); if (line == null) { throw new IllegalStateException("unable to read from standard input; is standard input open and a tty attached?"); } @@ -216,5 +277,11 @@ public String readText(String text) { public char[] readSecret(String text) { return readText(text).toCharArray(); } + + @Override + public char[] readSecret(String text, int maxLength) { + getErrorWriter().println(text); + return readLineToCharArray(getReader(), maxLength); + } } } diff --git a/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java b/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java index 4749b1b87b7aa..fd6ec7807a5d7 100644 --- a/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java +++ b/libs/cli/src/main/java/org/elasticsearch/cli/UserException.java @@ -19,6 +19,8 @@ package org.elasticsearch.cli; +import org.elasticsearch.common.Nullable; + /** * An exception representing a user fixable problem in {@link Command} usage. */ @@ -27,20 +29,26 @@ public class UserException extends Exception { /** The exist status the cli should use when catching this user error. */ public final int exitCode; - /** Constructs a UserException with an exit status and message to show the user. */ - public UserException(int exitCode, String msg) { + /** + * Constructs a UserException with an exit status and message to show the user. + *

+ * To suppress cli output on error, supply a null message. + */ + public UserException(int exitCode, @Nullable String msg) { super(msg); this.exitCode = exitCode; } /** * Constructs a new user exception with specified exit status, message, and underlying cause. + *

+ * To suppress cli output on error, supply a null message. * * @param exitCode the exit code * @param msg the message * @param cause the underlying cause */ - public UserException(final int exitCode, final String msg, final Throwable cause) { + public UserException(final int exitCode, @Nullable final String msg, final Throwable cause) { super(msg, cause); this.exitCode = exitCode; } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java index 1fcbd2d79e92f..fbe515e71f185 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java @@ -34,12 +34,8 @@ import java.nio.file.Paths; import java.util.stream.Stream; -import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; -import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; -import static org.elasticsearch.packaging.util.FileMatcher.file; -import static org.elasticsearch.packaging.util.FileMatcher.p660; import static org.elasticsearch.packaging.util.FileUtils.append; import static org.elasticsearch.packaging.util.FileUtils.cp; import static org.elasticsearch.packaging.util.FileUtils.getTempDir; @@ -106,43 +102,13 @@ public void test31BadJavaHome() throws Exception { } - public void test40CreateKeystoreManually() throws Exception { - final Installation.Executables bin = installation.executables(); - - Platforms.onLinux(() -> sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " create")); - - // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. - // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. - // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. - // when we run these commands as a role user we won't have to do this - Platforms.onWindows(() -> { - sh.run(bin.elasticsearchKeystore + " create"); - sh.chown(installation.config("elasticsearch.keystore")); - }); - - assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); - - Platforms.onLinux(() -> { - final Result r = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); - assertThat(r.stdout, containsString("keystore.seed")); - }); - - Platforms.onWindows(() -> { - final Result r = sh.run(bin.elasticsearchKeystore + " list"); - assertThat(r.stdout, containsString("keystore.seed")); - }); - } - public void test50StartAndStop() throws Exception { - // cleanup from previous test - rm(installation.config("elasticsearch.keystore")); - - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); assertTrue("gc logs exist", Files.exists(installation.logs.resolve("gc.log"))); ServerUtils.runElasticsearchTests(); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); } public void test51JavaHomeOverride() throws Exception { @@ -155,9 +121,9 @@ public void test51JavaHomeOverride() throws Exception { sh.getEnv().put("JAVA_HOME", systemJavaHome1); }); - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); ServerUtils.runElasticsearchTests(); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); String systemJavaHome1 = sh.getEnv().get("JAVA_HOME"); assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), @@ -179,9 +145,9 @@ public void test52BundledJdkRemoved() throws Exception { sh.getEnv().put("JAVA_HOME", systemJavaHome1); }); - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); ServerUtils.runElasticsearchTests(); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); String systemJavaHome1 = sh.getEnv().get("JAVA_HOME"); assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), @@ -202,9 +168,9 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception { sh.getEnv().put("JAVA_HOME", "C:\\Program Files (x86)\\java"); //verify ES can start, stop and run plugin list - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); String pluginListCommand = installation.bin + "/elasticsearch-plugin list"; Result result = sh.run(pluginListCommand); @@ -228,9 +194,9 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception { sh.getEnv().put("JAVA_HOME", testJavaHome); //verify ES can start, stop and run plugin list - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); String pluginListCommand = installation.bin + "/elasticsearch-plugin list"; Result result = sh.run(pluginListCommand); @@ -241,22 +207,6 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception { }); } - public void test60AutoCreateKeystore() throws Exception { - sh.chown(installation.config("elasticsearch.keystore")); - assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); - - final Installation.Executables bin = installation.executables(); - Platforms.onLinux(() -> { - final Result result = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); - assertThat(result.stdout, containsString("keystore.seed")); - }); - - Platforms.onWindows(() -> { - final Result result = sh.run(bin.elasticsearchKeystore + " list"); - assertThat(result.stdout, containsString("keystore.seed")); - }); - } - public void test70CustomPathConfAndJvmOptions() throws Exception { final Path tempConf = getTempDir().resolve("esconf-alternate"); @@ -281,13 +231,13 @@ public void test70CustomPathConfAndJvmOptions() throws Exception { sh.getEnv().put("ES_PATH_CONF", tempConf.toString()); sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops"); - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); } finally { rm(tempConf); @@ -314,12 +264,12 @@ public void test80RelativePathConf() throws Exception { sh.setWorkingDirectory(temp); sh.getEnv().put("ES_PATH_CONF", "config"); - Archives.runElasticsearch(installation, sh); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); assertThat(nodesResponse, containsString("\"name\":\"relative\"")); - Archives.stopElasticsearch(installation); + Archives.stopElasticsearch(installation, sh); } finally { rm(tempConf); @@ -384,8 +334,8 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex sh.setWorkingDirectory(getTempDir()); - Archives.runElasticsearch(installation, sh); - Archives.stopElasticsearch(installation); + awaitElasticsearchStartup(Archives.startElasticsearch(installation, sh)); + Archives.stopElasticsearch(installation, sh); Result result = sh.run("echo y | " + installation.executables().elasticsearchNode + " unsafe-bootstrap"); assertThat(result.stdout, containsString("Master node was successfully bootstrapped")); diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java new file mode 100644 index 0000000000000..c822c3922c9eb --- /dev/null +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/KeystoreManagementTests.java @@ -0,0 +1,313 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.test; + +import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.FileUtils; +import org.elasticsearch.packaging.util.Installation; +import org.elasticsearch.packaging.util.Platforms; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; +import static org.elasticsearch.packaging.util.Archives.installArchive; +import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; +import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; +import static org.elasticsearch.packaging.util.FileMatcher.file; +import static org.elasticsearch.packaging.util.FileMatcher.p660; +import static org.elasticsearch.packaging.util.FileUtils.rm; +import static org.elasticsearch.packaging.util.Packages.assertInstalled; +import static org.elasticsearch.packaging.util.Packages.assertRemoved; +import static org.elasticsearch.packaging.util.Packages.installPackage; +import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.junit.Assume.assumeThat; +import static org.junit.Assume.assumeTrue; + +public class KeystoreManagementTests extends PackagingTestCase { + + private static final String PASSWORD_ERROR_MESSAGE = "Provided keystore password was incorrect"; + + /** We need an initially installed package */ + public void test10InstallArchiveDistribution() throws Exception { + assumeTrue(distribution().isArchive()); + + installation = installArchive(distribution); + verifyArchiveInstallation(installation, distribution()); + + final Installation.Executables bin = installation.executables(); + Shell.Result r = sh.runIgnoreExitCode(bin.elasticsearchKeystore + " has-passwd"); + assertThat("has-passwd should fail", r.exitCode, not(is(0))); + assertThat("has-passwd should fail", r.stderr, containsString("ERROR: Elasticsearch keystore not found")); + } + + /** We need an initially installed package */ + public void test11InstallPackageDistribution() throws Exception { + assumeTrue(distribution().isPackage()); + + assertRemoved(distribution); + installation = installPackage(distribution); + assertInstalled(distribution); + verifyPackageInstallation(installation, distribution, sh); + + final Installation.Executables bin = installation.executables(); + Shell.Result r = sh.runIgnoreExitCode(bin.elasticsearchKeystore + " has-passwd"); + assertThat("has-passwd should fail", r.exitCode, not(is(0))); + assertThat("has-passwd should fail", r.stderr, containsString("ERROR: Keystore is not password-protected")); + } + + public void test20CreateKeystoreManually() throws Exception { + rmKeystoreIfExists(); + createKeystore(); + + final Installation.Executables bin = installation.executables(); + verifyKeystorePermissions(); + + String possibleSudo = distribution().isArchive() && Platforms.LINUX + ? "sudo -u " + ARCHIVE_OWNER + " " + : ""; + Shell.Result r = sh.run(possibleSudo + bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + } + + public void test30AutoCreateKeystore() throws Exception { + assumeTrue("RPMs and Debs install a keystore file", distribution.isArchive()); + rmKeystoreIfExists(); + + awaitElasticsearchStartup(startElasticsearch()); + stopElasticsearch(); + + Platforms.onWindows(() -> sh.chown(installation.config("elasticsearch.keystore"))); + + verifyKeystorePermissions(); + + final Installation.Executables bin = installation.executables(); + String possibleSudo = distribution().isArchive() && Platforms.LINUX + ? "sudo -u " + ARCHIVE_OWNER + " " + : ""; + Shell.Result r = sh.run(possibleSudo + bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + } + + public void test40KeystorePasswordOnStandardInput() throws Exception { + assumeTrue("packages will use systemd, which doesn't handle stdin", + distribution.isArchive()); + assumeThat(installation, is(notNullValue())); + + String password = "^|<>\\&exit"; // code insertion on Windows if special characters are not escaped + + rmKeystoreIfExists(); + createKeystore(); + setKeystorePassword(password); + + assertPasswordProtectedKeystore(); + + awaitElasticsearchStartup(startElasticsearchStandardInputPassword(password)); + ServerUtils.runElasticsearchTests(); + stopElasticsearch(); + } + + public void test41WrongKeystorePasswordOnStandardInput() { + assumeTrue("packages will use systemd, which doesn't handle stdin", + distribution.isArchive()); + assumeThat(installation, is(notNullValue())); + + assertPasswordProtectedKeystore(); + + Shell.Result result = startElasticsearchStandardInputPassword("wrong"); + assertElasticsearchFailure(result, PASSWORD_ERROR_MESSAGE); + } + + public void test42KeystorePasswordOnTty() throws Exception { + assumeTrue("expect command isn't on Windows", + distribution.platform != Distribution.Platform.WINDOWS); + assumeTrue("packages will use systemd, which doesn't handle stdin", + distribution.isArchive()); + assumeThat(installation, is(notNullValue())); + + String password = "keystorepass"; + + rmKeystoreIfExists(); + createKeystore(); + setKeystorePassword(password); + + assertPasswordProtectedKeystore(); + + awaitElasticsearchStartup(startElasticsearchTtyPassword(password)); + ServerUtils.runElasticsearchTests(); + stopElasticsearch(); + } + + public void test43WrongKeystorePasswordOnTty() throws Exception { + assumeTrue("expect command isn't on Windows", + distribution.platform != Distribution.Platform.WINDOWS); + assumeTrue("packages will use systemd, which doesn't handle stdin", + distribution.isArchive()); + assumeThat(installation, is(notNullValue())); + + assertPasswordProtectedKeystore(); + + Shell.Result result = startElasticsearchTtyPassword("wrong"); + // error will be on stdout for "expect" + assertThat(result.stdout, containsString(PASSWORD_ERROR_MESSAGE)); + } + + public void test50KeystorePasswordFromFile() throws Exception { + assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage()); + String password = "!@#$%^&*()|\\<>/?"; + Path esKeystorePassphraseFile = installation.config.resolve("eks"); + + rmKeystoreIfExists(); + createKeystore(); + setKeystorePassword(password); + + assertPasswordProtectedKeystore(); + + sh.getEnv().put("ES_KEYSTORE_PASSPHRASE_FILE", esKeystorePassphraseFile.toString()); + distribution().packagingConditional() + .forPackage( + () -> sh.run("sudo systemctl set-environment ES_KEYSTORE_PASSPHRASE_FILE=$ES_KEYSTORE_PASSPHRASE_FILE") + ) + .forArchive(Platforms.NO_ACTION) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + + Files.createFile(esKeystorePassphraseFile); + Files.write(esKeystorePassphraseFile, + (password + System.lineSeparator()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.WRITE); + + awaitElasticsearchStartup(startElasticsearch()); + ServerUtils.runElasticsearchTests(); + stopElasticsearch(); + + distribution().packagingConditional() + .forPackage( + () -> sh.run("sudo systemctl unset-environment ES_KEYSTORE_PASSPHRASE_FILE") + ) + .forArchive(Platforms.NO_ACTION) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + } + + public void test51WrongKeystorePasswordFromFile() throws Exception { + assumeTrue("only for systemd", Platforms.isSystemd() && distribution().isPackage()); + Path esKeystorePassphraseFile = installation.config.resolve("eks"); + + assertPasswordProtectedKeystore(); + + sh.getEnv().put("ES_KEYSTORE_PASSPHRASE_FILE", esKeystorePassphraseFile.toString()); + distribution().packagingConditional() + .forPackage( + () -> sh.run("sudo systemctl set-environment ES_KEYSTORE_PASSPHRASE_FILE=$ES_KEYSTORE_PASSPHRASE_FILE") + ) + .forArchive(Platforms.NO_ACTION) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + + if (Files.exists(esKeystorePassphraseFile)) { + rm(esKeystorePassphraseFile); + } + + Files.createFile(esKeystorePassphraseFile); + Files.write(esKeystorePassphraseFile, + ("wrongpassword" + System.lineSeparator()).getBytes(StandardCharsets.UTF_8), + StandardOpenOption.WRITE); + + Shell.Result result = startElasticsearch(); + assertElasticsearchFailure(result, PASSWORD_ERROR_MESSAGE); + + distribution().packagingConditional() + .forPackage( + () -> sh.run("sudo systemctl unset-environment ES_KEYSTORE_PASSPHRASE_FILE") + ) + .forArchive(Platforms.NO_ACTION) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + } + + private void createKeystore() throws Exception { + Path keystore = installation.config("elasticsearch.keystore"); + final Installation.Executables bin = installation.executables(); + Platforms.onLinux(() -> { + distribution().packagingConditional() + .forPackage(() -> sh.run(bin.elasticsearchKeystore + " create")) + .forArchive(() -> sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " create")) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + }); + + // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. + // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. + // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. + // when we run these commands as a role user we won't have to do this + Platforms.onWindows(() -> { + sh.run(bin.elasticsearchKeystore + " create"); + sh.chown(keystore); + }); + } + + private void rmKeystoreIfExists() { + Path keystore = installation.config("elasticsearch.keystore"); + if (Files.exists(keystore)) { + FileUtils.rm(keystore); + } + } + + private void setKeystorePassword(String password) throws Exception { + final Installation.Executables bin = installation.executables(); + + // set the password by passing it to stdin twice + Platforms.onLinux(() -> distribution().packagingConditional() + .forPackage(() -> sh.run("( echo \'" + password + "\' ; echo \'" + password + "\' ) | " + + bin.elasticsearchKeystore + " passwd")) + .forArchive(() -> sh.run("( echo \'" + password + "\' ; echo \'" + password + "\' ) | " + + "sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " passwd")) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run() + ); + Platforms.onWindows(() -> { + sh.run("Invoke-Command -ScriptBlock {echo \'" + password + "\'; echo \'" + password + "\'} | " + + bin.elasticsearchKeystore + " passwd"); + }); + } + + private void assertPasswordProtectedKeystore() { + Shell.Result r = sh.runIgnoreExitCode(installation.executables().elasticsearchKeystore.toString() + " has-passwd"); + assertThat("keystore should be password protected", r.exitCode, is(0)); + } + + private void verifyKeystorePermissions() throws Exception { + Path keystore = installation.config("elasticsearch.keystore"); + distribution().packagingConditional() + .forPackage(() -> assertThat(keystore, file(File, "root", "elasticsearch", p660))) + .forArchive(() -> assertThat(keystore, file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660))) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + } +} diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java index 386603dcce538..b0f46b7eea97e 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java @@ -50,8 +50,6 @@ import static org.elasticsearch.packaging.util.Packages.installPackage; import static org.elasticsearch.packaging.util.Packages.remove; import static org.elasticsearch.packaging.util.Packages.restartElasticsearch; -import static org.elasticsearch.packaging.util.Packages.startElasticsearch; -import static org.elasticsearch.packaging.util.Packages.stopElasticsearch; import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation; import static org.elasticsearch.packaging.util.Platforms.getOsRelease; import static org.elasticsearch.packaging.util.Platforms.isSystemd; @@ -100,14 +98,14 @@ public void assertRunsWithJavaHome() throws Exception { try { Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8), StandardOpenOption.APPEND); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); runElasticsearchTests(); - stopElasticsearch(sh); + stopElasticsearch(); } finally { Files.write(installation.envFile, originalEnvFile); } - assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"), + assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "elasticsearch*.log.gz"), containsString(systemJavaHome)); } @@ -128,9 +126,9 @@ public void test33RunsIfJavaNotOnPath() throws Exception { } try { - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); runElasticsearchTests(); - stopElasticsearch(sh); + stopElasticsearch(); } finally { if (Files.exists(Paths.get(backupPath))) { sh.run("sudo mv " + backupPath + " /usr/bin/java"); @@ -152,7 +150,7 @@ public void test42BundledJdkRemoved() throws Exception { public void test40StartServer() throws Exception { String start = sh.runIgnoreExitCode("date ").stdout.trim(); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); String journalEntries = sh.runIgnoreExitCode("journalctl _SYSTEMD_UNIT=elasticsearch.service " + "--since \"" + start + "\" --output cat | wc -l").stdout.trim(); @@ -163,6 +161,7 @@ public void test40StartServer() throws Exception { runElasticsearchTests(); verifyPackageInstallation(installation, distribution(), sh); // check startup script didn't change permissions + stopElasticsearch(); } public void test50Remove() throws Exception { @@ -230,10 +229,10 @@ public void test70RestartServer() throws Exception { installation = installPackage(distribution()); assertInstalled(distribution()); - startElasticsearch(sh, installation); - restartElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); + awaitElasticsearchStartup(restartElasticsearch(sh)); runElasticsearchTests(); - stopElasticsearch(sh); + stopElasticsearch(); } finally { cleanup(); } @@ -244,9 +243,9 @@ public void test72TestRuntimeDirectory() throws Exception { try { installation = installPackage(distribution()); FileUtils.rm(installation.pidDir); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); assertPathsExist(installation.pidDir); - stopElasticsearch(sh); + stopElasticsearch(); } finally { cleanup(); } @@ -254,10 +253,10 @@ public void test72TestRuntimeDirectory() throws Exception { public void test73gcLogsExist() throws Exception { installation = installPackage(distribution()); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); // it can be gc.log or gc.log.0.current assertThat(installation.logs, fileWithGlobExist("gc.log*")); - stopElasticsearch(sh); + stopElasticsearch(); } // TEST CASES FOR SYSTEMD ONLY @@ -276,13 +275,13 @@ public void test80DeletePID_DIRandRestart() throws Exception { sh.run("systemd-tmpfiles --create"); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); final Path pidFile = installation.pidDir.resolve("elasticsearch.pid"); assertTrue(Files.exists(pidFile)); - stopElasticsearch(sh); + stopElasticsearch(); } public void test81CustomPathConfAndJvmOptions() throws Exception { @@ -290,7 +289,7 @@ public void test81CustomPathConfAndJvmOptions() throws Exception { assertPathsExist(installation.envFile); - stopElasticsearch(sh); + stopElasticsearch(); // The custom config directory is not under /tmp or /var/tmp because // systemd's private temp directory functionally means different @@ -319,13 +318,13 @@ public void test81CustomPathConfAndJvmOptions() throws Exception { append(installation.envFile, "ES_PATH_CONF=" + tempConf + "\n"); append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops"); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); assertThat(nodesResponse, CoreMatchers.containsString("\"heap_init_in_bytes\":536870912")); assertThat(nodesResponse, CoreMatchers.containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); - stopElasticsearch(sh); + stopElasticsearch(); } finally { rm(installation.envFile); @@ -355,7 +354,7 @@ public void test83serviceFileSetsLimits() throws Exception { installation = installPackage(distribution()); - startElasticsearch(sh, installation); + awaitElasticsearchStartup(startElasticsearch()); final Path pidFile = installation.pidDir.resolve("elasticsearch.pid"); assertTrue(Files.exists(pidFile)); @@ -372,6 +371,6 @@ public void test83serviceFileSetsLimits() throws Exception { String maxAddressSpace = sh.run("cat /proc/%s/limits | grep \"Max address space\" | awk '{ print $4 }'", pid).stdout.trim(); assertThat(maxAddressSpace, equalTo("unlimited")); - stopElasticsearch(sh); + stopElasticsearch(); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java index 3efd1b36ddbdd..3bc00af0b2b6f 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/PackagingTestCase.java @@ -26,10 +26,15 @@ import com.carrotsearch.randomizedtesting.annotations.Timeout; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.packaging.util.Archives; import org.elasticsearch.packaging.util.Distribution; +import org.elasticsearch.packaging.util.Docker; +import org.elasticsearch.packaging.util.FileUtils; import org.elasticsearch.packaging.util.Installation; +import org.elasticsearch.packaging.util.Packages; import org.elasticsearch.packaging.util.Platforms; import org.elasticsearch.packaging.util.Shell; +import org.junit.After; import org.junit.Assert; import org.junit.Before; import org.junit.BeforeClass; @@ -40,9 +45,13 @@ import org.junit.runner.Description; import org.junit.runner.RunWith; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import static org.elasticsearch.packaging.util.Cleanup.cleanEverything; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.equalTo; import static org.junit.Assume.assumeFalse; import static org.junit.Assume.assumeTrue; @@ -114,6 +123,24 @@ public void setup() throws Exception { sh = newShell(); } + @After + public void teardown() throws Exception { + // move log file so we can avoid false positives when grepping for + // messages in logs during test + if (installation != null && Files.exists(installation.logs)) { + Path logFile = installation.logs.resolve("elasticsearch.log"); + String prefix = this.getClass().getSimpleName() + "." + testNameRule.getMethodName(); + if (Files.exists(logFile)) { + Path newFile = installation.logs.resolve(prefix + ".elasticsearch.log"); + FileUtils.mv(logFile, newFile); + } + for (Path rotatedLogFile : FileUtils.lsGlob(installation.logs, "elasticsearch*.tar.gz")) { + Path newRotatedLogFile = installation.logs.resolve(prefix + "." + rotatedLogFile.getFileName()); + FileUtils.mv(rotatedLogFile, newRotatedLogFile); + } + } + } + /** The {@link Distribution} that should be tested in this case */ protected static Distribution distribution() { return distribution; @@ -131,4 +158,76 @@ protected Shell newShell() throws Exception { } return sh; } + + public Shell.Result startElasticsearch() throws Exception { + if (distribution.isPackage()) { + return Packages.startElasticsearch(sh); + } else { + assertTrue(distribution.isArchive()); + return Archives.startElasticsearch(installation, sh); + } + } + + public Shell.Result startElasticsearchStandardInputPassword(String password) { + assertTrue("Only archives support passwords on standard input", distribution().isArchive()); + return Archives.startElasticsearch(installation, sh, password); + } + + public Shell.Result startElasticsearchTtyPassword(String password) throws Exception { + assertTrue("Only archives support passwords on TTY", distribution().isArchive()); + return Archives.startElasticsearchWithTty(installation, sh, password); + } + + public void stopElasticsearch() throws Exception { + distribution().packagingConditional() + .forPackage(() -> Packages.stopElasticsearch(sh)) + .forArchive(() -> Archives.stopElasticsearch(installation, sh)) + .forDocker(/* TODO */ Platforms.NO_ACTION) + .run(); + } + + public void awaitElasticsearchStartup(Shell.Result result) throws Exception { + assertThat("Startup command should succeed", result.exitCode, equalTo(0)); + distribution().packagingConditional() + .forPackage(() -> Packages.assertElasticsearchStarted(sh, installation)) + .forArchive(() -> Archives.assertElasticsearchStarted(installation, sh)) + .forDocker(Docker::waitForElasticsearchToStart) + .run(); + } + + public void assertElasticsearchFailure(Shell.Result result, String expectedMessage) { + + if (Files.exists(installation.logs.resolve("elasticsearch.log"))) { + + // If log file exists, then we have bootstrapped our logging and the + // error should be in the logs + assertTrue("log file exists", Files.exists(installation.logs.resolve("elasticsearch.log"))); + String logfile = FileUtils.slurp(installation.logs.resolve("elasticsearch.log")); + assertThat(logfile, containsString(expectedMessage)); + + } else if (distribution().isPackage() && Platforms.isSystemd()) { + + // For systemd, retrieve the error from journalctl + assertThat(result.stderr, containsString("Job for elasticsearch.service failed")); + Shell.Result error = sh.run("journalctl --boot --unit elasticsearch.service"); + assertThat(error.stdout, containsString(expectedMessage)); + + } else if (Platforms.WINDOWS == true) { + + // In Windows, we have written our stdout and stderr to files in order to run + // in the background + String wrapperPid = result.stdout.trim(); + sh.runIgnoreExitCode("Wait-Process -Timeout 15 -Id " + wrapperPid); + sh.runIgnoreExitCode("Get-EventSubscriber | " + + "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |" + + "Unregister-EventSubscriber -Force"); + assertThat(FileUtils.slurp(Archives.getPowershellErrorPath(installation)), containsString(expectedMessage)); + + } else { + + // Otherwise, error should be on shell stderr + assertThat(result.stderr, containsString(expectedMessage)); + + } + } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java index 56995bab1de0c..62a41baa0fa99 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Archives.java @@ -43,7 +43,7 @@ import static org.elasticsearch.packaging.util.FileUtils.slurp; import static org.elasticsearch.packaging.util.Platforms.isDPKG; import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.isEmptyOrNullString; +import static org.hamcrest.Matchers.emptyOrNullString; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; import static org.hamcrest.collection.IsEmptyCollection.empty; import static org.hamcrest.core.Is.is; @@ -63,6 +63,10 @@ public class Archives { ? System.getenv("username") : "elasticsearch"; + /** This is an arbitrarily chosen value that gives Elasticsearch time to log Bootstrap + * errors to the console if they occur before the logging framework is initialized. */ + private static final String ES_STARTUP_SLEEP_TIME_SECONDS = "10"; + public static Installation installArchive(Distribution distribution) throws Exception { return installArchive(distribution, getDefaultArchiveInstallPath(), getCurrentVersion()); } @@ -242,89 +246,130 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist ).forEach(configFile -> assertThat(es.config(configFile), file(File, owner, owner, p660))); } - public static void runElasticsearch(Installation installation, Shell sh) throws Exception { + public static Shell.Result startElasticsearch(Installation installation, Shell sh) { + return startElasticsearch(installation, sh, ""); + } + + public static Shell.Result startElasticsearchWithTty(Installation installation, Shell sh, String keystorePassword) throws Exception { + final Path pidFile = installation.home.resolve("elasticsearch.pid"); + final Installation.Executables bin = installation.executables(); + + // requires the "expect" utility to be installed + String script = "expect -c \"$(cat< { + if (Platforms.WINDOWS == false) { // If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try. // If it doesn't ignore it then Elasticsearch will fail to start because of security errors. // This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf if (Files.exists(Paths.get("/usr/share/java/jayatanaag.jar"))) { sh.getEnv().put("JAVA_TOOL_OPTIONS", "-javaagent:/usr/share/java/jayatanaag.jar"); } - sh.run("sudo -E -u " + ARCHIVE_OWNER + " " + - bin.elasticsearch + " -d -p " + installation.home.resolve("elasticsearch.pid")); - }); - Platforms.onWindows(() -> { - // this starts the server in the background. the -d flag is unsupported on windows - if (System.getenv("username").equals("vagrant")) { - // these tests run as Administrator in vagrant. - // we don't want to run the server as Administrator, so we provide the current user's - // username and password to the process which has the effect of starting it not as Administrator. - sh.run( - "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " + - "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + - "$processInfo.FileName = '" + bin.elasticsearch + "'; " + - "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + - "$processInfo.Username = 'vagrant'; " + - "$processInfo.Password = $password; " + - "$processInfo.RedirectStandardOutput = $true; " + - "$processInfo.RedirectStandardError = $true; " + - sh.env.entrySet().stream() - .map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ") - .collect(joining()) + - "$processInfo.UseShellExecute = $false; " + - "$process = New-Object System.Diagnostics.Process; " + - "$process.StartInfo = $processInfo; " + - "$process.Start() | Out-Null; " + - "$process.Id;" - ); - } else { - sh.run( - "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + - "$processInfo.FileName = '" + bin.elasticsearch + "'; " + - "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + - "$processInfo.RedirectStandardOutput = $true; " + - "$processInfo.RedirectStandardError = $true; " + - sh.env.entrySet().stream() - .map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ") - .collect(joining()) + - "$processInfo.UseShellExecute = $false; " + - "$process = New-Object System.Diagnostics.Process; " + - "$process.StartInfo = $processInfo; " + - "$process.Start() | Out-Null; " + - "$process.Id;" - ); - } - }); + // We need to give Elasticsearch enough time to print failures to stderr before exiting + sh.getEnv().put("ES_STARTUP_SLEEP_TIME", ES_STARTUP_SLEEP_TIME_SECONDS); + return sh.runIgnoreExitCode("sudo -E -u " + ARCHIVE_OWNER + " " + bin.elasticsearch + " -d -p " + pidFile + + " <<<'" + keystorePassword + "'"); + } + final Path stdout = getPowershellOutputPath(installation); + final Path stderr = getPowershellErrorPath(installation); + + String powerShellProcessUserSetup; + if (System.getenv("username").equals("vagrant")) { + // the tests will run as Administrator in vagrant. + // we don't want to run the server as Administrator, so we provide the current user's + // username and password to the process which has the effect of starting it not as Administrator. + powerShellProcessUserSetup = + "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " + + "$processInfo.Username = 'vagrant'; " + + "$processInfo.Password = $password; "; + } else { + powerShellProcessUserSetup = ""; + } + + // this starts the server in the background. the -d flag is unsupported on windows + return sh.run( + "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + + "$processInfo.FileName = '" + bin.elasticsearch + "'; " + + "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + + powerShellProcessUserSetup + + "$processInfo.RedirectStandardOutput = $true; " + + "$processInfo.RedirectStandardError = $true; " + + "$processInfo.RedirectStandardInput = $true; " + + sh.env.entrySet().stream() + .map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ") + .collect(joining()) + + "$processInfo.UseShellExecute = $false; " + + "$process = New-Object System.Diagnostics.Process; " + + "$process.StartInfo = $processInfo; " + + + // set up some asynchronous output handlers + "$outScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '" + stdout + "' }; " + + "$errScript = { $EventArgs.Data | Out-File -Encoding UTF8 -Append '" + stderr + "' }; " + + "$stdOutEvent = Register-ObjectEvent -InputObject $process " + + "-Action $outScript -EventName 'OutputDataReceived'; " + + "$stdErrEvent = Register-ObjectEvent -InputObject $process " + + "-Action $errScript -EventName 'ErrorDataReceived'; " + + + "$process.Start() | Out-Null; " + + "$process.BeginOutputReadLine(); " + + "$process.BeginErrorReadLine(); " + + "$process.StandardInput.WriteLine('" + keystorePassword + "'); " + + "Wait-Process -Timeout " + ES_STARTUP_SLEEP_TIME_SECONDS + " -Id $process.Id; " + + "$process.Id;" + ); + } + public static void assertElasticsearchStarted(Installation installation, Shell sh) throws Exception { + final Path pidFile = installation.home.resolve("elasticsearch.pid"); ServerUtils.waitForElasticsearch(installation); assertTrue("Starting Elasticsearch produced a pid file at " + pidFile, Files.exists(pidFile)); String pid = slurp(pidFile).trim(); - assertThat(pid, not(isEmptyOrNullString())); - - Platforms.onLinux(() -> sh.run("ps " + pid)); - Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid)); + assertThat(pid, is(not(emptyOrNullString()))); } - public static void stopElasticsearch(Installation installation) throws Exception { + public static void stopElasticsearch(Installation installation, Shell sh) throws Exception { Path pidFile = installation.home.resolve("elasticsearch.pid"); - assertTrue(Files.exists(pidFile)); + assertTrue("pid file should exist", Files.exists(pidFile)); String pid = slurp(pidFile).trim(); - assertThat(pid, not(isEmptyOrNullString())); + assertThat(pid, is(not(emptyOrNullString()))); - final Shell sh = new Shell(); Platforms.onLinux(() -> sh.run("kill -SIGTERM " + pid + "; tail --pid=" + pid + " -f /dev/null")); - Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid + " | Stop-Process -Force; Wait-Process -Id " + pid)); + Platforms.onWindows(() -> { + sh.run("Get-Process -Id " + pid + " | Stop-Process -Force; Wait-Process -Id " + pid); + + // Clear the asynchronous event handlers + sh.runIgnoreExitCode("Get-EventSubscriber | " + + "where {($_.EventName -eq 'OutputDataReceived' -Or $_.EventName -eq 'ErrorDataReceived' |" + + "Unregister-EventSubscriber -Force"); + }); if (Files.exists(pidFile)) { Files.delete(pidFile); } } + public static Path getPowershellErrorPath(Installation installation) { + return installation.logs.resolve("output.err"); + } + + private static Path getPowershellOutputPath(Installation installation) { + return installation.logs.resolve("output.out"); + } + } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java index f9b98d58ccacc..c62dc7427df50 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Cleanup.java @@ -35,6 +35,7 @@ public class Cleanup { private static final List ELASTICSEARCH_FILES_LINUX = Arrays.asList( "/usr/share/elasticsearch", + "/etc/elasticsearch/elasticsearch-keystore", "/etc/elasticsearch", "/var/lib/elasticsearch", "/var/log/elasticsearch", diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java index 13b2f31c7e4fd..e9d7052cd6755 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Distribution.java @@ -20,7 +20,11 @@ package org.elasticsearch.packaging.util; import java.nio.file.Path; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; import java.util.Locale; +import java.util.Map; public class Distribution { @@ -84,6 +88,46 @@ public enum Packaging { } } + public static class PackagingConditional { + Distribution distribution; + + public PackagingConditional(Distribution distribution) { + this.distribution = distribution; + } + + private final Map conditions = new HashMap<>(); + + public PackagingConditional forArchive(Platforms.PlatformAction action) { + conditions.put(Packaging.TAR, action); + conditions.put(Packaging.ZIP, action); + return this; + } + + public PackagingConditional forPackage(Platforms.PlatformAction action) { + conditions.put(Packaging.RPM, action); + conditions.put(Packaging.DEB, action); + return this; + } + + public PackagingConditional forDocker(Platforms.PlatformAction action) { + conditions.put(Packaging.DOCKER, action); + return this; + } + + public void run() throws Exception { + HashSet missingPackaging = new HashSet<>(Arrays.asList(Packaging.values())); + missingPackaging.removeAll(conditions.keySet()); + if (missingPackaging.isEmpty() == false) { + throw new IllegalArgumentException("No condition specified for " + missingPackaging); + } + conditions.get(this.distribution.packaging).run(); + } + } + + public PackagingConditional packagingConditional() { + return new PackagingConditional(this); + } + public enum Platform { LINUX, WINDOWS, diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java index d78b60236bc4a..fc001ec7f0830 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Docker.java @@ -133,7 +133,7 @@ public static Installation runContainer(Distribution distribution, Path configPa * Waits for the Elasticsearch process to start executing in the container. * This is called every time a container is started. */ - private static void waitForElasticsearchToStart() throws InterruptedException { + public static void waitForElasticsearchToStart() throws InterruptedException { boolean isElasticsearchRunning = false; int attempt = 0; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java index 16824ba4ae413..af3f12a73fa86 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/FileUtils.java @@ -126,7 +126,7 @@ public static void append(Path file, String text) { public static String slurp(Path file) { try { - return String.join("\n", Files.readAllLines(file, StandardCharsets.UTF_8)); + return String.join(System.lineSeparator(), Files.readAllLines(file, StandardCharsets.UTF_8)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java index c9fca10b5c8cf..373cf85cd9533 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java @@ -267,17 +267,14 @@ private static void verifyDefaultInstallation(Installation es) { ).forEach(configFile -> assertThat(es.config(configFile), file(File, "root", "elasticsearch", p660))); } - public static void startElasticsearch(Shell sh, Installation installation) throws IOException { + public static Shell.Result startElasticsearch(Shell sh) throws IOException { if (isSystemd()) { sh.run("systemctl daemon-reload"); sh.run("systemctl enable elasticsearch.service"); sh.run("systemctl is-enabled elasticsearch.service"); - sh.run("systemctl start elasticsearch.service"); - } else { - sh.run("service elasticsearch start"); + return sh.runIgnoreExitCode("systemctl start elasticsearch.service"); } - - assertElasticsearchStarted(sh, installation); + return sh.runIgnoreExitCode("service elasticsearch start"); } public static void assertElasticsearchStarted(Shell sh, Installation installation) throws IOException { @@ -299,13 +296,11 @@ public static void stopElasticsearch(Shell sh) throws IOException { } } - public static void restartElasticsearch(Shell sh, Installation installation) throws IOException { + public static Shell.Result restartElasticsearch(Shell sh) throws IOException { if (isSystemd()) { - sh.run("systemctl restart elasticsearch.service"); + return sh.run("systemctl restart elasticsearch.service"); } else { - sh.run("service elasticsearch restart"); + return sh.run("service elasticsearch restart"); } - - waitForElasticsearch(installation); } } diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java index b0778bf460ee6..fa324690bf6cc 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/Platforms.java @@ -28,6 +28,7 @@ public class Platforms { public static final boolean LINUX = OS_NAME.startsWith("Linux"); public static final boolean WINDOWS = OS_NAME.startsWith("Windows"); public static final boolean DARWIN = OS_NAME.startsWith("Mac OS X"); + public static final PlatformAction NO_ACTION = () -> {}; public static String getOsRelease() { if (LINUX) { diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java index fd34c48fa8ebf..b1bd99898f890 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java @@ -26,6 +26,9 @@ import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.Configurator; import org.apache.lucene.util.Constants; +import org.elasticsearch.cli.KeyStoreAwareCommand; +import org.elasticsearch.cli.Terminal; +import org.elasticsearch.common.settings.SecureString; import org.elasticsearch.core.internal.io.IOUtils; import org.apache.lucene.util.StringHelper; import org.elasticsearch.ElasticsearchException; @@ -51,9 +54,12 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.io.PrintStream; import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.security.NoSuchAlgorithmException; import java.util.Collections; @@ -234,14 +240,25 @@ static SecureSettings loadSecureSettings(Environment initialEnv) throws Bootstra throw new BootstrapException(e); } + SecureString password; try { + if (keystore != null && keystore.hasPassword()) { + password = readPassphrase(System.in, KeyStoreAwareCommand.MAX_PASSPHRASE_LENGTH); + } else { + password = new SecureString(new char[0]); + } + } catch (IOException e) { + throw new BootstrapException(e); + } + + try (password) { if (keystore == null) { final KeyStoreWrapper keyStoreWrapper = KeyStoreWrapper.create(); keyStoreWrapper.save(initialEnv.configFile(), new char[0]); return keyStoreWrapper; } else { - keystore.decrypt(new char[0] /* TODO: read password from stdin */); - KeyStoreWrapper.upgrade(keystore, initialEnv.configFile(), new char[0]); + keystore.decrypt(password.getChars()); + KeyStoreWrapper.upgrade(keystore, initialEnv.configFile(), password.getChars()); } } catch (Exception e) { throw new BootstrapException(e); @@ -249,6 +266,31 @@ static SecureSettings loadSecureSettings(Environment initialEnv) throws Bootstra return keystore; } + // visible for tests + /** + * Read from an InputStream up to the first carriage return or newline, + * returning no more than maxLength characters. + */ + static SecureString readPassphrase(InputStream stream, int maxLength) throws IOException { + SecureString passphrase; + + try(InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8)) { + passphrase = new SecureString(Terminal.readLineToCharArray(reader, maxLength)); + } catch (RuntimeException e) { + if (e.getMessage().startsWith("Input exceeded maximum length")) { + throw new IllegalStateException("Password exceeded maximum length of " + maxLength, e); + } + throw e; + } + + if (passphrase.length() == 0) { + passphrase.close(); + throw new IllegalStateException("Keystore passphrase required but none provided."); + } + + return passphrase; + } + private static Environment createEnvironment( final Path pidFile, final SecureSettings secureSettings, diff --git a/server/src/main/java/org/elasticsearch/cli/KeyStoreAwareCommand.java b/server/src/main/java/org/elasticsearch/cli/KeyStoreAwareCommand.java index cd9740bcd112e..dd26366f9b071 100644 --- a/server/src/main/java/org/elasticsearch/cli/KeyStoreAwareCommand.java +++ b/server/src/main/java/org/elasticsearch/cli/KeyStoreAwareCommand.java @@ -38,6 +38,9 @@ public KeyStoreAwareCommand(String description) { super(description); } + /** Arbitrarily chosen maximum passphrase length */ + public static final int MAX_PASSPHRASE_LENGTH = 128; + /** * Reads the keystore password from the {@link Terminal}, prompting for verification where applicable and returns it as a * {@link SecureString}. @@ -50,8 +53,10 @@ public KeyStoreAwareCommand(String description) { protected static SecureString readPassword(Terminal terminal, boolean withVerification) throws UserException { final char[] passwordArray; if (withVerification) { - passwordArray = terminal.readSecret("Enter new password for the elasticsearch keystore (empty for no password): "); - char[] passwordVerification = terminal.readSecret("Enter same password again: "); + passwordArray = terminal.readSecret("Enter new password for the elasticsearch keystore (empty for no password): ", + MAX_PASSPHRASE_LENGTH); + char[] passwordVerification = terminal.readSecret("Enter same password again: ", + MAX_PASSPHRASE_LENGTH); if (Arrays.equals(passwordArray, passwordVerification) == false) { throw new UserException(ExitCodes.DATA_ERROR, "Passwords are not equal, exiting."); } diff --git a/server/src/test/java/org/elasticsearch/cli/MultiCommandTests.java b/server/src/test/java/org/elasticsearch/cli/MultiCommandTests.java index 41fe851ed2561..71280520058b6 100644 --- a/server/src/test/java/org/elasticsearch/cli/MultiCommandTests.java +++ b/server/src/test/java/org/elasticsearch/cli/MultiCommandTests.java @@ -25,6 +25,10 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicBoolean; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; + public class MultiCommandTests extends CommandTestCase { static class DummyMultiCommand extends MultiCommand { @@ -173,4 +177,55 @@ public void testCloseWhenSubCommandCloseThrowsException() throws Exception { assertTrue("SubCommand2 was not closed when close method is invoked", subCommand2.closeCalled.get()); } + // Tests for multicommand error logging + + static class ErrorHandlingMultiCommand extends MultiCommand { + ErrorHandlingMultiCommand() { + super("error catching", () -> {}); + } + + @Override + protected boolean addShutdownHook() { + return false; + } + } + + static class ErrorThrowingSubCommand extends Command { + ErrorThrowingSubCommand() { + super("error throwing", () -> {}); + } + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + throw new UserException(1, "Dummy error"); + } + + @Override + protected boolean addShutdownHook() { + return false; + } + } + + public void testErrorDisplayedWithDefault() throws Exception { + MockTerminal terminal = new MockTerminal(); + MultiCommand mc = new ErrorHandlingMultiCommand(); + mc.subcommands.put("throw", new ErrorThrowingSubCommand()); + mc.main(new String[]{"throw", "--silent"}, terminal); + assertThat(terminal.getOutput(), is(emptyString())); + assertThat(terminal.getErrorOutput(), equalTo("ERROR: Dummy error\n")); + } + + public void testNullErrorMessageSuppressesErrorOutput() throws Exception { + MockTerminal terminal = new MockTerminal(); + MultiCommand mc = new ErrorHandlingMultiCommand(); + mc.subcommands.put("throw", new ErrorThrowingSubCommand() { + @Override + protected void execute(Terminal terminal, OptionSet options) throws Exception { + throw new UserException(1, null); + } + }); + mc.main(new String[]{"throw", "--silent"}, terminal); + assertThat(terminal.getOutput(), is(emptyString())); + assertThat(terminal.getErrorOutput(), is(emptyString())); + } + } diff --git a/server/src/test/java/org/elasticsearch/cli/TerminalTests.java b/server/src/test/java/org/elasticsearch/cli/TerminalTests.java index 99bbe9d618441..85b8ec5bf2684 100644 --- a/server/src/test/java/org/elasticsearch/cli/TerminalTests.java +++ b/server/src/test/java/org/elasticsearch/cli/TerminalTests.java @@ -21,7 +21,14 @@ import org.elasticsearch.test.ESTestCase; +import java.io.BufferedReader; +import java.io.StringReader; + +import static org.elasticsearch.cli.Terminal.readLineToCharArray; +import static org.hamcrest.Matchers.equalTo; + public class TerminalTests extends ESTestCase { + public void testVerbosity() throws Exception { MockTerminal terminal = new MockTerminal(); terminal.setVerbosity(Terminal.Verbosity.SILENT); @@ -95,6 +102,22 @@ public void testPromptYesNoCase() throws Exception { assertFalse(terminal.promptYesNo("Answer?", true)); } + public void testMaxSecretLength() throws Exception { + MockTerminal terminal = new MockTerminal(); + String secret = "A very long secret, too long in fact for our purposes."; + terminal.addSecretInput(secret); + + expectThrows(IllegalStateException.class, "Secret exceeded maximum length of ", + () -> terminal.readSecret("Secret? ", secret.length() - 1)); + } + + public void testTerminalReusesBufferedReaders() throws Exception { + Terminal.SystemTerminal terminal = new Terminal.SystemTerminal(); + BufferedReader reader1 = terminal.getReader(); + BufferedReader reader2 = terminal.getReader(); + assertSame("System terminal should not create multiple buffered readers", reader1, reader2); + } + private void assertPrinted(MockTerminal logTerminal, Terminal.Verbosity verbosity, String text) throws Exception { logTerminal.println(verbosity, text); String output = logTerminal.getOutput(); @@ -121,4 +144,47 @@ private void assertErrorNotPrinted(MockTerminal logTerminal, Terminal.Verbosity assertTrue(output, output.isEmpty()); } + public void testSystemTerminalReadsSingleLines() throws Exception { + assertRead("\n", ""); + assertRead("\r\n", ""); + + assertRead("hello\n", "hello"); + assertRead("hello\r\n", "hello"); + + assertRead("hellohello\n", "hellohello"); + assertRead("hellohello\r\n", "hellohello"); + } + + public void testSystemTerminalReadsMultipleLines() throws Exception { + assertReadLines("hello\nhello\n", "hello", "hello"); + assertReadLines("hello\r\nhello\r\n", "hello", "hello"); + + assertReadLines("one\ntwo\n\nthree", "one", "two", "", "three"); + assertReadLines("one\r\ntwo\r\n\r\nthree", "one", "two", "", "three"); + } + + public void testSystemTerminalLineExceedsMaxCharacters() throws Exception { + try (StringReader reader = new StringReader("hellohellohello!\n")) { + expectThrows(RuntimeException.class, "Input exceeded maximum length of 10", + () -> readLineToCharArray(reader, 10)); + } + } + + private void assertRead(String source, String expected) { + try (StringReader reader = new StringReader(source)) { + char[] result = readLineToCharArray(reader, 10); + assertThat(result, equalTo(expected.toCharArray())); + } + } + + private void assertReadLines(String source, String... expected) { + try (StringReader reader = new StringReader(source)) { + char[] result; + for (String exp : expected) { + result = readLineToCharArray(reader, 10); + assertThat(result, equalTo(exp.toCharArray())); + } + } + } + } diff --git a/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java index cff5c1b49fbc7..4959e6436f487 100644 --- a/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java +++ b/test/framework/src/main/java/org/elasticsearch/cli/MockTerminal.java @@ -85,7 +85,7 @@ public void addTextInput(String input) { textInput.add(input); } - /** Adds an an input that will be return from {@link #readText(String)}. Values are read in FIFO order. */ + /** Adds an an input that will be return from {@link #readSecret(String)}. Values are read in FIFO order. */ public void addSecretInput(String input) { secretInput.add(input); }