diff --git a/distribution/src/bin/elasticsearch-env b/distribution/src/bin/elasticsearch-env
index 78cb503ecef7c..e13eb824e77e9 100644
--- a/distribution/src/bin/elasticsearch-env
+++ b/distribution/src/bin/elasticsearch-env
@@ -38,6 +38,7 @@ ES_CLASSPATH="$ES_HOME/lib/*"
 # now set the path to java
 if [ ! -z "$JAVA_HOME" ]; then
   JAVA="$JAVA_HOME/bin/java"
+  JAVA_TYPE="JAVA_HOME"
 else
   if [ "$(uname -s)" = "Darwin" ]; then
     # OSX has a different structure
@@ -45,12 +46,13 @@ else
   else
     JAVA="$ES_HOME/jdk/bin/java"
   fi
+  JAVA_TYPE="bundled jdk"
 fi
 
 if [ ! -x "$JAVA" ]; then
-  echo "could not find java in JAVA_HOME or bundled at $JAVA" >&2
-  exit 1
-fi
+    echo "could not find java in $JAVA_TYPE at $JAVA" >&2
+    exit 1
+  fi
 
 # do not let JAVA_TOOL_OPTIONS slip in (as the JVM does by default)
 if [ ! -z "$JAVA_TOOL_OPTIONS" ]; then
diff --git a/distribution/src/bin/elasticsearch-env.bat b/distribution/src/bin/elasticsearch-env.bat
index 8ac141986a4a7..c974792088181 100644
--- a/distribution/src/bin/elasticsearch-env.bat
+++ b/distribution/src/bin/elasticsearch-env.bat
@@ -38,13 +38,15 @@ if "%1" == "nojava" (
 
 if defined JAVA_HOME (
   set JAVA="%JAVA_HOME%\bin\java.exe"
+  set JAVA_TYPE=JAVA_HOME
 ) else (
   set JAVA="%ES_HOME%\jdk\bin\java.exe"
   set JAVA_HOME="%ES_HOME%\jdk"
+  set JAVA_TYPE=bundled jdk
 )
 
-if not exist %JAVA% (
-  echo "could not find java in JAVA_HOME or bundled at %ES_HOME%\jdk" >&2
+if not exist !JAVA! (
+  echo "could not find java in !JAVA_TYPE! at !JAVA!" >&2
   exit /b 1
 )
 
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
index 83b15be74f387..7530ea9bed334 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java
@@ -74,7 +74,7 @@ public void test20PluginsListWithNoPlugins() throws Exception {
         assertThat(r.stdout, isEmptyString());
     }
 
-    public void test30NoJava() throws Exception {
+    public void test30MissingBundledJdk() throws Exception {
         final Installation.Executables bin = installation.executables();
         sh.getEnv().remove("JAVA_HOME");
 
@@ -87,7 +87,7 @@ public void test30NoJava() throws Exception {
             // ask for elasticsearch version to quickly exit if java is actually found (ie test failure)
             final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -v");
             assertThat(runResult.exitCode, is(1));
-            assertThat(runResult.stderr, containsString("could not find java in JAVA_HOME or bundled"));
+            assertThat(runResult.stderr, containsString("could not find java in bundled jdk"));
         } finally {
             if (distribution().hasJdk) {
                 mv(relocatedJdk, installation.bundledJdk);
@@ -95,6 +95,17 @@ public void test30NoJava() throws Exception {
         }
     }
 
+    public void test31BadJavaHome() throws Exception {
+        final Installation.Executables bin = installation.executables();
+        sh.getEnv().put("JAVA_HOME", "doesnotexist");
+
+        // ask for elasticsearch version to quickly exit if java is actually found (ie test failure)
+        final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString() + " -v");
+        assertThat(runResult.exitCode, is(1));
+        assertThat(runResult.stderr, containsString("could not find java in JAVA_HOME"));
+
+    }
+
     public void test40CreateKeystoreManually() throws Exception {
         final Installation.Executables bin = installation.executables();
 
@@ -174,7 +185,7 @@ public void test52BundledJdkRemoved() throws Exception {
 
     public void test53JavaHomeWithSpecialCharacters() throws Exception {
         Platforms.onWindows(() -> {
-            final Shell sh = new Shell();
+            final Shell sh = newShell();
             try {
                 // once windows 2012 is no longer supported and powershell 5.0 is always available we can change this command
                 sh.run("cmd /c mklink /D 'C:\\Program Files (x86)\\java' $Env:SYSTEM_JAVA_HOME");
@@ -197,7 +208,7 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception {
         });
 
         Platforms.onLinux(() -> {
-            final Shell sh = new Shell();
+            final Shell sh = newShell();
             // Create temporary directory with a space and link to real java home
             String testJavaHome = Paths.get("/tmp", "java home").toString();
             try {
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java
index 77e83c9522857..9d32e4b2b0256 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/WindowsServiceTests.java
@@ -117,20 +117,27 @@ public void test12InstallService() {
         sh.run(serviceScript + " remove");
     }
 
-    public void test13InstallMissingJava() throws IOException {
+    public void test13InstallMissingBundledJdk() throws IOException {
         final Path relocatedJdk = installation.bundledJdk.getParent().resolve("jdk.relocated");
 
         try {
             mv(installation.bundledJdk, relocatedJdk);
             Result result = sh.runIgnoreExitCode(serviceScript + " install");
             assertThat(result.exitCode, equalTo(1));
-            assertThat(result.stderr, containsString("could not find java in JAVA_HOME or bundled"));
+            assertThat(result.stderr, containsString("could not find java in bundled jdk"));
         } finally {
             mv(relocatedJdk, installation.bundledJdk);
         }
     }
 
-    public void test14RemoveNotInstalled() {
+    public void test14InstallBadJavaHome() throws IOException {
+        sh.getEnv().put("JAVA_HOME", "doesnotexist");
+        Result result = sh.runIgnoreExitCode(serviceScript + " install");
+        assertThat(result.exitCode, equalTo(1));
+        assertThat(result.stderr, containsString("could not find java in JAVA_HOME"));
+    }
+
+    public void test15RemoveNotInstalled() {
         Result result = assertFailure(serviceScript + " remove", 1);
         assertThat(result.stdout, containsString("Failed removing '" + DEFAULT_ID + "' service"));
     }
diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java
index 6331b4bf46e9a..ff006a34e6892 100644
--- a/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java
+++ b/qa/os/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java
@@ -72,7 +72,7 @@ public static void waitForElasticsearch(String status, String index) throws IOEx
 
             } catch (HttpHostConnectException e) {
                 // we want to retry if the connection is refused
-                LOG.debug("Got connection refused when waiting for cluster health", e);
+                LOG.info("Got connection refused when waiting for cluster health", e);
             }
 
             timeElapsed = System.currentTimeMillis() - startTime;