Refactor utility code in qa:os: tests (#49945)

This refactor bridges some gaps between a long-running feature branch (#49268) and the master branch.

First of all, this PR gives our PackagingTestCase class some methods to start and stop Elasticsearch that will switch on packaging type and delegate to the appropriate utility class for deb/RPM packages, archive installations, and Docker. These methods should be very useful as we continue group tests by function rather than by package or platform type.

Second, the password-protected keystore tests have a particular need to read the output of Elasticsearch startup commands. In order to make this easer to do, some commands now return Shell.Result objects so that tests can check over output to the shell. To that end, there's also an assertElasticsearchFailure method that will handle checking for startup failures for the various distribution types.

There is an update to the Powershell startup script for archives that asynchronously redirects the output of the Powershell process to files that we can read for errors.

Finally, we use the ES_STARTUP_SLEEP_TIME environment variable to make sure that our startup commands wait long enough before exiting for errors to make it to the standard output and error streams.

Author: williamrandolph

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -138,7 +138,7 @@ public void test50StartAndStop() throws Exception {
         rm(installation.config("elasticsearch.keystore"));
 
         try {
-            Archives.runElasticsearch(installation, sh);
+            startElasticsearch();
         } catch (Exception e ){
             if (Files.exists(installation.home.resolve("elasticsearch.pid"))) {
                 String pid = FileUtils.slurp(installation.home.resolve("elasticsearch.pid")).trim();
@@ -151,7 +151,7 @@ public void test50StartAndStop() throws Exception {
         assertTrue("gc logs exist", Files.exists(installation.logs.resolve("gc.log")));
         ServerUtils.runElasticsearchTests();
 
-        Archives.stopElasticsearch(installation);
+        stopElasticsearch();
     }
 
     public void test51JavaHomeOverride() throws Exception {
@@ -164,9 +164,9 @@ public void test51JavaHomeOverride() throws Exception {
             sh.getEnv().put("JAVA_HOME", systemJavaHome1);
         });
 
-        Archives.runElasticsearch(installation, sh);
+        startElasticsearch();
         ServerUtils.runElasticsearchTests();
-        Archives.stopElasticsearch(installation);
+        stopElasticsearch();
 
         String systemJavaHome1 = sh.getEnv().get("JAVA_HOME");
         assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"),
@@ -188,9 +188,9 @@ public void test52BundledJdkRemoved() throws Exception {
                 sh.getEnv().put("JAVA_HOME", systemJavaHome1);
             });
 
-            Archives.runElasticsearch(installation, sh);
+            startElasticsearch();
             ServerUtils.runElasticsearchTests();
-            Archives.stopElasticsearch(installation);
+            stopElasticsearch();
 
             String systemJavaHome1 = sh.getEnv().get("JAVA_HOME");
             assertThat(FileUtils.slurpAllLogs(installation.logs, "elasticsearch.log", "*.log.gz"),
@@ -211,9 +211,9 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception {
                 sh.getEnv().put("JAVA_HOME", "C:\\Program Files (x86)\\java");
 
                 //verify ES can start, stop and run plugin list
-                Archives.runElasticsearch(installation, sh);
+                startElasticsearch();
 
-                Archives.stopElasticsearch(installation);
+                stopElasticsearch();
 
                 String pluginListCommand = installation.bin + "/elasticsearch-plugin list";
                 Result result = sh.run(pluginListCommand);
@@ -237,9 +237,9 @@ public void test53JavaHomeWithSpecialCharacters() throws Exception {
                 sh.getEnv().put("JAVA_HOME", testJavaHome);
 
                 //verify ES can start, stop and run plugin list
-                Archives.runElasticsearch(installation, sh);
+                startElasticsearch();
 
-                Archives.stopElasticsearch(installation);
+                stopElasticsearch();
 
                 String pluginListCommand = installation.bin + "/elasticsearch-plugin list";
                 Result result = sh.run(pluginListCommand);
@@ -284,13 +284,12 @@ public void test70CustomPathConfAndJvmOptions() throws Exception {
                 "-Dlog4j2.disable.jmx=true\n";
             append(tempConf.resolve("jvm.options"), jvmOptions);
 
-            final Shell sh = newShell();
             sh.chown(tempConf);
 
             sh.getEnv().put("ES_PATH_CONF", tempConf.toString());
             sh.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops");
 
-            Archives.runElasticsearch(installation, sh);
+            startElasticsearch();
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
@@ -318,17 +317,16 @@ public void test80RelativePathConf() throws Exception {
 
             append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");
 
-            final Shell sh = newShell();
             sh.chown(temp);
 
             sh.setWorkingDirectory(temp);
             sh.getEnv().put("ES_PATH_CONF", "config");
-            Archives.runElasticsearch(installation, sh);
+            startElasticsearch();
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, containsString("\"name\":\"relative\""));
 
-            Archives.stopElasticsearch(installation);
+            stopElasticsearch();
 
         } finally {
             rm(tempConf);
@@ -393,7 +391,7 @@ public void test93ElasticsearchNodeCustomDataPathAndNotEsHomeWorkDir() throws Ex
 
         sh.setWorkingDirectory(getTempDir());
 
-        Archives.runElasticsearch(installation, sh);
+        startElasticsearch();
         Archives.stopElasticsearch(installation);
 
         Result result = sh.run("echo y | " + installation.executables().elasticsearchNode + " unsafe-bootstrap");

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -50,9 +50,6 @@
 import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.remove;
 import static org.elasticsearch.packaging.util.Packages.restartElasticsearch;
-import static org.elasticsearch.packaging.util.Packages.startElasticsearch;
-import static org.elasticsearch.packaging.util.Packages.startElasticsearchIgnoringFailure;
-import static org.elasticsearch.packaging.util.Packages.stopElasticsearch;
 import static org.elasticsearch.packaging.util.Packages.verifyPackageInstallation;
 import static org.elasticsearch.packaging.util.Platforms.getOsRelease;
 import static org.elasticsearch.packaging.util.Platforms.isSystemd;
@@ -101,9 +98,9 @@ private void assertRunsWithJavaHome() throws Exception {
         try {
             Files.write(installation.envFile, ("JAVA_HOME=" + systemJavaHome + "\n").getBytes(StandardCharsets.UTF_8),
                 StandardOpenOption.APPEND);
-            startElasticsearch(sh, installation);
+            startElasticsearch();
             runElasticsearchTests();
-            stopElasticsearch(sh);
+            stopElasticsearch();
         } finally {
             Files.write(installation.envFile, originalEnvFile);
         }
@@ -129,9 +126,9 @@ public void test33RunsIfJavaNotOnPath() throws Exception {
         }
 
         try {
-            startElasticsearch(sh, installation);
+            startElasticsearch();
             runElasticsearchTests();
-            stopElasticsearch(sh);
+            stopElasticsearch();
         } finally {
             if (Files.exists(Paths.get(backupPath))) {
                 sh.run("sudo mv " + backupPath + " /usr/bin/java");
@@ -153,7 +150,7 @@ public void test42BundledJdkRemoved() throws Exception {
 
     public void test40StartServer() throws Exception {
         String start = sh.runIgnoreExitCode("date ").stdout.trim();
-        startElasticsearch(sh, installation);
+        startElasticsearch();
 
         String journalEntries = sh.runIgnoreExitCode("journalctl _SYSTEMD_UNIT=elasticsearch.service " +
             "--since \"" + start + "\" --output cat | wc -l").stdout.trim();
@@ -218,7 +215,7 @@ public void test50Remove() throws Exception {
     }
 
     public void test60Reinstall() throws Exception {
-        installation = installPackage(distribution());
+        install();
         assertInstalled(distribution());
         verifyPackageInstallation(installation, distribution(), sh);
 
@@ -228,13 +225,13 @@ public void test60Reinstall() throws Exception {
 
     public void test70RestartServer() throws Exception {
         try {
-            installation = installPackage(distribution());
+            install();
             assertInstalled(distribution());
 
-            startElasticsearch(sh, installation);
+            startElasticsearch();
             restartElasticsearch(sh, installation);
             runElasticsearchTests();
-            stopElasticsearch(sh);
+            stopElasticsearch();
         } finally {
             cleanup();
         }
@@ -243,22 +240,22 @@ public void test70RestartServer() throws Exception {
 
     public void test72TestRuntimeDirectory() throws Exception {
         try {
-            installation = installPackage(distribution());
+            install();
             FileUtils.rm(installation.pidDir);
-            startElasticsearch(sh, installation);
+            startElasticsearch();
             assertPathsExist(installation.pidDir);
-            stopElasticsearch(sh);
+            stopElasticsearch();
         } finally {
             cleanup();
         }
     }
 
     public void test73gcLogsExist() throws Exception {
-        installation = installPackage(distribution());
-        startElasticsearch(sh, installation);
+        install();
+        startElasticsearch();
         // it can be gc.log or gc.log.0.current
         assertThat(installation.logs, fileWithGlobExist("gc.log*"));
-        stopElasticsearch(sh);
+        stopElasticsearch();
     }
 
     // TEST CASES FOR SYSTEMD ONLY
@@ -277,26 +274,26 @@ public void test80DeletePID_DIRandRestart() throws Exception {
 
         sh.run("systemd-tmpfiles --create");
 
-        startElasticsearch(sh, installation);
+        startElasticsearch();
 
         final Path pidFile = installation.pidDir.resolve("elasticsearch.pid");
 
         assertTrue(Files.exists(pidFile));
 
-        stopElasticsearch(sh);
+        stopElasticsearch();
     }
 
     public void test81CustomPathConfAndJvmOptions() throws Exception {
         withCustomConfig(tempConf -> {
             append(installation.envFile, "ES_JAVA_OPTS=-XX:-UseCompressedOops");
 
-            startElasticsearch(sh, installation);
+            startElasticsearch();
 
             final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes"));
             assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912"));
             assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\""));
 
-            stopElasticsearch(sh);
+            stopElasticsearch();
         });
     }
 
@@ -306,7 +303,7 @@ public void test82SystemdMask() throws Exception {
 
             sh.run("systemctl mask systemd-sysctl.service");
 
-            installation = installPackage(distribution());
+            install();
 
             sh.run("systemctl unmask systemd-sysctl.service");
         } finally {
@@ -318,9 +315,9 @@ public void test83serviceFileSetsLimits() throws Exception {
         // Limits are changed on systemd platforms only
         assumeTrue(isSystemd());
 
-        installation = installPackage(distribution());
+        install();
 
-        startElasticsearch(sh, installation);
+        startElasticsearch();
 
         final Path pidFile = installation.pidDir.resolve("elasticsearch.pid");
         assertTrue(Files.exists(pidFile));
@@ -337,7 +334,7 @@ public void test83serviceFileSetsLimits() throws Exception {
         String maxAddressSpace = sh.run("cat /proc/%s/limits | grep \"Max address space\" | awk '{ print $4 }'", pid).stdout.trim();
         assertThat(maxAddressSpace, equalTo("unlimited"));
 
-        stopElasticsearch(sh);
+        stopElasticsearch();
     }
 
     public void test90DoNotCloseStderrWhenQuiet() throws Exception {
@@ -347,7 +344,7 @@ public void test90DoNotCloseStderrWhenQuiet() throws Exception {
 
             // Make sure we don't pick up the journal entries for previous ES instances.
             clearJournal(sh);
-            startElasticsearchIgnoringFailure(sh);
+            runElasticsearchStartCommand();
 
             final Result logs = sh.run("journalctl -u elasticsearch.service");
 
@@ -365,7 +362,7 @@ private void withCustomConfig(CustomConfigConsumer pathConsumer) throws Exceptio
 
         assertPathsExist(installation.envFile);
 
-        stopElasticsearch(sh);
+        stopElasticsearch();
 
         // The custom config directory is not under /tmp or /var/tmp because
         // systemd's private temp directory functionally means different