Add test for dying with dignity

qa/die-with-dignity/build.gradle

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    description 'Out of memory plugin'
+    classname 'org.elasticsearch.DieWithDignityPlugin'
+}
+
+integTestRunner {
+    systemProperty 'tests.security.manager', 'false'
+    systemProperty 'tests.system_call_filter', 'false'
+    systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}"
+    systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}.log"
+    systemProperty 'runtime.java.home', "${project.runtimeJavaHome}"
+}
+
+test.enabled = false
+
+check.dependsOn integTest

qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class DieWithDignityPlugin extends Plugin implements ActionPlugin {
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+            final Settings settings,
+            final RestController restController,
+            final ClusterSettings clusterSettings,
+            final IndexScopedSettings indexScopedSettings,
+            final SettingsFilter settingsFilter,
+            final IndexNameExpressionResolver indexNameExpressionResolver,
+            final Supplier<DiscoveryNodes> nodesInCluster) {
+        return Collections.singletonList(new RestDieWithDignityAction(settings, restController));
+    }
+
+}

qa/die-with-dignity/src/main/java/org/elasticsearch/RestDieWithDignityAction.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+public class RestDieWithDignityAction extends BaseRestHandler {
+
+    RestDieWithDignityAction(final Settings settings, final RestController restController) {
+        super(settings);
+        restController.registerHandler(RestRequest.Method.GET, "/_die_with_dignity", this);
+    }
+
+    @Override
+    public String getName() {
+        return "die_with_dignity_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        throw new OutOfMemoryError("die with dignity");
+    }
+
+}

qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.qa.die_with_dignity;
+
+import org.apache.http.ConnectionClosedException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
+
+public class DieWithDignityIT extends ESRestTestCase {
+
+    public void testDieWithDignity() throws Exception {
+        // deleting the PID file prevents stopping the cluster from failing since it occurs if and only if the PID file exists
+        final Path pidFile = PathUtils.get(System.getProperty("pidfile"));
+        final List<String> pidFileLines = Files.readAllLines(pidFile);
+        assertThat(pidFileLines, hasSize(1));
+        final int pid = Integer.parseInt(pidFileLines.get(0));
+        Files.delete(pidFile);
+        expectThrows(ConnectionClosedException.class, () -> client().performRequest("GET", "/_die_with_dignity"));
+
+        // the Elasticsearch process should die and disappear from the output of jps
+        assertBusy(() -> {
+            final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
+            final Process process = new ProcessBuilder().command(jpsPath).start();
+            assertThat(process.waitFor(), equalTo(0));
+            try (InputStream is = process.getInputStream();
+                 BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) {
+                String line;
+                while ((line = in.readLine()) != null) {
+                    final int currentPid = Integer.parseInt(line.split("\\s+")[0]);
+                    assertThat(line, pid, not(equalTo(currentPid)));
+                }
+            }
+        });
+
+        // parse the logs and ensure that Elasticsearch died with the expected cause
+        final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
+
+        final Iterator<String> it = lines.iterator();
+
+        boolean fatalErrorOnTheNetworkLayer = false;
+        boolean fatalErrorInThreadExiting = false;
+
+        while (it.hasNext() && (fatalErrorOnTheNetworkLayer == false || fatalErrorInThreadExiting == false)) {
+            final String line = it.next();
+            if (line.contains("fatal error on the network layer")) {
+                fatalErrorOnTheNetworkLayer = true;
+            } else if (line.matches(".*\\[ERROR\\]\\[o.e.b.ElasticsearchUncaughtExceptionHandler\\] \\[node-0\\]"
+                    + " fatal error in thread \\[Thread-\\d+\\], exiting$")) {
+                fatalErrorInThreadExiting = true;
+                assertTrue(it.hasNext());
+                assertThat(it.next(), equalTo("java.lang.OutOfMemoryError: die with dignity"));
+            }
+        }
+
+        assertTrue(fatalErrorOnTheNetworkLayer);
+        assertTrue(fatalErrorInThreadExiting);
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        // as the cluster is dead its state can not be wiped successfully so we have to bypass wiping the cluster
+        return true;
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -78,7 +78,8 @@ public class BootstrapForTesting {
         }
 
         // just like bootstrap, initialize natives, then SM
-        Bootstrap.initializeNatives(javaTmpDir, true, true, true);
+        final boolean systemCallFilter = Booleans.parseBoolean(System.getProperty("tests.system_call_filter", "true"));
+        Bootstrap.initializeNatives(javaTmpDir, true, systemCallFilter, true);
 
         // initialize probes
         Bootstrap.initializeProbes();

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -145,9 +145,11 @@ public void initClient() throws IOException {
      */
     @After
     public final void cleanUpCluster() throws Exception {
-        wipeCluster();
-        waitForClusterStateUpdatesToFinish();
-        logIfThereAreRunningTasks();
+        if (preserveClusterUponCompletion() == false) {
+            wipeCluster();
+            waitForClusterStateUpdatesToFinish();
+            logIfThereAreRunningTasks();
+        }
     }
 
     @AfterClass
@@ -175,6 +177,17 @@ protected static RestClient adminClient() {
         return adminClient;
     }
 
+    /**
+     * Returns whether to preserve the state of the cluster upon completion of this test. Defaults to false. If true, overrides the value of
+     * {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()}, and
+     * {@link #preserveSnapshotsUponCompletion()}.
+     *
+     * @return true if the state of the cluster should be preserved
+     */
+    protected boolean preserveClusterUponCompletion() {
+        return false;
+    }
+
     /**
      * Returns whether to preserve the indices created during this test on completion of this test.
      * Defaults to {@code false}. Override this method if indices should be preserved after the test,