Die with dignity on the Lucene layer

When a fatal error tragically closes an index writer, such an error
never makes its way to the uncaught exception handler. This prevents the
node from being torn down if an out of memory error or other fatal error
is thrown in the Lucene layer. This commit ensures that such events
bubble their way up to the uncaught exception handler.

Relates #21721

Author: jasontedor

core/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -960,15 +960,21 @@ public IndexCommit acquireIndexCommit(final boolean flushFirst) throws EngineExc
         }
     }
 
+    @SuppressWarnings("finally")
     private void failOnTragicEvent(AlreadyClosedException ex) {
         // if we are already closed due to some tragic exception
         // we need to fail the engine. it might have already been failed before
         // but we are double-checking it's failed and closed
         if (indexWriter.isOpen() == false && indexWriter.getTragicException() != null) {
-            final Exception tragedy = indexWriter.getTragicException() instanceof Exception ?
-                (Exception) indexWriter.getTragicException() :
-                new Exception(indexWriter.getTragicException());
-            failEngine("already closed by tragic event on the index writer", tragedy);
+            if (indexWriter.getTragicException() instanceof Error) {
+                try {
+                    logger.error("tragic event in index writer", ex);
+                } finally {
+                    throw (Error) indexWriter.getTragicException();
+                }
+            } else {
+                failEngine("already closed by tragic event on the index writer", (Exception) indexWriter.getTragicException());
+            }
         } else if (translog.isOpen() == false && translog.getTragicException() != null) {
             failEngine("already closed by tragic event on the translog", translog.getTragicException());
         } else if (failedEngine.get() == null) { // we are closed but the engine is not failed yet?

core/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java

@@ -26,6 +26,8 @@
 import org.apache.logging.log4j.core.LogEvent;
 import org.apache.logging.log4j.core.appender.AbstractAppender;
 import org.apache.logging.log4j.core.filter.RegexFilter;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Tokenizer;
 import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.document.Field;
@@ -112,6 +114,7 @@
 import org.junit.After;
 import org.junit.Before;
 
+import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.charset.Charset;
@@ -208,8 +211,12 @@ public void setUp() throws Exception {
     }
 
     public EngineConfig copy(EngineConfig config, EngineConfig.OpenMode openMode) {
+        return copy(config, openMode, config.getAnalyzer());
+    }
+
+    public EngineConfig copy(EngineConfig config, EngineConfig.OpenMode openMode, Analyzer analyzer) {
         return new EngineConfig(openMode, config.getShardId(), config.getThreadPool(), config.getIndexSettings(), config.getWarmer(),
-            config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), config.getAnalyzer(), config.getSimilarity(),
+            config.getStore(), config.getDeletionPolicy(), config.getMergePolicy(), analyzer, config.getSimilarity(),
             new CodecService(null, logger), config.getEventListener(), config.getTranslogRecoveryPerformer(), config.getQueryCache(),
             config.getQueryCachingPolicy(), config.getTranslogConfig(), config.getFlushMergesAfter(), config.getRefreshListeners(),
             config.getMaxUnsafeAutoIdTimestamp());
@@ -2521,4 +2528,40 @@ public void afterRefresh(boolean didRefresh) throws IOException {
             assertTrue(internalEngine.failedEngine.get() instanceof MockDirectoryWrapper.FakeIOException);
         }
     }
+
+    public void testTragicEventErrorBubblesUp() throws IOException {
+        engine.close();
+        final AtomicBoolean failWithFatalError = new AtomicBoolean(true);
+        final VirtualMachineError error = randomFrom(
+            new InternalError(),
+            new OutOfMemoryError(),
+            new StackOverflowError(),
+            new UnknownError());
+        engine = new InternalEngine(copy(engine.config(), EngineConfig.OpenMode.OPEN_INDEX_AND_TRANSLOG, new Analyzer() {
+            @Override
+            protected TokenStreamComponents createComponents(String fieldName) {
+                return new TokenStreamComponents(new Tokenizer() {
+                    @Override
+                    public boolean incrementToken() throws IOException {
+                        if (failWithFatalError.get()) {
+                            throw error;
+                        } else {
+                            throw new AssertionError("should not get to this point");
+                        }
+                    }
+                });
+            }
+        }));
+        final Document document = testDocument();
+        document.add(new TextField("value", "test", Field.Store.YES));
+        final ParsedDocument firstDoc = testParsedDocument("1", "1", "test", null, -1, -1, document, B_1, null);
+        final Engine.Index first = new Engine.Index(newUid("1"), firstDoc);
+        expectThrows(error.getClass(), () -> engine.index(first));
+        failWithFatalError.set(false);
+        final ParsedDocument secondDoc = testParsedDocument("2", "2", "test", null, -1, -1, document, B_1, null);
+        final Engine.Index second = new Engine.Index(newUid("2"), secondDoc);
+        expectThrows(error.getClass(), () -> engine.index(second));
+        assertNull(engine.failedEngine.get());
+    }
+
 }