Add max file size bootstrap check

This commit adds a bootstrap check for the maximum file size, and
ensures the limit is set correctly when Elasticsearch is installed as a
service on systemd-based systems.

Relates #25974

Author: jasontedor

core/src/main/java/org/elasticsearch/bootstrap/Bootstrap.java

@@ -147,6 +147,7 @@ public boolean handle(int code) {
 
         Natives.trySetMaxNumberOfThreads();
         Natives.trySetMaxSizeVirtualMemory();
+        Natives.trySetMaxFileSize();
 
         // init lucene random seed. it will use /dev/urandom where available:
         StringHelper.randomId();

core/src/main/java/org/elasticsearch/bootstrap/BootstrapChecks.java

@@ -193,6 +193,9 @@ static List<BootstrapCheck> checks(final Settings settings) {
         if (Constants.LINUX || Constants.MAC_OS_X) {
             checks.add(new MaxSizeVirtualMemoryCheck());
         }
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            checks.add(new MaxFileSizeCheck());
+        }
         if (Constants.LINUX) {
             checks.add(new MaxMapCountCheck());
         }
@@ -367,6 +370,36 @@ long getMaxSizeVirtualMemory() {
 
     }
 
+    /**
+     * Bootstrap check that the maximum file size is unlimited (otherwise Elasticsearch could run in to an I/O exception writing files).
+     */
+    static class MaxFileSizeCheck implements BootstrapCheck {
+
+        @Override
+        public boolean check() {
+            final long maxFileSize = getMaxFileSize();
+            return maxFileSize != Long.MIN_VALUE && maxFileSize != getRlimInfinity();
+        }
+
+        @Override
+        public String errorMessage() {
+            return String.format(
+                    Locale.ROOT,
+                    "max file size [%d] for user [%s] is too low, increase to [unlimited]",
+                    getMaxFileSize(),
+                    BootstrapInfo.getSystemProperties().get("user.name"));
+        }
+
+        long getRlimInfinity() {
+            return JNACLibrary.RLIM_INFINITY;
+        }
+
+        long getMaxFileSize() {
+            return JNANatives.MAX_FILE_SIZE;
+        }
+
+    }
+
     static class MaxMapCountCheck implements BootstrapCheck {
 
         private static final long LIMIT = 1 << 18;

core/src/main/java/org/elasticsearch/bootstrap/JNACLibrary.java

@@ -40,6 +40,7 @@ final class JNACLibrary {
     public static final int ENOMEM = 12;
     public static final int RLIMIT_MEMLOCK = Constants.MAC_OS_X ? 6 : 8;
     public static final int RLIMIT_AS = Constants.MAC_OS_X ? 5 : 9;
+    public static final int RLIMIT_FSIZE = Constants.MAC_OS_X ? 1 : 1;
     public static final long RLIM_INFINITY = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
 
     static {

core/src/main/java/org/elasticsearch/bootstrap/JNANatives.java

@@ -55,6 +55,8 @@ private JNANatives() {}
 
     static long MAX_SIZE_VIRTUAL_MEMORY = Long.MIN_VALUE;
 
+    static long MAX_FILE_SIZE = Long.MIN_VALUE;
+
     static void tryMlockall() {
         int errno = Integer.MIN_VALUE;
         String errMsg = null;
@@ -138,6 +140,17 @@ static void trySetMaxSizeVirtualMemory() {
         }
     }
 
+    static void trySetMaxFileSize() {
+        if (Constants.LINUX || Constants.MAC_OS_X) {
+            final JNACLibrary.Rlimit rlimit = new JNACLibrary.Rlimit();
+            if (JNACLibrary.getrlimit(JNACLibrary.RLIMIT_FSIZE, rlimit) == 0) {
+                MAX_FILE_SIZE = rlimit.rlim_cur.longValue();
+            } else {
+                logger.warn("unable to retrieve max file size [" + JNACLibrary.strerror(Native.getLastError()) + "]");
+            }
+        }
+    }
+
     static String rlimitToString(long value) {
         assert Constants.LINUX || Constants.MAC_OS_X;
         if (value == JNACLibrary.RLIM_INFINITY) {

core/src/main/java/org/elasticsearch/bootstrap/Natives.java

@@ -129,6 +129,14 @@ static void trySetMaxSizeVirtualMemory() {
         JNANatives.trySetMaxSizeVirtualMemory();
     }
 
+    static void trySetMaxFileSize() {
+        if (!JNA_AVAILABLE) {
+            logger.warn("cannot getrlimit RLIMIT_FSIZE because JNA is not available");
+            return;
+        }
+        JNANatives.trySetMaxFileSize();
+    }
+
     static boolean isSystemCallFilterInstalled() {
         if (!JNA_AVAILABLE) {
             return false;

core/src/test/java/org/elasticsearch/bootstrap/BootstrapChecksTests.java

@@ -330,7 +330,6 @@ long getRlimInfinity() {
             }
         };
 
-
         final NodeValidationException e = expectThrows(
                 NodeValidationException.class,
                 () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory"));
@@ -340,12 +339,40 @@ long getRlimInfinity() {
 
         BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
 
-        // nothing should happen if max size virtual memory is not
-        // available
+        // nothing should happen if max size virtual memory is not available
         maxSizeVirtualMemory.set(Long.MIN_VALUE);
         BootstrapChecks.check(true, Collections.singletonList(check), "testMaxSizeVirtualMemory");
     }
 
+    public void testMaxFileSizeCheck() throws NodeValidationException {
+        final long rlimInfinity = Constants.MAC_OS_X ? 9223372036854775807L : -1L;
+        final AtomicLong maxFileSize = new AtomicLong(randomIntBetween(0, Integer.MAX_VALUE));
+        final BootstrapChecks.MaxFileSizeCheck check = new BootstrapChecks.MaxFileSizeCheck() {
+            @Override
+            long getMaxFileSize() {
+                return maxFileSize.get();
+            }
+
+            @Override
+            long getRlimInfinity() {
+                return rlimInfinity;
+            }
+        };
+
+        final NodeValidationException e = expectThrows(
+                NodeValidationException.class,
+                () -> BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize"));
+        assertThat(e.getMessage(), containsString("max file size"));
+
+        maxFileSize.set(rlimInfinity);
+
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize");
+
+        // nothing should happen if max file size is not available
+        maxFileSize.set(Long.MIN_VALUE);
+        BootstrapChecks.check(true, Collections.singletonList(check), "testMaxFileSize");
+    }
+
     public void testMaxMapCountCheck() throws NodeValidationException {
         final int limit = 1 << 18;
         final AtomicLong maxMapCount = new AtomicLong(randomIntBetween(1, limit - 1));

distribution/src/main/packaging/systemd/elasticsearch.service

@@ -32,6 +32,9 @@ LimitNOFILE=65536
 # Specifies the maximum number of processes
 LimitNPROC=4096
 
+# Specifies the maximum file size
+LimitFSIZE=infinity
+
 # Specifies the maximum number of bytes of memory that may be locked into RAM
 # Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
 # in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in ${path.env}

docs/reference/setup/bootstrap-checks.asciidoc

@@ -130,6 +130,19 @@ address space. This can be done via `/etc/security/limits.conf` using
 the `as` setting to `unlimited` (note that you might have to increase
 the limits for the `root` user too).
 
+=== Max file size check
+
+The segment files that are the components of individual shards and the translog
+generations that are components of the translog can get large (exceeding
+multiple gigabytes). On systems where the max size of files that can be created
+by the Elasticsearch process is limited, this can lead to failed
+writes. Therefore, the safest option here is that the max file size is unlimited
+and that is what the max file size bootstrap check enforces. To pass the max
+file check, you must configure your system to allow the Elasticsearch process
+the ability to write files of unlimited size. This can be done via
+`/etc/security/limits.conf` using the `fsize` setting to `unlimited` (note that
+you might have to increase the limits for the `root` user too).
+
 === Maximum map count check
 
 Continuing from the previous <<max-size-virtual-memory-check,point>>, to

qa/evil-tests/src/test/java/org/elasticsearch/bootstrap/EvilJNANativesTests.java

@@ -80,4 +80,27 @@ public void testSetMaxSizeVirtualMemory() throws IOException {
         }
     }
 
+    public void testSetMaxFileSize() throws IOException {
+        if (Constants.LINUX) {
+            final List<String> lines = Files.readAllLines(PathUtils.get("/proc/self/limits"));
+            for (final String line : lines) {
+                if (line != null && line.startsWith("Max file size")) {
+                    final String[] fields = line.split("\\s+");
+                    final String limit = fields[3];
+                    assertThat(
+                            JNANatives.rlimitToString(JNANatives.MAX_FILE_SIZE),
+                            equalTo(limit));
+                    return;
+                }
+            }
+            fail("should have read max file size from /proc/self/limits");
+        } else if (Constants.MAC_OS_X) {
+            assertThat(
+                    JNANatives.MAX_FILE_SIZE,
+                    anyOf(equalTo(Long.MIN_VALUE), greaterThanOrEqualTo(0L)));
+        } else {
+            assertThat(JNANatives.MAX_FILE_SIZE, equalTo(Long.MIN_VALUE));
+        }
+    }
+
 }