Allow total memory to be overridden (#78750)

Since #65905 Elasticsearch has determined the Java heap settings
from node roles and total system memory.

This change allows the total system memory used in that calculation
to be overridden with a user-specified value. This is intended to
be used when Elasticsearch is running on a machine where some other
software that consumes a non-negligible amount of memory is running.
For example, a user could tell Elasticsearch to assume it was
running on a machine with 3GB of RAM when actually it was running
on a machine with 4GB of RAM.

The system property is `es.total_memory_bytes`, so, for example,
could be specified using `-Des.total_memory_bytes=3221225472`.
(It is specified in bytes rather than using a unit, because it
needs to be parsed by startup code that does not have access to
the utility classes that interpret byte size units.)

Author: David Roberts

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java

@@ -123,7 +123,6 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         throws InterruptedException, IOException, JvmOptionsFileParserException {
 
         final List<String> jvmOptions = readJvmOptionsFiles(config);
-        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(new DefaultSystemMemoryInfo());
 
         if (esJavaOpts != null) {
             jvmOptions.addAll(
@@ -132,6 +131,9 @@ private List<String> jvmOptions(final Path config, Path plugins, final String es
         }
 
         final List<String> substitutedJvmOptions = substitutePlaceholders(jvmOptions, Collections.unmodifiableMap(substitutions));
+        final MachineDependentHeap machineDependentHeap = new MachineDependentHeap(
+            new OverridableSystemMemoryInfo(substitutedJvmOptions, new DefaultSystemMemoryInfo())
+        );
         substitutedJvmOptions.addAll(machineDependentHeap.determineHeapSettings(config, substitutedJvmOptions));
         final List<String> ergonomicJvmOptions = JvmErgonomics.choose(substitutedJvmOptions);
         final List<String> systemJvmOptions = SystemJvmOptions.systemJvmOptions();

distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfo.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * A {@link SystemMemoryInfo} which returns a user-overridden memory size if one
+ * has been specified using the {@code es.total_memory_bytes} system property, or
+ * else returns the value provided by a fallback provider.
+ */
+public final class OverridableSystemMemoryInfo implements SystemMemoryInfo {
+
+    private final List<String> userDefinedJvmOptions;
+    private final SystemMemoryInfo fallbackSystemMemoryInfo;
+
+    public OverridableSystemMemoryInfo(final List<String> userDefinedJvmOptions, SystemMemoryInfo fallbackSystemMemoryInfo) {
+        this.userDefinedJvmOptions = Objects.requireNonNull(userDefinedJvmOptions);
+        this.fallbackSystemMemoryInfo = Objects.requireNonNull(fallbackSystemMemoryInfo);
+    }
+
+    @Override
+    public long availableSystemMemory() throws SystemMemoryInfoException {
+
+        return userDefinedJvmOptions.stream()
+            .filter(option -> option.startsWith("-Des.total_memory_bytes="))
+            .map(totalMemoryBytesOption -> {
+                try {
+                    long bytes = Long.parseLong(totalMemoryBytesOption.split("=", 2)[1]);
+                    if (bytes < 0) {
+                        throw new IllegalArgumentException("Negative memory size specified in [" + totalMemoryBytesOption + "]");
+                    }
+                    return bytes;
+                } catch (NumberFormatException e) {
+                    throw new IllegalArgumentException("Unable to parse number of bytes from [" + totalMemoryBytesOption + "]", e);
+                }
+            })
+            .reduce((previous, current) -> current) // this is effectively findLast(), so that ES_JAVA_OPTS overrides jvm.options
+            .orElse(fallbackSystemMemoryInfo.availableSystemMemory());
+    }
+}

distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/OverridableSystemMemoryInfoTests.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.tools.launchers;
+
+import org.elasticsearch.tools.launchers.SystemMemoryInfo.SystemMemoryInfoException;
+
+import java.util.List;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.fail;
+
+public class OverridableSystemMemoryInfoTests extends LaunchersTestCase {
+
+    private static final long FALLBACK = -1L;
+
+    public void testNoOptions() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of(), fallbackSystemMemoryInfo());
+        assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
+    }
+
+    public void testNoOverrides() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(List.of("-Da=b", "-Dx=y"), fallbackSystemMemoryInfo());
+        assertThat(memoryInfo.availableSystemMemory(), is(FALLBACK));
+    }
+
+    public void testValidSingleOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Des.total_memory_bytes=123456789"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(123456789L));
+    }
+
+    public void testValidOverrideInList() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
+    }
+
+    public void testMultipleValidOverridesInList() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Des.total_memory_bytes=123456789", "-Da=b", "-Des.total_memory_bytes=987654321", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        assertThat(memoryInfo.availableSystemMemory(), is(987654321L));
+    }
+
+    public void testNegativeOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=-123", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        try {
+            memoryInfo.availableSystemMemory();
+            fail("expected to fail");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Negative memory size specified in [-Des.total_memory_bytes=-123]"));
+        }
+    }
+
+    public void testUnparsableOverride() throws SystemMemoryInfoException {
+        final SystemMemoryInfo memoryInfo = new OverridableSystemMemoryInfo(
+            List.of("-Da=b", "-Des.total_memory_bytes=invalid", "-Dx=y"),
+            fallbackSystemMemoryInfo()
+        );
+        try {
+            memoryInfo.availableSystemMemory();
+            fail("expected to fail");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), is("Unable to parse number of bytes from [-Des.total_memory_bytes=invalid]"));
+        }
+    }
+
+    private static SystemMemoryInfo fallbackSystemMemoryInfo() {
+        return () -> FALLBACK;
+    }
+}

docs/changelog/78750.yaml

@@ -0,0 +1,6 @@
+pr: 78750
+summary: Allow total memory to be overridden
+area: Packaging
+type: enhancement
+issues:
+ - 65905

docs/reference/cluster/nodes-stats.asciidoc

@@ -1036,6 +1036,18 @@ Total amount of physical memory.
 (integer)
 Total amount of physical memory in bytes.
 
+`adjusted_total`::
+(<<byte-units,byte value>>)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property then this reports the overridden value. Otherwise it reports the same
+value as `total`.
+
+`adjusted_total_in_bytes`::
+(integer)
+If the amount of physical memory has been overridden using the `es.total_memory_bytes`
+system property then this reports the overridden value in bytes. Otherwise it reports
+the same value as `total_in_bytes`.
+
 `free`::
 (<<byte-units,byte value>>)
 Amount of free physical memory.

docs/reference/cluster/stats.asciidoc

@@ -916,6 +916,18 @@ Total amount of physical memory across all selected nodes.
 (integer)
 Total amount, in bytes, of physical memory across all selected nodes.
 
+`adjusted_total`::
+(<<byte-units,byte value>>)
+Total amount of memory across all selected nodes, but using the value specified
+using the `es.total_memory_bytes` system property instead of measured total
+memory for those nodes where that system property was set.
+
+`adjusted_total_in_bytes`::
+(integer)
+Total amount, in bytes, of memory across all selected nodes, but using the
+value specified using the `es.total_memory_bytes` system property instead
+of measured total memory for those nodes where that system property was set.
+
 `free`::
 (<<byte-units, byte units>>)
 Amount of free physical memory across all selected nodes.
@@ -1399,6 +1411,8 @@ The API returns the following response:
          "mem" : {
             "total" : "16gb",
             "total_in_bytes" : 17179869184,
+            "adjusted_total" : "16gb",
+            "adjusted_total_in_bytes" : 17179869184,
             "free" : "78.1mb",
             "free_in_bytes" : 81960960,
             "used" : "15.9gb",

qa/os/src/test/java/org/elasticsearch/packaging/test/ArchiveTests.java

@@ -511,6 +511,27 @@ public void test73CustomJvmOptionsDirectoryFilesWithoutOptionsExtensionIgnored()
         }
     }
 
+    public void test74CustomJvmOptionsTotalMemoryOverride() throws Exception {
+        final Path heapOptions = installation.config(Paths.get("jvm.options.d", "total_memory.options"));
+        try {
+            setHeap(null); // delete default options
+            // Work as though total system memory is 850MB
+            append(heapOptions, "-Des.total_memory_bytes=891289600\n");
+
+            startElasticsearch();
+
+            final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats");
+            assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600"));
+            final String nodesResponse = makeRequest("https://localhost:9200/_nodes");
+            // 40% of 850MB
+            assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":356515840"));
+
+            stopElasticsearch();
+        } finally {
+            rm(heapOptions);
+        }
+    }
+
     public void test80RelativePathConf() throws Exception {
         withCustomConfig(tempConf -> {
             append(tempConf.resolve("elasticsearch.yml"), "node.name: relative");

qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java

@@ -883,22 +883,45 @@ public void test140CgroupOsStatsAreAvailable() throws Exception {
      * logic sets the correct heap size, based on the container limits.
      */
     public void test150MachineDependentHeap() throws Exception {
+        final List<String> xArgs = machineDependentHeapTest("942m", List.of());
+
+        // This is roughly 0.4 * 942
+        assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
+    }
+
+    /**
+     * Check that when available system memory is constrained by a total memory override as well as Docker,
+     * the machine-dependant heap sizing logic sets the correct heap size, preferring the override to the
+     * container limits.
+     */
+    public void test151MachineDependentHeapWithSizeOverride() throws Exception {
+        final List<String> xArgs = machineDependentHeapTest(
+            "942m",
+            // 799014912 = 762m
+            List.of("-Des.total_memory_bytes=799014912")
+        );
+
+        // This is roughly 0.4 * 762, in particular it's NOT 0.4 * 942
+        assertThat(xArgs, hasItems("-Xms304m", "-Xmx304m"));
+    }
+
+    private List<String> machineDependentHeapTest(final String containerMemory, final List<String> extraJvmOptions) throws Exception {
         // Start by ensuring `jvm.options` doesn't define any heap options
         final Path jvmOptionsPath = tempDir.resolve("jvm.options");
         final Path containerJvmOptionsPath = installation.config("jvm.options");
         copyFromContainer(containerJvmOptionsPath, jvmOptionsPath);
 
-        final List<String> jvmOptions = Files.readAllLines(jvmOptionsPath)
-            .stream()
-            .filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false)
-            .collect(Collectors.toList());
+        final List<String> jvmOptions = Stream.concat(
+            Files.readAllLines(jvmOptionsPath).stream().filter(line -> (line.startsWith("-Xms") || line.startsWith("-Xmx")) == false),
+            extraJvmOptions.stream()
+        ).collect(Collectors.toList());
 
         Files.writeString(jvmOptionsPath, String.join("\n", jvmOptions));
 
         // Now run the container, being explicit about the available memory
         runContainer(
             distribution(),
-            builder().memory("942m").volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
+            builder().memory(containerMemory).volume(jvmOptionsPath, containerJvmOptionsPath).envVar("ELASTIC_PASSWORD", PASSWORD)
         );
 
         waitForElasticsearch(installation, "elastic", PASSWORD);
@@ -913,12 +936,9 @@ public void test150MachineDependentHeap() throws Exception {
         final JsonNode jsonNode = new ObjectMapper().readTree(jvmArgumentsLine.get());
 
         final String argsStr = jsonNode.get("message").textValue();
-        final List<String> xArgs = Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
+        return Arrays.stream(argsStr.substring(1, argsStr.length() - 1).split(",\\s*"))
             .filter(arg -> arg.startsWith("-X"))
             .collect(Collectors.toList());
-
-        // This is roughly 0.4 * 942
-        assertThat(xArgs, hasItems("-Xms376m", "-Xmx376m"));
     }
 
     /**

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -237,6 +237,34 @@ public void test70RestartServer() throws Exception {
         }
     }
 
+    public void test71JvmOptionsTotalMemoryOverride() throws Exception {
+        try {
+            install();
+            assertPathsExist(installation.envFile);
+            setHeap(null);
+
+            // Recreate file realm users that have been deleted in earlier tests
+            setFileSuperuser("test_superuser", "test_superuser_password");
+
+            withCustomConfig(tempConf -> {
+                // Work as though total system memory is 850MB
+                append(installation.envFile, "ES_JAVA_OPTS=\"-Des.total_memory_bytes=891289600\"");
+
+                startElasticsearch();
+
+                final String nodesStatsResponse = makeRequest("https://localhost:9200/_nodes/stats");
+                assertThat(nodesStatsResponse, containsString("\"adjusted_total_in_bytes\":891289600"));
+
+                // 40% of 850MB
+                assertThat(sh.run("ps auwwx").stdout, containsString("-Xms340m -Xmx340m"));
+
+                stopElasticsearch();
+            });
+        } finally {
+            cleanup();
+        }
+    }
+
     public void test72TestRuntimeDirectory() throws Exception {
         try {
             install();

server/src/main/java/org/elasticsearch/action/admin/cluster/stats/ClusterStatsNodes.java

@@ -266,20 +266,26 @@ private OsStats(List<NodeInfo> nodeInfos, List<NodeStats> nodeStatsList) {
             this.allocatedProcessors = allocatedProcessors;
 
             long totalMemory = 0;
+            long adjustedTotalMemory = 0;
             long freeMemory = 0;
             for (NodeStats nodeStats : nodeStatsList) {
                 if (nodeStats.getOs() != null) {
-                    long total = nodeStats.getOs().getMem().getTotal().getBytes();
+                    org.elasticsearch.monitor.os.OsStats.Mem mem = nodeStats.getOs().getMem();
+                    long total = mem.getTotal().getBytes();
                     if (total > 0) {
                         totalMemory += total;
                     }
-                    long free = nodeStats.getOs().getMem().getFree().getBytes();
+                    long adjustedTotal = mem.getAdjustedTotal().getBytes();
+                    if (adjustedTotal > 0) {
+                        adjustedTotalMemory += adjustedTotal;
+                    }
+                    long free = mem.getFree().getBytes();
                     if (free > 0) {
                         freeMemory += free;
                     }
                 }
             }
-            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, freeMemory);
+            this.mem = new org.elasticsearch.monitor.os.OsStats.Mem(totalMemory, adjustedTotalMemory, freeMemory);
         }
 
         public int getAvailableProcessors() {