[7.x] Workaround for JDK bug with total mem on Debian8 (#68542) (#68698)

Author: danhermann

server/src/main/java/org/elasticsearch/monitor/os/OsProbe.java

@@ -100,11 +100,16 @@ public long getTotalPhysicalMemorySize() {
             return 0;
         }
         try {
-            final long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
+            long totalMem = (long) getTotalPhysicalMemorySize.invoke(osMxBean);
             if (totalMem < 0) {
                 logger.debug("OS reported a negative total memory value [{}]", totalMem);
                 return 0;
             }
+            if (totalMem == 0 && isDebian8()) {
+                // workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
+                totalMem = getTotalMemFromProcMeminfo();
+            }
+
             return totalMem;
         } catch (Exception e) {
             logger.warn("exception retrieving total physical memory", e);
@@ -641,6 +646,55 @@ List<String> readOsRelease() throws IOException {
         }
     }
 
+    /**
+     * Returns the lines from /proc/meminfo as a workaround for JDK bugs that prevent retrieval of total system memory
+     * on some Linux variants such as Debian8.
+     */
+    @SuppressForbidden(reason = "access /proc/meminfo")
+    List<String> readProcMeminfo() throws IOException {
+        final List<String> lines;
+        if (Files.exists(PathUtils.get("/proc/meminfo"))) {
+            lines = Files.readAllLines(PathUtils.get("/proc/meminfo"));
+            assert lines != null && lines.isEmpty() == false;
+            return lines;
+        } else {
+            return Collections.emptyList();
+        }
+    }
+
+    /**
+     * Retrieves system total memory in bytes from /proc/meminfo
+     */
+    long getTotalMemFromProcMeminfo() throws IOException {
+        List<String> meminfoLines = readProcMeminfo();
+        final List<String> memTotalLines = meminfoLines.stream().filter(line -> line.startsWith("MemTotal")).collect(Collectors.toList());
+        assert memTotalLines.size() <= 1 : memTotalLines;
+        if (memTotalLines.size() == 1) {
+            final String memTotalLine = memTotalLines.get(0);
+            int beginIdx = memTotalLine.indexOf("MemTotal:");
+            int endIdx = memTotalLine.lastIndexOf(" kB");
+            if (beginIdx + 9 < endIdx) {
+                final String memTotalString = memTotalLine.substring(beginIdx + 9, endIdx).trim();
+                try {
+                    long memTotalInKb = Long.parseLong(memTotalString);
+                    return memTotalInKb * 1024;
+                } catch (NumberFormatException e) {
+                    logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
+                    return 0;
+                }
+            } else {
+                logger.warn("Unable to retrieve total memory from meminfo line [" + memTotalLine + "]");
+                return 0;
+            }
+        } else {
+            return 0;
+        }
+    }
+
+    boolean isDebian8() throws IOException {
+        return Constants.LINUX && getPrettyName().equals("Debian GNU/Linux 8 (jessie)");
+    }
+
     public OsStats osStats() {
         final OsStats.Cpu cpu = new OsStats.Cpu(getSystemCpuPercent(), getSystemLoadAverage());
         final OsStats.Mem mem = new OsStats.Mem(getTotalPhysicalMemorySize(), getFreePhysicalMemorySize());

server/src/test/java/org/elasticsearch/monitor/os/OsProbeTests.java

@@ -255,6 +255,71 @@ public void testCgroupProbeWithMissingMemory() {
         assertNull(cgroup);
     }
 
+    public void testGetTotalMemFromProcMeminfo() throws Exception {
+        // missing MemTotal line
+        List<String> meminfoLines = Arrays.asList(
+            "MemFree:         8467692 kB",
+            "MemAvailable:   39646240 kB",
+            "Buffers:         4699504 kB",
+            "Cached:         23290380 kB",
+            "SwapCached:            0 kB",
+            "Active:         43637908 kB",
+            "Inactive:        8130280 kB"
+        );
+        OsProbe probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
+        assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
+
+        // MemTotal line with invalid value
+        meminfoLines = Arrays.asList(
+            "MemTotal:        invalid kB",
+            "MemFree:         8467692 kB",
+            "MemAvailable:   39646240 kB",
+            "Buffers:         4699504 kB",
+            "Cached:         23290380 kB",
+            "SwapCached:            0 kB",
+            "Active:         43637908 kB",
+            "Inactive:        8130280 kB"
+        );
+        probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
+        assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
+
+        // MemTotal line with invalid unit
+        meminfoLines = Arrays.asList(
+            "MemTotal:       39646240 MB",
+            "MemFree:         8467692 kB",
+            "MemAvailable:   39646240 kB",
+            "Buffers:         4699504 kB",
+            "Cached:         23290380 kB",
+            "SwapCached:            0 kB",
+            "Active:         43637908 kB",
+            "Inactive:        8130280 kB"
+        );
+        probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
+        assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(0L));
+
+        // MemTotal line with random valid value
+        long memTotalInKb = randomLongBetween(1, Long.MAX_VALUE / 1024L);
+        meminfoLines = Arrays.asList(
+            "MemTotal:        " + memTotalInKb + " kB",
+            "MemFree:         8467692 kB",
+            "MemAvailable:   39646240 kB",
+            "Buffers:         4699504 kB",
+            "Cached:         23290380 kB",
+            "SwapCached:            0 kB",
+            "Active:         43637908 kB",
+            "Inactive:        8130280 kB"
+        );
+        probe = buildStubOsProbe(true, "", org.elasticsearch.common.collect.List.of(), meminfoLines);
+        assertThat(probe.getTotalMemFromProcMeminfo(), equalTo(memTotalInKb * 1024L));
+    }
+
+    public void testGetTotalMemoryOnDebian8() throws Exception {
+        // tests the workaround for JDK bug on debian8: https://github.com/elastic/elasticsearch/issues/67089#issuecomment-756114654
+        final OsProbe osProbe = new OsProbe();
+        assumeTrue("runs only on Debian 8", osProbe.isDebian8());
+        assertThat(osProbe.getTotalPhysicalMemorySize(), greaterThan(0L));
+    }
+
     private static List<String> getProcSelfGroupLines(String hierarchy) {
         return Arrays.asList(
             "10:freezer:/",
@@ -283,12 +348,14 @@ private static OsProbe buildStubOsProbe(final boolean areCgroupStatsAvailable, f
      * @param areCgroupStatsAvailable whether or not cgroup data is available. Normally OsProbe establishes this for itself.
      * @param hierarchy a mock value used to generate a cgroup hierarchy.
      * @param procSelfCgroupLines the lines that will be used as the content of <code>/proc/self/cgroup</code>
+     * @param procMeminfoLines lines that will be used as the content of <code>/proc/meminfo</code>
      * @return a test instance
      */
     private static OsProbe buildStubOsProbe(
         final boolean areCgroupStatsAvailable,
         final String hierarchy,
-        List<String> procSelfCgroupLines
+        List<String> procSelfCgroupLines,
+        List<String> procMeminfoLines
     ) {
         return new OsProbe() {
             @Override
@@ -339,6 +406,20 @@ String readSysFsCgroupMemoryUsageInBytes(String controlGroup) {
             boolean areCgroupStatsAvailable() {
                 return areCgroupStatsAvailable;
             }
+
+            @Override
+            List<String> readProcMeminfo() throws IOException {
+                return procMeminfoLines;
+            }
         };
     }
+
+    private static OsProbe buildStubOsProbe(
+        final boolean areCgroupStatsAvailable,
+        final String hierarchy,
+        List<String> procSelfCgroupLines
+    ) {
+        return buildStubOsProbe(areCgroupStatsAvailable, hierarchy, procSelfCgroupLines, org.elasticsearch.common.collect.List.of());
+    }
+
 }