From 44525807a42c37ba87c3ab1fd9dd950a20618180 Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Thu, 17 May 2018 15:38:36 +0200 Subject: [PATCH 1/2] Choose JVM options ergonomically With this commit we add the possibility to define further JVM options (and system properties) based on the current environment. As a proof of concept, it chooses Netty's allocator ergonomically based on the maximum defined heap size. --- .../tools/launchers/JvmErgonomics.java | 106 ++++++++++++++++++ .../tools/launchers/JvmOptionsParser.java | 2 + .../tools/launchers/JvmErgonomicsTests.java | 81 +++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java create mode 100644 distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java new file mode 100644 index 0000000000000..593676a09a77f --- /dev/null +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java @@ -0,0 +1,106 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Tunes Elasticsearch JVM settings based on inspection of provided JVM options. + */ +final class JvmErgonomics { + private static final long KB = 1024L; + + private static final long MB = 1024L * 1024L; + + private static final long GB = 1024L * 1024L * 1024L; + + + private JvmErgonomics() { + throw new AssertionError("No instances intended"); + } + + /** + * Chooses additional JVM options for Elasticsearch. + * + * @param userDefinedJvmOptions A list of JVM options that have been defined by the user. + * @return A list of additional JVM options to set. + */ + static List choose(List userDefinedJvmOptions) { + List ergonomicChoices = new ArrayList<>(); + Long heapSize = extractHeapSize(userDefinedJvmOptions); + Map systemProperties = extractSystemProperties(userDefinedJvmOptions); + if (heapSize != null) { + if (heapSize < 1 * GB) { + if (systemProperties.containsKey("io.netty.allocator.type") == false) { + ergonomicChoices.add("-Dio.netty.allocator.type=unpooled"); + } + } + } + return ergonomicChoices; + } + + private static final Pattern MAX_HEAP_SIZE = Pattern.compile("^(-Xmx|-XX:MaxHeapSize=)(?\\d+)(?\\w)?$"); + + // package private for testing + static Long extractHeapSize(List userDefinedJvmOptions) { + for (String jvmOption : userDefinedJvmOptions) { + final Matcher matcher = MAX_HEAP_SIZE.matcher(jvmOption); + if (matcher.matches()) { + final long size = Long.parseLong(matcher.group("size")); + final String unit = matcher.group("unit"); + if (unit == null) { + return size; + } else { + switch (unit.toLowerCase(Locale.ROOT)) { + case "k": + return size * KB; + case "m": + return size * MB; + case "g": + return size * GB; + default: + throw new IllegalArgumentException("Unknown unit [" + unit + "] for max heap size in [" + jvmOption + "]"); + } + } + } + } + return null; + } + + private static final Pattern SYSTEM_PROPERTY = Pattern.compile("^-D(?[\\w+].*?)=(?.*)$"); + + // package private for testing + static Map extractSystemProperties(List userDefinedJvmOptions) { + Map systemProperties = new HashMap<>(); + for (String jvmOption : userDefinedJvmOptions) { + final Matcher matcher = SYSTEM_PROPERTY.matcher(jvmOption); + if (matcher.matches()) { + systemProperties.put(matcher.group("key"), matcher.group("value")); + } + } + return systemProperties; + } +} diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java index 8cd401e53229d..03344017c8c31 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmOptionsParser.java @@ -76,6 +76,8 @@ public void accept(final int lineNumber, final String line) { } if (invalidLines.isEmpty()) { + List ergonomicJvmOptions = JvmErgonomics.choose(jvmOptions); + jvmOptions.addAll(ergonomicJvmOptions); final String spaceDelimitedJvmOptions = spaceDelimitJvmOptions(jvmOptions); Launchers.outPrintln(spaceDelimitedJvmOptions); Launchers.exit(0); diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java new file mode 100644 index 0000000000000..1815f0cb6b475 --- /dev/null +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java @@ -0,0 +1,81 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.tools.launchers; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +public class JvmErgonomicsTests extends LaunchersTestCase { + public void testExtractValidHeapSize() { + assertEquals(Long.valueOf(1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx1024"))); + assertEquals(Long.valueOf(2L * 1024 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx2g"))); + assertEquals(Long.valueOf(32 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx32M"))); + assertEquals(Long.valueOf(32 * 1024 * 1024), JvmErgonomics.extractHeapSize(Collections.singletonList("-XX:MaxHeapSize=32M"))); + } + + public void testExtractInvalidHeapSize() { + try { + JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx2T")); + fail("Expected IllegalArgumentException to be raised"); + } catch (IllegalArgumentException expected) { + assertEquals("Unknown unit [T] for max heap size in [-Xmx2T]", expected.getMessage()); + } + } + + public void testExtractNoHeapSize() { + assertNull("No spaces allowed", JvmErgonomics.extractHeapSize(Collections.singletonList("-Xmx 1024"))); + assertNull("JVM option is not present", JvmErgonomics.extractHeapSize(Collections.singletonList(""))); + assertNull("Multiple JVM options per line", JvmErgonomics.extractHeapSize(Collections.singletonList("-Xms2g -Xmx2g"))); + } + + public void testExtractSystemProperties() { + Map expectedSystemProperties = new HashMap<>(); + expectedSystemProperties.put("file.encoding", "UTF-8"); + expectedSystemProperties.put("kv.setting", "ABC=DEF"); + + Map parsedSystemProperties = JvmErgonomics.extractSystemProperties( + Arrays.asList("-Dfile.encoding=UTF-8", "-Dkv.setting=ABC=DEF")); + + assertEquals(expectedSystemProperties, parsedSystemProperties); + } + + public void testExtractNoSystemProperties() { + Map parsedSystemProperties = JvmErgonomics.extractSystemProperties(Arrays.asList("-Xms1024M", "-Xmx1024M")); + assertTrue(parsedSystemProperties.isEmpty()); + } + + public void testLittleMemoryErgonomicChoices() { + List expectedChoices = Collections.singletonList("-Dio.netty.allocator.type=unpooled"); + assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms512M", "-Xmx512M"))); + } + + public void testPlentyMemoryErgonomicChoices() { + List expectedChoices = Collections.emptyList(); + assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms8G", "-Xmx8G"))); + } +} From 8f162f2a80118a01cf15075333d48c4f9107eeac Mon Sep 17 00:00:00 2001 From: Daniel Mitterdorfer Date: Mon, 18 Jun 2018 13:10:04 +0200 Subject: [PATCH 2/2] Switch allocator at 1GB heap and be explicit With this commit we switch to the unpooled allocator at 1GB heap size (value determined experimentally determined, see PR for more details). We are also explicit about the choice of the allocator in either case. --- .../org/elasticsearch/tools/launchers/JvmErgonomics.java | 6 ++++-- .../elasticsearch/tools/launchers/JvmErgonomicsTests.java | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java index 593676a09a77f..761cd9e1be5db 100644 --- a/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java +++ b/distribution/tools/launchers/src/main/java/org/elasticsearch/tools/launchers/JvmErgonomics.java @@ -53,9 +53,11 @@ static List choose(List userDefinedJvmOptions) { Long heapSize = extractHeapSize(userDefinedJvmOptions); Map systemProperties = extractSystemProperties(userDefinedJvmOptions); if (heapSize != null) { - if (heapSize < 1 * GB) { - if (systemProperties.containsKey("io.netty.allocator.type") == false) { + if (systemProperties.containsKey("io.netty.allocator.type") == false) { + if (heapSize <= 1 * GB) { ergonomicChoices.add("-Dio.netty.allocator.type=unpooled"); + } else { + ergonomicChoices.add("-Dio.netty.allocator.type=pooled"); } } } diff --git a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java index 1815f0cb6b475..4b075d78b70a8 100644 --- a/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java +++ b/distribution/tools/launchers/src/test/java/org/elasticsearch/tools/launchers/JvmErgonomicsTests.java @@ -70,12 +70,14 @@ public void testExtractNoSystemProperties() { } public void testLittleMemoryErgonomicChoices() { + String smallHeap = randomFrom(Arrays.asList("64M", "512M", "1024M", "1G")); List expectedChoices = Collections.singletonList("-Dio.netty.allocator.type=unpooled"); - assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms512M", "-Xmx512M"))); + assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms" + smallHeap, "-Xmx" + smallHeap))); } public void testPlentyMemoryErgonomicChoices() { - List expectedChoices = Collections.emptyList(); - assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms8G", "-Xmx8G"))); + String largeHeap = randomFrom(Arrays.asList("1025M", "2048M", "2G", "8G")); + List expectedChoices = Collections.singletonList("-Dio.netty.allocator.type=pooled"); + assertEquals(expectedChoices, JvmErgonomics.choose(Arrays.asList("-Xms" + largeHeap, "-Xmx" + largeHeap))); } }