From d057d3c933c5f97c07c85c79efc46c81b04db3ed Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 12 Dec 2018 14:54:16 -0500 Subject: [PATCH 1/7] Override the JVM DNS cache policy When a security manager is present, the JVM will cache positive hostname lookups indefinitely. This can be problematic, especially in the modern world with cloud services where DNS addresses can change, or environments using Docker containers where IP addresses could be considered ephemeral. This behavior impacts cluster discovery, cross-cluster replication and cross-cluster search, reindex from remote, snapshot repositories, webhooks in Watcher, external authentication mechanisms, and the Elastic Stack Monitoring Service. The experience of watching a DNS lookup change yet not be reflected within Elasticsearch is a poor experience for users. The reason the JVM has this is guard against DNS cache posioning attacks. Yet, there is already a defense in the modern world against such attacks: TLS. With proper certificate validation, even if a resolver falls prey to a DNS cache poisoning attack, using TLS would neuter the attack. Therefore we have a policy with dubious security value that significantly impacts usability. As such we make the usability/security tradeoff towards usability, since the security risks are very low. This commit introduces new system properties that Elasticsearch observes to override the JVM DNS cache policy. --- distribution/src/config/jvm.options | 8 +++++++ .../setup/sysconfig/dns-cache.asciidoc | 23 ++++++++++--------- .../bootstrap/Elasticsearch.java | 18 +++++++++++++-- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index d8b651231cb46..467305c7f2974 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -45,6 +45,14 @@ # 10-:-XX:+UseG1GC # 10-:-XX:InitiatingHeapOccupancyPercent=75 +## DNS cache policy +# cache ttl for positive DNS lookups noting that this overrides the JDK security +# property networkaddress.cache.ttl; set to -1 to cache forever +-Des.networkaddress.cache.ttl=60 +# cache ttl for negative DNS lookups noting that this overrides the JDK security +# property networkaddress.cache.negative ttl; set to -1 to cache forever +-Des.networkaddress.cache.negative.ttl=10 + ## optimizations # pre-touch memory pages used by the JVM during initialization diff --git a/docs/reference/setup/sysconfig/dns-cache.asciidoc b/docs/reference/setup/sysconfig/dns-cache.asciidoc index 76ef4f5204210..db2945b5258dd 100644 --- a/docs/reference/setup/sysconfig/dns-cache.asciidoc +++ b/docs/reference/setup/sysconfig/dns-cache.asciidoc @@ -2,17 +2,18 @@ === DNS cache settings Elasticsearch runs with a security manager in place. With a security manager in -place, the JVM defaults to caching positive hostname resolutions -indefinitely. If your Elasticsearch nodes rely on DNS in an environment where -DNS resolutions vary with time (e.g., for node-to-node discovery) then you might -want to modify the default JVM behavior. This can be modified by adding +place, the JVM defaults to caching positive hostname resolutions indefinitely +and defaults to caching negative hostname resolutions for ten +seconds. Elasticsearch overrides this behavior with default values to cache +positive lookups for sixty seconds, and to cache negative lookups for ten +seconds. These values should be suitable for most environments, including +environments where DNS resolutions vary with time. If not, you can edit the +values `es.networkaddress.cache.ttl` and `es.networkaddress.cache.negative.ttl` +in the <>. Note that the values http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.ttl=`] -to your -http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java -security policy]. Any hosts that fail to resolve will be logged. Note also that -with the Java security manager in place, the JVM defaults to caching negative -hostname resolutions for ten seconds. This can be modified by adding +and http://docs.oracle.com/javase/8/docs/technotes/guides/net/properties.html[`networkaddress.cache.negative.ttl=`] -to your +in the http://docs.oracle.com/javase/8/docs/technotes/guides/security/PolicyFiles.html[Java -security policy]. \ No newline at end of file +security policy] are ignored by Elasticsearch unless you remove the settings for +`es.networkaddress.cache.ttl` and `es.networkaddress.cache.negative.ttl`. diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 1cf88c6245fa4..80c3fb38d64f1 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -36,6 +36,7 @@ import java.io.IOException; import java.nio.file.Path; import java.security.Permission; +import java.security.Security; import java.util.Arrays; import java.util.Locale; @@ -72,8 +73,21 @@ class Elasticsearch extends EnvironmentAwareCommand { * Main entry point for starting elasticsearch */ public static void main(final String[] args) throws Exception { - // we want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the - // presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy) + final String esNetworkAddressCacheTtl = System.getProperty("es.networkaddress.cache.ttl"); + if (esNetworkAddressCacheTtl != null) { + // round-trip the property to an integer and back to a string to ensure that it parses properly + Security.setProperty("networkaddress.cache.ttl", Integer.toString(Integer.valueOf(esNetworkAddressCacheTtl))); + } + final String esNetworkAddressCacheNegativeTtl = System.getProperty("es.networkaddress.cache.negative.ttl"); + if (esNetworkAddressCacheNegativeTtl != null) { + // round-trip the property to an integer and back to a string to ensure that it parses properly + Security.setProperty("networkaddress.cache.negative.ttl", Integer.toString(Integer.valueOf(esNetworkAddressCacheNegativeTtl))); + } + /* + * We want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the + * presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). This + * forces such policies to take effect immediately. + */ System.setSecurityManager(new SecurityManager() { @Override public void checkPermission(Permission perm) { From 9147bcce2a2e7b77009c05847003c9a26205ca63 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 12 Dec 2018 15:32:02 -0500 Subject: [PATCH 2/7] Remove repetition, add better error message --- .../bootstrap/Elasticsearch.java | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index 80c3fb38d64f1..cb2df46ac632c 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -73,16 +73,8 @@ class Elasticsearch extends EnvironmentAwareCommand { * Main entry point for starting elasticsearch */ public static void main(final String[] args) throws Exception { - final String esNetworkAddressCacheTtl = System.getProperty("es.networkaddress.cache.ttl"); - if (esNetworkAddressCacheTtl != null) { - // round-trip the property to an integer and back to a string to ensure that it parses properly - Security.setProperty("networkaddress.cache.ttl", Integer.toString(Integer.valueOf(esNetworkAddressCacheTtl))); - } - final String esNetworkAddressCacheNegativeTtl = System.getProperty("es.networkaddress.cache.negative.ttl"); - if (esNetworkAddressCacheNegativeTtl != null) { - // round-trip the property to an integer and back to a string to ensure that it parses properly - Security.setProperty("networkaddress.cache.negative.ttl", Integer.toString(Integer.valueOf(esNetworkAddressCacheNegativeTtl))); - } + overrideDnsCachePolicyProperty("networkaddress.cache.ttl"); + overrideDnsCachePolicyProperty("networkaddress.cache.negative.ttl"); /* * We want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the * presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). This @@ -102,6 +94,18 @@ public void checkPermission(Permission perm) { } } + private static void overrideDnsCachePolicyProperty(final String property) { + final String override = System.getProperty("es." + property); + if (override != null) { + // round-trip the property to an integer and back to a string to ensure that it parses properly + try { + Security.setProperty(property, Integer.toString(Integer.valueOf(override))); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException("failed to parse [es." + property + "] with value [" + override + "]", e); + } + } + } + static int main(final String[] args, final Elasticsearch elasticsearch, final Terminal terminal) throws Exception { return elasticsearch.main(args, terminal); } From 9c38f870310590990b84ea167b7dbf32304dbb30 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 12 Dec 2018 15:40:49 -0500 Subject: [PATCH 3/7] More simplification --- .../bootstrap/Elasticsearch.java | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index cb2df46ac632c..aec4fca3f62ae 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -73,8 +73,7 @@ class Elasticsearch extends EnvironmentAwareCommand { * Main entry point for starting elasticsearch */ public static void main(final String[] args) throws Exception { - overrideDnsCachePolicyProperty("networkaddress.cache.ttl"); - overrideDnsCachePolicyProperty("networkaddress.cache.negative.ttl"); + overrideDnsCachePolicyProperties(); /* * We want the JVM to think there is a security manager installed so that if internal policy decisions that would be based on the * presence of a security manager or lack thereof act as if there is a security manager present (e.g., DNS cache policy). This @@ -94,14 +93,16 @@ public void checkPermission(Permission perm) { } } - private static void overrideDnsCachePolicyProperty(final String property) { - final String override = System.getProperty("es." + property); - if (override != null) { - // round-trip the property to an integer and back to a string to ensure that it parses properly - try { - Security.setProperty(property, Integer.toString(Integer.valueOf(override))); - } catch (final NumberFormatException e) { - throw new IllegalArgumentException("failed to parse [es." + property + "] with value [" + override + "]", e); + private static void overrideDnsCachePolicyProperties() { + for (final String property : new String[] {"networkaddress.cache.ttl", "networkaddress.cache.negative.ttl" }) { + final String override = System.getProperty("es." + property); + if (override != null) { + // round-trip the property to an integer and back to a string to ensure that it parses properly + try { + Security.setProperty(property, Integer.toString(Integer.valueOf(override))); + } catch (final NumberFormatException e) { + throw new IllegalArgumentException("failed to parse [es." + property + "] with value [" + override + "]", e); + } } } } From ed22e7f7b8362e6712340593c45a993f1c530a9d Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Wed, 12 Dec 2018 15:42:08 -0500 Subject: [PATCH 4/7] Add detail to comment in jvm.options --- distribution/src/config/jvm.options | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/distribution/src/config/jvm.options b/distribution/src/config/jvm.options index 467305c7f2974..2b30d6a87b4a1 100644 --- a/distribution/src/config/jvm.options +++ b/distribution/src/config/jvm.options @@ -46,11 +46,12 @@ # 10-:-XX:InitiatingHeapOccupancyPercent=75 ## DNS cache policy -# cache ttl for positive DNS lookups noting that this overrides the JDK security -# property networkaddress.cache.ttl; set to -1 to cache forever +# cache ttl in seconds for positive DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.ttl; set to -1 to cache forever -Des.networkaddress.cache.ttl=60 -# cache ttl for negative DNS lookups noting that this overrides the JDK security -# property networkaddress.cache.negative ttl; set to -1 to cache forever +# cache ttl in seconds for negative DNS lookups noting that this overrides the +# JDK security property networkaddress.cache.negative ttl; set to -1 to cache +# forever -Des.networkaddress.cache.negative.ttl=10 ## optimizations From 7719d3870e7ea786c49094af5f41d956f41cc6c5 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Dec 2018 06:50:22 -0500 Subject: [PATCH 5/7] Fix doc typo --- docs/reference/setup/sysconfig/dns-cache.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/setup/sysconfig/dns-cache.asciidoc b/docs/reference/setup/sysconfig/dns-cache.asciidoc index db2945b5258dd..54a1e20a15aea 100644 --- a/docs/reference/setup/sysconfig/dns-cache.asciidoc +++ b/docs/reference/setup/sysconfig/dns-cache.asciidoc @@ -4,7 +4,7 @@ Elasticsearch runs with a security manager in place. With a security manager in place, the JVM defaults to caching positive hostname resolutions indefinitely and defaults to caching negative hostname resolutions for ten -seconds. Elasticsearch overrides this behavior with default values to cache +seconds. Elasticsearch overrides this behavior with default values to cache positive lookups for sixty seconds, and to cache negative lookups for ten seconds. These values should be suitable for most environments, including environments where DNS resolutions vary with time. If not, you can edit the From bbef1b2a4df94a0966eae69edb62d32b83aea900 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Dec 2018 06:52:10 -0500 Subject: [PATCH 6/7] Tweaks --- .../org/elasticsearch/bootstrap/Elasticsearch.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index aec4fca3f62ae..db2c271e9f0bb 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -95,13 +95,15 @@ public void checkPermission(Permission perm) { private static void overrideDnsCachePolicyProperties() { for (final String property : new String[] {"networkaddress.cache.ttl", "networkaddress.cache.negative.ttl" }) { - final String override = System.getProperty("es." + property); - if (override != null) { - // round-trip the property to an integer and back to a string to ensure that it parses properly + final String overrideProperty = "es." + property; + final String overrideValue = System.getProperty(overrideProperty); + if (overrideValue != null) { try { - Security.setProperty(property, Integer.toString(Integer.valueOf(override))); + // round-trip the property to an integer and back to a string to ensure that it parses properly + Security.setProperty(property, Integer.toString(Integer.valueOf(overrideValue))); } catch (final NumberFormatException e) { - throw new IllegalArgumentException("failed to parse [es." + property + "] with value [" + override + "]", e); + throw new IllegalArgumentException( + "failed to parse [" + overrideProperty + "] with value [" + overrideValue + "]", e); } } } From 916b22f0c1336866aabb4d02848ead61ed2da1b1 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Thu, 13 Dec 2018 06:52:38 -0500 Subject: [PATCH 7/7] Another tweak --- .../main/java/org/elasticsearch/bootstrap/Elasticsearch.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java index db2c271e9f0bb..48cf00164558f 100644 --- a/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java +++ b/server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java @@ -80,10 +80,12 @@ public static void main(final String[] args) throws Exception { * forces such policies to take effect immediately. */ System.setSecurityManager(new SecurityManager() { + @Override public void checkPermission(Permission perm) { // grant all permissions so that we can later set the security manager to the one that we want } + }); LogConfigurator.registerErrorListener(); final Elasticsearch elasticsearch = new Elasticsearch();